From 8a6fd8df16b8f9dcf18210f6057b79ee4815c868 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Wed, 28 Apr 2021 02:42:27 +0000 Subject: [PATCH 001/157] Refactoring service_manager in crash_gen to use stronger types --- tests/pytest/crash_gen/__init__.py | 5 + tests/pytest/crash_gen/crash_gen_main.py | 33 ++-- tests/pytest/crash_gen/service_manager.py | 175 ++++++++++++---------- tests/pytest/crash_gen/settings.py | 13 +- 4 files changed, 128 insertions(+), 98 deletions(-) create mode 100644 tests/pytest/crash_gen/__init__.py diff --git a/tests/pytest/crash_gen/__init__.py b/tests/pytest/crash_gen/__init__.py new file mode 100644 index 0000000000..71855bc5a9 --- /dev/null +++ b/tests/pytest/crash_gen/__init__.py @@ -0,0 +1,5 @@ +# Helpful Ref: https://stackoverflow.com/questions/24100558/how-can-i-split-a-module-into-multiple-files-without-breaking-a-backwards-compa/24100645 +from crash_gen.service_manager import ServiceManager, TdeInstance, TdeSubProcess +from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress +from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager +from crash_gen.settings import Settings diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 44295e8bee..39475a32d0 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -32,21 +32,22 @@ import getopt import sys import os +import io import signal import traceback import resource # from guppy import hpy import gc -from crash_gen.service_manager import ServiceManager, TdeInstance -from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress -from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager -import crash_gen.settings +# from crash_gen import ServiceManager, TdeInstance, TdeSubProcess +from crash_gen import ServiceManager, Settings, DbConn, DbConnNative, Dice, DbManager, Status, Logging, Helper, \ + CrashGenError, Progress, MyTDSql, \ + TdeInstance import taos import requests -crash_gen.settings.init() +Settings.init() # Require Python 3 if sys.version_info[0] < 3: @@ -89,9 +90,9 @@ class WorkerThread: self._dbConn = DbConn.createRest(tInst.getDbTarget()) elif gConfig.connector_type == 'mixed': if Dice.throw(2) == 0: # 1/2 chance - self._dbConn = DbConn.createNative() + self._dbConn = DbConn.createNative(tInst.getDbTarget()) else: - self._dbConn = DbConn.createRest() + self._dbConn = DbConn.createRest(tInst.getDbTarget()) else: raise RuntimeError("Unexpected connector type: {}".format(gConfig.connector_type)) @@ -1370,13 +1371,13 @@ class Task(): self._err = e self._aborted = True traceback.print_exc() - except BaseException as e: + except BaseException as e2: self.logInfo("Python base exception encountered") - self._err = e + # self._err = e2 # Exception/BaseException incompatible! self._aborted = True traceback.print_exc() - except BaseException: # TODO: what is this again??!! - raise RuntimeError("Punt") + # except BaseException: # TODO: what is this again??!! + # raise RuntimeError("Punt") # self.logDebug( # "[=] Unexpected exception, SQL: {}".format( # wt.getDbConn().getLastSql())) @@ -1980,8 +1981,8 @@ class TaskAddData(StateTransitionTask): activeTable: Set[int] = set() # We use these two files to record operations to DB, useful for power-off tests - fAddLogReady = None # type: TextIOWrapper - fAddLogDone = None # type: TextIOWrapper + fAddLogReady = None # type: io.TextIOWrapper + fAddLogDone = None # type: io.TextIOWrapper @classmethod def prepToRecordOps(cls): @@ -2025,7 +2026,7 @@ class TaskAddData(StateTransitionTask): self.prepToRecordOps() self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) self.fAddLogReady.flush() - os.fsync(self.fAddLogReady) + os.fsync(self.fAddLogReady.fileno()) # TODO: too ugly trying to lock the table reliably, refactor... fullTableName = db.getName() + '.' + regTableName @@ -2088,7 +2089,7 @@ class TaskAddData(StateTransitionTask): if gConfig.record_ops: self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName)) self.fAddLogDone.flush() - os.fsync(self.fAddLogDone) + os.fsync(self.fAddLogDone.fileno()) def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): # ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access @@ -2468,7 +2469,7 @@ class MainExec: global gConfig gConfig = parser.parse_args() - crash_gen.settings.gConfig = gConfig # TODO: fix this hack, consolidate this global var + Settings.setConfig(gConfig) # TODO: fix this hack, consolidate this global var # Sanity check for arguments if gConfig.use_shadow_db and gConfig.max_dbs>1 : diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py index cdbf2db4da..146215f2bc 100644 --- a/tests/pytest/crash_gen/service_manager.py +++ b/tests/pytest/crash_gen/service_manager.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import io import sys @@ -5,9 +7,9 @@ import threading import signal import logging import time -import subprocess +from subprocess import PIPE, Popen, TimeoutExpired -from typing import IO, List +from typing import IO, List, NewType, Optional try: import psutil @@ -170,6 +172,7 @@ quorum 2 if crash_gen.settings.gConfig.track_memory_leaks: Logging.info("Invoking VALGRIND on service...") cmdLine = ['valgrind', '--leak-check=yes'] + # TODO: move "exec -c" into Popen(), we can both "use shell" and NOT fork so ask to lose kill control cmdLine += ["exec " + self.getExecFile(), '-c', self.getCfgDir()] # used in subproce.Popen() return cmdLine @@ -225,41 +228,46 @@ class TdeSubProcess: It takes a TdeInstance object as its parameter, with the rationale being "a sub process runs an instance". + + We aim to ensure that this object has exactly the same life-cycle as the + underlying sub process. """ # RET_ALREADY_STOPPED = -1 # RET_TIME_OUT = -3 # RET_SUCCESS = -4 - def __init__(self): - self.subProcess = None # type: subprocess.Popen + def __init__(self, po: Popen): + self._popen = po # type: Popen # if tInst is None: # raise CrashGenError("Empty instance not allowed in TdeSubProcess") # self._tInst = tInst # Default create at ServiceManagerThread def __repr__(self): - if self.subProcess is None: - return '[TdeSubProc: Empty]' + # if self.subProcess is None: + # return '[TdeSubProc: Empty]' return '[TdeSubProc: pid = {}]'.format(self.getPid()) def getStdOut(self): - return self.subProcess.stdout + return self._popen.stdout def getStdErr(self): - return self.subProcess.stderr + return self._popen.stderr - def isRunning(self): - return self.subProcess is not None + # Now it's always running, since we matched the life cycle + # def isRunning(self): + # return self.subProcess is not None def getPid(self): - return self.subProcess.pid + return self._popen.pid - def start(self, cmdLine): + @classmethod + def start(cls, cmdLine): ON_POSIX = 'posix' in sys.builtin_module_names # Sanity check - if self.subProcess: # already there - raise RuntimeError("Corrupt process state") + # if self.subProcess: # already there + # raise RuntimeError("Corrupt process state") # Prepare environment variables for coverage information # Ref: https://stackoverflow.com/questions/2231227/python-subprocess-popen-with-a-modified-environment @@ -270,23 +278,22 @@ class TdeSubProcess: # print("Starting TDengine with env: ", myEnv.items()) # print("Starting TDengine via Shell: {}".format(cmdLineStr)) - useShell = True # Needed to pass environments into it - self.subProcess = subprocess.Popen( - # ' '.join(cmdLine) if useShell else cmdLine, - # shell=useShell, - ' '.join(cmdLine), - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - # bufsize=1, # not supported in binary mode + # useShell = True # Needed to pass environments into it + popen = Popen( + ' '.join(cmdLine), # ' '.join(cmdLine) if useShell else cmdLine, + shell=True, # Always use shell, since we need to pass ENV vars + stdout=PIPE, + stderr=PIPE, close_fds=ON_POSIX, env=myEnv ) # had text=True, which interferred with reading EOF + return cls(popen) STOP_SIGNAL = signal.SIGINT # signal.SIGKILL/SIGINT # What signal to use (in kill) to stop a taosd process? SIG_KILL_RETCODE = 137 # ref: https://stackoverflow.com/questions/43268156/process-finished-with-exit-code-137-in-pycharm - def stop(self): + @classmethod + def stop(cls, tsp: TdeSubProcess): """ Stop a sub process, DO NOT return anything, process all conditions INSIDE @@ -306,29 +313,30 @@ class TdeSubProcess: SIGSEGV 11 SIGUSR2 12 """ - if not self.subProcess: - Logging.error("Sub process already stopped") - return + # self._popen should always be valid. - retCode = self.subProcess.poll() # ret -N means killed with signal N, otherwise it's from exit(N) + # if not self.subProcess: + # Logging.error("Sub process already stopped") + # return + + retCode = tsp._popen.poll() # ret -N means killed with signal N, otherwise it's from exit(N) if retCode: # valid return code, process ended # retCode = -retCode # only if valid Logging.warning("TSP.stop(): process ended itself") - self.subProcess = None + # self.subProcess = None return # process still alive, let's interrupt it - self._stopForSure(self.subProcess, self.STOP_SIGNAL) # success if no exception - self.subProcess = None + cls._stopForSure(tsp._popen, cls.STOP_SIGNAL) # success if no exception # sub process should end, then IPC queue should end, causing IO thread to end @classmethod - def _stopForSure(cls, proc: subprocess.Popen, sig: int): + def _stopForSure(cls, proc: Popen, sig: int): ''' Stop a process and all sub processes with a singal, and SIGKILL if necessary ''' - def doKillTdService(proc: subprocess.Popen, sig: int): + def doKillTdService(proc: Popen, sig: int): Logging.info("Killing sub-sub process {} with signal {}".format(proc.pid, sig)) proc.send_signal(sig) try: @@ -340,7 +348,7 @@ class TdeSubProcess: else: Logging.warning("TD service terminated, EXPECTING ret code {}, got {}".format(sig, -retCode)) return True # terminated successfully - except subprocess.TimeoutExpired as err: + except TimeoutExpired as err: Logging.warning("Failed to kill sub-sub process {} with signal {}".format(proc.pid, sig)) return False # failed to terminate @@ -361,10 +369,10 @@ class TdeSubProcess: Logging.warning("Failed to kill sub-sub process {} with signal {}".format(child.pid, sig)) return False # did not terminate - def doKill(proc: subprocess.Popen, sig: int): + def doKill(proc: Popen, sig: int): pid = proc.pid try: - topSubProc = psutil.Process(pid) + topSubProc = psutil.Process(pid) # Now that we are doing "exec -c", should not have children any more for child in topSubProc.children(recursive=True): # or parent.children() for recursive=False Logging.warning("Unexpected child to be killed") doKillChild(child, sig) @@ -389,17 +397,15 @@ class TdeSubProcess: return doKill(proc, sig) def hardKill(proc): - return doKill(proc, signal.SIGKILL) - - + return doKill(proc, signal.SIGKILL) pid = proc.pid Logging.info("Terminate running processes under {}, with SIG #{} and wait...".format(pid, sig)) if softKill(proc, sig): - return# success + return # success if sig != signal.SIGKILL: # really was soft above if hardKill(proc): - return + return raise CrashGenError("Failed to stop process, pid={}".format(pid)) class ServiceManager: @@ -657,10 +663,9 @@ class ServiceManagerThread: Logging.info("Attempting to start TAOS service: {}".format(self)) self._status.set(Status.STATUS_STARTING) - self._tdeSubProcess = TdeSubProcess() - self._tdeSubProcess.start(cmdLine) # TODO: verify process is running + self._tdeSubProcess = TdeSubProcess.start(cmdLine) # TODO: verify process is running - self._ipcQueue = Queue() + self._ipcQueue = Queue() # type: Queue self._thread = threading.Thread( # First thread captures server OUTPUT target=self.svcOutputReader, args=(self._tdeSubProcess.getStdOut(), self._ipcQueue, logDir)) @@ -738,21 +743,15 @@ class ServiceManagerThread: raise RuntimeError("sub process object missing") self._status.set(Status.STATUS_STOPPING) - # retCode = self._tdeSubProcess.stop() - # try: - # retCode = self._tdeSubProcess.stop() - # # print("Attempted to stop sub process, got return code: {}".format(retCode)) - # if retCode == signal.SIGSEGV : # SGV - # Logging.error("[[--ERROR--]]: TDengine service SEGV fault (check core file!)") - # except subprocess.TimeoutExpired as err: - # Logging.info("Time out waiting for TDengine service process to exit") - if not self._tdeSubProcess.stop(): # everything withing - if self._tdeSubProcess.isRunning(): # still running, should now never happen - Logging.error("FAILED to stop sub process, it is still running... pid = {}".format( - self._tdeSubProcess.getPid())) - else: - self._tdeSubProcess = None # not running any more - self.join() # stop the thread, change the status, etc. + TdeSubProcess.stop(self._tdeSubProcess) # must stop, no matter what + self._tdeSubProcess = None + # if not self._tdeSubProcess.stop(): # everything withing + # if self._tdeSubProcess.isRunning(): # still running, should now never happen + # Logging.error("FAILED to stop sub process, it is still running... pid = {}".format( + # self._tdeSubProcess.getPid())) + # else: + # self._tdeSubProcess = None # not running any more + self.join() # stop the thread, change the status, etc. # Check if it's really stopped outputLines = 10 # for last output @@ -827,6 +826,19 @@ class ServiceManagerThread: print(pBar, end="", flush=True) print('\b\b\b\b', end="", flush=True) + BinaryLine = NewType('BinaryLine', bytes) # line with binary data, directly from STDOUT, etc. + TextLine = NewType('TextLine', str) # properly decoded, suitable for printing, etc. + x = TextLine('xyz') + + @classmethod + def _decodeBinLine(cls, bLine: BinaryLine) -> Optional[TextLine] : + try: + tLine = bLine.decode("utf-8").rstrip() + return cls.TextLine(tLine) + except UnicodeError: + print("\nNon-UTF8 server output: {}\n".format(bLine.decode('cp437'))) + return None + def svcOutputReader(self, out: IO, queue, logDir: str): ''' The infinite routine that processes the STDOUT stream for the sub process being managed. @@ -841,32 +853,37 @@ class ServiceManagerThread: # Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python # print("This is the svcOutput Reader...") # for line in out : - for line in iter(out.readline, b''): - fOut.write(line) + out.readline() + for bLine in iter(out.readline, b''): + fOut.write(bLine) # print("Finished reading a line: {}".format(line)) # print("Adding item to queue...") - try: - line = line.decode("utf-8").rstrip() - except UnicodeError: - print("\nNon-UTF8 server output: {}\n".format(line)) - # This might block, and then causing "out" buffer to block - queue.put(line) - self._printProgress("_i") + # Moved to above + # try: + # line = line.decode("utf-8").rstrip() + # except UnicodeError: + # print("\nNon-UTF8 server output: {}\n".format(line)) + tLine = self._decodeBinLine(bLine) - if self._status.isStarting(): # we are starting, let's see if we have started - if line.find(self.TD_READY_MSG) != -1: # found - Logging.info("Waiting for the service to become FULLY READY") - time.sleep(1.0) # wait for the server to truly start. TODO: remove this - Logging.info("Service is now FULLY READY") # TODO: more ID info here? - self._status.set(Status.STATUS_RUNNING) + if tLine is not None: + # This might block, and then causing "out" buffer to block + queue.put(tLine) + self._printProgress("_i") - # Trim the queue if necessary: TODO: try this 1 out of 10 times - self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10) # trim to 90% size + if self._status.isStarting(): # we are starting, let's see if we have started + if tLine.find(self.TD_READY_MSG) != -1: # found + Logging.info("Waiting for the service to become FULLY READY") + time.sleep(1.0) # wait for the server to truly start. TODO: remove this + Logging.info("Service is now FULLY READY") # TODO: more ID info here? + self._status.set(Status.STATUS_RUNNING) - if self._status.isStopping(): # TODO: use thread status instead - # WAITING for stopping sub process to finish its outptu - print("_w", end="", flush=True) + # Trim the queue if necessary: TODO: try this 1 out of 10 times + self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10) # trim to 90% size + + if self._status.isStopping(): # TODO: use thread status instead + # WAITING for stopping sub process to finish its outptu + print("_w", end="", flush=True) # queue.put(line) # meaning sub process must have died diff --git a/tests/pytest/crash_gen/settings.py b/tests/pytest/crash_gen/settings.py index 3c4c91e6e0..ae0132378d 100644 --- a/tests/pytest/crash_gen/settings.py +++ b/tests/pytest/crash_gen/settings.py @@ -3,6 +3,13 @@ import argparse gConfig: argparse.Namespace -def init(): - global gConfig - gConfig = [] \ No newline at end of file +class Settings: + @classmethod + def init(cls): + global gConfig + gConfig = [] + + @classmethod + def setConfig(cls, config): + global gConfig + gConfig = config \ No newline at end of file From 5d1d5cadc293b971404c6a230f2da6f16f1b9a0c Mon Sep 17 00:00:00 2001 From: Steven Li Date: Wed, 28 Apr 2021 08:36:56 +0000 Subject: [PATCH 002/157] Refactored crash_gen to have the TdeSubProcess own the SvcMgrThread object, also switched to Pylance --- .../python/linux/python3/taos/__init__.py | 1 + tests/pytest/crash_gen/__init__.py | 1 + tests/pytest/crash_gen/crash_gen_main.py | 167 ++++++------ tests/pytest/crash_gen/db.py | 39 +-- tests/pytest/crash_gen/misc.py | 17 +- tests/pytest/crash_gen/service_manager.py | 243 +++++++++++------- tests/pytest/crash_gen/settings.py | 26 +- tests/pytest/crash_gen/types.py | 5 + 8 files changed, 287 insertions(+), 212 deletions(-) create mode 100644 tests/pytest/crash_gen/types.py diff --git a/src/connector/python/linux/python3/taos/__init__.py b/src/connector/python/linux/python3/taos/__init__.py index 9732635738..1b086f36ec 100644 --- a/src/connector/python/linux/python3/taos/__init__.py +++ b/src/connector/python/linux/python3/taos/__init__.py @@ -1,6 +1,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor +from .error import Error # Globals threadsafety = 0 diff --git a/tests/pytest/crash_gen/__init__.py b/tests/pytest/crash_gen/__init__.py index 71855bc5a9..2509d7d76d 100644 --- a/tests/pytest/crash_gen/__init__.py +++ b/tests/pytest/crash_gen/__init__.py @@ -3,3 +3,4 @@ from crash_gen.service_manager import ServiceManager, TdeInstance, TdeSubProcess from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager from crash_gen.settings import Settings +from crash_gen.types import DirPath \ No newline at end of file diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 39475a32d0..8d34a7555b 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -15,7 +15,7 @@ # https://stackoverflow.com/questions/33533148/how-do-i-specify-that-the-return-type-of-a-method-is-the-same-as-the-class-itsel from __future__ import annotations -from typing import Set +from typing import Any, Set, Tuple from typing import Dict from typing import List from typing import Optional # Type hinting, ref: https://stackoverflow.com/questions/19202633/python-3-type-hinting-for-none @@ -57,8 +57,8 @@ if sys.version_info[0] < 3: # Command-line/Environment Configurations, will set a bit later # ConfigNameSpace = argparse.Namespace -gConfig: argparse.Namespace -gSvcMgr: ServiceManager # TODO: refactor this hack, use dep injection +# gConfig: argparse.Namespace +gSvcMgr: Optional[ServiceManager] # TODO: refactor this hack, use dep injection # logger: logging.Logger gContainer: Container @@ -81,20 +81,20 @@ class WorkerThread: self._stepGate = threading.Event() # Let us have a DB connection of our own - if (gConfig.per_thread_db_connection): # type: ignore + if (Settings.getConfig().per_thread_db_connection): # type: ignore # print("connector_type = {}".format(gConfig.connector_type)) tInst = gContainer.defTdeInstance - if gConfig.connector_type == 'native': + if Settings.getConfig().connector_type == 'native': self._dbConn = DbConn.createNative(tInst.getDbTarget()) - elif gConfig.connector_type == 'rest': + elif Settings.getConfig().connector_type == 'rest': self._dbConn = DbConn.createRest(tInst.getDbTarget()) - elif gConfig.connector_type == 'mixed': + elif Settings.getConfig().connector_type == 'mixed': if Dice.throw(2) == 0: # 1/2 chance self._dbConn = DbConn.createNative(tInst.getDbTarget()) else: self._dbConn = DbConn.createRest(tInst.getDbTarget()) else: - raise RuntimeError("Unexpected connector type: {}".format(gConfig.connector_type)) + raise RuntimeError("Unexpected connector type: {}".format(Settings.getConfig().connector_type)) # self._dbInUse = False # if "use db" was executed already @@ -123,14 +123,14 @@ class WorkerThread: # self.isSleeping = False Logging.info("Starting to run thread: {}".format(self._tid)) - if (gConfig.per_thread_db_connection): # type: ignore + if (Settings.getConfig().per_thread_db_connection): # type: ignore Logging.debug("Worker thread openning database connection") self._dbConn.open() self._doTaskLoop() # clean up - if (gConfig.per_thread_db_connection): # type: ignore + if (Settings.getConfig().per_thread_db_connection): # type: ignore if self._dbConn.isOpen: #sometimes it is not open self._dbConn.close() else: @@ -158,7 +158,7 @@ class WorkerThread: # Before we fetch the task and run it, let's ensure we properly "use" the database (not needed any more) try: - if (gConfig.per_thread_db_connection): # most likely TRUE + if (Settings.getConfig().per_thread_db_connection): # most likely TRUE if not self._dbConn.isOpen: # might have been closed during server auto-restart self._dbConn.open() # self.useDb() # might encounter exceptions. TODO: catch @@ -232,7 +232,7 @@ class WorkerThread: return self.getDbConn().getQueryResult() def getDbConn(self) -> DbConn : - if (gConfig.per_thread_db_connection): + if (Settings.getConfig().per_thread_db_connection): return self._dbConn else: return self._tc.getDbManager().getDbConn() @@ -254,7 +254,7 @@ class ThreadCoordinator: self._pool = pool # self._wd = wd self._te = None # prepare for every new step - self._dbManager = dbManager + self._dbManager = dbManager # type: Optional[DbManager] # may be freed self._executedTasks: List[Task] = [] # in a given step self._lock = threading.RLock() # sync access for a few things @@ -266,9 +266,13 @@ class ThreadCoordinator: self._stepStartTime = None # Track how long it takes to execute each step def getTaskExecutor(self): + if self._te is None: + raise CrashGenError("Unexpected empty TE") return self._te def getDbManager(self) -> DbManager: + if self._dbManager is None: + raise ChildProcessError("Unexpected empty _dbManager") return self._dbManager def crossStepBarrier(self, timeout=None): @@ -279,7 +283,7 @@ class ThreadCoordinator: self._execStats.registerFailure("User Interruption") def _runShouldEnd(self, transitionFailed, hasAbortedTask, workerTimeout): - maxSteps = gConfig.max_steps # type: ignore + maxSteps = Settings.getConfig().max_steps # type: ignore if self._curStep >= (maxSteps - 1): # maxStep==10, last curStep should be 9 return True if self._runStatus != Status.STATUS_RUNNING: @@ -384,7 +388,7 @@ class ThreadCoordinator: hasAbortedTask = False workerTimeout = False while not self._runShouldEnd(transitionFailed, hasAbortedTask, workerTimeout): - if not gConfig.debug: # print this only if we are not in debug mode + if not Settings.getConfig().debug: # print this only if we are not in debug mode Progress.emit(Progress.STEP_BOUNDARY) # print(".", end="", flush=True) # if (self._curStep % 2) == 0: # print memory usage once every 10 steps @@ -469,7 +473,7 @@ class ThreadCoordinator: self._pool = None self._te = None self._dbManager = None - self._executedTasks = None + self._executedTasks = [] self._lock = None self._stepBarrier = None self._execStats = None @@ -508,18 +512,18 @@ class ThreadCoordinator: ''' Initialize multiple databases, invoked at __ini__() time ''' self._dbs = [] # type: List[Database] dbc = self.getDbManager().getDbConn() - if gConfig.max_dbs == 0: + if Settings.getConfig().max_dbs == 0: self._dbs.append(Database(0, dbc)) else: baseDbNumber = int(datetime.datetime.now().timestamp( # Don't use Dice/random, as they are deterministic - )*333) % 888 if gConfig.dynamic_db_table_names else 0 - for i in range(gConfig.max_dbs): + )*333) % 888 if Settings.getConfig().dynamic_db_table_names else 0 + for i in range(Settings.getConfig().max_dbs): self._dbs.append(Database(baseDbNumber + i, dbc)) def pickDatabase(self): idxDb = 0 - if gConfig.max_dbs != 0 : - idxDb = Dice.throw(gConfig.max_dbs) # 0 to N-1 + if Settings.getConfig().max_dbs != 0 : + idxDb = Dice.throw(Settings.getConfig().max_dbs) # 0 to N-1 db = self._dbs[idxDb] # type: Database return db @@ -563,7 +567,7 @@ class ThreadPool: workerThread._thread.join() def cleanup(self): - self.threadList = None # maybe clean up each? + self.threadList = [] # maybe clean up each? # A queue of continguous POSITIVE integers, used by DbManager to generate continuous numbers # for new table names @@ -673,7 +677,7 @@ class AnyState: # Each sub state tells us the "info", about itself, so we can determine # on things like canDropDB() - def getInfo(self): + def getInfo(self) -> List[Any]: raise RuntimeError("Must be overriden by child classes") def equals(self, other): @@ -701,7 +705,7 @@ class AnyState: def canDropDb(self): # If user requests to run up to a number of DBs, # we'd then not do drop_db operations any more - if gConfig.max_dbs > 0 or gConfig.use_shadow_db : + if Settings.getConfig().max_dbs > 0 or Settings.getConfig().use_shadow_db : return False return self._info[self.CAN_DROP_DB] @@ -709,7 +713,7 @@ class AnyState: return self._info[self.CAN_CREATE_FIXED_SUPER_TABLE] def canDropFixedSuperTable(self): - if gConfig.use_shadow_db: # duplicate writes to shaddow DB, in which case let's disable dropping s-table + if Settings.getConfig().use_shadow_db: # duplicate writes to shaddow DB, in which case let's disable dropping s-table return False return self._info[self.CAN_DROP_FIXED_SUPER_TABLE] @@ -911,7 +915,7 @@ class StateMechine: # May be slow, use cautionsly... def getTaskTypes(self): # those that can run (directly/indirectly) from the current state - def typesToStrings(types): + def typesToStrings(types) -> List: ss = [] for t in types: ss.append(t.__name__) @@ -1030,13 +1034,14 @@ class StateMechine: # ref: # https://eli.thegreenplace.net/2010/01/22/weighted-random-generation-in-python/ - def _weighted_choice_sub(self, weights): + def _weighted_choice_sub(self, weights) -> int: # TODO: use our dice to ensure it being determinstic? rnd = random.random() * sum(weights) for i, w in enumerate(weights): rnd -= w if rnd < 0: return i + raise CrashGenError("Unexpected no choice") class Database: ''' We use this to represent an actual TDengine database inside a service instance, @@ -1048,8 +1053,8 @@ class Database: ''' _clsLock = threading.Lock() # class wide lock _lastInt = 101 # next one is initial integer - _lastTick = 0 - _lastLaggingTick = 0 # lagging tick, for out-of-sequence (oos) data insertions + _lastTick = None # Optional[datetime] + _lastLaggingTick = None # Optional[datetime] # lagging tick, for out-of-sequence (oos) data insertions def __init__(self, dbNum: int, dbc: DbConn): # TODO: remove dbc self._dbNum = dbNum # we assign a number to databases, for our testing purpose @@ -1114,14 +1119,14 @@ class Database: Fetch a timestamp tick, with some random factor, may not be unique. ''' with cls._clsLock: # prevent duplicate tick - if cls._lastLaggingTick==0 or cls._lastTick==0 : # not initialized + if cls._lastLaggingTick is None or cls._lastTick is None : # not initialized # 10k at 1/20 chance, should be enough to avoid overlaps tick = cls.setupLastTick() cls._lastTick = tick cls._lastLaggingTick = tick + datetime.timedelta(0, -60*2) # lagging behind 2 minutes, should catch up fast # if : # should be quite a bit into the future - if gConfig.mix_oos_data and Dice.throw(20) == 0: # if asked to do so, and 1 in 20 chance, return lagging tick + if Settings.getConfig().mix_oos_data and Dice.throw(20) == 0: # if asked to do so, and 1 in 20 chance, return lagging tick cls._lastLaggingTick += datetime.timedelta(0, 1) # pick the next sequence from the lagging tick sequence return cls._lastLaggingTick else: # regular @@ -1303,10 +1308,10 @@ class Task(): ]: return True # These are the ALWAYS-ACCEPTABLE ones # This case handled below already. - # elif (errno in [ 0x0B ]) and gConfig.auto_start_service: + # elif (errno in [ 0x0B ]) and Settings.getConfig().auto_start_service: # return True # We may get "network unavilable" when restarting service - elif gConfig.ignore_errors: # something is specified on command line - moreErrnos = [int(v, 0) for v in gConfig.ignore_errors.split(',')] + elif Settings.getConfig().ignore_errors: # something is specified on command line + moreErrnos = [int(v, 0) for v in Settings.getConfig().ignore_errors.split(',')] if errno in moreErrnos: return True elif errno == 0x200 : # invalid SQL, we need to div in a bit more @@ -1342,7 +1347,7 @@ class Task(): self._executeInternal(te, wt) # TODO: no return value? except taos.error.ProgrammingError as err: errno2 = Helper.convertErrno(err.errno) - if (gConfig.continue_on_exception): # user choose to continue + if (Settings.getConfig().continue_on_exception): # user choose to continue self.logDebug("[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format( errno2, err, wt.getDbConn().getLastSql())) self._err = err @@ -1357,7 +1362,7 @@ class Task(): self.__class__.__name__, errno2, err, wt.getDbConn().getLastSql()) self.logDebug(errMsg) - if gConfig.debug: + if Settings.getConfig().debug: # raise # so that we see full stack traceback.print_exc() print( @@ -1422,11 +1427,11 @@ class Task(): class ExecutionStats: def __init__(self): # total/success times for a task - self._execTimes: Dict[str, [int, int]] = {} + self._execTimes: Dict[str, List[int]] = {} self._tasksInProgress = 0 self._lock = threading.Lock() - self._firstTaskStartTime = None - self._execStartTime = None + self._firstTaskStartTime = 0.0 + self._execStartTime = 0.0 self._errors = {} self._elapsedTime = 0.0 # total elapsed time self._accRunTime = 0.0 # accumulated run time @@ -1471,7 +1476,7 @@ class ExecutionStats: self._tasksInProgress -= 1 if self._tasksInProgress == 0: # all tasks have stopped self._accRunTime += (time.time() - self._firstTaskStartTime) - self._firstTaskStartTime = None + self._firstTaskStartTime = 0.0 def registerFailure(self, reason): self._failed = True @@ -1555,7 +1560,7 @@ class StateTransitionTask(Task): def getRegTableName(cls, i): if ( StateTransitionTask._baseTableNumber is None): # Set it one time StateTransitionTask._baseTableNumber = Dice.throw( - 999) if gConfig.dynamic_db_table_names else 0 + 999) if Settings.getConfig().dynamic_db_table_names else 0 return "reg_table_{}".format(StateTransitionTask._baseTableNumber + i) def execute(self, wt: WorkerThread): @@ -1575,14 +1580,14 @@ class TaskCreateDb(StateTransitionTask): def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): # was: self.execWtSql(wt, "create database db") repStr = "" - if gConfig.num_replicas != 1: - # numReplica = Dice.throw(gConfig.max_replicas) + 1 # 1,2 ... N - numReplica = gConfig.num_replicas # fixed, always + if Settings.getConfig().num_replicas != 1: + # numReplica = Dice.throw(Settings.getConfig().max_replicas) + 1 # 1,2 ... N + numReplica = Settings.getConfig().num_replicas # fixed, always repStr = "replica {}".format(numReplica) - updatePostfix = "update 1" if gConfig.verify_data else "" # allow update only when "verify data" is active + updatePostfix = "update 1" if Settings.getConfig().verify_data else "" # allow update only when "verify data" is active dbName = self._db.getName() self.execWtSql(wt, "create database {} {} {} ".format(dbName, repStr, updatePostfix ) ) - if dbName == "db_0" and gConfig.use_shadow_db: + if dbName == "db_0" and Settings.getConfig().use_shadow_db: self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix ) ) class TaskDropDb(StateTransitionTask): @@ -1887,7 +1892,7 @@ class TaskDropSuperTable(StateTransitionTask): if Dice.throw(2) == 0: # print("_7_", end="", flush=True) tblSeq = list(range( - 2 + (self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES))) + 2 + (self.LARGE_NUMBER_OF_TABLES if Settings.getConfig().larger_data else self.SMALL_NUMBER_OF_TABLES))) random.shuffle(tblSeq) tickOutput = False # if we have spitted out a "d" character for "drop regular table" isSuccess = True @@ -1953,13 +1958,13 @@ class TaskRestartService(StateTransitionTask): @classmethod def canBeginFrom(cls, state: AnyState): - if gConfig.auto_start_service: + if Settings.getConfig().auto_start_service: return state.canDropFixedSuperTable() # Basicallly when we have the super table return False # don't run this otherwise CHANCE_TO_RESTART_SERVICE = 200 def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - if not gConfig.auto_start_service: # only execute when we are in -a mode + if not Settings.getConfig().auto_start_service: # only execute when we are in -a mode print("_a", end="", flush=True) return @@ -1981,12 +1986,12 @@ class TaskAddData(StateTransitionTask): activeTable: Set[int] = set() # We use these two files to record operations to DB, useful for power-off tests - fAddLogReady = None # type: io.TextIOWrapper - fAddLogDone = None # type: io.TextIOWrapper + fAddLogReady = None # type: Optional[io.TextIOWrapper] + fAddLogDone = None # type: Optional[io.TextIOWrapper] @classmethod def prepToRecordOps(cls): - if gConfig.record_ops: + if Settings.getConfig().record_ops: if (cls.fAddLogReady is None): Logging.info( "Recording in a file operations to be performed...") @@ -2004,7 +2009,7 @@ class TaskAddData(StateTransitionTask): return state.canAddData() def _addDataInBatch(self, db, dbc, regTableName, te: TaskExecutor): - numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS + numRecords = self.LARGE_NUMBER_OF_RECORDS if Settings.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS fullTableName = db.getName() + '.' + regTableName sql = "INSERT INTO {} VALUES ".format(fullTableName) @@ -2016,21 +2021,23 @@ class TaskAddData(StateTransitionTask): dbc.execute(sql) def _addData(self, db: Database, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches - numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS + numRecords = self.LARGE_NUMBER_OF_RECORDS if Settings.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS for j in range(numRecords): # number of records per table nextInt = db.getNextInt() nextTick = db.getNextTick() nextColor = db.getNextColor() - if gConfig.record_ops: + if Settings.getConfig().record_ops: self.prepToRecordOps() + if self.fAddLogReady is None: + raise CrashGenError("Unexpected empty fAddLogReady") self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) self.fAddLogReady.flush() os.fsync(self.fAddLogReady.fileno()) # TODO: too ugly trying to lock the table reliably, refactor... fullTableName = db.getName() + '.' + regTableName - if gConfig.verify_data: + if Settings.getConfig().verify_data: self.lockTable(fullTableName) # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written @@ -2043,7 +2050,7 @@ class TaskAddData(StateTransitionTask): dbc.execute(sql) # Quick hack, attach an update statement here. TODO: create an "update" task - if (not gConfig.use_shadow_db) and Dice.throw(5) == 0: # 1 in N chance, plus not using shaddow DB + if (not Settings.getConfig().use_shadow_db) and Dice.throw(5) == 0: # 1 in N chance, plus not using shaddow DB nextInt = db.getNextInt() nextColor = db.getNextColor() sql = "INSERt INTO {} VALUES ('{}', {}, '{}');".format( # "INSERt" means "update" here @@ -2054,12 +2061,12 @@ class TaskAddData(StateTransitionTask): dbc.execute(sql) except: # Any exception at all - if gConfig.verify_data: + if Settings.getConfig().verify_data: self.unlockTable(fullTableName) raise # Now read it back and verify, we might encounter an error if table is dropped - if gConfig.verify_data: # only if command line asks for it + if Settings.getConfig().verify_data: # only if command line asks for it try: readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts='{}'". format(db.getName(), regTableName, nextTick)) @@ -2086,7 +2093,9 @@ class TaskAddData(StateTransitionTask): # Successfully wrote the data into the DB, let's record it somehow te.recordDataMark(nextInt) - if gConfig.record_ops: + if Settings.getConfig().record_ops: + if self.fAddLogDone is None: + raise CrashGenError("Unexpected empty fAddLogDone") self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName)) self.fAddLogDone.flush() os.fsync(self.fAddLogDone.fileno()) @@ -2095,8 +2104,8 @@ class TaskAddData(StateTransitionTask): # ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access db = self._db dbc = wt.getDbConn() - numTables = self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES - numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS + numTables = self.LARGE_NUMBER_OF_TABLES if Settings.getConfig().larger_data else self.SMALL_NUMBER_OF_TABLES + numRecords = self.LARGE_NUMBER_OF_RECORDS if Settings.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS tblSeq = list(range(numTables )) random.shuffle(tblSeq) # now we have random sequence for i in tblSeq: @@ -2126,7 +2135,9 @@ class ThreadStacks: # stack info for all threads def __init__(self): self._allStacks = {} allFrames = sys._current_frames() - for th in threading.enumerate(): + for th in threading.enumerate(): + if th.ident is None: + continue stack = traceback.extract_stack(allFrames[th.ident]) self._allStacks[th.native_id] = stack @@ -2247,14 +2258,15 @@ class ClientManager: def run(self, svcMgr): # self._printLastNumbers() - global gConfig + # global gConfig # Prepare Tde Instance global gContainer tInst = gContainer.defTdeInstance = TdeInstance() # "subdir to hold the instance" - dbManager = DbManager(gConfig.connector_type, tInst.getDbTarget()) # Regular function - thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps) + cfg = Settings.getConfig() + dbManager = DbManager(cfg.connector_type, tInst.getDbTarget()) # Regular function + thPool = ThreadPool(cfg.num_threads, cfg.max_steps) self.tc = ThreadCoordinator(thPool, dbManager) Logging.info("Starting client instance: {}".format(tInst)) @@ -2267,7 +2279,8 @@ class ClientManager: # Release global variables - gConfig = None + # gConfig = None + Settings.clearConfig() gSvcMgr = None logger = None @@ -2298,7 +2311,7 @@ class ClientManager: class MainExec: def __init__(self): self._clientMgr = None - self._svcMgr = None # type: ServiceManager + self._svcMgr = None # type: Optional[ServiceManager] signal.signal(signal.SIGTERM, self.sigIntHandler) signal.signal(signal.SIGINT, self.sigIntHandler) @@ -2318,7 +2331,7 @@ class MainExec: def runClient(self): global gSvcMgr - if gConfig.auto_start_service: + if Settings.getConfig().auto_start_service: gSvcMgr = self._svcMgr = ServiceManager(1) # hack alert gSvcMgr.startTaosServices() # we start, don't run @@ -2327,13 +2340,13 @@ class MainExec: try: ret = self._clientMgr.run(self._svcMgr) # stop TAOS service inside except requests.exceptions.ConnectionError as err: - Logging.warning("Failed to open REST connection to DB: {}".format(err.getMessage())) + Logging.warning("Failed to open REST connection to DB: {}".format(err)) # don't raise return ret def runService(self): global gSvcMgr - gSvcMgr = self._svcMgr = ServiceManager(gConfig.num_dnodes) # save it in a global variable TODO: hack alert + gSvcMgr = self._svcMgr = ServiceManager(Settings.getConfig().num_dnodes) # save it in a global variable TODO: hack alert gSvcMgr.run() # run to some end state gSvcMgr = self._svcMgr = None @@ -2467,20 +2480,20 @@ class MainExec: action='store_true', help='Continue execution after encountering unexpected/disallowed errors/exceptions (default: false)') - global gConfig - gConfig = parser.parse_args() - Settings.setConfig(gConfig) # TODO: fix this hack, consolidate this global var + # global gConfig + config = parser.parse_args() + Settings.setConfig(config) # TODO: fix this hack, consolidate this global var # Sanity check for arguments - if gConfig.use_shadow_db and gConfig.max_dbs>1 : + if Settings.getConfig().use_shadow_db and Settings.getConfig().max_dbs>1 : raise CrashGenError("Cannot combine use-shadow-db with max-dbs of more than 1") - Logging.clsInit(gConfig) + Logging.clsInit(Settings.getConfig()) Dice.seed(0) # initial seeding of dice def run(self): - if gConfig.run_tdengine: # run server + if Settings.getConfig().run_tdengine: # run server try: self.runService() return 0 # success diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/db.py index 62a369c41a..6c85da0051 100644 --- a/tests/pytest/crash_gen/db.py +++ b/tests/pytest/crash_gen/db.py @@ -5,6 +5,7 @@ import time import threading import requests from requests.auth import HTTPBasicAuth +from crash_gen.types import QueryResult import taos from util.sql import * @@ -18,7 +19,7 @@ import datetime import traceback # from .service_manager import TdeInstance -import crash_gen.settings +from crash_gen.settings import Settings class DbConn: TYPE_NATIVE = "native-c" @@ -79,7 +80,7 @@ class DbConn: raise RuntimeError("Cannot query database until connection is open") nRows = self.query(sql) if nRows != 1: - raise taos.error.ProgrammingError( + raise CrashGenError( "Unexpected result for query: {}, rows = {}".format(sql, nRows), (CrashGenError.INVALID_EMPTY_RESULT if nRows==0 else CrashGenError.INVALID_MULTIPLE_RESULT) ) @@ -115,7 +116,7 @@ class DbConn: try: self.execute(sql) return True # ignore num of results, return success - except taos.error.ProgrammingError as err: + except taos.error.Error as err: return False # failed, for whatever TAOS reason # Not possile to reach here, non-TAOS exception would have been thrown @@ -126,7 +127,7 @@ class DbConn: def openByType(self): raise RuntimeError("Unexpected execution, should be overriden") - def getQueryResult(self): + def getQueryResult(self) -> QueryResult : raise RuntimeError("Unexpected execution, should be overriden") def getResultRows(self): @@ -221,7 +222,7 @@ class DbConnRest(DbConn): class MyTDSql: # Class variables _clsLock = threading.Lock() # class wide locking - longestQuery = None # type: str + longestQuery = '' # type: str longestQueryTime = 0.0 # seconds lqStartTime = 0.0 # lqEndTime = 0.0 # Not needed, as we have the two above already @@ -261,7 +262,7 @@ class MyTDSql: cls.lqStartTime = startTime # Now write to the shadow database - if crash_gen.settings.gConfig.use_shadow_db: + if Settings.getConfig().use_shadow_db: if sql[:11] == "INSERT INTO": if sql[:16] == "INSERT INTO db_0": sql2 = "INSERT INTO db_s" + sql[16:] @@ -453,31 +454,11 @@ class DbManager(): ''' Release the underlying DB connection upon deletion of DbManager ''' self.cleanUp() - def getDbConn(self): + def getDbConn(self) -> DbConn : + if self._dbConn is None: + raise CrashGenError("Unexpected empty DbConn") return self._dbConn - # TODO: not used any more, to delete - def pickAndAllocateTable(self): # pick any table, and "use" it - return self.tableNumQueue.pickAndAllocate() - - # TODO: Not used any more, to delete - def addTable(self): - with self._lock: - tIndex = self.tableNumQueue.push() - return tIndex - - # Not used any more, to delete - def releaseTable(self, i): # return the table back, so others can use it - self.tableNumQueue.release(i) - - # TODO: not used any more, delete - def getTableNameToDelete(self): - tblNum = self.tableNumQueue.pop() # TODO: race condition! - if (not tblNum): # maybe false - return False - - return "table_{}".format(tblNum) - def cleanUp(self): if self._dbConn: self._dbConn.close() diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py index 9774ec5455..cb98bc7c5b 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/misc.py @@ -3,6 +3,7 @@ import random import logging import os import sys +from typing import Optional import taos @@ -39,11 +40,11 @@ class MyLoggingAdapter(logging.LoggerAdapter): class Logging: - logger = None + logger = None # type: Optional[MyLoggingAdapter] @classmethod def getLogger(cls): - return logger + return cls.logger @classmethod def clsInit(cls, gConfig): # TODO: refactor away gConfig @@ -60,7 +61,7 @@ class Logging: # Logging adapter, to be used as a logger # print("setting logger variable") # global logger - cls.logger = MyLoggingAdapter(_logger, []) + cls.logger = MyLoggingAdapter(_logger, {}) if (gConfig.debug): cls.logger.setLevel(logging.DEBUG) # default seems to be INFO @@ -84,6 +85,7 @@ class Logging: cls.logger.error(msg) class Status: + STATUS_EMPTY = 99 STATUS_STARTING = 1 STATUS_RUNNING = 2 STATUS_STOPPING = 3 @@ -95,12 +97,16 @@ class Status: def __repr__(self): return "[Status: v={}]".format(self._status) - def set(self, status): + def set(self, status: int): self._status = status def get(self): return self._status + def isEmpty(self): + ''' Empty/Undefined ''' + return self._status == Status.STATUS_EMPTY + def isStarting(self): return self._status == Status.STATUS_STARTING @@ -117,6 +123,9 @@ class Status: def isStable(self): return self.isRunning() or self.isStopped() + def isActive(self): + return self.isStarting() or self.isRunning() or self.isStopping() + # Deterministic random number generator class Dice(): seeded = False # static, uninitialized diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py index 146215f2bc..4afd7e4c78 100644 --- a/tests/pytest/crash_gen/service_manager.py +++ b/tests/pytest/crash_gen/service_manager.py @@ -3,12 +3,12 @@ from __future__ import annotations import os import io import sys +from enum import Enum import threading import signal import logging import time from subprocess import PIPE, Popen, TimeoutExpired - from typing import IO, List, NewType, Optional try: @@ -16,12 +16,12 @@ try: except: print("Psutil module needed, please install: sudo pip3 install psutil") sys.exit(-1) - from queue import Queue, Empty -from .misc import Logging, Status, CrashGenError, Dice, Helper, Progress -from .db import DbConn, DbTarget -import crash_gen.settings +from crash_gen.misc import CrashGenError, Dice, Helper, Logging, Progress, Status +from crash_gen.db import DbConn, DbTarget +from crash_gen.settings import Settings +from crash_gen.types import DirPath class TdeInstance(): """ @@ -70,7 +70,10 @@ class TdeInstance(): self._fepPort = fepPort self._tInstNum = tInstNum - self._smThread = ServiceManagerThread() + + # An "Tde Instance" will *contain* a "sub process" object, with will/may use a thread internally + # self._smThread = ServiceManagerThread() + self._subProcess = None # type: Optional[TdeSubProcess] def getDbTarget(self): return DbTarget(self.getCfgDir(), self.getHostAddr(), self._port) @@ -155,21 +158,21 @@ quorum 2 def getExecFile(self): # .../taosd return self._buildDir + "/build/bin/taosd" - def getRunDir(self): # TODO: rename to "root dir" ?! - return self._buildDir + self._subdir + def getRunDir(self) -> DirPath : # TODO: rename to "root dir" ?! + return DirPath(self._buildDir + self._subdir) - def getCfgDir(self): # path, not file - return self.getRunDir() + "/cfg" + def getCfgDir(self) -> DirPath : # path, not file + return DirPath(self.getRunDir() + "/cfg") - def getLogDir(self): - return self.getRunDir() + "/log" + def getLogDir(self) -> DirPath : + return DirPath(self.getRunDir() + "/log") def getHostAddr(self): return "127.0.0.1" def getServiceCmdLine(self): # to start the instance cmdLine = [] - if crash_gen.settings.gConfig.track_memory_leaks: + if Settings.getConfig().track_memory_leaks: Logging.info("Invoking VALGRIND on service...") cmdLine = ['valgrind', '--leak-check=yes'] # TODO: move "exec -c" into Popen(), we can both "use shell" and NOT fork so ask to lose kill control @@ -199,27 +202,46 @@ quorum 2 dbc.close() def getStatus(self): - return self._smThread.getStatus() + # return self._smThread.getStatus() + if self._subProcess is None: + return Status(Status.STATUS_EMPTY) + return self._subProcess.getStatus() - def getSmThread(self): - return self._smThread + # def getSmThread(self): + # return self._smThread def start(self): - if not self.getStatus().isStopped(): + if self.getStatus().isActive(): raise CrashGenError("Cannot start instance from status: {}".format(self.getStatus())) Logging.info("Starting TDengine instance: {}".format(self)) self.generateCfgFile() # service side generates config file, client does not self.rotateLogs() - self._smThread.start(self.getServiceCmdLine(), self.getLogDir()) # May raise exceptions + # self._smThread.start(self.getServiceCmdLine(), self.getLogDir()) # May raise exceptions + self._subProcess = TdeSubProcess(self.getServiceCmdLine(), self.getLogDir()) def stop(self): - self._smThread.stop() + self._subProcess.stop() + self._subProcess = None def isFirst(self): return self._tInstNum == 0 + def printFirst10Lines(self): + if self._subProcess is None: + Logging.warning("Incorrect TI status for procIpcBatch-10 operation") + return + self._subProcess.procIpcBatch(trimToTarget=10, forceOutput=True) + + def procIpcBatch(self): + if self._subProcess is None: + Logging.warning("Incorrect TI status for procIpcBatch operation") + return + self._subProcess.procIpcBatch() # may enounter EOF and change status to STOPPED + if self._subProcess.getStatus().isStopped(): + self._subProcess.stop() + self._subProcess = None class TdeSubProcess: """ @@ -237,16 +259,21 @@ class TdeSubProcess: # RET_TIME_OUT = -3 # RET_SUCCESS = -4 - def __init__(self, po: Popen): - self._popen = po # type: Popen - # if tInst is None: - # raise CrashGenError("Empty instance not allowed in TdeSubProcess") - # self._tInst = tInst # Default create at ServiceManagerThread + def __init__(self, cmdLine: List[str], logDir: DirPath): + # Create the process + managing thread immediately + + Logging.info("Attempting to start TAOS sub process...") + self._popen = self._start(cmdLine) # the actual sub process + self._smThread = ServiceManagerThread(self, logDir) # A thread to manage the sub process, mostly to process the IO + Logging.info("Successfully started TAOS process: {}".format(self)) + + def __repr__(self): # if self.subProcess is None: # return '[TdeSubProc: Empty]' - return '[TdeSubProc: pid = {}]'.format(self.getPid()) + return '[TdeSubProc: pid = {}, status = {}]'.format( + self.getPid(), self.getStatus() ) def getStdOut(self): return self._popen.stdout @@ -261,14 +288,14 @@ class TdeSubProcess: def getPid(self): return self._popen.pid - @classmethod - def start(cls, cmdLine): + def _start(self, cmdLine) -> Popen : ON_POSIX = 'posix' in sys.builtin_module_names # Sanity check # if self.subProcess: # already there # raise RuntimeError("Corrupt process state") + # Prepare environment variables for coverage information # Ref: https://stackoverflow.com/questions/2231227/python-subprocess-popen-with-a-modified-environment myEnv = os.environ.copy() @@ -279,7 +306,7 @@ class TdeSubProcess: # print("Starting TDengine via Shell: {}".format(cmdLineStr)) # useShell = True # Needed to pass environments into it - popen = Popen( + return Popen( ' '.join(cmdLine), # ' '.join(cmdLine) if useShell else cmdLine, shell=True, # Always use shell, since we need to pass ENV vars stdout=PIPE, @@ -287,15 +314,15 @@ class TdeSubProcess: close_fds=ON_POSIX, env=myEnv ) # had text=True, which interferred with reading EOF - return cls(popen) STOP_SIGNAL = signal.SIGINT # signal.SIGKILL/SIGINT # What signal to use (in kill) to stop a taosd process? SIG_KILL_RETCODE = 137 # ref: https://stackoverflow.com/questions/43268156/process-finished-with-exit-code-137-in-pycharm - @classmethod - def stop(cls, tsp: TdeSubProcess): + def stop(self): """ - Stop a sub process, DO NOT return anything, process all conditions INSIDE + Stop a sub process, DO NOT return anything, process all conditions INSIDE. + + Calling function should immediately delete/unreference the object Common POSIX signal values (from man -7 signal): SIGHUP 1 @@ -315,11 +342,17 @@ class TdeSubProcess: """ # self._popen should always be valid. - # if not self.subProcess: - # Logging.error("Sub process already stopped") - # return + Logging.info("Terminating TDengine service running as the sub process...") + if self.getStatus().isStopped(): + Logging.info("Service already stopped") + return + if self.getStatus().isStopping(): + Logging.info("Service is already being stopped, pid: {}".format(self.getPid())) + return - retCode = tsp._popen.poll() # ret -N means killed with signal N, otherwise it's from exit(N) + self.setStatus(Status.STATUS_STOPPING) + + retCode = self._popen.poll() # ret -N means killed with signal N, otherwise it's from exit(N) if retCode: # valid return code, process ended # retCode = -retCode # only if valid Logging.warning("TSP.stop(): process ended itself") @@ -327,9 +360,12 @@ class TdeSubProcess: return # process still alive, let's interrupt it - cls._stopForSure(tsp._popen, cls.STOP_SIGNAL) # success if no exception + self._stopForSure(self._popen, self.STOP_SIGNAL) # success if no exception - # sub process should end, then IPC queue should end, causing IO thread to end + # sub process should end, then IPC queue should end, causing IO thread to end + self._smThread.stop() # stop for sure too + + self.setStatus(Status.STATUS_STOPPED) @classmethod def _stopForSure(cls, proc: Popen, sig: int): @@ -357,13 +393,13 @@ class TdeSubProcess: Logging.info("Killing sub-sub process {} with signal {}".format(child.pid, sig)) child.send_signal(sig) try: - retCode = child.wait(20) - if (- retCode) == signal.SIGSEGV: # Crashed + retCode = child.wait(20) # type: ignore + if (- retCode) == signal.SIGSEGV: # type: ignore # Crashed Logging.warning("Process {} CRASHED, please check CORE file!".format(child.pid)) - elif (- retCode) == sig : + elif (- retCode) == sig : # type: ignore Logging.info("Sub-sub process terminated with expected return code {}".format(sig)) else: - Logging.warning("Process terminated, EXPECTING ret code {}, got {}".format(sig, -retCode)) + Logging.warning("Process terminated, EXPECTING ret code {}, got {}".format(sig, -retCode)) # type: ignore return True # terminated successfully except psutil.TimeoutExpired as err: Logging.warning("Failed to kill sub-sub process {} with signal {}".format(child.pid, sig)) @@ -408,6 +444,15 @@ class TdeSubProcess: return raise CrashGenError("Failed to stop process, pid={}".format(pid)) + def getStatus(self): + return self._smThread.getStatus() + + def setStatus(self, status): + self._smThread.setStatus(status) + + def procIpcBatch(self, trimToTarget=0, forceOutput=False): + self._smThread.procIpcBatch(trimToTarget, forceOutput) + class ServiceManager: PAUSE_BETWEEN_IPC_CHECK = 1.2 # seconds between checks on STDOUT of sub process @@ -504,10 +549,10 @@ class ServiceManager: def isActive(self): """ Determine if the service/cluster is active at all, i.e. at least - one thread is not "stopped". + one instance is active """ for ti in self._tInsts: - if not ti.getStatus().isStopped(): + if ti.getStatus().isActive(): return True return False @@ -545,10 +590,10 @@ class ServiceManager: # while self.isRunning() or self.isRestarting() : # for as long as the svc mgr thread is still here status = ti.getStatus() if status.isRunning(): - th = ti.getSmThread() - th.procIpcBatch() # regular processing, + # th = ti.getSmThread() + ti.procIpcBatch() # regular processing, if status.isStopped(): - th.procIpcBatch() # one last time? + ti.procIpcBatch() # one last time? # self._updateThreadStatus() time.sleep(self.PAUSE_BETWEEN_IPC_CHECK) # pause, before next round @@ -578,7 +623,8 @@ class ServiceManager: if not ti.isFirst(): tFirst = self._getFirstInstance() tFirst.createDnode(ti.getDbTarget()) - ti.getSmThread().procIpcBatch(trimToTarget=10, forceOutput=True) # for printing 10 lines + ti.printFirst10Lines() + # ti.getSmThread().procIpcBatch(trimToTarget=10, forceOutput=True) # for printing 10 lines def stopTaosServices(self): with self._lock: @@ -624,21 +670,24 @@ class ServiceManagerThread: """ MAX_QUEUE_SIZE = 10000 - def __init__(self): + def __init__(self, subProc: TdeSubProcess, logDir: str): # Set the sub process - self._tdeSubProcess = None # type: TdeSubProcess + # self._tdeSubProcess = None # type: TdeSubProcess # Arrange the TDengine instance # self._tInstNum = tInstNum # instance serial number in cluster, ZERO based # self._tInst = tInst or TdeInstance() # Need an instance - self._thread = None # The actual thread, # type: threading.Thread - self._thread2 = None # watching stderr + # self._thread = None # type: Optional[threading.Thread] # The actual thread, # type: threading.Thread + # self._thread2 = None # type: Optional[threading.Thread] Thread # watching stderr self._status = Status(Status.STATUS_STOPPED) # The status of the underlying service, actually. + self._start(subProc, logDir) + def __repr__(self): - return "[SvcMgrThread: status={}, subProc={}]".format( - self.getStatus(), self._tdeSubProcess) + raise CrashGenError("SMT status moved to TdeSubProcess") + # return "[SvcMgrThread: status={}, subProc={}]".format( + # self.getStatus(), self._tdeSubProcess) def getStatus(self): ''' @@ -646,29 +695,33 @@ class ServiceManagerThread: ''' return self._status + def setStatus(self, statusVal: int): + self._status.set(statusVal) + # Start the thread (with sub process), and wait for the sub service # to become fully operational - def start(self, cmdLine : str, logDir: str): + def _start(self, subProc :TdeSubProcess, logDir: str): ''' Request the manager thread to start a new sub process, and manage it. :param cmdLine: the command line to invoke :param logDir: the logging directory, to hold stdout/stderr files ''' - if self._thread: - raise RuntimeError("Unexpected _thread") - if self._tdeSubProcess: - raise RuntimeError("TDengine sub process already created/running") + # if self._thread: + # raise RuntimeError("Unexpected _thread") + # if self._tdeSubProcess: + # raise RuntimeError("TDengine sub process already created/running") - Logging.info("Attempting to start TAOS service: {}".format(self)) + # Moved to TdeSubProcess + # Logging.info("Attempting to start TAOS service: {}".format(self)) self._status.set(Status.STATUS_STARTING) - self._tdeSubProcess = TdeSubProcess.start(cmdLine) # TODO: verify process is running + # self._tdeSubProcess = TdeSubProcess.start(cmdLine) # TODO: verify process is running self._ipcQueue = Queue() # type: Queue self._thread = threading.Thread( # First thread captures server OUTPUT target=self.svcOutputReader, - args=(self._tdeSubProcess.getStdOut(), self._ipcQueue, logDir)) + args=(subProc.getStdOut(), self._ipcQueue, logDir)) self._thread.daemon = True # thread dies with the program self._thread.start() time.sleep(0.01) @@ -680,7 +733,7 @@ class ServiceManagerThread: self._thread2 = threading.Thread( # 2nd thread captures server ERRORs target=self.svcErrorReader, - args=(self._tdeSubProcess.getStdErr(), self._ipcQueue, logDir)) + args=(subProc.getStdErr(), self._ipcQueue, logDir)) self._thread2.daemon = True # thread dies with the program self._thread2.start() time.sleep(0.01) @@ -695,14 +748,14 @@ class ServiceManagerThread: Progress.emit(Progress.SERVICE_START_NAP) # print("_zz_", end="", flush=True) if self._status.isRunning(): - Logging.info("[] TDengine service READY to process requests") - Logging.info("[] TAOS service started: {}".format(self)) + Logging.info("[] TDengine service READY to process requests: pid={}".format(subProc.getPid())) + # Logging.info("[] TAOS service started: {}".format(self)) # self._verifyDnode(self._tInst) # query and ensure dnode is ready # Logging.debug("[] TAOS Dnode verified: {}".format(self)) return # now we've started # TODO: handle failure-to-start better? self.procIpcBatch(100, True) # display output before cronking out, trim to last 20 msgs, force output - raise RuntimeError("TDengine service did not start successfully: {}".format(self)) + raise RuntimeError("TDengine service DID NOT achieve READY status: pid={}".format(subProc.getPid())) def _verifyDnode(self, tInst: TdeInstance): dbc = DbConn.createNative(tInst.getDbTarget()) @@ -722,29 +775,23 @@ class ServiceManagerThread: break if not isValid: print("Failed to start dnode, sleep for a while") - time.sleep(600) + time.sleep(10.0) raise RuntimeError("Failed to start Dnode, expected port not found: {}". format(tInst.getPort())) dbc.close() def stop(self): # can be called from both main thread or signal handler - Logging.info("Terminating TDengine service running as the sub process...") - if self.getStatus().isStopped(): - Logging.info("Service already stopped") - return - if self.getStatus().isStopping(): - Logging.info("Service is already being stopped, pid: {}".format(self._tdeSubProcess.getPid())) - return + # Linux will send Control-C generated SIGINT to the TDengine process # already, ref: # https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes - if not self._tdeSubProcess: - raise RuntimeError("sub process object missing") + # if not self._tdeSubProcess: + # raise RuntimeError("sub process object missing") - self._status.set(Status.STATUS_STOPPING) - TdeSubProcess.stop(self._tdeSubProcess) # must stop, no matter what - self._tdeSubProcess = None + # self._status.set(Status.STATUS_STOPPING) + # TdeSubProcess.stop(self._tdeSubProcess) # must stop, no matter what + # self._tdeSubProcess = None # if not self._tdeSubProcess.stop(): # everything withing # if self._tdeSubProcess.isRunning(): # still running, should now never happen # Logging.error("FAILED to stop sub process, it is still running... pid = {}".format( @@ -757,29 +804,28 @@ class ServiceManagerThread: outputLines = 10 # for last output if self.getStatus().isStopped(): self.procIpcBatch(outputLines) # one last time - Logging.debug("End of TDengine Service Output: {}".format(self)) + Logging.debug("End of TDengine Service Output") Logging.info("----- TDengine Service (managed by SMT) is now terminated -----\n") else: - print("WARNING: SMT did not terminate as expected: {}".format(self)) + print("WARNING: SMT did not terminate as expected") def join(self): # TODO: sanity check - if not self.getStatus().isStopping(): + s = self.getStatus() + if s.isStopping() or s.isStopped(): # we may be stopping ourselves, or have been stopped/killed by others + if self._thread or self._thread2 : + if self._thread: + self._thread.join() + self._thread = None + if self._thread2: # STD ERR thread + self._thread2.join() + self._thread2 = None + else: + Logging.warning("Joining empty thread, doing nothing") + else: raise RuntimeError( "SMT.Join(): Unexpected status: {}".format(self._status)) - if self._thread or self._thread2 : - if self._thread: - self._thread.join() - self._thread = None - if self._thread2: # STD ERR thread - self._thread2.join() - self._thread2 = None - else: - print("Joining empty thread, doing nothing") - - self._status.set(Status.STATUS_STOPPED) - def _trimQueue(self, targetSize): if targetSize <= 0: return # do nothing @@ -798,6 +844,10 @@ class ServiceManagerThread: TD_READY_MSG = "TDengine is initialized successfully" def procIpcBatch(self, trimToTarget=0, forceOutput=False): + ''' + Process a batch of STDOUT/STDERR data, until we read EMPTY from + the pipe. + ''' self._trimQueue(trimToTarget) # trim if necessary # Process all the output generated by the underlying sub process, # managed by IO thread @@ -887,7 +937,8 @@ class ServiceManagerThread: # queue.put(line) # meaning sub process must have died - Logging.info("EOF for TDengine STDOUT: {}".format(self)) + Logging.info("EOF found TDengine STDOUT, marking the process as terminated") + self.setStatus(Status.STATUS_STOPPED) out.close() # Close the stream fOut.close() # Close the output file @@ -898,6 +949,6 @@ class ServiceManagerThread: for line in iter(err.readline, b''): fErr.write(line) Logging.info("TDengine STDERR: {}".format(line)) - Logging.info("EOF for TDengine STDERR: {}".format(self)) + Logging.info("EOF for TDengine STDERR") err.close() fErr.close() \ No newline at end of file diff --git a/tests/pytest/crash_gen/settings.py b/tests/pytest/crash_gen/settings.py index ae0132378d..82214930b4 100644 --- a/tests/pytest/crash_gen/settings.py +++ b/tests/pytest/crash_gen/settings.py @@ -1,15 +1,29 @@ from __future__ import annotations import argparse +from typing import Optional -gConfig: argparse.Namespace +from crash_gen.misc import CrashGenError + +# gConfig: Optional[argparse.Namespace] class Settings: + _config = None # type Optional[argparse.Namespace] + @classmethod def init(cls): - global gConfig - gConfig = [] + cls._config = None @classmethod - def setConfig(cls, config): - global gConfig - gConfig = config \ No newline at end of file + def setConfig(cls, config: argparse.Namespace): + cls._config = config + + @classmethod + # TODO: check items instead of exposing everything + def getConfig(cls) -> argparse.Namespace: + if cls._config is None: + raise CrashGenError("invalid state") + return cls._config + + @classmethod + def clearConfig(cls): + cls._config = None \ No newline at end of file diff --git a/tests/pytest/crash_gen/types.py b/tests/pytest/crash_gen/types.py new file mode 100644 index 0000000000..fdf8ea2478 --- /dev/null +++ b/tests/pytest/crash_gen/types.py @@ -0,0 +1,5 @@ +from typing import Any, List, NewType + +DirPath = NewType('DirPath', str) + +QueryResult = NewType('QueryResult', List[List[Any]]) \ No newline at end of file From 34eb7e255a021a58d7dae26bb75e9bf799fabac7 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Wed, 28 Apr 2021 23:11:19 +0000 Subject: [PATCH 003/157] Used Python generator to simplify crash_gen IPC processing of STDOUT/STDERR --- tests/pytest/crash_gen/service_manager.py | 127 +++++++++++----------- 1 file changed, 66 insertions(+), 61 deletions(-) diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py index 4afd7e4c78..f6784f17e3 100644 --- a/tests/pytest/crash_gen/service_manager.py +++ b/tests/pytest/crash_gen/service_manager.py @@ -9,7 +9,8 @@ import signal import logging import time from subprocess import PIPE, Popen, TimeoutExpired -from typing import IO, List, NewType, Optional +from typing import BinaryIO, Generator, IO, List, NewType, Optional +import typing try: import psutil @@ -275,11 +276,16 @@ class TdeSubProcess: return '[TdeSubProc: pid = {}, status = {}]'.format( self.getPid(), self.getStatus() ) - def getStdOut(self): - return self._popen.stdout + def getStdOut(self) -> BinaryIO : + if self._popen.universal_newlines : # alias of text_mode + raise CrashGenError("We need binary mode for STDOUT IPC") + # Logging.info("Type of stdout is: {}".format(type(self._popen.stdout))) + return typing.cast(BinaryIO, self._popen.stdout) - def getStdErr(self): - return self._popen.stderr + def getStdErr(self) -> BinaryIO : + if self._popen.universal_newlines : # alias of text_mode + raise CrashGenError("We need binary mode for STDERR IPC") + return typing.cast(BinaryIO, self._popen.stderr) # Now it's always running, since we matched the life cycle # def isRunning(self): @@ -846,7 +852,7 @@ class ServiceManagerThread: def procIpcBatch(self, trimToTarget=0, forceOutput=False): ''' Process a batch of STDOUT/STDERR data, until we read EMPTY from - the pipe. + the queue. ''' self._trimQueue(trimToTarget) # trim if necessary # Process all the output generated by the underlying sub process, @@ -876,79 +882,78 @@ class ServiceManagerThread: print(pBar, end="", flush=True) print('\b\b\b\b', end="", flush=True) - BinaryLine = NewType('BinaryLine', bytes) # line with binary data, directly from STDOUT, etc. - TextLine = NewType('TextLine', str) # properly decoded, suitable for printing, etc. - x = TextLine('xyz') - + BinaryChunk = NewType('BinaryChunk', bytes) # line with binary data, directly from STDOUT, etc. + TextChunk = NewType('TextChunk', str) # properly decoded, suitable for printing, etc. + @classmethod - def _decodeBinLine(cls, bLine: BinaryLine) -> Optional[TextLine] : + def _decodeBinaryChunk(cls, bChunk: bytes) -> Optional[TextChunk] : try: - tLine = bLine.decode("utf-8").rstrip() - return cls.TextLine(tLine) + tChunk = bChunk.decode("utf-8").rstrip() + return cls.TextChunk(tChunk) except UnicodeError: - print("\nNon-UTF8 server output: {}\n".format(bLine.decode('cp437'))) + print("\nNon-UTF8 server output: {}\n".format(bChunk.decode('cp437'))) return None - def svcOutputReader(self, out: IO, queue, logDir: str): + def _textChunkGenerator(self, streamIn: BinaryIO, logDir: str, logFile: str + ) -> Generator[TextChunk, None, None]: + ''' + Take an input stream with binary data, produced a generator of decoded + "text chunks", and also save the original binary data in a log file. + ''' + os.makedirs(logDir, exist_ok=True) + logF = open(os.path.join(logDir, logFile), 'wb') + for bChunk in iter(streamIn.readline, b''): + logF.write(bChunk) # Write to log file immediately + tChunk = self._decodeBinaryChunk(bChunk) # decode + if tChunk is not None: + yield tChunk # TODO: split into actual text lines + + # At the end... + streamIn.close() # Close the stream + logF.close() # Close the output file + + def svcOutputReader(self, stdOut: BinaryIO, queue, logDir: str): ''' The infinite routine that processes the STDOUT stream for the sub process being managed. - :param out: the IO stream object used to fetch the data from - :param queue: the queue where we dump the roughly parsed line-by-line data + :param stdOut: the IO stream object used to fetch the data from + :param queue: the queue where we dump the roughly parsed chunk-by-chunk text data :param logDir: where we should dump a verbatim output file ''' - os.makedirs(logDir, exist_ok=True) - logFile = os.path.join(logDir,'stdout.log') - fOut = open(logFile, 'wb') + # Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python # print("This is the svcOutput Reader...") - # for line in out : - out.readline() - for bLine in iter(out.readline, b''): - fOut.write(bLine) - # print("Finished reading a line: {}".format(line)) - # print("Adding item to queue...") + # stdOut.readline() # Skip the first output? TODO: remove? + for tChunk in self._textChunkGenerator(stdOut, logDir, 'stdout.log') : + queue.put(tChunk) # tChunk garanteed not to be None + self._printProgress("_i") - # Moved to above - # try: - # line = line.decode("utf-8").rstrip() - # except UnicodeError: - # print("\nNon-UTF8 server output: {}\n".format(line)) - tLine = self._decodeBinLine(bLine) + if self._status.isStarting(): # we are starting, let's see if we have started + if tChunk.find(self.TD_READY_MSG) != -1: # found + Logging.info("Waiting for the service to become FULLY READY") + time.sleep(1.0) # wait for the server to truly start. TODO: remove this + Logging.info("Service is now FULLY READY") # TODO: more ID info here? + self._status.set(Status.STATUS_RUNNING) - if tLine is not None: - # This might block, and then causing "out" buffer to block - queue.put(tLine) - self._printProgress("_i") + # Trim the queue if necessary: TODO: try this 1 out of 10 times + self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10) # trim to 90% size - if self._status.isStarting(): # we are starting, let's see if we have started - if tLine.find(self.TD_READY_MSG) != -1: # found - Logging.info("Waiting for the service to become FULLY READY") - time.sleep(1.0) # wait for the server to truly start. TODO: remove this - Logging.info("Service is now FULLY READY") # TODO: more ID info here? - self._status.set(Status.STATUS_RUNNING) - - # Trim the queue if necessary: TODO: try this 1 out of 10 times - self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10) # trim to 90% size - - if self._status.isStopping(): # TODO: use thread status instead - # WAITING for stopping sub process to finish its outptu - print("_w", end="", flush=True) + if self._status.isStopping(): # TODO: use thread status instead + # WAITING for stopping sub process to finish its outptu + print("_w", end="", flush=True) # queue.put(line) - # meaning sub process must have died + # stdOut has no more data, meaning sub process must have died Logging.info("EOF found TDengine STDOUT, marking the process as terminated") self.setStatus(Status.STATUS_STOPPED) - out.close() # Close the stream - fOut.close() # Close the output file - def svcErrorReader(self, err: IO, queue, logDir: str): - os.makedirs(logDir, exist_ok=True) - logFile = os.path.join(logDir,'stderr.log') - fErr = open(logFile, 'wb') - for line in iter(err.readline, b''): - fErr.write(line) - Logging.info("TDengine STDERR: {}".format(line)) + def svcErrorReader(self, stdErr: BinaryIO, queue, logDir: str): + # os.makedirs(logDir, exist_ok=True) + # logFile = os.path.join(logDir,'stderr.log') + # fErr = open(logFile, 'wb') + # for line in iter(err.readline, b''): + for tChunk in self._textChunkGenerator(stdErr, logDir, 'stderr.log') : + queue.put(tChunk) # tChunk garanteed not to be None + # fErr.write(line) + Logging.info("TDengine STDERR: {}".format(tChunk)) Logging.info("EOF for TDengine STDERR") - err.close() - fErr.close() \ No newline at end of file From 073d7b1f0841efbfd8c2098f0a25ad625cb3947f Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 29 Apr 2021 08:40:28 +0000 Subject: [PATCH 004/157] [TD-3963]tsdbRepo config hot change --- src/tsdb/inc/tsdbint.h | 5 +++ src/tsdb/src/tsdbCommitQueue.c | 25 ++++++++++++ src/tsdb/src/tsdbMain.c | 72 ++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+) diff --git a/src/tsdb/inc/tsdbint.h b/src/tsdb/inc/tsdbint.h index 074ff20f22..4d62164df9 100644 --- a/src/tsdb/inc/tsdbint.h +++ b/src/tsdb/inc/tsdbint.h @@ -71,6 +71,11 @@ struct STsdbRepo { uint8_t state; STsdbCfg config; + + STsdbCfg save_config; // save apply config + bool config_changed; // config changed flag + pthread_mutex_t save_mutex; // protect save config + STsdbAppH appH; STsdbStat stat; STsdbMeta* tsdbMeta; diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c index 9e8e4acd7e..7a1b225849 100644 --- a/src/tsdb/src/tsdbCommitQueue.c +++ b/src/tsdb/src/tsdbCommitQueue.c @@ -112,6 +112,24 @@ int tsdbScheduleCommit(STsdbRepo *pRepo) { return 0; } +static void tsdbApplyRepoConfig(STsdbRepo *pRepo) { + pRepo->config_changed = false; + STsdbCfg * pSaveCfg = &pRepo->save_config; + + pRepo->config.compression = pRepo->save_config.compression; + pRepo->config.keep = pRepo->save_config.keep; + pRepo->config.keep1 = pRepo->save_config.keep1; + pRepo->config.keep2 = pRepo->save_config.keep2; + pRepo->config.cacheLastRow = pRepo->save_config.cacheLastRow; + pRepo->config.update = pRepo->save_config.update; + + tsdbInfo("vgId:%d apply new config: compression(%d), keep(%d,%d,%d), totalBlocks(%d), cacheLastRow(%d), update(%d)", + REPO_ID(pRepo), + pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, + pSaveCfg->totalBlocks, pSaveCfg->cacheLastRow, pSaveCfg->update); + +} + static void *tsdbLoopCommit(void *arg) { SCommitQueue *pQueue = &tsCommitQueue; SListNode * pNode = NULL; @@ -138,6 +156,13 @@ static void *tsdbLoopCommit(void *arg) { pRepo = ((SCommitReq *)pNode->data)->pRepo; + // check if need to apply new config + if (pRepo->config_changed) { + pthread_mutex_lock(&pRepo->save_mutex); + tsdbApplyRepoConfig(pRepo); + pthread_mutex_unlock(&pRepo->save_mutex); + } + tsdbCommitData(pRepo); listNodeFree(pNode); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 99929f3542..16804c7cf9 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -203,6 +203,70 @@ void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { // TODO: think about multithread cases + if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; + + STsdbCfg * pRCfg = &repo->config; + + ASSERT(pRCfg->tsdbId == pCfg->tsdbId); + ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); + ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); + ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); + ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); + ASSERT(pRCfg->precision == pCfg->precision); + + bool configChanged = false; + if (pRCfg->compression != pCfg->compression) { + configChanged = true; + } + if (pRCfg->keep != pCfg->keep) { + configChanged = true; + } + if (pRCfg->keep1 != pCfg->keep1) { + configChanged = true; + } + if (pRCfg->keep2 != pCfg->keep2) { + configChanged = true; + } + if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { + configChanged = true; + } + if (pRCfg->update != pCfg->update) { + configChanged = true; + } + + if (!configChanged) { + tsdbError("vgId:%d no config changed", REPO_ID(repo)); + } + + int code = pthread_mutex_lock(&repo->save_mutex); + if (code != 0) { + tsdbError("vgId:%d failed to lock tsdb save config mutex since %s", REPO_ID(repo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + + STsdbCfg * pSaveCfg = &repo->save_config; + *pSaveCfg = repo->config; + + pSaveCfg->compression = pCfg->compression; + pSaveCfg->keep = pCfg->keep; + pSaveCfg->keep1 = pCfg->keep1; + pSaveCfg->keep2 = pCfg->keep2; + pSaveCfg->cacheLastRow = pCfg->cacheLastRow; + pSaveCfg->update = pCfg->update; + + tsdbInfo("vgId:%d old config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d), update(%d)", + REPO_ID(repo), + pRCfg->compression, pRCfg->keep, pRCfg->keep1,pRCfg->keep2, + pRCfg->cacheLastRow, pRCfg->update); + tsdbInfo("vgId:%d new config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d), update(%d)", + REPO_ID(repo), + pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, + pSaveCfg->cacheLastRow, pSaveCfg->update); + + repo->config_changed = true; + + pthread_mutex_unlock(&repo->save_mutex); return 0; #if 0 STsdbRepo *pRepo = (STsdbRepo *)repo; @@ -474,6 +538,14 @@ static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { return NULL; } + code = pthread_mutex_init(&(pRepo->save_mutex), NULL); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + tsdbFreeRepo(pRepo); + return NULL; + } + pRepo->config_changed = false; + code = tsem_init(&(pRepo->readyToCommit), 0, 1); if (code != 0) { code = errno; From 2742159fde19020be9dec4d28b94aecaa4ffbdf7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 29 Apr 2021 23:16:33 +0800 Subject: [PATCH 005/157] [td-3967]1). fix escape char error in sql. 2) refactor. --- src/client/inc/tschemautil.h | 2 +- src/client/src/tscParseInsert.c | 33 ++++++++++++------- src/client/src/tscPrepare.c | 2 +- src/client/src/tscSQLParser.c | 14 ++++---- src/client/src/tscSql.c | 2 +- src/client/src/tscUtil.c | 8 ++--- src/client/tests/timeParseTest.cpp | 2 +- src/common/inc/tname.h | 2 +- src/common/inc/tvariant.h | 2 +- src/common/src/tname.c | 2 +- src/common/src/tvariant.c | 14 ++++---- src/os/tests/test.cpp | 2 +- src/query/inc/qSqlparser.h | 2 +- src/query/src/qSqlParser.c | 6 ++-- src/query/src/sql.c | 6 ++-- src/query/tests/tsBufTest.cpp | 4 +-- src/query/tests/unitTest.cpp | 14 ++++---- src/util/inc/{tstoken.h => ttoken.h} | 29 ++++++++-------- .../qTokenizer.c => util/src/ttokenizer.c} | 32 +++++++++++++----- 19 files changed, 102 insertions(+), 76 deletions(-) rename src/util/inc/{tstoken.h => ttoken.h} (93%) rename src/{query/src/qTokenizer.c => util/src/ttokenizer.c} (92%) diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h index a9dcd230a6..0026a27e19 100644 --- a/src/client/inc/tschemautil.h +++ b/src/client/inc/tschemautil.h @@ -21,8 +21,8 @@ extern "C" { #endif #include "taosmsg.h" -#include "tstoken.h" #include "tsclient.h" +#include "ttoken.h" /** * get the number of tags of this table diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 79ceb81cbc..a9d8afd09e 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -29,8 +29,7 @@ #include "taosdef.h" #include "tscLog.h" -#include "tscSubquery.h" -#include "tstoken.h" +#include "ttoken.h" #include "tdataformat.h" @@ -463,23 +462,33 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1 // Remove quotation marks if (TK_STRING == sToken.type) { // delete escape character: \\, \', \" - char delim = sToken.z[0]; +// char delim = sToken.z[0]; int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < sToken.n - 1; ++k) { - if (sToken.z[k] == delim || sToken.z[k] == '\\') { - if (sToken.z[k + 1] == delim) { - cnt++; - tmpTokenBuf[j] = sToken.z[k + 1]; - j++; - k++; - continue; - } + if (sToken.z[k] == '\\') { + cnt++; + + tmpTokenBuf[j] = GET_ESCAPE_CHAR(sToken.z[k+1]); + j++; + k++; + continue; } +// if (sToken.z[k] == delim || sToken.z[k] == '\\') { +// if (sToken.z[k + 1] == delim) { +// cnt++; +// tmpTokenBuf[j] = sToken.z[k + 1]; +// j++;s +// k++; +// continue; +// } +// } + tmpTokenBuf[j] = sToken.z[k]; j++; } + tmpTokenBuf[j] = 0; sToken.z = tmpTokenBuf; sToken.n -= 2 + cnt; @@ -1005,7 +1014,7 @@ int validateTableName(char *tblName, int len, SStrToken* psTblToken) { psTblToken->n = len; psTblToken->type = TK_ID; - tSQLGetToken(psTblToken->z, &psTblToken->type); + tGetToken(psTblToken->z, &psTblToken->type); return tscValidateName(psTblToken); } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 4efaf7c2b5..73e39b471b 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -151,7 +151,7 @@ static int normalStmtPrepare(STscStmt* stmt) { while (sql[i] != 0) { SStrToken token = {0}; - token.n = tSQLGetToken(sql + i, &token.type); + token.n = tGetToken(sql + i, &token.type); if (token.type == TK_QUESTION) { sql[i] = 0; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4cf1423c43..e33b8ef33a 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -21,19 +21,19 @@ #endif // __APPLE__ #include "os.h" -#include "ttype.h" -#include "texpr.h" #include "taos.h" #include "taosmsg.h" #include "tcompare.h" +#include "texpr.h" #include "tname.h" #include "tscLog.h" #include "tscUtil.h" #include "tschemautil.h" #include "tsclient.h" -#include "tstoken.h" #include "tstrbuild.h" +#include "ttoken.h" #include "ttokendef.h" +#include "ttype.h" #include "qUtil.h" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" @@ -4629,7 +4629,7 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t } } else { SStrToken token = {.z = pRight->value.pz, .n = pRight->value.nLen, .type = TK_ID}; - int32_t len = tSQLGetToken(pRight->value.pz, &token.type); + int32_t len = tGetToken(pRight->value.pz, &token.type); if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->value.nLen) { return TSDB_CODE_TSC_INVALID_SQL; @@ -5498,13 +5498,13 @@ int32_t validateLocalConfig(SMiscInfo* pOptions) { } int32_t validateColumnName(char* name) { - bool ret = isKeyWord(name, (int32_t)strlen(name)); + bool ret = taosIsKeyWordToken(name, (int32_t)strlen(name)); if (ret) { return TSDB_CODE_TSC_INVALID_SQL; } SStrToken token = {.z = name}; - token.n = tSQLGetToken(name, &token.type); + token.n = tGetToken(name, &token.type); if (token.type != TK_STRING && token.type != TK_ID) { return TSDB_CODE_TSC_INVALID_SQL; @@ -5515,7 +5515,7 @@ int32_t validateColumnName(char* name) { strntolower(token.z, token.z, token.n); token.n = (uint32_t)strtrim(token.z); - int32_t k = tSQLGetToken(token.z, &token.type); + int32_t k = tGetToken(token.z, &token.type); if (k != token.n) { return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 36d5cec06f..02cd2bd5ef 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -962,7 +962,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t len = (int32_t)strtrim(tblName); SStrToken sToken = {.n = len, .type = TK_ID, .z = tblName}; - tSQLGetToken(tblName, &sToken.type); + tGetToken(tblName, &sToken.type); // Check if the table name available or not if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index e8d7d5d03c..a9826f2fdf 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1519,7 +1519,7 @@ void tscColumnListDestroy(SArray* pColumnList) { static int32_t validateQuoteToken(SStrToken* pToken) { tscDequoteAndTrimToken(pToken); - int32_t k = tSQLGetToken(pToken->z, &pToken->type); + int32_t k = tGetToken(pToken->z, &pToken->type); if (pToken->type == TK_STRING) { return tscValidateName(pToken); @@ -1587,7 +1587,7 @@ int32_t tscValidateName(SStrToken* pToken) { tscStrToLower(pToken->z, pToken->n); //pToken->n = (uint32_t)strtrim(pToken->z); - int len = tSQLGetToken(pToken->z, &pToken->type); + int len = tGetToken(pToken->z, &pToken->type); // single token, validate it if (len == pToken->n) { @@ -1613,7 +1613,7 @@ int32_t tscValidateName(SStrToken* pToken) { pToken->n = (uint32_t)strtrim(pToken->z); } - pToken->n = tSQLGetToken(pToken->z, &pToken->type); + pToken->n = tGetToken(pToken->z, &pToken->type); if (pToken->z[pToken->n] != TS_PATH_DELIMITER[0]) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -1630,7 +1630,7 @@ int32_t tscValidateName(SStrToken* pToken) { pToken->z = sep + 1; pToken->n = (uint32_t)(oldLen - (sep - pStr) - 1); - int32_t len = tSQLGetToken(pToken->z, &pToken->type); + int32_t len = tGetToken(pToken->z, &pToken->type); if (len != pToken->n || (pToken->type != TK_STRING && pToken->type != TK_ID)) { return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/client/tests/timeParseTest.cpp b/src/client/tests/timeParseTest.cpp index d7325430cd..ba06a6b9aa 100644 --- a/src/client/tests/timeParseTest.cpp +++ b/src/client/tests/timeParseTest.cpp @@ -4,7 +4,7 @@ #include #include "taos.h" -#include "tstoken.h" +#include "ttoken.h" #include "tutil.h" int main(int argc, char** argv) { diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 465b298973..cacd6d2ae7 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -18,7 +18,7 @@ #include "os.h" #include "taosmsg.h" -#include "tstoken.h" +#include "ttoken.h" #include "tvariant.h" typedef struct SDataStatis { diff --git a/src/common/inc/tvariant.h b/src/common/inc/tvariant.h index f8f715c6ca..21b7fd8223 100644 --- a/src/common/inc/tvariant.h +++ b/src/common/inc/tvariant.h @@ -16,8 +16,8 @@ #ifndef TDENGINE_TVARIANT_H #define TDENGINE_TVARIANT_H -#include "tstoken.h" #include "tarray.h" +#include "ttoken.h" #ifdef __cplusplus extern "C" { diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 787aa1e95b..65725455e8 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -2,7 +2,7 @@ #include "tutil.h" #include "tname.h" -#include "tstoken.h" +#include "ttoken.h" #include "tvariant.h" #define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index c872d8731b..9988450c30 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -14,14 +14,14 @@ */ #include "os.h" -#include "tvariant.h" #include "hash.h" #include "taos.h" #include "taosdef.h" -#include "tstoken.h" +#include "ttoken.h" #include "ttokendef.h" -#include "tutil.h" #include "ttype.h" +#include "tutil.h" +#include "tvariant.h" void tVariantCreate(tVariant *pVar, SStrToken *token) { int32_t ret = 0; @@ -49,7 +49,7 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) { ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, true); if (ret != 0) { SStrToken t = {0}; - tSQLGetToken(token->z, &t.type); + tGetToken(token->z, &t.type); if (t.type == TK_MINUS) { // it is a signed number which is greater than INT64_MAX or less than INT64_MIN pVar->nType = -1; // -1 means error type return; @@ -460,7 +460,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result *result = (int64_t) pVariant->dKey; } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { SStrToken token = {.z = pVariant->pz, .n = pVariant->nLen}; - /*int32_t n = */tSQLGetToken(pVariant->pz, &token.type); + /*int32_t n = */tGetToken(pVariant->pz, &token.type); if (token.type == TK_NULL) { if (releaseVariantPtr) { @@ -495,10 +495,10 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result wchar_t *endPtr = NULL; SStrToken token = {0}; - token.n = tSQLGetToken(pVariant->pz, &token.type); + token.n = tGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { - token.n = tSQLGetToken(pVariant->pz + token.n, &token.type); + token.n = tGetToken(pVariant->pz + token.n, &token.type); } if (token.type == TK_FLOAT) { diff --git a/src/os/tests/test.cpp b/src/os/tests/test.cpp index 600e5d71a7..bdd3114363 100644 --- a/src/os/tests/test.cpp +++ b/src/os/tests/test.cpp @@ -4,7 +4,7 @@ #include #include "taos.h" -#include "tstoken.h" +#include "ttoken.h" #include "tutil.h" int main(int argc, char** argv) { diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index 0a0587f701..70ba48e551 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -22,8 +22,8 @@ extern "C" { #include "taos.h" #include "taosmsg.h" -#include "tstoken.h" #include "tstrbuild.h" +#include "ttoken.h" #include "tvariant.h" #define ParseTOKENTYPE SStrToken diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index 6b38536b15..56d395e498 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -13,13 +13,13 @@ * along with this program. If not, see . */ -#include "os.h" #include "qSqlparser.h" +#include "os.h" #include "taosdef.h" #include "taosmsg.h" #include "tcmdtype.h" -#include "tstoken.h" #include "tstrbuild.h" +#include "ttoken.h" #include "ttokendef.h" #include "tutil.h" @@ -38,7 +38,7 @@ SSqlInfo qSqlParse(const char *pStr) { goto abort_parse; } - t0.n = tSQLGetToken((char *)&pStr[i], &t0.type); + t0.n = tGetToken((char *)&pStr[i], &t0.type); t0.z = (char *)(pStr + i); i += t0.n; diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 96d33a8ed6..8f36638dc9 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -26,14 +26,14 @@ #include /************ Begin %include sections from the grammar ************************/ +#include +#include #include #include #include -#include -#include #include "qSqlparser.h" #include "tcmdtype.h" -#include "tstoken.h" +#include "ttoken.h" #include "ttokendef.h" #include "tutil.h" #include "tvariant.h" diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index 8ca636b834..dd7f03a494 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -3,10 +3,10 @@ #include #include +#include "qTsbuf.h" #include "taos.h" #include "tsdb.h" -#include "qTsbuf.h" -#include "tstoken.h" +#include "ttoken.h" #include "tutil.h" namespace { diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index 3406d83090..d2b058cf7c 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -21,7 +21,7 @@ int32_t testValidateName(char* name) { token.n = strlen(name); token.type = 0; - tSQLGetToken(name, &token.type); + tGetToken(name, &token.type); return tscValidateName(&token); } } @@ -691,32 +691,32 @@ TEST(testCase, tGetToken_Test) { char* s = ".123 "; uint32_t type = 0; - int32_t len = tSQLGetToken(s, &type); + int32_t len = tGetToken(s, &type); EXPECT_EQ(type, TK_FLOAT); EXPECT_EQ(len, strlen(s) - 1); char s1[] = "1.123e10 "; - len = tSQLGetToken(s1, &type); + len = tGetToken(s1, &type); EXPECT_EQ(type, TK_FLOAT); EXPECT_EQ(len, strlen(s1) - 1); char s4[] = "0xff "; - len = tSQLGetToken(s4, &type); + len = tGetToken(s4, &type); EXPECT_EQ(type, TK_HEX); EXPECT_EQ(len, strlen(s4) - 1); // invalid data type char s2[] = "e10 "; - len = tSQLGetToken(s2, &type); + len = tGetToken(s2, &type); EXPECT_FALSE(type == TK_FLOAT); char s3[] = "1.1.1.1"; - len = tSQLGetToken(s3, &type); + len = tGetToken(s3, &type); EXPECT_EQ(type, TK_IPTOKEN); EXPECT_EQ(len, strlen(s3)); char s5[] = "0x "; - len = tSQLGetToken(s5, &type); + len = tGetToken(s5, &type); EXPECT_FALSE(type == TK_HEX); } diff --git a/src/util/inc/tstoken.h b/src/util/inc/ttoken.h similarity index 93% rename from src/util/inc/tstoken.h rename to src/util/inc/ttoken.h index ab1ef7b279..c1e2170ac3 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/ttoken.h @@ -37,13 +37,25 @@ typedef struct SStrToken { char *z; } SStrToken; +extern const char escapeChar[]; + +/** + * check if it is a number or not + * @param pToken + * @return + */ +#define isNumber(tk) \ +((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN) + +#define GET_ESCAPE_CHAR(c) (escapeChar[(uint8_t)(c)]) + /** * tokenizer for sql string * @param z * @param tokenType * @return */ -uint32_t tSQLGetToken(char *z, uint32_t *tokenType); +uint32_t tGetToken(char *z, uint32_t *tokenType); /** * enhanced tokenizer for sql string. @@ -61,21 +73,12 @@ SStrToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr); * @param len * @return */ -bool isKeyWord(const char *z, int32_t len); - -/** - * check if it is a number or not - * @param pToken - * @return - */ -#define isNumber(tk) \ -((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN) - +bool taosIsKeyWordToken(const char *z, int32_t len); /** * check if it is a token or not - * @param pToken - * @return token type, if it is not a number, TK_ILLEGAL will return + * @param pToken + * @return token type, if it is not a number, TK_ILLEGAL will return */ static FORCE_INLINE int32_t tGetNumericStringType(const SStrToken* pToken) { const char* z = pToken->z; diff --git a/src/query/src/qTokenizer.c b/src/util/src/ttokenizer.c similarity index 92% rename from src/query/src/qTokenizer.c rename to src/util/src/ttokenizer.c index 7869e27707..794420d55b 100644 --- a/src/query/src/qTokenizer.c +++ b/src/util/src/ttokenizer.c @@ -18,7 +18,7 @@ #include "hash.h" #include "hashfunc.h" #include "taosdef.h" -#include "tstoken.h" +#include "ttoken.h" #include "ttokendef.h" #include "tutil.h" @@ -232,6 +232,18 @@ static const char isIdChar[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ }; +const char escapeChar[] = { + /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, /* 0x */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, /* 1x */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, /* 2x */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, /* 3x */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F,/* 4x */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,/* 5x */ + 0x60, 0x07, 0x08, 0x63, 0x64, 0x65, 0x0C, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x0A, 0x6F,/* 6x */ + 0x70, 0x71, 0x0D, 0x73, 0x09, 0x75, 0x0B, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F,/* 7x */ +}; + static void* keywordHashTable = NULL; static void doInitKeywordsTable(void) { @@ -247,7 +259,7 @@ static void doInitKeywordsTable(void) { static pthread_once_t keywordsHashTableInit = PTHREAD_ONCE_INIT; -int tSQLKeywordCode(const char* z, int n) { +static int32_t tKeywordCode(const char* z, int n) { pthread_once(&keywordsHashTableInit, doInitKeywordsTable); char key[512] = {0}; @@ -271,7 +283,7 @@ int tSQLKeywordCode(const char* z, int n) { * Return the length of the token that begins at z[0]. * Store the token type in *type before returning. */ -uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { +uint32_t tGetToken(char* z, uint32_t* tokenId) { uint32_t i; switch (*z) { case ' ': @@ -408,7 +420,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { continue; } - if (z[i] == delim ) { + if (z[i] == delim) { if (z[i + 1] == delim) { i++; } else { @@ -551,7 +563,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { } for (i = 1; ((z[i] & 0x80) == 0) && isIdChar[(uint8_t) z[i]]; i++) { } - *tokenId = tSQLKeywordCode(z, i); + *tokenId = tKeywordCode(z, i); return i; } } @@ -584,7 +596,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { t = str[++(*i)]; } - t0.n = tSQLGetToken(&str[*i], &t0.type); + t0.n = tGetToken(&str[*i], &t0.type); break; // not support user specfied ignored symbol list @@ -613,7 +625,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { // support parse the 'db.tbl' format, notes: There should be no space on either side of the dot! if ('.' == str[*i + t0.n]) { - len = tSQLGetToken(&str[*i + t0.n + 1], &type); + len = tGetToken(&str[*i + t0.n + 1], &type); // only id and string are valid if ((TK_STRING != t0.type) && (TK_ID != t0.type)) { @@ -628,7 +640,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { } else { // support parse the -/+number format if ((isPrevOptr) && (t0.type == TK_MINUS || t0.type == TK_PLUS)) { - len = tSQLGetToken(&str[*i + t0.n], &type); + len = tGetToken(&str[*i + t0.n], &type); if (type == TK_INTEGER || type == TK_FLOAT) { t0.type = type; t0.n += len; @@ -642,7 +654,9 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { return t0; } -bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); } +bool taosIsKeyWordToken(const char* z, int32_t len) { + return (tKeywordCode((char*)z, len) != TK_ID); +} void taosCleanupKeywordsTable() { void* m = keywordHashTable; From c64fd41be7f4dba7f929565acdf033a89bd07042 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Apr 2021 10:46:13 +0800 Subject: [PATCH 006/157] [td-3967]refactor. --- src/client/src/tscParseInsert.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index a9d8afd09e..35dc7bca55 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -475,16 +475,6 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1 continue; } -// if (sToken.z[k] == delim || sToken.z[k] == '\\') { -// if (sToken.z[k + 1] == delim) { -// cnt++; -// tmpTokenBuf[j] = sToken.z[k + 1]; -// j++;s -// k++; -// continue; -// } -// } - tmpTokenBuf[j] = sToken.z[k]; j++; } From bdc887741ece56dc7e1afa9789338d55b652c35c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Apr 2021 10:47:42 +0800 Subject: [PATCH 007/157] [td-3967] --- src/client/src/tscParseInsert.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 35dc7bca55..298f1245e7 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -462,7 +462,6 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1 // Remove quotation marks if (TK_STRING == sToken.type) { // delete escape character: \\, \', \" -// char delim = sToken.z[0]; int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < sToken.n - 1; ++k) { @@ -704,6 +703,8 @@ static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, STableDataBlock } code = TSDB_CODE_TSC_INVALID_SQL; + + // todo the size should be limited to the current sql length char *tmpTokenBuf = calloc(1, 16*1024); // used for deleting Escape character: \\, \', \" if (NULL == tmpTokenBuf) { return TSDB_CODE_TSC_OUT_OF_MEMORY; From d78ac9848c3efd37107b886bd1f6e1780b805d86 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Apr 2021 17:12:20 +0800 Subject: [PATCH 008/157] [td-3967]1) refactor. --- src/client/src/tscParseInsert.c | 12 +++++++++--- .../script/general/parser/binary_escapeCharacter.sim | 9 +++++++++ tests/script/general/parser/testSuite.sim | 1 + 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 298f1245e7..9130118f80 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -462,13 +462,19 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1 // Remove quotation marks if (TK_STRING == sToken.type) { // delete escape character: \\, \', \" + char delim = sToken.z[0]; + int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < sToken.n - 1; ++k) { - if (sToken.z[k] == '\\') { - cnt++; + if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) { + if (sToken.z[k] == '\\') { + tmpTokenBuf[j] = GET_ESCAPE_CHAR(sToken.z[k+1]); + } else { + tmpTokenBuf[j] = sToken.z[k + 1]; + } - tmpTokenBuf[j] = GET_ESCAPE_CHAR(sToken.z[k+1]); + cnt++; j++; k++; continue; diff --git a/tests/script/general/parser/binary_escapeCharacter.sim b/tests/script/general/parser/binary_escapeCharacter.sim index f0589d154f..1c6bc5b37f 100644 --- a/tests/script/general/parser/binary_escapeCharacter.sim +++ b/tests/script/general/parser/binary_escapeCharacter.sim @@ -93,5 +93,14 @@ if $data41 != @udp005@ then print "[ERROR] expect: udp005, act:$data41" endi +print ---------------------> TD-3967 +sql insert into tb values(now, '\\abc\\\\'); +sql insert into tb values(now, '\\abc\\\\'); +sql insert into tb values(now, '\\\\'); + +print ------------->sim bug +# sql_error insert into tb values(now, '\\\'); +# sql_error insert into tb values(now, '\\'); +# sql_error insert into tb values(now, '\\n'); system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index f05474d158..b605a04df7 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -55,4 +55,5 @@ run general/parser/sliding.sim run general/parser/function.sim run general/parser/stableOp.sim run general/parser/slimit_alter_tags.sim +run general/parser/binary_escapeCharacter.sim From b0ab4573bba52b91df50edde52819ecb7e6c1468 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Apr 2021 17:14:57 +0800 Subject: [PATCH 009/157] [td-3967]update the sim. --- tests/script/general/parser/binary_escapeCharacter.sim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/script/general/parser/binary_escapeCharacter.sim b/tests/script/general/parser/binary_escapeCharacter.sim index 1c6bc5b37f..b5bb10284b 100644 --- a/tests/script/general/parser/binary_escapeCharacter.sim +++ b/tests/script/general/parser/binary_escapeCharacter.sim @@ -100,7 +100,8 @@ sql insert into tb values(now, '\\\\'); print ------------->sim bug # sql_error insert into tb values(now, '\\\'); -# sql_error insert into tb values(now, '\\'); -# sql_error insert into tb values(now, '\\n'); +sql_error insert into tb values(now, '\'); +#sql_error insert into tb values(now, '\\\n'); +sql insert into tb values(now, '\n'); system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 968917f31264d380d4d2d6e6dd9f3607bf5333ab Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 6 May 2021 09:57:22 +0800 Subject: [PATCH 010/157] [TD-3963]remove change update config --- src/tsdb/src/tsdbCommitQueue.c | 6 ++--- src/tsdb/src/tsdbMain.c | 12 ++++------ src/vnode/src/vnodeMain.c | 40 ++++++++++++++++++---------------- 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c index 7a1b225849..2e2cc74159 100644 --- a/src/tsdb/src/tsdbCommitQueue.c +++ b/src/tsdb/src/tsdbCommitQueue.c @@ -121,12 +121,12 @@ static void tsdbApplyRepoConfig(STsdbRepo *pRepo) { pRepo->config.keep1 = pRepo->save_config.keep1; pRepo->config.keep2 = pRepo->save_config.keep2; pRepo->config.cacheLastRow = pRepo->save_config.cacheLastRow; - pRepo->config.update = pRepo->save_config.update; + //pRepo->config.update = pRepo->save_config.update; - tsdbInfo("vgId:%d apply new config: compression(%d), keep(%d,%d,%d), totalBlocks(%d), cacheLastRow(%d), update(%d)", + tsdbInfo("vgId:%d apply new config: compression(%d), keep(%d,%d,%d), totalBlocks(%d), cacheLastRow(%d)", REPO_ID(pRepo), pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, - pSaveCfg->totalBlocks, pSaveCfg->cacheLastRow, pSaveCfg->update); + pSaveCfg->totalBlocks, pSaveCfg->cacheLastRow); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 16804c7cf9..c5f1052b63 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -230,9 +230,6 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { configChanged = true; } - if (pRCfg->update != pCfg->update) { - configChanged = true; - } if (!configChanged) { tsdbError("vgId:%d no config changed", REPO_ID(repo)); @@ -253,16 +250,15 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { pSaveCfg->keep1 = pCfg->keep1; pSaveCfg->keep2 = pCfg->keep2; pSaveCfg->cacheLastRow = pCfg->cacheLastRow; - pSaveCfg->update = pCfg->update; - tsdbInfo("vgId:%d old config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d), update(%d)", + tsdbInfo("vgId:%d old config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d)", REPO_ID(repo), pRCfg->compression, pRCfg->keep, pRCfg->keep1,pRCfg->keep2, - pRCfg->cacheLastRow, pRCfg->update); - tsdbInfo("vgId:%d new config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d), update(%d)", + pRCfg->cacheLastRow); + tsdbInfo("vgId:%d new config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d)", REPO_ID(repo), pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, - pSaveCfg->cacheLastRow, pSaveCfg->update); + pSaveCfg->cacheLastRow); repo->config_changed = true; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 0921c5ce48..e471fb7e18 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -170,29 +170,31 @@ static int32_t vnodeAlterImp(SVnodeObj *pVnode, SCreateVnodeMsg *pVnodeCfg) { vDebug("vgId:%d, tsdbchanged:%d syncchanged:%d while alter vnode", pVnode->vgId, tsdbCfgChanged, syncCfgChanged); - if (/*tsdbCfgChanged || */syncCfgChanged) { + if (tsdbCfgChanged || syncCfgChanged) { // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS // dbCfgVersion can be corrected by status msg - if (!vnodeSetUpdatingStatus(pVnode)) { - vDebug("vgId:%d, vnode is not ready, do alter operation later", pVnode->vgId); - pVnode->dbCfgVersion = dbCfgVersion; - pVnode->vgCfgVersion = vgCfgVersion; - pVnode->syncCfg = syncCfg; - pVnode->tsdbCfg = tsdbCfg; - return TSDB_CODE_SUCCESS; + if (syncCfgChanged) { + if (!vnodeSetUpdatingStatus(pVnode)) { + vDebug("vgId:%d, vnode is not ready, do alter operation later", pVnode->vgId); + pVnode->dbCfgVersion = dbCfgVersion; + pVnode->vgCfgVersion = vgCfgVersion; + pVnode->syncCfg = syncCfg; + pVnode->tsdbCfg = tsdbCfg; + return TSDB_CODE_SUCCESS; + } + + code = syncReconfig(pVnode->sync, &pVnode->syncCfg); + if (code != TSDB_CODE_SUCCESS) { + pVnode->dbCfgVersion = dbCfgVersion; + pVnode->vgCfgVersion = vgCfgVersion; + pVnode->syncCfg = syncCfg; + pVnode->tsdbCfg = tsdbCfg; + vnodeSetReadyStatus(pVnode); + return code; + } } - code = syncReconfig(pVnode->sync, &pVnode->syncCfg); - if (code != TSDB_CODE_SUCCESS) { - pVnode->dbCfgVersion = dbCfgVersion; - pVnode->vgCfgVersion = vgCfgVersion; - pVnode->syncCfg = syncCfg; - pVnode->tsdbCfg = tsdbCfg; - vnodeSetReadyStatus(pVnode); - return code; - } - - if (pVnode->tsdb) { + if (tsdbCfgChanged && pVnode->tsdb) { code = tsdbConfigRepo(pVnode->tsdb, &pVnode->tsdbCfg); if (code != TSDB_CODE_SUCCESS) { pVnode->dbCfgVersion = dbCfgVersion; From 344aff2087fdffef25179178177667aa7a46fc01 Mon Sep 17 00:00:00 2001 From: lichuang Date: Fri, 7 May 2021 10:36:51 +0800 Subject: [PATCH 011/157] [TD-3963]change totalBlocks config --- src/tsdb/inc/tsdbBuffer.h | 5 ++++- src/tsdb/src/tsdbBuffer.c | 41 +++++++++++++++++++++++++++++++++- src/tsdb/src/tsdbCommitQueue.c | 10 ++++++--- src/tsdb/src/tsdbMain.c | 12 ++++++---- src/tsdb/src/tsdbMemTable.c | 23 +++++++++++++------ 5 files changed, 75 insertions(+), 16 deletions(-) diff --git a/src/tsdb/inc/tsdbBuffer.h b/src/tsdb/inc/tsdbBuffer.h index 414ace0009..c6aabeaab9 100644 --- a/src/tsdb/inc/tsdbBuffer.h +++ b/src/tsdb/inc/tsdbBuffer.h @@ -28,8 +28,9 @@ typedef struct { int bufBlockSize; int tBufBlocks; int nBufBlocks; + int nRecycleBlocks; int64_t index; - SList* bufBlockList; + SList* bufBlockList; } STsdbBufPool; #define TSDB_BUFFER_RESERVE 1024 // Reseve 1K as commit threshold @@ -39,5 +40,7 @@ void tsdbFreeBufPool(STsdbBufPool* pBufPool); int tsdbOpenBufPool(STsdbRepo* pRepo); void tsdbCloseBufPool(STsdbRepo* pRepo); SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); +void tsdbExpendPool(STsdbRepo* pRepo, int32_t oldTotalBlocks); +void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode); #endif /* _TD_TSDB_BUFFER_H_ */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index 1798a21b99..266f2a45d8 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -70,6 +70,7 @@ int tsdbOpenBufPool(STsdbRepo *pRepo) { pPool->tBufBlocks = pCfg->totalBlocks; pPool->nBufBlocks = 0; pPool->index = 0; + pPool->nRecycleBlocks = 0; for (int i = 0; i < pCfg->totalBlocks; i++) { STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); @@ -156,4 +157,42 @@ _err: return NULL; } -static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } \ No newline at end of file +static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } + +void tsdbExpendPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) { + if (oldTotalBlocks == pRepo->config.totalBlocks) { + return; + } + + if (tsdbLockRepo(pRepo) < 0) return; + STsdbBufPool* pPool = pRepo->pPool; + + if (pRepo->config.totalBlocks > oldTotalBlocks) { + for (int i = 0; i < pRepo->config.totalBlocks - oldTotalBlocks; i++) { + STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); + if (pBufBlock == NULL) goto err; + + if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { + tsdbFreeBufBlock(pBufBlock); + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto err; + } + + pPool->nBufBlocks++; + } + pthread_cond_signal(&pPool->poolNotEmpty); + } else { + pPool->nRecycleBlocks = oldTotalBlocks - pRepo->config.totalBlocks; + } + +err: + tsdbUnlockRepo(pRepo); +} + +void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode) { + STsdbBufBlock *pBufBlock = NULL; + tdListNodeGetData(pPool->bufBlockList, pNode, (void *)(&pBufBlock)); + tsdbFreeBufBlock(pBufBlock); + free(pNode); + pPool->nBufBlocks--; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c index 2e2cc74159..86712db957 100644 --- a/src/tsdb/src/tsdbCommitQueue.c +++ b/src/tsdb/src/tsdbCommitQueue.c @@ -116,17 +116,21 @@ static void tsdbApplyRepoConfig(STsdbRepo *pRepo) { pRepo->config_changed = false; STsdbCfg * pSaveCfg = &pRepo->save_config; + int32_t oldTotalBlocks = pRepo->config.totalBlocks; + pRepo->config.compression = pRepo->save_config.compression; pRepo->config.keep = pRepo->save_config.keep; pRepo->config.keep1 = pRepo->save_config.keep1; pRepo->config.keep2 = pRepo->save_config.keep2; pRepo->config.cacheLastRow = pRepo->save_config.cacheLastRow; - //pRepo->config.update = pRepo->save_config.update; + pRepo->config.totalBlocks = pRepo->save_config.totalBlocks; - tsdbInfo("vgId:%d apply new config: compression(%d), keep(%d,%d,%d), totalBlocks(%d), cacheLastRow(%d)", + tsdbInfo("vgId:%d apply new config: compression(%d), keep(%d,%d,%d), totalBlocks(%d), cacheLastRow(%d),totalBlocks(%d)", REPO_ID(pRepo), pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, - pSaveCfg->totalBlocks, pSaveCfg->cacheLastRow); + pSaveCfg->totalBlocks, pSaveCfg->cacheLastRow, pSaveCfg->totalBlocks); + + tsdbExpendPool(pRepo, oldTotalBlocks); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index c5f1052b63..fd02a3c8b9 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -230,6 +230,9 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { configChanged = true; } + if (pRCfg->totalBlocks != pCfg->totalBlocks) { + configChanged = true; + } if (!configChanged) { tsdbError("vgId:%d no config changed", REPO_ID(repo)); @@ -250,15 +253,16 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { pSaveCfg->keep1 = pCfg->keep1; pSaveCfg->keep2 = pCfg->keep2; pSaveCfg->cacheLastRow = pCfg->cacheLastRow; + pSaveCfg->totalBlocks = pCfg->totalBlocks; - tsdbInfo("vgId:%d old config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d)", + tsdbInfo("vgId:%d old config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)", REPO_ID(repo), pRCfg->compression, pRCfg->keep, pRCfg->keep1,pRCfg->keep2, - pRCfg->cacheLastRow); - tsdbInfo("vgId:%d new config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d)", + pRCfg->cacheLastRow, pRCfg->totalBlocks); + tsdbInfo("vgId:%d new config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)", REPO_ID(repo), pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, - pSaveCfg->cacheLastRow); + pSaveCfg->cacheLastRow,pSaveCfg->totalBlocks); repo->config_changed = true; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 20ec426018..776cc07d2f 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -98,17 +98,26 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { STsdbBufPool *pBufPool = pRepo->pPool; SListNode *pNode = NULL; + bool recycleBlocks = pBufPool->nRecycleBlocks > 0; if (tsdbLockRepo(pRepo) < 0) return -1; while ((pNode = tdListPopHead(pMemTable->bufBlockList)) != NULL) { - tdListAppendNode(pBufPool->bufBlockList, pNode); + if (pBufPool->nRecycleBlocks > 0) { + tsdbRecycleBufferBlock(pBufPool, pNode); + pBufPool->nRecycleBlocks -= 1; + } else { + tdListAppendNode(pBufPool->bufBlockList, pNode); + } } - int code = pthread_cond_signal(&pBufPool->poolNotEmpty); - if (code != 0) { - if (tsdbUnlockRepo(pRepo) < 0) return -1; - tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; + if (!recycleBlocks) { + int code = pthread_cond_signal(&pBufPool->poolNotEmpty); + if (code != 0) { + if (tsdbUnlockRepo(pRepo) < 0) return -1; + tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } } + if (tsdbUnlockRepo(pRepo) < 0) return -1; for (int i = 0; i < pMemTable->maxTables; i++) { From b6e04604cb268e731e24f0141fac529752c80ab9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 7 May 2021 17:29:00 +0800 Subject: [PATCH 012/157] [td-3967] --- src/client/src/tscParseInsert.c | 4 ---- src/util/src/ttokenizer.c | 2 +- tests/pytest/insert/nchar-unicode.py | 5 ++++- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 3882be56dc..923d95c888 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -468,11 +468,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1 int32_t j = 0; for (uint32_t k = 1; k < sToken.n - 1; ++k) { if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) { - if (sToken.z[k] == '\\') { - tmpTokenBuf[j] = GET_ESCAPE_CHAR(sToken.z[k+1]); - } else { tmpTokenBuf[j] = sToken.z[k + 1]; - } cnt++; j++; diff --git a/src/util/src/ttokenizer.c b/src/util/src/ttokenizer.c index 794420d55b..54da75cae0 100644 --- a/src/util/src/ttokenizer.c +++ b/src/util/src/ttokenizer.c @@ -415,7 +415,7 @@ uint32_t tGetToken(char* z, uint32_t* tokenId) { int delim = z[0]; bool strEnd = false; for (i = 1; z[i]; i++) { - if (z[i] == '\\') { + if (z[i] == '\\') { // ignore the escaped character that follows this backslash i++; continue; } diff --git a/tests/pytest/insert/nchar-unicode.py b/tests/pytest/insert/nchar-unicode.py index c417a6bca2..4afcf5b760 100644 --- a/tests/pytest/insert/nchar-unicode.py +++ b/tests/pytest/insert/nchar-unicode.py @@ -57,12 +57,15 @@ class TDTestCase: # https://www.ltg.ed.ac.uk/~richard/unicode-sample.html # Basic Latin - data = r'! # $ % & ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~' + data = r'! # $ % & ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \\ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~' tdLog.info("insert Basic Latin %d length data: %s" % (len(data), data)) tdSql.execute("insert into tb values (now, '%s')" % data) tdSql.query("select * from tb") tdSql.checkRows(3) + + data = data.replace('\\\\', '\\') tdSql.checkData(2, 1, data) + # tdSql.execute("insert into tb values(now, 'abc')") # Latin-1 Supplement data = ' ¡ ¢ £ ¤ ¥ ¦ § ¨ © ª « ¬ ­ ® ¯ ° ± ² ³ ´ µ ¶ · ¸ ¹ º » ¼ ½ ¾ ¿ À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß à á â ã ä å æ ç è é ê ë ì í î ï ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ' From a0020b15ef900c68f23d7b15d9476c9b495e15d6 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 8 May 2021 16:25:03 +0800 Subject: [PATCH 013/157] add assert info --- src/client/src/tscServer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d9405f300a..2035d6261f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1973,6 +1973,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { return TSDB_CODE_TSC_INVALID_VALUE; } + assert(pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE || pTableMeta->tableType == TSDB_NORMAL_TABLE || pTableMeta->tableType == TSDB_STREAM_TABLE); + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { // check if super table hashmap or not int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); From 75172ef7bbe21d1a5634d4d562e16801e6b92538 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 8 May 2021 20:44:29 +0800 Subject: [PATCH 014/157] [TD-4096] --- src/client/src/tscSQLParser.c | 45 +++++++++++++---------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4cf1423c43..4251695d7b 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -64,7 +64,7 @@ static char* getAccountId(SSqlObj* pSql); static bool has(SArray* pFieldList, int32_t startIdx, const char* name); static char* cloneCurrentDBName(SSqlObj* pSql); -static bool hasSpecifyDB(SStrToken* pTableName); +static int32_t getDelimiterIndex(SStrToken* pTableName); static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd); static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd); @@ -426,17 +426,11 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_DESCRIBE_TABLE: { const char* msg1 = "invalid table name"; - const char* msg2 = "table name too long"; SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - - if (!tscValidateTableNameLength(pToken->n)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - // additional msg has been attached already code = tscSetTableFullName(pTableMetaInfo, pToken, pSql); if (code != TSDB_CODE_SUCCESS) { @@ -447,17 +441,12 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_SHOW_CREATE_TABLE: { const char* msg1 = "invalid table name"; - const char* msg2 = "table name is too long"; SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (!tscValidateTableNameLength(pToken->n)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - code = tscSetTableFullName(pTableMetaInfo, pToken, pSql); if (code != TSDB_CODE_SUCCESS) { return code; @@ -987,11 +976,13 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam const char* msg1 = "name too long"; const char* msg2 = "acctId too long"; const char* msg3 = "no acctId"; + const char* msg4 = "db name too long"; + const char* msg5 = "table name too long"; SSqlCmd* pCmd = &pSql->cmd; int32_t code = TSDB_CODE_SUCCESS; - - if (hasSpecifyDB(pTableName)) { // db has been specified in sql string so we ignore current db path + int32_t idx = getDelimiterIndex(pTableName); + if (idx != -1) { // db has been specified in sql string so we ignore current db path char* acctId = getAccountId(pSql); if (acctId == NULL || strlen(acctId) <= 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -1001,6 +992,13 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam if (code != 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } + if (idx >= TSDB_DB_NAME_LEN) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + if (pTableName->n - 1 - idx >= TSDB_TABLE_NAME_LEN) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + } + char name[TSDB_TABLE_FNAME_LEN] = {0}; strncpy(name, pTableName->z, pTableName->n); @@ -1345,14 +1343,13 @@ static char* cloneCurrentDBName(SSqlObj* pSql) { } /* length limitation, strstr cannot be applied */ -static bool hasSpecifyDB(SStrToken* pTableName) { - for (uint32_t i = 0; i < pTableName->n; ++i) { - if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { - return true; +static int32_t getDelimiterIndex(SStrToken* pTableName) { + for (uint32_t i = 0; i < pTableName->n; ++i) { + if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { + return i; } } - - return false; + return -1; } int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* xlen) { @@ -7483,11 +7480,3 @@ bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) { return false; } - - - - - - - - From 9f3845e97cfbc249819562d53592195fe6d84cd0 Mon Sep 17 00:00:00 2001 From: davkor Date: Sat, 8 May 2021 16:30:13 +0100 Subject: [PATCH 015/157] tests: add fuzz setup. --- tests/fuzz/sql-fuzzer.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/fuzz/sql-fuzzer.c diff --git a/tests/fuzz/sql-fuzzer.c b/tests/fuzz/sql-fuzzer.c new file mode 100644 index 0000000000..3eb0eed2e7 --- /dev/null +++ b/tests/fuzz/sql-fuzzer.c @@ -0,0 +1,15 @@ +#include "qSqlparser.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size){ + char *new_str = (char *)malloc(size+1); + if (new_str == NULL){ + return 0; + } + memcpy(new_str, data, size); + new_str[size] = '\0'; + + qSqlParse(new_str); + + free(new_str); + return 0; +} From 1729bc75ab163c2e7a1192333b45338ea8f7dbbd Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 9 May 2021 00:34:48 +0800 Subject: [PATCH 016/157] [TD-4119] --- src/mnode/inc/mnodeDef.h | 2 +- src/mnode/src/mnodeShow.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index ed1de1b87a..e052f34a33 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -249,7 +249,7 @@ typedef struct SAcctObj { } SAcctObj; typedef struct { - char db[TSDB_DB_NAME_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; int8_t type; int16_t numOfColumns; int32_t index; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 03772f2724..c0fa6368f3 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -129,7 +129,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) { SShowObj *pShow = calloc(1, showObjSize); pShow->type = pShowMsg->type; pShow->payloadLen = htons(pShowMsg->payloadLen); - tstrncpy(pShow->db, pShowMsg->db, TSDB_DB_NAME_LEN); + tstrncpy(pShow->db, pShowMsg->db, TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN); memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen); pShow = mnodePutShowObj(pShow); From ee4f7ffa07fb2cdcb631a4f0cf4e4daff111d1e3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 10 May 2021 04:08:23 +0800 Subject: [PATCH 017/157] [TD-4133] --- src/client/src/tscSubquery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index cc4eb751ff..3cb2b60ce2 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2865,7 +2865,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR tscDebug("0x%"PRIx64" sub:%p retrieve numOfRows:%d totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", pParentSql->self, pSql, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); - if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { + if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0) && !(tscGetQueryInfoDetail(&pParentSql->cmd, 0)->distinctTag)) { tscError("0x%"PRIx64" sub:0x%"PRIx64" num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, pParentSql->self, pSql->self, tsMaxNumOfOrderedResults, num); tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_SORTED_RES_TOO_MANY); From 8e7381d70e56afcb17cb161bf34040e68890d742 Mon Sep 17 00:00:00 2001 From: lichuang Date: Mon, 10 May 2021 10:14:44 +0800 Subject: [PATCH 018/157] free lastRow while cacheLastRow config reset --- src/tsdb/src/tsdbMemTable.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 776cc07d2f..c6fcf55686 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -967,6 +967,15 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row) { STsdbCfg *pCfg = &pRepo->config; + // if cacheLastRow config has been reset, free the lastRow + if (!pCfg->cacheLastRow && pTable->lastRow != NULL) { + taosTZfree(pTable->lastRow); + TSDB_WLOCK_TABLE(pTable); + pTable->lastRow = NULL; + pTable->lastKey = TSKEY_INITIAL_VAL; + TSDB_WUNLOCK_TABLE(pTable); + } + if (tsdbGetTableLastKeyImpl(pTable) < dataRowKey(row)) { if (pCfg->cacheLastRow || pTable->lastRow != NULL) { SDataRow nrow = pTable->lastRow; From 7570894fbb68b43cfb583ae5c015894441fb3e35 Mon Sep 17 00:00:00 2001 From: wu champion Date: Mon, 10 May 2021 15:30:00 +0800 Subject: [PATCH 019/157] [TD-3960] add case for TD-3960 --- tests/pytest/fulltest.sh | 1 + .../functions/showOfflineThresholdIs864000.py | 36 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/pytest/functions/showOfflineThresholdIs864000.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index b5aae6fcef..a748c9dd2d 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -305,6 +305,7 @@ python3 ./test.py -f functions/function_top.py -r 1 python3 ./test.py -f functions/function_twa.py -r 1 python3 ./test.py -f functions/function_twa_test2.py python3 ./test.py -f functions/function_stddev_td2555.py +python3 ./test.py -f functions/showOfflineThresholdIs864000.py python3 ./test.py -f insert/metadataUpdate.py python3 ./test.py -f query/last_cache.py python3 ./test.py -f query/last_row_cache.py diff --git a/tests/pytest/functions/showOfflineThresholdIs864000.py b/tests/pytest/functions/showOfflineThresholdIs864000.py new file mode 100644 index 0000000000..6cce869bf2 --- /dev/null +++ b/tests/pytest/functions/showOfflineThresholdIs864000.py @@ -0,0 +1,36 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to execute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.query("show variables") + tdSql.checkData(51, 1, 864000) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 79b422390f9174ea23c56584e0dbefc2c79345ee Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 10 May 2021 16:17:46 +0800 Subject: [PATCH 020/157] fix bug --- src/client/src/tscServer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 2035d6261f..33325c7fd7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2542,6 +2542,7 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } pTableMetaInfo->pTableMeta = (STableMeta *)tmp; + memset(pTableMetaInfo->pTableMeta, 0, size); pTableMetaInfo->tableMetaSize = size; } else { //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); From f2fa15156543728d68e3502970bcf4e2ee55381a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 May 2021 17:01:46 +0800 Subject: [PATCH 021/157] [td-3967]update the test case. --- .../taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java index 9014e82a9e..a3c46dc232 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java @@ -12,7 +12,7 @@ public class InsertSpecialCharacterJniTest { private static String tbname1 = "test"; private static String tbname2 = "weather"; private static String special_character_str_1 = "$asd$$fsfsf$"; - private static String special_character_str_2 = "\\asdfsfsf\\\\"; + private static String special_character_str_2 = "\\\\asdfsfsf\\\\"; private static String special_character_str_3 = "\\\\asdfsfsf\\"; private static String special_character_str_4 = "?asd??fsf?sf?"; private static String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$"; @@ -70,7 +70,7 @@ public class InsertSpecialCharacterJniTest { String f1 = new String(rs.getBytes(2)); //TODO: bug to be fixed // Assert.assertEquals(special_character_str_2, f1); - Assert.assertEquals(special_character_str_2.substring(0, special_character_str_1.length() - 2), f1); + Assert.assertEquals(special_character_str_2.substring(1, special_character_str_1.length() - 1), f1); String f2 = rs.getString(3); Assert.assertNull(f2); } From a9c5957165255f58b8fa56a274249de482033b5c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 11 May 2021 00:08:30 +0800 Subject: [PATCH 022/157] [TD-4095]: taosdump long tablename. (#6076) Co-authored-by: Shuduo Sang --- src/kit/taosdump/taosdump.c | 115 +++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 46 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index e706ddefd6..f80ac069a0 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -110,14 +110,14 @@ typedef struct { } SColDes; typedef struct { - char name[TSDB_COL_NAME_LEN + 1]; + char name[TSDB_TABLE_NAME_LEN]; SColDes cols[]; } STableDef; extern char version[]; typedef struct { - char name[TSDB_DB_NAME_LEN + 1]; + char name[TSDB_DB_NAME_LEN]; char create_time[32]; int32_t ntables; int32_t vgroups; @@ -142,8 +142,8 @@ typedef struct { } SDbInfo; typedef struct { - char name[TSDB_TABLE_NAME_LEN + 1]; - char metric[TSDB_TABLE_NAME_LEN + 1]; + char name[TSDB_TABLE_NAME_LEN]; + char metric[TSDB_TABLE_NAME_LEN]; } STableRecord; typedef struct { @@ -155,7 +155,7 @@ typedef struct { pthread_t threadID; int32_t threadIndex; int32_t totalThreads; - char dbName[TSDB_TABLE_NAME_LEN + 1]; + char dbName[TSDB_DB_NAME_LEN]; void *taosCon; int64_t rowsOfDumpOut; int64_t tablesOfDumpOut; @@ -214,13 +214,13 @@ static struct argp_option options[] = { {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, // dump unit options {"all-databases", 'A', 0, 0, "Dump all databases.", 2}, - {"databases", 'B', 0, 0, "Dump assigned databases", 2}, + {"databases", 'D', 0, 0, "Dump assigned databases", 2}, // dump format options {"schemaonly", 's', 0, 0, "Only dump schema.", 3}, - {"with-property", 'M', 0, 0, "Dump schema with properties.", 3}, + {"without-property", 'N', 0, 0, "Dump schema without properties.", 3}, {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, {"end-time", 'E', "END_TIME", 0, "End time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, - {"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, + {"data-batch", 'B', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, @@ -341,15 +341,15 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'A': arguments->all_databases = true; break; - case 'B': + case 'D': arguments->databases = true; break; // dump format option case 's': arguments->schemaonly = true; break; - case 'M': - arguments->with_property = true; + case 'N': + arguments->with_property = false; break; case 'S': // parse time here. @@ -358,7 +358,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'E': arguments->end_time = atol(arg); break; - case 'N': + case 'B': arguments->data_batch = atoi(arg); if (arguments->data_batch >= INT16_MAX) { arguments->data_batch = INT16_MAX - 1; @@ -402,17 +402,17 @@ static resultStatistics g_resultStatistics = {0}; static FILE *g_fpOfResult = NULL; static int g_numOfCores = 1; -int taosDumpOut(struct arguments *arguments); -int taosDumpIn(struct arguments *arguments); -void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); -int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon); -int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName); -void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName); -void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName); -int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName); -int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName); -int taosCheckParam(struct arguments *arguments); -void taosFreeDbInfos(); +static int taosDumpOut(struct arguments *arguments); +static int taosDumpIn(struct arguments *arguments); +static void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); +static int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon); +static int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName); +static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName); +static void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName); +static int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName); +static int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName); +static int taosCheckParam(struct arguments *arguments); +static void taosFreeDbInfos(); static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName); struct arguments g_args = { @@ -436,8 +436,8 @@ struct arguments g_args = { false, false, // dump format option - false, - false, + false, // schemeonly + true, // with_property 0, INT64_MAX, 1, @@ -959,7 +959,8 @@ int taosDumpOut(struct arguments *arguments) { goto _exit_failure; } - strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes); + strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], + fields[TSDB_SHOW_DB_NAME_INDEX].bytes); if (arguments->with_property) { dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); @@ -967,7 +968,8 @@ int taosDumpOut(struct arguments *arguments) { dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); - strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); + strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], + fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); //dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]); //dbInfos[count]->daysToKeep1; //dbInfos[count]->daysToKeep2; @@ -980,7 +982,8 @@ int taosDumpOut(struct arguments *arguments) { dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); - strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); + strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], + fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); //dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]); dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); } @@ -1095,7 +1098,9 @@ _exit_failure: return -1; } -int taosGetTableDes(char* dbName, char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) { +int taosGetTableDes( + char* dbName, char *table, + STableDef *tableDes, TAOS* taosCon, bool isSuperTable) { TAOS_ROW row = NULL; TAOS_RES* res = NULL; int count = 0; @@ -1113,7 +1118,7 @@ int taosGetTableDes(char* dbName, char *table, STableDef *tableDes, TAOS* taosCo TAOS_FIELD *fields = taos_fetch_fields(res); - tstrncpy(tableDes->name, table, TSDB_COL_NAME_LEN); + tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN); while ((row = taos_fetch_row(res)) != NULL) { strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], @@ -1232,7 +1237,9 @@ int taosGetTableDes(char* dbName, char *table, STableDef *tableDes, TAOS* taosCo return count; } -int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName) { +int32_t taosDumpTable( + char *table, char *metric, struct arguments *arguments, + FILE *fp, TAOS* taosCon, char* dbName) { int count = 0; STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); @@ -1346,14 +1353,17 @@ void* taosDumpOutWorkThreadFp(void *arg) ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); if (readLen <= 0) break; - int ret = taosDumpTable(tableRecord.name, tableRecord.metric, &g_args, fp, pThread->taosCon, pThread->dbName); + int ret = taosDumpTable( + tableRecord.name, tableRecord.metric, &g_args, + fp, pThread->taosCon, pThread->dbName); if (ret >= 0) { // TODO: sum table count and table rows by self pThread->tablesOfDumpOut++; pThread->rowsOfDumpOut += ret; if (pThread->rowsOfDumpOut >= lastRowsPrint) { - printf(" %"PRId64 " rows already be dumpout from database %s\n", pThread->rowsOfDumpOut, pThread->dbName); + printf(" %"PRId64 " rows already be dumpout from database %s\n", + pThread->rowsOfDumpOut, pThread->dbName); lastRowsPrint += 5000000; } @@ -1364,9 +1374,12 @@ void* taosDumpOutWorkThreadFp(void *arg) memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); if (g_args.outpath[0] != 0) { - sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", g_args.outpath, pThread->dbName, pThread->threadIndex, fileNameIndex); + sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", + g_args.outpath, pThread->dbName, + pThread->threadIndex, fileNameIndex); } else { - sprintf(tmpBuf, "%s.tables.%d-%d.sql", pThread->dbName, pThread->threadIndex, fileNameIndex); + sprintf(tmpBuf, "%s.tables.%d-%d.sql", + pThread->dbName, pThread->threadIndex, fileNameIndex); } fileNameIndex++; @@ -1391,14 +1404,15 @@ void* taosDumpOutWorkThreadFp(void *arg) static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName) { pthread_attr_t thattr; - SThreadParaObj *threadObj = (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); + SThreadParaObj *threadObj = + (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); for (int t = 0; t < numOfThread; ++t) { SThreadParaObj *pThread = threadObj + t; pThread->rowsOfDumpOut = 0; pThread->tablesOfDumpOut = 0; pThread->threadIndex = t; pThread->totalThreads = numOfThread; - tstrncpy(pThread->dbName, dbName, TSDB_TABLE_NAME_LEN); + tstrncpy(pThread->dbName, dbName, TSDB_DB_NAME_LEN); pThread->taosCon = taosCon; pthread_attr_init(&thattr); @@ -1487,7 +1501,8 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp) while ((row = taos_fetch_row(res)) != NULL) { memset(&tableRecord, 0, sizeof(STableRecord)); - strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); taosWrite(fd, &tableRecord, sizeof(STableRecord)); } @@ -1557,8 +1572,10 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao int32_t numOfTable = 0; while ((row = taos_fetch_row(res)) != NULL) { memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); - tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); + tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], + fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); taosWrite(fd, &tableRecord, sizeof(STableRecord)); @@ -1643,15 +1660,18 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, cha char* pstr = sqlstr; - pstr += sprintf(sqlstr, "CREATE TABLE IF NOT EXISTS %s.%s", dbName, tableDes->name); + pstr += sprintf(sqlstr, "CREATE TABLE IF NOT EXISTS %s.%s", + dbName, tableDes->name); for (; counter < numOfCols; counter++) { if (tableDes->cols[counter].note[0] != '\0') break; if (counter == 0) { - pstr += sprintf(pstr, " (%s %s", tableDes->cols[counter].field, tableDes->cols[counter].type); + pstr += sprintf(pstr, " (%s %s", + tableDes->cols[counter].field, tableDes->cols[counter].type); } else { - pstr += sprintf(pstr, ", %s %s", tableDes->cols[counter].field, tableDes->cols[counter].type); + pstr += sprintf(pstr, ", %s %s", + tableDes->cols[counter].field, tableDes->cols[counter].type); } if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || @@ -1664,9 +1684,11 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, cha for (; counter < numOfCols; counter++) { if (counter == count_temp) { - pstr += sprintf(pstr, ") TAGS (%s %s", tableDes->cols[counter].field, tableDes->cols[counter].type); + pstr += sprintf(pstr, ") TAGS (%s %s", + tableDes->cols[counter].field, tableDes->cols[counter].type); } else { - pstr += sprintf(pstr, ", %s %s", tableDes->cols[counter].field, tableDes->cols[counter].type); + pstr += sprintf(pstr, ", %s %s", + tableDes->cols[counter].field, tableDes->cols[counter].type); } if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || @@ -1693,7 +1715,8 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols char *pstr = NULL; pstr = tmpBuf; - pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", dbName, tableDes->name, dbName, metric); + pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", + dbName, tableDes->name, dbName, metric); for (; counter < numOfCols; counter++) { if (tableDes->cols[counter].note[0] != '\0') break; From 456ea712b357b69f0ec79a58a50439977abdbf40 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Tue, 11 May 2021 01:48:13 +0000 Subject: [PATCH 023/157] Refactored crash_gen tool with stronger typing --- tests/pytest/crash_gen/__init__.py | 4 - tests/pytest/crash_gen/crash_gen_main.py | 184 +++++++++--------- tests/pytest/crash_gen/service_manager.py | 31 ++- .../{settings.py => shared/config.py} | 23 ++- tests/pytest/crash_gen/{ => shared}/db.py | 19 +- tests/pytest/crash_gen/{ => shared}/misc.py | 10 +- tests/pytest/crash_gen/shared/types.py | 28 +++ tests/pytest/crash_gen/types.py | 5 - 8 files changed, 168 insertions(+), 136 deletions(-) rename tests/pytest/crash_gen/{settings.py => shared/config.py} (54%) rename tests/pytest/crash_gen/{ => shared}/db.py (97%) rename tests/pytest/crash_gen/{ => shared}/misc.py (96%) create mode 100644 tests/pytest/crash_gen/shared/types.py delete mode 100644 tests/pytest/crash_gen/types.py diff --git a/tests/pytest/crash_gen/__init__.py b/tests/pytest/crash_gen/__init__.py index 2509d7d76d..fe03bde354 100644 --- a/tests/pytest/crash_gen/__init__.py +++ b/tests/pytest/crash_gen/__init__.py @@ -1,6 +1,2 @@ # Helpful Ref: https://stackoverflow.com/questions/24100558/how-can-i-split-a-module-into-multiple-files-without-breaking-a-backwards-compa/24100645 from crash_gen.service_manager import ServiceManager, TdeInstance, TdeSubProcess -from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress -from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager -from crash_gen.settings import Settings -from crash_gen.types import DirPath \ No newline at end of file diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 8d34a7555b..644aa79916 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1,6 +1,6 @@ # -----!/usr/bin/python3.7 ################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. +# Copyright (c) 2016-2021 by TAOS Technologies, Inc. # All rights reserved. # # This file is proprietary and confidential to TAOS Technologies. @@ -24,30 +24,34 @@ import textwrap import time import datetime import random -import logging import threading -import copy import argparse -import getopt import sys import os import io import signal import traceback -import resource +import requests # from guppy import hpy import gc +import taos + +from .shared.types import TdColumns, TdTags # from crash_gen import ServiceManager, TdeInstance, TdeSubProcess -from crash_gen import ServiceManager, Settings, DbConn, DbConnNative, Dice, DbManager, Status, Logging, Helper, \ - CrashGenError, Progress, MyTDSql, \ - TdeInstance +# from crash_gen import ServiceManager, Config, DbConn, DbConnNative, Dice, DbManager, Status, Logging, Helper, \ +# CrashGenError, Progress, MyTDSql, \ +# TdeInstance -import taos -import requests +from .service_manager import ServiceManager, TdeInstance -Settings.init() +from .shared.config import Config +from .shared.db import DbConn, DbManager, DbConnNative, MyTDSql +from .shared.misc import Dice, Logging, Helper, Status, CrashGenError, Progress +from .shared.types import TdDataType + +# Config.init() # Require Python 3 if sys.version_info[0] < 3: @@ -81,20 +85,20 @@ class WorkerThread: self._stepGate = threading.Event() # Let us have a DB connection of our own - if (Settings.getConfig().per_thread_db_connection): # type: ignore + if (Config.getConfig().per_thread_db_connection): # type: ignore # print("connector_type = {}".format(gConfig.connector_type)) tInst = gContainer.defTdeInstance - if Settings.getConfig().connector_type == 'native': + if Config.getConfig().connector_type == 'native': self._dbConn = DbConn.createNative(tInst.getDbTarget()) - elif Settings.getConfig().connector_type == 'rest': + elif Config.getConfig().connector_type == 'rest': self._dbConn = DbConn.createRest(tInst.getDbTarget()) - elif Settings.getConfig().connector_type == 'mixed': + elif Config.getConfig().connector_type == 'mixed': if Dice.throw(2) == 0: # 1/2 chance self._dbConn = DbConn.createNative(tInst.getDbTarget()) else: self._dbConn = DbConn.createRest(tInst.getDbTarget()) else: - raise RuntimeError("Unexpected connector type: {}".format(Settings.getConfig().connector_type)) + raise RuntimeError("Unexpected connector type: {}".format(Config.getConfig().connector_type)) # self._dbInUse = False # if "use db" was executed already @@ -123,14 +127,14 @@ class WorkerThread: # self.isSleeping = False Logging.info("Starting to run thread: {}".format(self._tid)) - if (Settings.getConfig().per_thread_db_connection): # type: ignore + if (Config.getConfig().per_thread_db_connection): # type: ignore Logging.debug("Worker thread openning database connection") self._dbConn.open() self._doTaskLoop() # clean up - if (Settings.getConfig().per_thread_db_connection): # type: ignore + if (Config.getConfig().per_thread_db_connection): # type: ignore if self._dbConn.isOpen: #sometimes it is not open self._dbConn.close() else: @@ -158,7 +162,7 @@ class WorkerThread: # Before we fetch the task and run it, let's ensure we properly "use" the database (not needed any more) try: - if (Settings.getConfig().per_thread_db_connection): # most likely TRUE + if (Config.getConfig().per_thread_db_connection): # most likely TRUE if not self._dbConn.isOpen: # might have been closed during server auto-restart self._dbConn.open() # self.useDb() # might encounter exceptions. TODO: catch @@ -232,7 +236,7 @@ class WorkerThread: return self.getDbConn().getQueryResult() def getDbConn(self) -> DbConn : - if (Settings.getConfig().per_thread_db_connection): + if (Config.getConfig().per_thread_db_connection): return self._dbConn else: return self._tc.getDbManager().getDbConn() @@ -283,7 +287,7 @@ class ThreadCoordinator: self._execStats.registerFailure("User Interruption") def _runShouldEnd(self, transitionFailed, hasAbortedTask, workerTimeout): - maxSteps = Settings.getConfig().max_steps # type: ignore + maxSteps = Config.getConfig().max_steps # type: ignore if self._curStep >= (maxSteps - 1): # maxStep==10, last curStep should be 9 return True if self._runStatus != Status.STATUS_RUNNING: @@ -388,7 +392,7 @@ class ThreadCoordinator: hasAbortedTask = False workerTimeout = False while not self._runShouldEnd(transitionFailed, hasAbortedTask, workerTimeout): - if not Settings.getConfig().debug: # print this only if we are not in debug mode + if not Config.getConfig().debug: # print this only if we are not in debug mode Progress.emit(Progress.STEP_BOUNDARY) # print(".", end="", flush=True) # if (self._curStep % 2) == 0: # print memory usage once every 10 steps @@ -512,18 +516,18 @@ class ThreadCoordinator: ''' Initialize multiple databases, invoked at __ini__() time ''' self._dbs = [] # type: List[Database] dbc = self.getDbManager().getDbConn() - if Settings.getConfig().max_dbs == 0: + if Config.getConfig().max_dbs == 0: self._dbs.append(Database(0, dbc)) else: baseDbNumber = int(datetime.datetime.now().timestamp( # Don't use Dice/random, as they are deterministic - )*333) % 888 if Settings.getConfig().dynamic_db_table_names else 0 - for i in range(Settings.getConfig().max_dbs): + )*333) % 888 if Config.getConfig().dynamic_db_table_names else 0 + for i in range(Config.getConfig().max_dbs): self._dbs.append(Database(baseDbNumber + i, dbc)) def pickDatabase(self): idxDb = 0 - if Settings.getConfig().max_dbs != 0 : - idxDb = Dice.throw(Settings.getConfig().max_dbs) # 0 to N-1 + if Config.getConfig().max_dbs != 0 : + idxDb = Dice.throw(Config.getConfig().max_dbs) # 0 to N-1 db = self._dbs[idxDb] # type: Database return db @@ -705,7 +709,7 @@ class AnyState: def canDropDb(self): # If user requests to run up to a number of DBs, # we'd then not do drop_db operations any more - if Settings.getConfig().max_dbs > 0 or Settings.getConfig().use_shadow_db : + if Config.getConfig().max_dbs > 0 or Config.getConfig().use_shadow_db : return False return self._info[self.CAN_DROP_DB] @@ -713,7 +717,7 @@ class AnyState: return self._info[self.CAN_CREATE_FIXED_SUPER_TABLE] def canDropFixedSuperTable(self): - if Settings.getConfig().use_shadow_db: # duplicate writes to shaddow DB, in which case let's disable dropping s-table + if Config.getConfig().use_shadow_db: # duplicate writes to shaddow DB, in which case let's disable dropping s-table return False return self._info[self.CAN_DROP_FIXED_SUPER_TABLE] @@ -1110,7 +1114,7 @@ class Database: t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec2) # see explanation above - Logging.info("Setting up TICKS to start from: {}".format(t4)) + Logging.debug("Setting up TICKS to start from: {}".format(t4)) return t4 @classmethod @@ -1126,7 +1130,7 @@ class Database: cls._lastLaggingTick = tick + datetime.timedelta(0, -60*2) # lagging behind 2 minutes, should catch up fast # if : # should be quite a bit into the future - if Settings.getConfig().mix_oos_data and Dice.throw(20) == 0: # if asked to do so, and 1 in 20 chance, return lagging tick + if Config.isSet('mix_oos_data') and Dice.throw(20) == 0: # if asked to do so, and 1 in 20 chance, return lagging tick cls._lastLaggingTick += datetime.timedelta(0, 1) # pick the next sequence from the lagging tick sequence return cls._lastLaggingTick else: # regular @@ -1310,8 +1314,8 @@ class Task(): # This case handled below already. # elif (errno in [ 0x0B ]) and Settings.getConfig().auto_start_service: # return True # We may get "network unavilable" when restarting service - elif Settings.getConfig().ignore_errors: # something is specified on command line - moreErrnos = [int(v, 0) for v in Settings.getConfig().ignore_errors.split(',')] + elif Config.getConfig().ignore_errors: # something is specified on command line + moreErrnos = [int(v, 0) for v in Config.getConfig().ignore_errors.split(',')] if errno in moreErrnos: return True elif errno == 0x200 : # invalid SQL, we need to div in a bit more @@ -1347,7 +1351,7 @@ class Task(): self._executeInternal(te, wt) # TODO: no return value? except taos.error.ProgrammingError as err: errno2 = Helper.convertErrno(err.errno) - if (Settings.getConfig().continue_on_exception): # user choose to continue + if (Config.getConfig().continue_on_exception): # user choose to continue self.logDebug("[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format( errno2, err, wt.getDbConn().getLastSql())) self._err = err @@ -1362,7 +1366,7 @@ class Task(): self.__class__.__name__, errno2, err, wt.getDbConn().getLastSql()) self.logDebug(errMsg) - if Settings.getConfig().debug: + if Config.getConfig().debug: # raise # so that we see full stack traceback.print_exc() print( @@ -1560,7 +1564,7 @@ class StateTransitionTask(Task): def getRegTableName(cls, i): if ( StateTransitionTask._baseTableNumber is None): # Set it one time StateTransitionTask._baseTableNumber = Dice.throw( - 999) if Settings.getConfig().dynamic_db_table_names else 0 + 999) if Config.getConfig().dynamic_db_table_names else 0 return "reg_table_{}".format(StateTransitionTask._baseTableNumber + i) def execute(self, wt: WorkerThread): @@ -1580,14 +1584,14 @@ class TaskCreateDb(StateTransitionTask): def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): # was: self.execWtSql(wt, "create database db") repStr = "" - if Settings.getConfig().num_replicas != 1: + if Config.getConfig().num_replicas != 1: # numReplica = Dice.throw(Settings.getConfig().max_replicas) + 1 # 1,2 ... N - numReplica = Settings.getConfig().num_replicas # fixed, always + numReplica = Config.getConfig().num_replicas # fixed, always repStr = "replica {}".format(numReplica) - updatePostfix = "update 1" if Settings.getConfig().verify_data else "" # allow update only when "verify data" is active + updatePostfix = "update 1" if Config.getConfig().verify_data else "" # allow update only when "verify data" is active dbName = self._db.getName() self.execWtSql(wt, "create database {} {} {} ".format(dbName, repStr, updatePostfix ) ) - if dbName == "db_0" and Settings.getConfig().use_shadow_db: + if dbName == "db_0" and Config.getConfig().use_shadow_db: self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix ) ) class TaskDropDb(StateTransitionTask): @@ -1620,10 +1624,11 @@ class TaskCreateSuperTable(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - sTable.create(wt.getDbConn(), - {'ts':'TIMESTAMP', 'speed':'INT', 'color':'BINARY(16)'}, {'b':'BINARY(200)', 'f':'FLOAT'}, - dropIfExists = True - ) + sTable.create(wt.getDbConn(), + {'ts': TdDataType.TIMESTAMP, 'speed': TdDataType.INT, 'color': TdDataType.BINARY16}, { + 'b': TdDataType.BINARY200, 'f': TdDataType.FLOAT}, + dropIfExists=True + ) # self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName)) # No need to create the regular tables, INSERT will do that # automatically @@ -1651,9 +1656,7 @@ class TdSuperTable: return dbc.existsSuperTable(self._stName) # TODO: odd semantic, create() method is usually static? - def create(self, dbc, cols: dict, tags: dict, - dropIfExists = False - ): + def create(self, dbc, cols: TdColumns, tags: TdTags, dropIfExists = False): '''Creating a super table''' dbName = self._dbName @@ -1664,17 +1667,17 @@ class TdSuperTable: dbc.execute("DROP TABLE {}".format(fullTableName)) else: # error raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName)) - + # Now let's create sql = "CREATE TABLE {} ({})".format( fullTableName, - ",".join(['%s %s'%(k,v) for (k,v) in cols.items()])) - if tags is None : - sql += " TAGS (dummy int) " - else: + ",".join(['%s %s'%(k,v.value) for (k,v) in cols.items()])) + if tags : sql += " TAGS ({})".format( - ",".join(['%s %s'%(k,v) for (k,v) in tags.items()]) - ) + ",".join(['%s %s'%(k,v.value) for (k,v) in tags.items()]) + ) + else: + sql += " TAGS (dummy int) " dbc.execute(sql) def getRegTables(self, dbc: DbConn): @@ -1692,7 +1695,7 @@ class TdSuperTable: def hasRegTables(self, dbc: DbConn): return dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) > 0 - def ensureTable(self, task: Task, dbc: DbConn, regTableName: str): + def ensureRegTable(self, task: Optional[Task], dbc: DbConn, regTableName: str): dbName = self._dbName sql = "select tbname from {}.{} where tbname in ('{}')".format(dbName, self._stName, regTableName) if dbc.query(sql) >= 1 : # reg table exists already @@ -1700,7 +1703,7 @@ class TdSuperTable: # acquire a lock first, so as to be able to *verify*. More details in TD-1471 fullTableName = dbName + '.' + regTableName - if task is not None: # optional lock + if task is not None: # TODO: what happens if we don't lock the table task.lockTable(fullTableName) Progress.emit(Progress.CREATE_TABLE_ATTEMPT) # ATTEMPT to create a new table # print("(" + fullTableName[-3:] + ")", end="", flush=True) @@ -1892,7 +1895,7 @@ class TaskDropSuperTable(StateTransitionTask): if Dice.throw(2) == 0: # print("_7_", end="", flush=True) tblSeq = list(range( - 2 + (self.LARGE_NUMBER_OF_TABLES if Settings.getConfig().larger_data else self.SMALL_NUMBER_OF_TABLES))) + 2 + (self.LARGE_NUMBER_OF_TABLES if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_TABLES))) random.shuffle(tblSeq) tickOutput = False # if we have spitted out a "d" character for "drop regular table" isSuccess = True @@ -1958,13 +1961,13 @@ class TaskRestartService(StateTransitionTask): @classmethod def canBeginFrom(cls, state: AnyState): - if Settings.getConfig().auto_start_service: + if Config.getConfig().auto_start_service: return state.canDropFixedSuperTable() # Basicallly when we have the super table return False # don't run this otherwise CHANCE_TO_RESTART_SERVICE = 200 def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - if not Settings.getConfig().auto_start_service: # only execute when we are in -a mode + if not Config.getConfig().auto_start_service: # only execute when we are in -a mode print("_a", end="", flush=True) return @@ -1991,7 +1994,7 @@ class TaskAddData(StateTransitionTask): @classmethod def prepToRecordOps(cls): - if Settings.getConfig().record_ops: + if Config.getConfig().record_ops: if (cls.fAddLogReady is None): Logging.info( "Recording in a file operations to be performed...") @@ -2009,7 +2012,7 @@ class TaskAddData(StateTransitionTask): return state.canAddData() def _addDataInBatch(self, db, dbc, regTableName, te: TaskExecutor): - numRecords = self.LARGE_NUMBER_OF_RECORDS if Settings.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS + numRecords = self.LARGE_NUMBER_OF_RECORDS if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS fullTableName = db.getName() + '.' + regTableName sql = "INSERT INTO {} VALUES ".format(fullTableName) @@ -2021,13 +2024,13 @@ class TaskAddData(StateTransitionTask): dbc.execute(sql) def _addData(self, db: Database, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches - numRecords = self.LARGE_NUMBER_OF_RECORDS if Settings.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS + numRecords = self.LARGE_NUMBER_OF_RECORDS if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS for j in range(numRecords): # number of records per table nextInt = db.getNextInt() nextTick = db.getNextTick() nextColor = db.getNextColor() - if Settings.getConfig().record_ops: + if Config.getConfig().record_ops: self.prepToRecordOps() if self.fAddLogReady is None: raise CrashGenError("Unexpected empty fAddLogReady") @@ -2037,7 +2040,7 @@ class TaskAddData(StateTransitionTask): # TODO: too ugly trying to lock the table reliably, refactor... fullTableName = db.getName() + '.' + regTableName - if Settings.getConfig().verify_data: + if Config.getConfig().verify_data: self.lockTable(fullTableName) # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written @@ -2050,7 +2053,7 @@ class TaskAddData(StateTransitionTask): dbc.execute(sql) # Quick hack, attach an update statement here. TODO: create an "update" task - if (not Settings.getConfig().use_shadow_db) and Dice.throw(5) == 0: # 1 in N chance, plus not using shaddow DB + if (not Config.getConfig().use_shadow_db) and Dice.throw(5) == 0: # 1 in N chance, plus not using shaddow DB nextInt = db.getNextInt() nextColor = db.getNextColor() sql = "INSERt INTO {} VALUES ('{}', {}, '{}');".format( # "INSERt" means "update" here @@ -2061,12 +2064,12 @@ class TaskAddData(StateTransitionTask): dbc.execute(sql) except: # Any exception at all - if Settings.getConfig().verify_data: + if Config.getConfig().verify_data: self.unlockTable(fullTableName) raise # Now read it back and verify, we might encounter an error if table is dropped - if Settings.getConfig().verify_data: # only if command line asks for it + if Config.getConfig().verify_data: # only if command line asks for it try: readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts='{}'". format(db.getName(), regTableName, nextTick)) @@ -2093,7 +2096,7 @@ class TaskAddData(StateTransitionTask): # Successfully wrote the data into the DB, let's record it somehow te.recordDataMark(nextInt) - if Settings.getConfig().record_ops: + if Config.getConfig().record_ops: if self.fAddLogDone is None: raise CrashGenError("Unexpected empty fAddLogDone") self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName)) @@ -2104,8 +2107,8 @@ class TaskAddData(StateTransitionTask): # ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access db = self._db dbc = wt.getDbConn() - numTables = self.LARGE_NUMBER_OF_TABLES if Settings.getConfig().larger_data else self.SMALL_NUMBER_OF_TABLES - numRecords = self.LARGE_NUMBER_OF_RECORDS if Settings.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS + numTables = self.LARGE_NUMBER_OF_TABLES if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_TABLES + numRecords = self.LARGE_NUMBER_OF_RECORDS if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS tblSeq = list(range(numTables )) random.shuffle(tblSeq) # now we have random sequence for i in tblSeq: @@ -2120,7 +2123,7 @@ class TaskAddData(StateTransitionTask): regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) fullTableName = dbName + '.' + regTableName # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked" - sTable.ensureTable(self, wt.getDbConn(), regTableName) # Ensure the table exists + sTable.ensureRegTable(self, wt.getDbConn(), regTableName) # Ensure the table exists # self._unlockTable(fullTableName) if Dice.throw(1) == 0: # 1 in 2 chance @@ -2264,7 +2267,7 @@ class ClientManager: global gContainer tInst = gContainer.defTdeInstance = TdeInstance() # "subdir to hold the instance" - cfg = Settings.getConfig() + cfg = Config.getConfig() dbManager = DbManager(cfg.connector_type, tInst.getDbTarget()) # Regular function thPool = ThreadPool(cfg.num_threads, cfg.max_steps) self.tc = ThreadCoordinator(thPool, dbManager) @@ -2280,7 +2283,7 @@ class ClientManager: # Release global variables # gConfig = None - Settings.clearConfig() + Config.clearConfig() gSvcMgr = None logger = None @@ -2331,7 +2334,7 @@ class MainExec: def runClient(self): global gSvcMgr - if Settings.getConfig().auto_start_service: + if Config.getConfig().auto_start_service: gSvcMgr = self._svcMgr = ServiceManager(1) # hack alert gSvcMgr.startTaosServices() # we start, don't run @@ -2346,20 +2349,12 @@ class MainExec: def runService(self): global gSvcMgr - gSvcMgr = self._svcMgr = ServiceManager(Settings.getConfig().num_dnodes) # save it in a global variable TODO: hack alert + gSvcMgr = self._svcMgr = ServiceManager(Config.getConfig().num_dnodes) # save it in a global variable TODO: hack alert gSvcMgr.run() # run to some end state gSvcMgr = self._svcMgr = None - def init(self): # TODO: refactor - global gContainer - gContainer = Container() # micky-mouse DI - - global gSvcMgr # TODO: refactor away - gSvcMgr = None - - # Super cool Python argument library: - # https://docs.python.org/3/library/argparse.html + def _buildCmdLineParser(self): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent('''\ @@ -2480,20 +2475,29 @@ class MainExec: action='store_true', help='Continue execution after encountering unexpected/disallowed errors/exceptions (default: false)') - # global gConfig - config = parser.parse_args() - Settings.setConfig(config) # TODO: fix this hack, consolidate this global var + return parser + + + def init(self): # TODO: refactor + global gContainer + gContainer = Container() # micky-mouse DI + + global gSvcMgr # TODO: refactor away + gSvcMgr = None + + parser = self._buildCmdLineParser() + Config.init(parser) # Sanity check for arguments - if Settings.getConfig().use_shadow_db and Settings.getConfig().max_dbs>1 : + if Config.getConfig().use_shadow_db and Config.getConfig().max_dbs>1 : raise CrashGenError("Cannot combine use-shadow-db with max-dbs of more than 1") - Logging.clsInit(Settings.getConfig()) + Logging.clsInit(Config.getConfig().debug) Dice.seed(0) # initial seeding of dice def run(self): - if Settings.getConfig().run_tdengine: # run server + if Config.getConfig().run_tdengine: # run server try: self.runService() return 0 # success diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py index f6784f17e3..95507f0142 100644 --- a/tests/pytest/crash_gen/service_manager.py +++ b/tests/pytest/crash_gen/service_manager.py @@ -19,10 +19,15 @@ except: sys.exit(-1) from queue import Queue, Empty -from crash_gen.misc import CrashGenError, Dice, Helper, Logging, Progress, Status -from crash_gen.db import DbConn, DbTarget -from crash_gen.settings import Settings -from crash_gen.types import DirPath +from .shared.config import Config +from .shared.db import DbTarget, DbConn +from .shared.misc import Logging, Helper, CrashGenError, Status, Progress, Dice +from .shared.types import DirPath + +# from crash_gen.misc import CrashGenError, Dice, Helper, Logging, Progress, Status +# from crash_gen.db import DbConn, DbTarget +# from crash_gen.settings import Config +# from crash_gen.types import DirPath class TdeInstance(): """ @@ -173,7 +178,7 @@ quorum 2 def getServiceCmdLine(self): # to start the instance cmdLine = [] - if Settings.getConfig().track_memory_leaks: + if Config.getConfig().track_memory_leaks: Logging.info("Invoking VALGRIND on service...") cmdLine = ['valgrind', '--leak-check=yes'] # TODO: move "exec -c" into Popen(), we can both "use shell" and NOT fork so ask to lose kill control @@ -789,22 +794,10 @@ class ServiceManagerThread: def stop(self): # can be called from both main thread or signal handler - # Linux will send Control-C generated SIGINT to the TDengine process - # already, ref: + # Linux will send Control-C generated SIGINT to the TDengine process already, ref: # https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes - # if not self._tdeSubProcess: - # raise RuntimeError("sub process object missing") - # self._status.set(Status.STATUS_STOPPING) - # TdeSubProcess.stop(self._tdeSubProcess) # must stop, no matter what - # self._tdeSubProcess = None - # if not self._tdeSubProcess.stop(): # everything withing - # if self._tdeSubProcess.isRunning(): # still running, should now never happen - # Logging.error("FAILED to stop sub process, it is still running... pid = {}".format( - # self._tdeSubProcess.getPid())) - # else: - # self._tdeSubProcess = None # not running any more - self.join() # stop the thread, change the status, etc. + self.join() # stop the thread, status change moved to TdeSubProcess # Check if it's really stopped outputLines = 10 # for last output diff --git a/tests/pytest/crash_gen/settings.py b/tests/pytest/crash_gen/shared/config.py similarity index 54% rename from tests/pytest/crash_gen/settings.py rename to tests/pytest/crash_gen/shared/config.py index 82214930b4..7b9f7c3873 100644 --- a/tests/pytest/crash_gen/settings.py +++ b/tests/pytest/crash_gen/shared/config.py @@ -1,17 +1,23 @@ from __future__ import annotations import argparse + from typing import Optional -from crash_gen.misc import CrashGenError +from .misc import CrashGenError + +# from crash_gen.misc import CrashGenError # gConfig: Optional[argparse.Namespace] -class Settings: +class Config: _config = None # type Optional[argparse.Namespace] @classmethod - def init(cls): - cls._config = None + def init(cls, parser: argparse.ArgumentParser): + if cls._config is not None: + raise CrashGenError("Config can only be initialized once") + cls._config = parser.parse_args() + # print(cls._config) @classmethod def setConfig(cls, config: argparse.Namespace): @@ -26,4 +32,11 @@ class Settings: @classmethod def clearConfig(cls): - cls._config = None \ No newline at end of file + cls._config = None + + @classmethod + def isSet(cls, cfgKey): + cfg = cls.getConfig() + if cfgKey not in cfg: + return False + return cfg.__getattribute__(cfgKey) \ No newline at end of file diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/shared/db.py similarity index 97% rename from tests/pytest/crash_gen/db.py rename to tests/pytest/crash_gen/shared/db.py index 6c85da0051..75931ace48 100644 --- a/tests/pytest/crash_gen/db.py +++ b/tests/pytest/crash_gen/shared/db.py @@ -1,11 +1,13 @@ from __future__ import annotations import sys +import os +import datetime import time import threading import requests from requests.auth import HTTPBasicAuth -from crash_gen.types import QueryResult + import taos from util.sql import * @@ -13,13 +15,12 @@ from util.cases import * from util.dnodes import * from util.log import * -from .misc import Logging, CrashGenError, Helper, Dice -import os -import datetime import traceback # from .service_manager import TdeInstance -from crash_gen.settings import Settings +from .config import Config +from .misc import Logging, CrashGenError, Helper +from .types import QueryResult class DbConn: TYPE_NATIVE = "native-c" @@ -250,7 +251,13 @@ class MyTDSql: def _execInternal(self, sql): startTime = time.time() # Logging.debug("Executing SQL: " + sql) + # ret = None # TODO: use strong type here + # try: # Let's not capture the error, and let taos.error.ProgrammingError pass through ret = self._cursor.execute(sql) + # except taos.error.ProgrammingError as err: + # Logging.warning("Taos SQL execution error: {}, SQL: {}".format(err.msg, sql)) + # raise CrashGenError(err.msg) + # print("\nSQL success: {}".format(sql)) queryTime = time.time() - startTime # Record the query time @@ -262,7 +269,7 @@ class MyTDSql: cls.lqStartTime = startTime # Now write to the shadow database - if Settings.getConfig().use_shadow_db: + if Config.isSet('use_shadow_db'): if sql[:11] == "INSERT INTO": if sql[:16] == "INSERT INTO db_0": sql2 = "INSERT INTO db_s" + sql[16:] diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/shared/misc.py similarity index 96% rename from tests/pytest/crash_gen/misc.py rename to tests/pytest/crash_gen/shared/misc.py index cb98bc7c5b..90ad802ff1 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/shared/misc.py @@ -47,7 +47,7 @@ class Logging: return cls.logger @classmethod - def clsInit(cls, gConfig): # TODO: refactor away gConfig + def clsInit(cls, debugMode: bool): if cls.logger: return @@ -62,12 +62,8 @@ class Logging: # print("setting logger variable") # global logger cls.logger = MyLoggingAdapter(_logger, {}) - - if (gConfig.debug): - cls.logger.setLevel(logging.DEBUG) # default seems to be INFO - else: - cls.logger.setLevel(logging.INFO) - + cls.logger.setLevel(logging.DEBUG if debugMode else logging.INFO) # default seems to be INFO + @classmethod def info(cls, msg): cls.logger.info(msg) diff --git a/tests/pytest/crash_gen/shared/types.py b/tests/pytest/crash_gen/shared/types.py new file mode 100644 index 0000000000..814a821917 --- /dev/null +++ b/tests/pytest/crash_gen/shared/types.py @@ -0,0 +1,28 @@ +from typing import Any, List, Dict, NewType +from enum import Enum + +DirPath = NewType('DirPath', str) + +QueryResult = NewType('QueryResult', List[List[Any]]) + +class TdDataType(Enum): + ''' + Use a Python Enum types of represent all the data types in TDengine. + + Ref: https://www.taosdata.com/cn/documentation/taos-sql#data-type + ''' + TIMESTAMP = 'TIMESTAMP' + INT = 'INT' + BIGINT = 'BIGINT' + FLOAT = 'FLOAT' + DOUBLE = 'DOUBLE' + BINARY = 'BINARY' + BINARY16 = 'BINARY(16)' # TODO: get rid of this hack + BINARY200 = 'BINARY(200)' + SMALLINT = 'SMALLINT' + TINYINT = 'TINYINT' + BOOL = 'BOOL' + NCHAR = 'NCHAR' + +TdColumns = Dict[str, TdDataType] +TdTags = Dict[str, TdDataType] diff --git a/tests/pytest/crash_gen/types.py b/tests/pytest/crash_gen/types.py deleted file mode 100644 index fdf8ea2478..0000000000 --- a/tests/pytest/crash_gen/types.py +++ /dev/null @@ -1,5 +0,0 @@ -from typing import Any, List, NewType - -DirPath = NewType('DirPath', str) - -QueryResult = NewType('QueryResult', List[List[Any]]) \ No newline at end of file From 1d7fd5aec416ea1b6be6286abea1c5e3c9e4ac5a Mon Sep 17 00:00:00 2001 From: Steven Li Date: Tue, 11 May 2021 01:48:58 +0000 Subject: [PATCH 024/157] Added initial code for the new perf_gen tool --- tests/pytest/perf_gen.py | 480 +++++++++++++++++++++++++++++++++++++++ tests/pytest/perf_gen.sh | 60 +++++ 2 files changed, 540 insertions(+) create mode 100755 tests/pytest/perf_gen.py create mode 100755 tests/pytest/perf_gen.sh diff --git a/tests/pytest/perf_gen.py b/tests/pytest/perf_gen.py new file mode 100755 index 0000000000..03a09a2da5 --- /dev/null +++ b/tests/pytest/perf_gen.py @@ -0,0 +1,480 @@ +#!/usr/bin/python3.8 + +from abc import abstractmethod + +import time +from datetime import datetime + +from influxdb_client import InfluxDBClient, Point, WritePrecision, BucketsApi +from influxdb_client.client.write_api import SYNCHRONOUS + +import argparse +import textwrap +import subprocess +import sys + +import taos + +from crash_gen.crash_gen_main import Database, TdSuperTable +from crash_gen.service_manager import TdeInstance + +from crash_gen.shared.config import Config +from crash_gen.shared.db import DbConn +from crash_gen.shared.misc import Dice, Logging, Helper +from crash_gen.shared.types import TdDataType + + +# NUM_PROCESSES = 10 +# NUM_REPS = 1000 + +tick = int(time.time() - 5000000.0) # for now we will create max 5M record +value = 101 + +DB_NAME = 'mydb' +TIME_SERIES_NAME = 'widget' + +MAX_SHELF = 500 # shelf number runs up to this, non-inclusive +ITEMS_PER_SHELF = 5 +BATCH_SIZE = 2000 # Number of data points per request + +# None_RW: +# INFLUX_TOKEN='RRzVQZs8ERCpV9cS2RXqgtM_Y6FEZuJ7Tuk0aHtZItFTfcM9ajixtGDhW8HzqNIBmG3hmztw-P4sHOstfJvjFA==' +# DevOrg_RW: +# INFLUX_TOKEN='o1P8sEhBmXKhxBmNuiCyOUKv8d7qm5wUjMff9AbskBu2LcmNPQzU77NrAn5hDil8hZ0-y1AGWpzpL-4wqjFdkA==' +# DevOrg_All_Access +INFLUX_TOKEN='T2QTr4sloJhINH_oSrwSS-WIIZYjDfD123NK4ou3b7ajRs0c0IphCh3bNc0OsDZQRW1HyCby7opdEndVYFGTWQ==' +INFLUX_ORG="DevOrg" +INFLUX_BUCKET="Bucket01" + +def writeTaosBatch(dbc, tblName): + # Database.setupLastTick() + global value, tick + + data = [] + for i in range(0, 100): + data.append("('{}', {})".format(Database.getNextTick(), value) ) + value += 1 + + sql = "INSERT INTO {} VALUES {}".format(tblName, ''.join(data)) + dbc.execute(sql) + +class PerfGenError(taos.error.ProgrammingError): + pass + +class Benchmark(): + + # @classmethod + # def create(cls, dbType): + # if dbType == 'taos': + # return TaosBenchmark() + # elif dbType == 'influx': + # return InfluxBenchmark() + # else: + # raise RuntimeError("Unknown DB type: {}".format(dbType)) + + def __init__(self, dbType, loopCount = 0): + self._dbType = dbType + self._setLoopCount(loopCount) + + def _setLoopCount(self, loopCount): + cfgLoopCount = Config.getConfig().loop_count + if loopCount == 0: # use config + self._loopCount = cfgLoopCount + else: + if cfgLoopCount : + Logging.warning("Ignoring loop count for fixed-loop-count benchmarks: {}".format(cfgLoopCount)) + self._loopCount = loopCount + + @abstractmethod + def doIterate(self): + ''' + Execute the benchmark directly, without invoking sub processes, + effectively using one execution thread. + ''' + pass + + @abstractmethod + def prepare(self): + ''' + Preparation needed to run a certain benchmark + ''' + pass + + @abstractmethod + def execute(self): + ''' + Actually execute the benchmark + ''' + Logging.warning("Unexpected execution") + + @property + def name(self): + return self.__class__.__name__ + + def run(self): + print("Running benchmark: {}, class={} ...".format(self.name, self.__class__)) + startTime = time.time() + + # Prepare to execute the benchmark + self.prepare() + + # Actually execute the benchmark + self.execute() + + # if Config.getConfig().iterate_directly: # execute directly + # Logging.debug("Iterating...") + # self.doIterate() + # else: + # Logging.debug("Executing via sub process...") + # startTime = time.time() + # self.prepare() + # self.spawnProcesses() + # self.waitForProcecess() + # duration = time.time() - startTime + # Logging.info("Benchmark execution completed in {:.3f} seconds".format(duration)) + Logging.info("Benchmark {} finished in {:.3f} seconds".format( + self.name, time.time()-startTime)) + + def spawnProcesses(self): + self._subProcs = [] + for j in range(0, Config.getConfig().subprocess_count): + ON_POSIX = 'posix' in sys.builtin_module_names + tblName = 'cars_reg_{}'.format(j) + cmdLineStr = './perf_gen.sh -t {} -i -n {} -l {}'.format( + self._dbType, + tblName, + Config.getConfig().loop_count + ) + if Config.getConfig().debug: + cmdLineStr += ' -d' + subProc = subprocess.Popen(cmdLineStr, + shell = True, + close_fds = ON_POSIX) + self._subProcs.append(subProc) + + def waitForProcecess(self): + for sp in self._subProcs: + sp.wait(300) + + +class TaosBenchmark(Benchmark): + + def __init__(self, loopCount): + super().__init__('taos', loopCount) + # self._dbType = 'taos' + tInst = TdeInstance() + self._dbc = DbConn.createNative(tInst.getDbTarget()) + self._dbc.open() + self._sTable = TdSuperTable(TIME_SERIES_NAME + '_s', DB_NAME) + + def doIterate(self): + tblName = Config.getConfig().target_table_name + print("Benchmarking TAOS database (1 pass) for: {}".format(tblName)) + self._dbc.execute("USE {}".format(DB_NAME)) + + self._sTable.ensureRegTable(None, self._dbc, tblName) + try: + lCount = Config.getConfig().loop_count + print("({})".format(lCount)) + for i in range(0, lCount): + writeTaosBatch(self._dbc, tblName) + except taos.error.ProgrammingError as err: + Logging.error("Failed to write batch") + + def prepare(self): + self._dbc.execute("CREATE DATABASE IF NOT EXISTS {}".format(DB_NAME)) + self._dbc.execute("USE {}".format(DB_NAME)) + # Create the super table + self._sTable.drop(self._dbc, True) + self._sTable.create(self._dbc, + {'ts': TdDataType.TIMESTAMP, + 'temperature': TdDataType.INT, + 'pressure': TdDataType.INT, + 'notes': TdDataType.BINARY200 + }, + {'rack': TdDataType.INT, + 'shelf': TdDataType.INT, + 'barcode': TdDataType.BINARY16 + }) + + def execSql(self, sql): + try: + self._dbc.execute(sql) + except taos.error.ProgrammingError as err: + Logging.warning("SQL Error: 0x{:X}, {}, SQL: {}".format( + Helper.convertErrno(err.errno), err.msg, sql)) + raise + + def executeWrite(self): + # Sample: INSERT INTO t1 USING st TAGS(1) VALUES(now, 1) t2 USING st TAGS(2) VALUES(now, 2) + sqlPrefix = "INSERT INTO " + dataTemplate = "{} USING {} TAGS({},{},'barcode_{}') VALUES('{}',{},{},'{}') " + + stName = self._sTable.getName() + BATCH_SIZE = 2000 # number of items per request batch + ITEMS_PER_SHELF = 5 + + # rackSize = 10 # shelves per rack + # shelfSize = 100 # items per shelf + batchCount = self._loopCount // BATCH_SIZE + lastRack = 0 + for i in range(batchCount): + sql = sqlPrefix + for j in range(BATCH_SIZE): + n = i*BATCH_SIZE + j # serial number + # values first + # rtName = 'rt_' + str(n) # table name contains serial number, has info + temperature = 20 + (n % 10) + pressure = 70 + (n % 10) + # tags + shelf = (n // ITEMS_PER_SHELF) % MAX_SHELF # shelf number + rack = n // (ITEMS_PER_SHELF * MAX_SHELF) # rack number + barcode = rack + shelf + # table name + tableName = "reg_" + str(rack) + '_' + str(shelf) + # now the SQL + sql += dataTemplate.format(tableName, stName,# table name + rack, shelf, barcode, # tags + Database.getNextTick(), temperature, pressure, 'xxx') # values + lastRack = rack + self.execSql(sql) + Logging.info("Last Rack: {}".format(lastRack)) + +class TaosWriteBenchmark(TaosBenchmark): + def execute(self): + self.executeWrite() + +class Taos100kWriteBenchmark(TaosWriteBenchmark): + def __init__(self): + super().__init__(100*1000) + +class Taos10kWriteBenchmark(TaosWriteBenchmark): + def __init__(self): + super().__init__(10*1000) + +class Taos1mWriteBenchmark(TaosWriteBenchmark): + def __init__(self): + super().__init__(1000*1000) + +class Taos5mWriteBenchmark(TaosWriteBenchmark): + def __init__(self): + super().__init__(5*1000*1000) + +class Taos1kQueryBenchmark(TaosBenchmark): + def __init__(self): + super().__init__(1000) + +class Taos1MCreationBenchmark(TaosBenchmark): + def __init__(self): + super().__init__(1000000) + + +class InfluxBenchmark(Benchmark): + def __init__(self, loopCount): + super().__init__('influx', loopCount) + # self._dbType = 'influx' + + + # self._client = InfluxDBClient(host='localhost', port=8086) + + # def _writeBatch(self, tblName): + # global value, tick + # data = [] + # for i in range(0, 100): + # line = "{},device={} value={} {}".format( + # TIME_SERIES_NAME, + # tblName, + # value, + # tick*1000000000) + # # print(line) + # data.append(line) + # value += 1 + # tick +=1 + + # self._client.write(data, {'db':DB_NAME}, protocol='line') + + def executeWrite(self): + global tick # influx tick #TODO refactor + + lineTemplate = TIME_SERIES_NAME + ",rack={},shelf={},barcode='barcode_{}' temperature={},pressure={} {}" + + batchCount = self._loopCount // BATCH_SIZE + for i in range(batchCount): + lineBatch = [] + for j in range(BATCH_SIZE): + n = i*BATCH_SIZE + j # serial number + # values first + # rtName = 'rt_' + str(n) # table name contains serial number, has info + temperature = 20 + (n % 10) + pressure = 70 + (n % 10) + # tags + shelf = (n // ITEMS_PER_SHELF) % MAX_SHELF # shelf number + rack = n // (ITEMS_PER_SHELF * MAX_SHELF) # rack number + barcode = rack + shelf + # now the SQL + line = lineTemplate.format( + rack, shelf, barcode, # tags + temperature, pressure, # values + tick * 1000000000 ) + tick += 1 + lineBatch.append(line) + write_api = self._client.write_api(write_options=SYNCHRONOUS) + write_api.write(INFLUX_BUCKET, INFLUX_ORG, lineBatch) + # self._client.write(lineBatch, {'db':DB_NAME}, protocol='line') + + # def doIterate(self): + # tblName = Config.getConfig().target_table_name + # print("Benchmarking INFLUX database (1 pass) for: {}".format(tblName)) + + # for i in range(0, Config.getConfig().loop_count): + # self._writeBatch(tblName) + + def _getOrgIdByName(self, orgName): + """Find org by name. + + """ + orgApi = self._client.organizations_api() + orgs = orgApi.find_organizations() + for org in orgs: + if org.name == orgName: + return org.id + raise PerfGenError("Org not found with name: {}".format(orgName)) + + def _fetchAuth(self): + authApi = self._client.authorizations_api() + auths = authApi.find_authorizations() + for auth in auths: + if auth.token == INFLUX_TOKEN : + return auth + raise PerfGenError("No proper auth found") + + def _verifyPermissions(self, perms: list): + if list: + return #OK + raise PerfGenError("No permission found") + + def prepare(self): + self._client = InfluxDBClient( + url="http://127.0.0.1:8086", + token=INFLUX_TOKEN, + org=INFLUX_ORG) + + auth = self._fetchAuth() + + self._verifyPermissions(auth.permissions) + + bktApi = self._client.buckets_api() + # Delete + bkt = bktApi.find_bucket_by_name(INFLUX_BUCKET) + if bkt: + bktApi.delete_bucket(bkt) + # Recreate + + orgId = self._getOrgIdByName(INFLUX_ORG) + bktApi.create_bucket(bucket=None, bucket_name=INFLUX_BUCKET, org_id=orgId) + + # self._client.drop_database(DB_NAME) + # self._client.create_database(DB_NAME) + # self._client.switch_database(DB_NAME) + +class InfluxWriteBenchmark(InfluxBenchmark): + def execute(self): + return self.executeWrite() + +class Influx10kWriteBenchmark(InfluxWriteBenchmark): + def __init__(self): + super().__init__(10*1000) + +class Influx100kWriteBenchmark(InfluxWriteBenchmark): + def __init__(self): + super().__init__(100*1000) + +class Influx1mWriteBenchmark(InfluxWriteBenchmark): + def __init__(self): + super().__init__(1000*1000) + +class Influx5mWriteBenchmark(InfluxWriteBenchmark): + def __init__(self): + super().__init__(5*1000*1000) + +def _buildCmdLineParser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=textwrap.dedent('''\ + TDengine Performance Benchmarking Tool + --------------------------------------------------------------------- + + ''')) + + parser.add_argument( + '-b', + '--benchmark-name', + action='store', + default='Taos1kQuery', + type=str, + help='Benchmark to use (default: Taos1kQuery)') + + parser.add_argument( + '-d', + '--debug', + action='store_true', + help='Turn on DEBUG mode for more logging (default: false)') + + parser.add_argument( + '-i', + '--iterate-directly', + action='store_true', + help='Execution operations directly without sub-process (default: false)') + + parser.add_argument( + '-l', + '--loop-count', + action='store', + default=1000, + type=int, + help='Number of loops to perform, 100 operations per loop. (default: 1000)') + + parser.add_argument( + '-n', + '--target-table-name', + action='store', + default=None, + type=str, + help='Regular table name in target DB (default: None)') + + parser.add_argument( + '-s', + '--subprocess-count', + action='store', + default=4, + type=int, + help='Number of sub processes to spawn. (default: 10)') + + parser.add_argument( + '-t', + '--target-database', + action='store', + default='taos', + type=str, + help='Benchmark target: taos, influx (default: taos)') + + return parser + +def main(): + parser = _buildCmdLineParser() + Config.init(parser) + Logging.clsInit(Config.getConfig().debug) + Dice.seed(0) # initial seeding of dice + + bName = Config.getConfig().benchmark_name + bClass = globals()[bName + 'Benchmark'] + bm = bClass() # Benchmark object + bm.run() + + # bm = Benchmark.create(Config.getConfig().target_database) + # bm.run() + +if __name__ == "__main__": + main() + + diff --git a/tests/pytest/perf_gen.sh b/tests/pytest/perf_gen.sh new file mode 100755 index 0000000000..fcedd2d407 --- /dev/null +++ b/tests/pytest/perf_gen.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# This is the script for us to try to cause the TDengine server or client to crash +# +# PREPARATION +# +# 1. Build an compile the TDengine source code that comes with this script, in the same directory tree +# 2. Please follow the direction in our README.md, and build TDengine in the build/ directory +# 3. Adjust the configuration file if needed under build/test/cfg/taos.cfg +# 4. Run the TDengine server instance: cd build; ./build/bin/taosd -c test/cfg +# 5. Make sure you have a working Python3 environment: run /usr/bin/python3 --version, and you should get 3.6 or above +# 6. Make sure you have the proper Python packages: # sudo apt install python3-setuptools python3-pip python3-distutils +# +# RUNNING THIS SCRIPT +# +# This script assumes the source code directory is intact, and that the binaries has been built in the +# build/ directory, as such, will will load the Python libraries in the directory tree, and also load +# the TDengine client shared library (so) file, in the build/directory, as evidenced in the env +# variables below. +# +# Running the script is simple, no parameter is needed (for now, but will change in the future). +# +# Happy Crashing... + + +# Due to the heavy path name assumptions/usage, let us require that the user be in the current directory +EXEC_DIR=`dirname "$0"` +if [[ $EXEC_DIR != "." ]] +then + echo "ERROR: Please execute `basename "$0"` in its own directory (for now anyway, pardon the dust)" + exit -1 +fi + +CURR_DIR=`pwd` +IN_TDINTERNAL="community" +if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + TAOS_DIR=$CURR_DIR/../../.. + TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1` + LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6,7|rev`/lib +else + TAOS_DIR=$CURR_DIR/../.. + TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1` + LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib +fi + +# Now getting ready to execute Python +# The following is the default of our standard dev env (Ubuntu 20.04), modify/adjust at your own risk +PYTHON_EXEC=python3.8 + +# First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. +export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3:$(pwd) + +# Then let us set up the library path so that our compiled SO file can be loaded by Python +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR + +# Now we are all let, and let's see if we can find a crash. Note we pass all params +PERF_GEN_EXEC=perf_gen.py +$PYTHON_EXEC $PERF_GEN_EXEC $@ + + From 12228d0d9636608e17ded5257361adb75f139432 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 11 May 2021 09:54:03 +0800 Subject: [PATCH 025/157] [TD-4073]: taosdemo rest query output to result file. (#6072) * [TD-4073]: taosdemo rest query output to result file. * [TD-4073]: taosdemo rest query output to result file. fix insecure strcat(). * [TD-4073]: taosdemo rest query output to result file. fix postSql command mistake * [TD-4073]: taosdemo rest query output to result file. fix append function bug. * [TD-4073]: taosdemo rest query output to result file. prevent potential null file passed. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 154 ++++++++++++++++++++---------------- 1 file changed, 84 insertions(+), 70 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 99befbaf31..935aa358c5 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -76,6 +76,7 @@ enum QUERY_MODE { #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) +#define COND_BUF_LEN BUFFER_SIZE - 30 #define MAX_USERNAME_SIZE 64 #define MAX_PASSWORD_SIZE 64 #define MAX_DB_NAME_SIZE 64 @@ -522,6 +523,8 @@ static int taosRandom() static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); +static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, + char* sqlstr, char *resultFile); /* ************ Global variables ************ */ @@ -1090,27 +1093,33 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { return 0; } -static void getResult(TAOS_RES *res, char* resultFileName) { +static void appendResultBufToFile(char *resultBuf, char *resultFile) +{ + FILE *fp = NULL; + if (resultFile[0] != 0) { + fp = fopen(resultFile, "at"); + if (fp == NULL) { + errorPrint( + "%s() LN%d, failed to open result file: %s, result will not save to file\n", + __func__, __LINE__, resultFile); + return; + } + } + + fprintf(fp, "%s", resultBuf); + tmfclose(fp); +} + +static void appendResultToFile(TAOS_RES *res, char* resultFile) { TAOS_ROW row = NULL; int num_rows = 0; int num_fields = taos_field_count(res); TAOS_FIELD *fields = taos_fetch_fields(res); - FILE *fp = NULL; - if (resultFileName[0] != 0) { - fp = fopen(resultFileName, "at"); - if (fp == NULL) { - errorPrint("%s() LN%d, failed to open result file: %s, result will not save to file\n", - __func__, __LINE__, resultFileName); - } - } - char* databuf = (char*) calloc(1, 100*1024*1024); if (databuf == NULL) { errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n", __func__, __LINE__); - if (fp) - fclose(fp); return ; } @@ -1120,7 +1129,7 @@ static void getResult(TAOS_RES *res, char* resultFileName) { // fetch the records row by row while((row = taos_fetch_row(res))) { if (totalLen >= 100*1024*1024 - 32000) { - if (fp) fprintf(fp, "%s", databuf); + appendResultBufToFile(databuf, resultFile); totalLen = 0; memset(databuf, 0, 100*1024*1024); } @@ -1132,22 +1141,39 @@ static void getResult(TAOS_RES *res, char* resultFileName) { totalLen += len; } - if (fp) fprintf(fp, "%s", databuf); - tmfclose(fp); + appendResultBufToFile(databuf, resultFile); free(databuf); } -static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) { - TAOS_RES *res = taos_query(taos, command); - if (res == NULL || taos_errno(res) != 0) { - errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n", - __func__, __LINE__, command, taos_errstr(res)); - taos_free_result(res); - return; - } +static void selectAndGetResult(threadInfo *pThreadInfo, char *command, char* resultFile) +{ + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) { + TAOS_RES *res = taos_query(pThreadInfo->taos, command); + if (res == NULL || taos_errno(res) != 0) { + errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n", + __func__, __LINE__, command, taos_errstr(res)); + taos_free_result(res); + return; + } - getResult(res, resultFileName); - taos_free_result(res); + if ((resultFile) && (strlen(resultFile))) { + appendResultToFile(res, resultFile); + } + taos_free_result(res); + + } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { + int retCode = postProceSql( + g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port, + command, + resultFile); + if (0 != retCode) { + printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID); + } + + } else { + errorPrint("%s() LN%d, unknown query mode: %s\n", + __func__, __LINE__, g_queryInfo.queryMode); + } } static int32_t rand_bool(){ @@ -1940,13 +1966,13 @@ static void printfQuerySystemInfo(TAOS * taos) { // show variables res = taos_query(taos, "show variables;"); - //getResult(res, filename); + //appendResultToFile(res, filename); xDumpResultToFile(filename, res); // show dnodes res = taos_query(taos, "show dnodes;"); xDumpResultToFile(filename, res); - //getResult(res, filename); + //appendResultToFile(res, filename); // show databases res = taos_query(taos, "show databases;"); @@ -1981,7 +2007,8 @@ static void printfQuerySystemInfo(TAOS * taos) { free(dbInfos); } -static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, char* sqlstr) +static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, + char* sqlstr, char *resultFile) { char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s"; @@ -2117,6 +2144,10 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port response_buf[RESP_BUF_LEN - 1] = '\0'; printf("Response:\n%s\n", response_buf); + if (resultFile) { + appendResultBufToFile(response_buf, resultFile); + } + free(request_buf); #ifdef WINDOWS closesocket(sockfd); @@ -4688,7 +4719,8 @@ static int64_t execInsert(threadInfo *pThreadInfo, char *buffer, uint64_t k) if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) { affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false); } else if (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest"))) { - if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port, buffer)) { + if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port, + buffer, NULL /* not set result file */)) { affectedRows = -1; printf("========restful return fail, threadID[%d]\n", pThreadInfo->threadID); @@ -5936,7 +5968,7 @@ static void *readMetric(void *sarg) { fprintf(fp, "Querying On %d records:\n", totalData); for (int j = 0; j < n; j++) { - char condition[BUFFER_SIZE - 30] = "\0"; + char condition[COND_BUF_LEN] = "\0"; char tempS[64] = "\0"; int m = 10 < num_of_tables ? 10 : num_of_tables; @@ -5947,7 +5979,7 @@ static void *readMetric(void *sarg) { } else { sprintf(tempS, " or t1 = %d ", i); } - strcat(condition, tempS); + strncat(condition, tempS, COND_BUF_LEN - 1); sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); @@ -6125,43 +6157,24 @@ static void *specifiedTableQuery(void *sarg) { taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms } - st = taosGetTimestampMs(); - - if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) { - int64_t t1 = taosGetTimestampMs(); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(tmpFile, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - } - selectAndGetResult(pThreadInfo->taos, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], tmpFile); - int64_t t2 = taosGetTimestampMs(); - printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %10.3f s\n", - taosGetSelfPthreadId(), (t2 - t1)/1000.0); - } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { - int64_t t1 = taosGetTimestampMs(); - int retCode = postProceSql(g_queryInfo.host, &(g_queryInfo.serv_addr), - g_queryInfo.port, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]); - if (0 != retCode) { - printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID); - return NULL; - } - int64_t t2 = taosGetTimestampMs(); - printf("=[restful] thread[%"PRId64"] complete one sql, Spent %10.3f s\n", - taosGetSelfPthreadId(), (t2 - t1)/1000.0); - - } else { - errorPrint("%s() LN%d, unknown query mode: %s\n", - __func__, __LINE__, g_queryInfo.queryMode); - return NULL; } - totalQueried ++; - g_queryInfo.specifiedQueryInfo.totalQueried ++; + + st = taosGetTimestampMs(); + + selectAndGetResult(pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], tmpFile); et = taosGetTimestampMs(); + printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n", + taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0); + + totalQueried ++; + g_queryInfo.specifiedQueryInfo.totalQueried ++; uint64_t currentPrintTime = taosGetTimestampMs(); uint64_t endTs = taosGetTimestampMs(); @@ -6194,14 +6207,14 @@ static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) { tstrncpy(outSql, inSql, pos - inSql + 1); //printf("1: %s\n", outSql); - strcat(outSql, subTblName); + strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1); //printf("2: %s\n", outSql); - strcat(outSql, pos+strlen(sourceString)); + strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1); //printf("3: %s\n", outSql); } static void *superTableQuery(void *sarg) { - char sqlstr[1024]; + char sqlstr[MAX_QUERY_SQL_LENGTH]; threadInfo *pThreadInfo = (threadInfo *)sarg; if (pThreadInfo->taos == NULL) { @@ -6246,7 +6259,7 @@ static void *superTableQuery(void *sarg) { g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } - selectAndGetResult(pThreadInfo->taos, sqlstr, tmpFile); + selectAndGetResult(pThreadInfo, sqlstr, tmpFile); totalQueried++; g_queryInfo.superQueryInfo.totalQueried ++; @@ -6447,7 +6460,8 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c return; } - getResult(res, (char*)param); + if (param) + appendResultToFile(res, (char*)param); // tao_unscribe() will free result. } @@ -6476,7 +6490,7 @@ static TAOS_SUB* subscribeImpl( static void *superSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; - char subSqlstr[1024]; + char subSqlstr[MAX_QUERY_SQL_LENGTH]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; if (g_queryInfo.superQueryInfo.sqlCount == 0) @@ -6551,8 +6565,8 @@ static void *superSubscribe(void *sarg) { sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], pThreadInfo->threadID); + appendResultToFile(res, tmpFile); } - getResult(res, tmpFile); } } } @@ -6639,8 +6653,8 @@ static void *specifiedSubscribe(void *sarg) { if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); + appendResultToFile(res, tmpFile); } - getResult(res, tmpFile); } } } From 731eadb80d76696c7471c4c988fb9c32e151dc5e Mon Sep 17 00:00:00 2001 From: Steven Li Date: Tue, 11 May 2021 02:18:05 +0000 Subject: [PATCH 026/157] Minor tweak of the new Perf_gen tool --- tests/pytest/perf_gen.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/pytest/perf_gen.py b/tests/pytest/perf_gen.py index 03a09a2da5..f0402fbb6b 100755 --- a/tests/pytest/perf_gen.py +++ b/tests/pytest/perf_gen.py @@ -467,9 +467,14 @@ def main(): Dice.seed(0) # initial seeding of dice bName = Config.getConfig().benchmark_name - bClass = globals()[bName + 'Benchmark'] - bm = bClass() # Benchmark object - bm.run() + bClassName = bName + 'Benchmark' + x = globals() + if bClassName in globals(): + bClass = globals()[bClassName] + bm = bClass() # Benchmark object + bm.run() + else: + raise PerfGenError("No such benchmark: {}".format(bName)) # bm = Benchmark.create(Config.getConfig().target_database) # bm.run() From edbc123e2e0b4265e2ac89d85f10bb82bb34c39e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 May 2021 10:43:53 +0800 Subject: [PATCH 027/157] [td-225]update the test case. --- .../jdbc/cases/InsertSpecialCharacterRestfulTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java index 0cbbe76716..ea0d1aec41 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java @@ -13,7 +13,7 @@ public class InsertSpecialCharacterRestfulTest { private static String tbname1 = "test"; private static String tbname2 = "weather"; private static String special_character_str_1 = "$asd$$fsfsf$"; - private static String special_character_str_2 = "\\asdfsfsf\\\\"; + private static String special_character_str_2 = "\\\\asdfsfsf\\\\"; private static String special_character_str_3 = "\\\\asdfsfsf\\"; private static String special_character_str_4 = "?asd??fsf?sf?"; private static String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$"; @@ -49,7 +49,7 @@ public class InsertSpecialCharacterRestfulTest { @Test public void testCase02() throws SQLException { //TODO: - // Expected :\asdfsfsf\\ + // Expected :\asdfsfsf\ // Actual :\asdfsfsf\ final long now = System.currentTimeMillis(); @@ -71,7 +71,7 @@ public class InsertSpecialCharacterRestfulTest { String f1 = new String(rs.getBytes(2)); //TODO: bug to be fixed // Assert.assertEquals(special_character_str_2, f1); - Assert.assertEquals(special_character_str_2.substring(0, special_character_str_1.length() - 2), f1); + Assert.assertEquals(special_character_str_2.substring(1, special_character_str_1.length() - 1), f1); String f2 = rs.getString(3); Assert.assertNull(f2); } From 055919a7a71f233bbc77517ded850f197061993c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 11 May 2021 15:38:05 +0800 Subject: [PATCH 028/157] Hotfix/sangshuduo/td 3992 taosdemo subscribe (#6062) * [TD-3902]: taosdemo subscribe. * [TD-3992]: taosdemo subscribe. refactor sync/async mode. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 128 ++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 71 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 935aa358c5..3c83cb7a5e 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -68,12 +68,6 @@ enum TEST_MODE { INVAID_TEST }; -enum QUERY_MODE { - SYNC_QUERY_MODE, // 0 - ASYNC_QUERY_MODE, // 1 - INVALID_MODE -}; - #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) #define COND_BUF_LEN BUFFER_SIZE - 30 @@ -119,8 +113,8 @@ typedef enum TALBE_EXISTS_EN { } TALBE_EXISTS_EN; enum MODE { - SYNC, - ASYNC, + SYNC_MODE, + ASYNC_MODE, MODE_BUT }; @@ -206,7 +200,7 @@ typedef struct SArguments_S { bool verbose_print; bool performance_print; char * output_file; - uint32_t query_mode; + bool async_mode; char * datatype[MAX_NUM_DATATYPE + 1]; uint32_t len_of_binary; uint32_t num_of_CPR; @@ -344,7 +338,7 @@ typedef struct SDbs_S { bool use_metric; bool insert_only; bool do_aggreFunc; - bool queryMode; + bool asyncMode; uint32_t threadCount; uint32_t threadCountByCreateTbl; @@ -361,7 +355,7 @@ typedef struct SpecifiedQueryInfo_S { uint64_t queryInterval; // 0: unlimit > 0 loop/s uint64_t concurrent; uint64_t sqlCount; - uint32_t mode; // 0: sync, 1: async + uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms uint64_t queryTimes; int subscribeRestart; @@ -376,7 +370,7 @@ typedef struct SuperQueryInfo_S { char sTblName[MAX_TB_NAME_SIZE+1]; uint64_t queryInterval; // 0: unlimit > 0 loop/s uint32_t threadCnt; - uint32_t mode; // 0: sync, 1: async + uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms int subscribeRestart; int subscribeKeepProgress; @@ -774,49 +768,48 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->sqlFile = argv[++i]; } else if (strcmp(argv[i], "-q") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n"); exit(EXIT_FAILURE); } - arguments->query_mode = atoi(argv[++i]); + arguments->async_mode = atoi(argv[++i]); } else if (strcmp(argv[i], "-T") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-T need a number following!\n"); exit(EXIT_FAILURE); } arguments->num_of_threads = atoi(argv[++i]); } else if (strcmp(argv[i], "-i") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-i need a number following!\n"); exit(EXIT_FAILURE); } arguments->insert_interval = atoi(argv[++i]); } else if (strcmp(argv[i], "-qt") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1])) - || (atoi(argv[i+1]) <= 0)) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { printHelp(); - errorPrint("%s", "\n\t-qt need a valid (>0) number following!\n"); + errorPrint("%s", "\n\t-qt need a number following!\n"); exit(EXIT_FAILURE); } arguments->query_times = atoi(argv[++i]); } else if (strcmp(argv[i], "-B") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-B need a number following!\n"); exit(EXIT_FAILURE); } arguments->interlace_rows = atoi(argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-r need a number following!\n"); exit(EXIT_FAILURE); @@ -1076,7 +1069,7 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { if (code != 0) { if (!quiet) { debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command); - errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res)); + errorPrint("Failed to run %s, reason: %s\n", command, taos_errstr(res)); } taos_free_result(res); //taos_close(taos); @@ -1672,7 +1665,7 @@ static void printfQueryMeta() { printf("concurrent: \033[33m%"PRIu64"\033[0m\n", g_queryInfo.specifiedQueryInfo.concurrent); printf("mod: \033[33m%s\033[0m\n", - (g_queryInfo.specifiedQueryInfo.mode)?"async":"sync"); + (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync"); printf("interval: \033[33m%"PRIu64"\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeInterval); printf("restart: \033[33m%d\033[0m\n", @@ -1704,7 +1697,7 @@ static void printfQueryMeta() { g_queryInfo.superQueryInfo.queryTimes); printf("mod: \033[33m%s\033[0m\n", - (g_queryInfo.superQueryInfo.mode)?"async":"sync"); + (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync"); printf("interval: \033[33m%"PRIu64"\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval); printf("restart: \033[33m%d\033[0m\n", @@ -4072,9 +4065,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times"); if (gQueryTimes && gQueryTimes->type == cJSON_Number) { - if (gQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", - __func__, __LINE__, gQueryTimes->valueint); + if (gQueryTimes->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } g_args.query_times = gQueryTimes->valueint; @@ -4123,9 +4116,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, "query_times"); if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { - if (specifiedQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", - __func__, __LINE__, specifiedQueryTimes->valueint); + if (specifiedQueryTimes->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } @@ -4152,20 +4145,20 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.specifiedQueryInfo.concurrent = 1; } - cJSON* mode = cJSON_GetObjectItem(specifiedQuery, "mode"); - if (mode && mode->type == cJSON_String - && mode->valuestring != NULL) { - if (0 == strcmp("sync", mode->valuestring)) { - g_queryInfo.specifiedQueryInfo.mode = SYNC_QUERY_MODE; - } else if (0 == strcmp("async", mode->valuestring)) { - g_queryInfo.specifiedQueryInfo.mode = ASYNC_QUERY_MODE; + cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode"); + if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String + && specifiedAsyncMode->valuestring != NULL) { + if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) { + g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE; + } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) { + g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE; } else { - errorPrint("%s() LN%d, failed to read json, query mode input error\n", + errorPrint("%s() LN%d, failed to read json, async mode input error\n", __func__, __LINE__); goto PARSE_OVER; } } else { - g_queryInfo.specifiedQueryInfo.mode = SYNC_QUERY_MODE; + g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE; } cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval"); @@ -4267,9 +4260,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times"); if (superQueryTimes && superQueryTimes->type == cJSON_Number) { - if (superQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", - __func__, __LINE__, superQueryTimes->valueint); + if (superQueryTimes->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint; @@ -4312,20 +4305,20 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON* submode = cJSON_GetObjectItem(superQuery, "mode"); - if (submode && submode->type == cJSON_String - && submode->valuestring != NULL) { - if (0 == strcmp("sync", submode->valuestring)) { - g_queryInfo.superQueryInfo.mode = SYNC_QUERY_MODE; - } else if (0 == strcmp("async", submode->valuestring)) { - g_queryInfo.superQueryInfo.mode = ASYNC_QUERY_MODE; + cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode"); + if (superAsyncMode && superAsyncMode->type == cJSON_String + && superAsyncMode->valuestring != NULL) { + if (0 == strcmp("sync", superAsyncMode->valuestring)) { + g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE; + } else if (0 == strcmp("async", superAsyncMode->valuestring)) { + g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE; } else { - errorPrint("%s() LN%d, failed to read json, query mode input error\n", + errorPrint("%s() LN%d, failed to read json, async mode input error\n", __func__, __LINE__); goto PARSE_OVER; } } else { - g_queryInfo.superQueryInfo.mode = SYNC_QUERY_MODE; + g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE; } cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval"); @@ -5233,13 +5226,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { startTs = taosGetTimestampMs(); - if (recOfBatch == 0) { - errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n", - pThreadInfo->threadID, __func__, __LINE__, - recOfBatch); - errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n"); - goto free_of_interlace; - } int64_t affectedRows = execInsert(pThreadInfo, buffer, recOfBatch); endTs = taosGetTimestampMs(); @@ -5780,10 +5766,10 @@ static void startMultiThreadInsertData(int threads, char* db_name, } */ tsem_init(&(t_info->lock_sem), 0, 0); - if (SYNC == g_Dbs.queryMode) { - pthread_create(pids + i, NULL, syncWrite, t_info); - } else { + if (ASYNC_MODE == g_Dbs.asyncMode) { pthread_create(pids + i, NULL, asyncWrite, t_info); + } else { + pthread_create(pids + i, NULL, syncWrite, t_info); } } @@ -6469,7 +6455,7 @@ static TAOS_SUB* subscribeImpl( TAOS *taos, char *sql, char* topic, char* resultFileName) { TAOS_SUB* tsub = NULL; - if (ASYNC_QUERY_MODE == g_queryInfo.specifiedQueryInfo.mode) { + if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { tsub = taos_subscribe(taos, g_queryInfo.specifiedQueryInfo.subscribeRestart, topic, sql, subscribe_callback, (void*)resultFileName, @@ -6554,7 +6540,7 @@ static void *superSubscribe(void *sarg) { TAOS_RES* res = NULL; while(1) { for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - if (ASYNC_QUERY_MODE == g_queryInfo.superQueryInfo.mode) { + if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { continue; } @@ -6643,7 +6629,7 @@ static void *specifiedSubscribe(void *sarg) { TAOS_RES* res = NULL; while(1) { for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - if (ASYNC_QUERY_MODE == g_queryInfo.specifiedQueryInfo.mode) { + if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { continue; } @@ -6866,7 +6852,7 @@ static void setParaFromArg(){ g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; g_Dbs.threadCount = g_args.num_of_threads; g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; - g_Dbs.queryMode = g_args.query_mode; + g_Dbs.asyncMode = g_args.async_mode; g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; From af33e1184063a047de7eb26805fd4bd314dce579 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 11 May 2021 19:06:02 +0800 Subject: [PATCH 029/157] [TD-4136]: taosdemo max records per req < 32767 (#6082) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 12 +++++++++++- .../tools/taosdemoAllTest/speciQueryRestful.json | 4 ++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3c83cb7a5e..a3a8968017 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -68,6 +68,14 @@ enum TEST_MODE { INVAID_TEST }; +enum QUERY_MODE { + SYNC_QUERY_MODE, // 0 + ASYNC_QUERY_MODE, // 1 + INVALID_MODE +}; + +#define MAX_RECORDS_PER_REQ 32766 + #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) #define COND_BUF_LEN BUFFER_SIZE - 30 @@ -3436,10 +3444,12 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n", __func__, __LINE__); goto PARSE_OVER; + } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) { + numRecPerReq->valueint = MAX_RECORDS_PER_REQ; } g_args.num_of_RPR = numRecPerReq->valueint; } else if (!numRecPerReq) { - g_args.num_of_RPR = UINT64_MAX; + g_args.num_of_RPR = MAX_RECORDS_PER_REQ; } else { errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", __func__, __LINE__); diff --git a/tests/pytest/tools/taosdemoAllTest/speciQueryRestful.json b/tests/pytest/tools/taosdemoAllTest/speciQueryRestful.json index 98e9b7a4e8..bc3a66f43c 100644 --- a/tests/pytest/tools/taosdemoAllTest/speciQueryRestful.json +++ b/tests/pytest/tools/taosdemoAllTest/speciQueryRestful.json @@ -8,7 +8,7 @@ "confirm_parameter_prompt": "no", "databases": "db", "query_times": 2, - "query_mode": "restful", + "query_mode": "rest", "specified_table_query": { "query_interval": 1, "concurrent": 3, @@ -35,4 +35,4 @@ ] } } - \ No newline at end of file + From 854ae073fbfc3d1f0b48eef7e6bb8eec725ad3c4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 May 2021 21:20:44 +0800 Subject: [PATCH 030/157] [td-4147]: report error if the column name is too long. --- src/query/src/qSqlParser.c | 14 +++++++------- tests/script/general/parser/alter_stable.sim | 2 +- tests/script/general/parser/create_tb.sim | 5 ++++- tests/script/general/parser/dbtbnameValidate.sim | 4 ++++ 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index 56d395e498..78c531ce49 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -588,14 +588,14 @@ void tSetDbName(SStrToken *pCpxName, SStrToken *pDb) { void tSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); - - // truncate the column name - if ((int32_t)pName->n >= maxLen) { - pName->n = maxLen - 1; - } - strncpy(pField->name, pName->z, pName->n); - pField->name[pName->n] = 0; + // column name is too long, set the it to be invalid. + if ((int32_t) pName->n >= maxLen) { + pName->n = -1; + } else { + strncpy(pField->name, pName->z, pName->n); + pField->name[pName->n] = 0; + } pField->type = pType->type; if(!isValidDataType(pField->type)){ diff --git a/tests/script/general/parser/alter_stable.sim b/tests/script/general/parser/alter_stable.sim index 8a7f4fa924..afdd7d3edf 100644 --- a/tests/script/general/parser/alter_stable.sim +++ b/tests/script/general/parser/alter_stable.sim @@ -22,7 +22,7 @@ sql_error alter table mt1 change tag a 1 sql_error create table mtx1 (ts timestamp, c1 int) tags (123 int) -sql create table mt2 (ts timestamp, c1 int) tags (abc012345678901234567890123456789012345678901234567890123456789def int) +sql_error create table mt2 (ts timestamp, c1 int) tags (abc012345678901234567890123456789012345678901234567890123456789def int) sql create table mt3 (ts timestamp, c1 int) tags (abc012345678901234567890123456789012345678901234567890123456789 int) sql_error alter table mt3 change tag abc012345678901234567890123456789012345678901234567890123456789 abcdefg012345678901234567890123456789012345678901234567890123456789 sql alter table mt3 change tag abc012345678901234567890123456789012345678901234567890123456789 abcdefg0123456789012345678901234567890123456789 diff --git a/tests/script/general/parser/create_tb.sim b/tests/script/general/parser/create_tb.sim index eb6e4f71c3..9593cbda12 100644 --- a/tests/script/general/parser/create_tb.sim +++ b/tests/script/general/parser/create_tb.sim @@ -114,7 +114,10 @@ sql_error create table $tb (ts timestamp, $tag int) sql_error create table $tb (ts timestamp, $tags int) sql_error create table $tb (ts timestamp, $sint int) sql_error create table $tb (ts timestamp, $tint int) -sql_error create table $tb (ts timestamp, $nchar int) +sql_error create table $tb (ts timestamp, $nchar int) + +# too long column name +sql_error create table $tb (ts timestamp, abcde_123456789_123456789_123456789_123456789_123456789_123456789 int) print illegal_column_names test passed # case5: chinese_char_in_table_support diff --git a/tests/script/general/parser/dbtbnameValidate.sim b/tests/script/general/parser/dbtbnameValidate.sim index f2e6de81f1..bc3bfefafb 100644 --- a/tests/script/general/parser/dbtbnameValidate.sim +++ b/tests/script/general/parser/dbtbnameValidate.sim @@ -119,4 +119,8 @@ if $rows != 4 then return -1 endi +print ================>td-4147 +sql_error create table tx(ts timestamp, a1234_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789 int) + + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 95139cf4a1c03cd6f9bb60fe0daca48bbd84e4e9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 11 May 2021 21:36:48 +0800 Subject: [PATCH 031/157] [TD-4130]: taosdemo subscribe super table. (#6086) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 134 +++++++++++++++++------------------- 1 file changed, 63 insertions(+), 71 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index a3a8968017..8866bf2607 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -68,12 +68,6 @@ enum TEST_MODE { INVAID_TEST }; -enum QUERY_MODE { - SYNC_QUERY_MODE, // 0 - ASYNC_QUERY_MODE, // 1 - INVALID_MODE -}; - #define MAX_RECORDS_PER_REQ 32766 #define MAX_SQL_SIZE 65536 @@ -1107,6 +1101,7 @@ static void appendResultBufToFile(char *resultBuf, char *resultFile) } } + fprintf(fp, "%s", resultBuf); tmfclose(fp); } @@ -1142,6 +1137,7 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { totalLen += len; } + verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile); appendResultBufToFile(databuf, resultFile); free(databuf); } @@ -6517,59 +6513,63 @@ static void *superSubscribe(void *sarg) { return NULL; } - //int64_t st = 0; - //int64_t et = 0; - do { - //if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < g_queryInfo.specifiedQueryInfo.queryInterval) { - // taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval- (et - st)); // ms - // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to); - //} - - //st = taosGetTimestampMs(); - char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - sprintf(topic, "taosdemo-subscribe-%d", i); + char topic[32] = {0}; + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to; i++) { + for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { + sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%d", i, j); memset(subSqlstr,0,sizeof(subSqlstr)); - replaceChildTblName(g_queryInfo.superQueryInfo.sql[i], subSqlstr, i); + replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], subSqlstr, i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[i][0] != 0) { + if (g_queryInfo.superQueryInfo.result[j][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[i], pThreadInfo->threadID); + g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } - tsub[i] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile); - if (NULL == tsub[i]) { + + uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", + __func__, __LINE__, subSeq, subSqlstr); + tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile); + if (NULL == tsub[subSeq]) { taos_close(pThreadInfo->taos); return NULL; } } - //et = taosGetTimestampMs(); - //printf("========thread[%"PRIu64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); - } while(0); + } // start loop to consume result TAOS_RES* res = NULL; while(1) { - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { - continue; - } + for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { + for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { + if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { + continue; + } - res = taos_consume(tsub[i]); - if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[i], + uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + taosMsleep(100); // ms + res = taos_consume(tsub[subSeq]); + if (res) { + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.superQueryInfo.result[j][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); - appendResultToFile(res, tmpFile); + appendResultToFile(res, tmpFile); + } } } } } taos_free_result(res); - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - taos_unsubscribe(tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to; i++) { + for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { + uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + taos_unsubscribe(tsub[subSeq], + g_queryInfo.superQueryInfo.subscribeKeepProgress); + } } taos_close(pThreadInfo->taos); @@ -6607,17 +6607,8 @@ static void *specifiedSubscribe(void *sarg) { return NULL; } - //int64_t st = 0; - //int64_t et = 0; - do { - //if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < g_queryInfo.specifiedQueryInfo.queryInterval) { - // taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval- (et - st)); // ms - // //printf("========sleep duration:%"PRIu64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to); - //} - - //st = taosGetTimestampMs(); - char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + char topic[32] = {0}; + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { sprintf(topic, "taosdemo-subscribe-%d", i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { @@ -6630,11 +6621,7 @@ static void *specifiedSubscribe(void *sarg) { taos_close(pThreadInfo->taos); return NULL; } - } - //et = taosGetTimestampMs(); - //printf("========thread[%"PRIu64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); - } while(0); - + } // start loop to consume result TAOS_RES* res = NULL; while(1) { @@ -6643,6 +6630,7 @@ static void *specifiedSubscribe(void *sarg) { continue; } + taosMsleep(1000); // ms res = taos_consume(tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; @@ -6699,31 +6687,35 @@ static int subscribeTestProcess() { pthread_t *pids = NULL; threadInfo *infos = NULL; - //==== create sub threads for query from super table - if ((g_queryInfo.specifiedQueryInfo.sqlCount <= 0) || - (g_queryInfo.specifiedQueryInfo.concurrent <= 0)) { - errorPrint("%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n", + //==== create sub threads for query for specified table + if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { + printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", __func__, __LINE__, - g_queryInfo.specifiedQueryInfo.sqlCount, - g_queryInfo.specifiedQueryInfo.concurrent); - exit(-1); - } + g_queryInfo.specifiedQueryInfo.sqlCount); + } else { + if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) { + errorPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", + __func__, __LINE__, + g_queryInfo.specifiedQueryInfo.sqlCount); + exit(-1); + } - pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); - infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); - if ((NULL == pids) || (NULL == infos)) { - errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); - exit(-1); - } + pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); + if ((NULL == pids) || (NULL == infos)) { + errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); + exit(-1); + } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; t_info->taos = NULL; // TODO: workaround to use separate taos connection; pthread_create(pids + i, NULL, specifiedSubscribe, t_info); + } } - //==== create sub threads for query from sub table + //==== create sub threads for super table query pthread_t *pidsOfSub = NULL; threadInfo *infosOfSub = NULL; if ((g_queryInfo.superQueryInfo.sqlCount > 0) From 5131e7b12a664ddcdfc48a1d11125d71f92cf64a Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Wed, 12 May 2021 09:41:03 +0800 Subject: [PATCH 032/157] Fix/td 3986 (#6088) * [TD-3986]: fix subscribe test case error * change jdbc version number --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 2 +- .../com/taosdata/jdbc/TSDBJNIConnector.java | 15 ++----- .../java/com/taosdata/jdbc/TSDBResultSet.java | 13 +++--- .../java/com/taosdata/jdbc/SubscribeTest.java | 44 +++++++++++++------ 7 files changed, 45 insertions(+), 35 deletions(-) diff --git a/cmake/install.inc b/cmake/install.inc index 9e325531d5..f8b3b7c3c6 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.28-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.29.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index de4b8f6bfb..61e976cb18 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.28-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.29.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index a31796ffde..968a9bf470 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.28 + 2.0.29 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 3400a82e73..d94d28d9fa 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.28 + 2.0.29 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 5e3ffffa4f..7f0cf7de8d 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -32,7 +32,7 @@ public class TSDBJNIConnector { // Connection pointer used in C private long taos = TSDBConstants.JNI_NULL_POINTER; // result set status in current connection - private boolean isResultsetClosed = true; + private boolean isResultsetClosed; private int affectedRows = -1; static { @@ -135,6 +135,7 @@ public class TSDBJNIConnector { // Try retrieving result set for the executed SQL using the current connection pointer. pSql = this.getResultSetImp(this.taos, pSql); + // if pSql == 0L that means resultset is closed isResultsetClosed = (pSql == TSDBConstants.JNI_NULL_POINTER); return pSql; @@ -172,16 +173,7 @@ public class TSDBJNIConnector { * Free resultset operation from C to release resultset pointer by JNI */ public int freeResultSet(long pSql) { - int res = TSDBConstants.JNI_SUCCESS; -// if (result != taosResultSetPointer && taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { -// throw new RuntimeException("Invalid result set pointer"); -// } - -// if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - res = this.freeResultSetImp(this.taos, pSql); -// taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; -// } - + int res = this.freeResultSetImp(this.taos, pSql); isResultsetClosed = true; return res; } @@ -199,7 +191,6 @@ public class TSDBJNIConnector { // } // return resCode; // } - private native int freeResultSetImp(long connection, long result); /** diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java index 2576a25f0d..aba29d602b 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java @@ -109,6 +109,8 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { public void close() throws SQLException { if (isClosed) return; + if (this.statement == null) + return; if (this.jniConnector != null) { int code = this.jniConnector.freeResultSet(this.resultSetPointer); if (code == TSDBConstants.JNI_CONNECTION_NULL) { @@ -461,12 +463,13 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { } public boolean isClosed() throws SQLException { - if (isClosed) - return true; - if (jniConnector != null) { - isClosed = jniConnector.isResultsetClosed(); - } return isClosed; +// if (isClosed) +// return true; +// if (jniConnector != null) { +// isClosed = jniConnector.isResultsetClosed(); +// } +// return isClosed; } public String getNString(int columnIndex) throws SQLException { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 3a223ed981..24c73fdd5c 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -1,6 +1,7 @@ package com.taosdata.jdbc; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -19,6 +20,7 @@ public class SubscribeTest { String tName = "t0"; String host = "127.0.0.1"; String topic = "test"; + private long ts; @Test public void subscribe() { @@ -27,26 +29,40 @@ public class SubscribeTest { TSDBConnection conn = connection.unwrap(TSDBConnection.class); TSDBSubscribe subscribe = conn.subscribe(topic, rawSql, false); - int a = 0; - while (true) { - TimeUnit.MILLISECONDS.sleep(1000); + for (int j = 0; j < 10; j++) { + TimeUnit.SECONDS.sleep(1); TSDBResultSet resSet = subscribe.consume(); + + int rowCnt = 0; while (resSet.next()) { - for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { - System.out.printf(i + ": " + resSet.getString(i) + "\t"); + if (rowCnt == 0) { + long cur_ts = resSet.getTimestamp(1).getTime(); + int k = resSet.getInt(2); + int v = resSet.getInt(3); + Assert.assertEquals(ts, cur_ts); + Assert.assertEquals(100, k); + Assert.assertEquals(1, v); } - System.out.println("\n======" + a + "=========="); - } - a++; - if (a >= 2) { - break; + if (rowCnt == 1) { + long cur_ts = resSet.getTimestamp(1).getTime(); + int k = resSet.getInt(2); + int v = resSet.getInt(3); + Assert.assertEquals(ts + 1, cur_ts); + Assert.assertEquals(101, k); + Assert.assertEquals(2, v); + + } + rowCnt++; } + if (j == 0) + Assert.assertEquals(2, rowCnt); resSet.close(); } - subscribe.close(true); - } catch (Exception e) { - e.printStackTrace(); + + + } catch (SQLException | InterruptedException throwables) { + throwables.printStackTrace(); } } @@ -62,7 +78,7 @@ public class SubscribeTest { statement.execute("drop database if exists " + dbName); statement.execute("create database if not exists " + dbName); statement.execute("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); - long ts = System.currentTimeMillis(); + ts = System.currentTimeMillis(); statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", 100, 1)"); statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + (ts + 1) + ", 101, 2)"); } From 15a7d1431d49395ae28dacd5bf36c874bec94bed Mon Sep 17 00:00:00 2001 From: lichuang Date: Wed, 12 May 2021 10:32:10 +0800 Subject: [PATCH 033/157] [TD-4145]fix compile in ningsi60 --- deps/rmonotonic/src/monotonic.c | 9 ++++ src/client/src/tscSubquery.c | 6 +++ src/common/inc/texpr.h | 3 -- src/plugins/http/src/httpServer.c | 4 ++ src/query/src/qAggMain.c | 72 ++++++++++++++++++++++++++++--- 5 files changed, 85 insertions(+), 9 deletions(-) diff --git a/deps/rmonotonic/src/monotonic.c b/deps/rmonotonic/src/monotonic.c index 1470f91b56..c6d2df9097 100644 --- a/deps/rmonotonic/src/monotonic.c +++ b/deps/rmonotonic/src/monotonic.c @@ -36,6 +36,15 @@ static char monotonic_info_string[32]; static long mono_ticksPerMicrosecond = 0; +#ifdef _TD_NINGSI_60 +// implement __rdtsc in ningsi60 +uint64_t __rdtsc(){ + unsigned int lo,hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return ((uint64_t)hi << 32) | lo; +} +#endif + static monotime getMonotonicUs_x86() { return __rdtsc() / mono_ticksPerMicrosecond; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 67eea432e6..e9f87b3115 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -637,7 +637,13 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); // set the tag column id for executor to extract correct tag value +#ifndef _TD_NINGSI_60 pExpr->base.param[0] = (tVariant) {.i64 = colId, .nType = TSDB_DATA_TYPE_BIGINT, .nLen = sizeof(int64_t)}; +#else + pExpr->base.param[0].i64 = colId; + pExpr->base.param[0].nType = TSDB_DATA_TYPE_BIGINT; + pExpr->base.param[0].nLen = sizeof(int64_t); +#endif pExpr->base.numOfParams = 1; } diff --git a/src/common/inc/texpr.h b/src/common/inc/texpr.h index a0854ce81b..d67de9ff69 100644 --- a/src/common/inc/texpr.h +++ b/src/common/inc/texpr.h @@ -89,9 +89,6 @@ tExprNode* exprdup(tExprNode* pTree); bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param); -typedef void (*_arithmetic_operator_fn_t)(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, - int32_t rightType, void *output, int32_t order); - void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, const char*, int32_t)); diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 4dcf3d5501..9d98d3f113 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -269,7 +269,11 @@ static void *httpAcceptHttpConnection(void *arg) { sprintf(pContext->ipstr, "%s:%u", taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); struct epoll_event event; +#ifndef _TD_NINGSI_60 event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP | EPOLLERR | EPOLLHUP | EPOLLRDHUP; +#else + event.events = EPOLLIN | EPOLLPRI | EPOLLERR | EPOLLHUP | EPOLLRDHUP; +#endif event.data.ptr = pContext; if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd, diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 3b1ffa46d9..7b656d473a 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -4009,7 +4009,13 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si continue; } +#ifndef _TD_NINGSI_60 SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4021,8 +4027,14 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - + +#ifndef _TD_NINGSI_60 SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4034,8 +4046,14 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - + +#ifndef _TD_NINGSI_60 SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4047,8 +4065,14 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - + +#ifndef _TD_NINGSI_60 SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = (double)val[i]; +#endif pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4060,8 +4084,14 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - + +#ifndef _TD_NINGSI_60 SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = (double)val[i]; +#endif pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4073,8 +4103,14 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - + +#ifndef _TD_NINGSI_60 SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4087,7 +4123,13 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si continue; } +#ifndef _TD_NINGSI_60 SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4100,7 +4142,13 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si continue; } +#ifndef _TD_NINGSI_60 SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4113,7 +4161,13 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si continue; } +#ifndef _TD_NINGSI_60 SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4125,8 +4179,14 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - + +#ifndef _TD_NINGSI_60 SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = (double) val[i]; +#endif pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } From 13c8c0049c1da05110ba7655710b873a85b1bbc8 Mon Sep 17 00:00:00 2001 From: zyyang-taosdata Date: Wed, 12 May 2021 10:52:34 +0800 Subject: [PATCH 034/157] [TD-4144]: add subscribe test cases --- .../com/taosdata/jdbc/cases/TD4144Test.java | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4144Test.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4144Test.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4144Test.java new file mode 100644 index 0000000000..6f29f64111 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4144Test.java @@ -0,0 +1,105 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBConnection; +import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.TSDBResultSet; +import com.taosdata.jdbc.TSDBSubscribe; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.DriverManager; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +public class TD4144Test { + + private static TSDBConnection connection; + private static final String host = "127.0.0.1"; + + private static final String topic = "topic-meter-current-bg-10"; + private static final String sql = "select * from meters where current > 10"; + private static final String sql2 = "select * from meters where ts >= '2020-08-15 12:20:00.000'"; + + + @Test + public void test() throws SQLException { + TSDBSubscribe subscribe = null; + TSDBResultSet res = null; + boolean hasNext = false; + + try { + subscribe = connection.subscribe(topic, sql, false); + int count = 0; + while (true) { + // 等待1秒，避免频繁调用 consume，给服务端造成压力 + TimeUnit.SECONDS.sleep(1); + if (res == null) { + // 消费数据 + res = subscribe.consume(); + hasNext = res.next(); + } + + if (res == null) { + continue; + } + ResultSetMetaData metaData = res.getMetaData(); + int number = 0; + while (hasNext) { + int columnCount = metaData.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + System.out.print(metaData.getColumnLabel(i) + ": " + res.getString(i) + "\t"); + } + System.out.println(); + count++; + number++; + hasNext = res.next(); + if (!hasNext) { + res.close(); + res = null; + System.out.println("rows： " + count); + } + if (hasNext == true && number >= 10) { + System.out.println("batch" + number); + break; + } + } + + } + + } catch (SQLException | InterruptedException throwables) { + throwables.printStackTrace(); + } finally { + if (subscribe != null) + subscribe.close(true); + } + } + + @BeforeClass + public static void beforeClass() throws SQLException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + connection = (DriverManager.getConnection(url, properties)).unwrap(TSDBConnection.class); + try (Statement stmt = connection.createStatement()) { + stmt.execute("drop database if exists power"); + stmt.execute("create database if not exists power"); + stmt.execute("use power"); + stmt.execute("create table meters(ts timestamp, current float, voltage int, phase int) tags(location binary(64), groupId int)"); + stmt.execute("create table d1001 using meters tags(\"Beijing.Chaoyang\", 2)"); + stmt.execute("create table d1002 using meters tags(\"Beijing.Haidian\", 2)"); + stmt.execute("insert into d1001 values(\"2020-08-15 12:00:00.000\", 12, 220, 1),(\"2020-08-15 12:10:00.000\", 12.3, 220, 2),(\"2020-08-15 12:20:00.000\", 12.2, 220, 1)"); + stmt.execute("insert into d1002 values(\"2020-08-15 12:00:00.000\", 9.9, 220, 1),(\"2020-08-15 12:10:00.000\", 10.3, 220, 1),(\"2020-08-15 12:20:00.000\", 11.2, 220, 1)"); + } + } + + @AfterClass + public static void afterClass() throws SQLException { + if (connection != null) + connection.close(); + } +} From 85d1b0aaf8ff679481301e3a7126b6aacac1ad6f Mon Sep 17 00:00:00 2001 From: zyyang-taosdata Date: Wed, 12 May 2021 11:17:53 +0800 Subject: [PATCH 035/157] escape TD4144Test case --- src/connector/jdbc/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index d94d28d9fa..ef353d1d19 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -122,6 +122,7 @@ **/FailOverTest.java **/InvalidResultSetPointerTest.java **/RestfulConnectionTest.java + **/TD4144Test.java true From 6d8f5000ff3def90266efb05b2380f33a7021183 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 May 2021 11:24:23 +0800 Subject: [PATCH 036/157] [td-4147]update the test case. --- src/common/inc/texpr.h | 3 --- tests/pytest/table/column_name.py | 5 ++--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/common/inc/texpr.h b/src/common/inc/texpr.h index acfbffc01e..9addea412b 100644 --- a/src/common/inc/texpr.h +++ b/src/common/inc/texpr.h @@ -89,9 +89,6 @@ void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); bool exprTreeApplayFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param); -typedef void (*_arithmetic_operator_fn_t)(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, - int32_t rightType, void *output, int32_t order); - void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, const char*, int32_t)); diff --git a/tests/pytest/table/column_name.py b/tests/pytest/table/column_name.py index a180d3f752..0f24b98f3a 100644 --- a/tests/pytest/table/column_name.py +++ b/tests/pytest/table/column_name.py @@ -88,10 +88,9 @@ class TDTestCase: # TSIM: # TSIM: print =============== step4 tdLog.info('=============== step4') - # TSIM: sql create table $tb (ts timestamp, - # a0123456789012345678901234567890123456789 int) + # TSIM: sql create table $tb (ts timestamp, a0123456789012345678901234567890123456789 int) getMaxColNum = "grep -w '#define TSDB_COL_NAME_LEN' ../../src/inc/taosdef.h|awk '{print $3}'" - boundary = int(subprocess.check_output(getMaxColNum, shell=True)) + boundary = int(subprocess.check_output(getMaxColNum, shell=True)) - 1 tdLog.info("get max column name length is %d" % boundary) chars = string.ascii_uppercase + string.ascii_lowercase From 64ae1f9a364efecf88243bead434016582d6b4e6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 May 2021 11:29:00 +0800 Subject: [PATCH 037/157] [td-4147]update the test script. --- tests/script/general/parser/create_tb.sim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/general/parser/create_tb.sim b/tests/script/general/parser/create_tb.sim index 9593cbda12..ca57f401b9 100644 --- a/tests/script/general/parser/create_tb.sim +++ b/tests/script/general/parser/create_tb.sim @@ -118,6 +118,7 @@ sql_error create table $tb (ts timestamp, $nchar int) # too long column name sql_error create table $tb (ts timestamp, abcde_123456789_123456789_123456789_123456789_123456789_123456789 int) +sql_error create table tx(ts timestamp, k int) tags(abcd5_123456789_123456789_123456789_123456789_123456789_123456789 int) print illegal_column_names test passed # case5: chinese_char_in_table_support From d9127c05017c022f474a6174fa4950f1674ac6e6 Mon Sep 17 00:00:00 2001 From: lichuang Date: Wed, 12 May 2021 14:41:33 +0800 Subject: [PATCH 038/157] [TD-3963]handle oom in tsdbExpendPool --- src/tsdb/inc/tsdbBuffer.h | 2 +- src/tsdb/src/tsdbBuffer.c | 10 +++++++--- src/tsdb/src/tsdbCommitQueue.c | 6 +++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/tsdb/inc/tsdbBuffer.h b/src/tsdb/inc/tsdbBuffer.h index c6aabeaab9..4e18ac711a 100644 --- a/src/tsdb/inc/tsdbBuffer.h +++ b/src/tsdb/inc/tsdbBuffer.h @@ -40,7 +40,7 @@ void tsdbFreeBufPool(STsdbBufPool* pBufPool); int tsdbOpenBufPool(STsdbRepo* pRepo); void tsdbCloseBufPool(STsdbRepo* pRepo); SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); -void tsdbExpendPool(STsdbRepo* pRepo, int32_t oldTotalBlocks); +int tsdbExpendPool(STsdbRepo* pRepo, int32_t oldTotalBlocks); void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode); #endif /* _TD_TSDB_BUFFER_H_ */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index 266f2a45d8..429ea8e0ce 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -159,12 +159,14 @@ _err: static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } -void tsdbExpendPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) { +int tsdbExpendPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) { if (oldTotalBlocks == pRepo->config.totalBlocks) { - return; + return TSDB_CODE_SUCCESS; } - if (tsdbLockRepo(pRepo) < 0) return; + int err = TSDB_CODE_SUCCESS; + + if (tsdbLockRepo(pRepo) < 0) return terrno; STsdbBufPool* pPool = pRepo->pPool; if (pRepo->config.totalBlocks > oldTotalBlocks) { @@ -175,6 +177,7 @@ void tsdbExpendPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) { if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { tsdbFreeBufBlock(pBufBlock); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + err = TSDB_CODE_TDB_OUT_OF_MEMORY; goto err; } @@ -187,6 +190,7 @@ void tsdbExpendPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) { err: tsdbUnlockRepo(pRepo); + return err; } void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode) { diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c index 86712db957..e753a3211e 100644 --- a/src/tsdb/src/tsdbCommitQueue.c +++ b/src/tsdb/src/tsdbCommitQueue.c @@ -130,7 +130,11 @@ static void tsdbApplyRepoConfig(STsdbRepo *pRepo) { pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, pSaveCfg->totalBlocks, pSaveCfg->cacheLastRow, pSaveCfg->totalBlocks); - tsdbExpendPool(pRepo, oldTotalBlocks); + int err = tsdbExpendPool(pRepo, oldTotalBlocks); + if (!TAOS_SUCCEEDED(err)) { + tsdbError("vgId:%d expand pool from %d to %d fail,reason:%s", + REPO_ID(pRepo), oldTotalBlocks, pSaveCfg->totalBlocks, tstrerror(err)); + } } From 46546c9d56c8006226126d37e375e79e2c1094d2 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Wed, 12 May 2021 14:41:59 +0800 Subject: [PATCH 039/157] support user specify column --- src/client/src/tscPrepare.c | 69 ++++- tests/script/api/batchprepare.c | 523 ++++++++++++++++++++++++++++++++ 2 files changed, 588 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 611cb604c4..9e1624c633 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -276,6 +276,60 @@ static char* normalStmtBuildSql(STscStmt* stmt) { return taosStringBuilderGetResult(&sb, NULL); } +static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { + SParsedDataColInfo* spd = &pBlock->boundColumnInfo; + int32_t offset = 0; + SSchema *schema = (SSchema*)pBlock->pTableMeta->schema; + + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (!spd->cols[i].hasVal) { // current column do not have any value to insert, set it to null + for (int32_t n = 0; n < rowNum; ++n) { + char *ptr = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * n + offset; + + if (schema[i].type == TSDB_DATA_TYPE_BINARY) { + varDataSetLen(ptr, sizeof(int8_t)); + *(uint8_t*) varDataVal(ptr) = TSDB_DATA_BINARY_NULL; + } else if (schema[i].type == TSDB_DATA_TYPE_NCHAR) { + varDataSetLen(ptr, sizeof(int32_t)); + *(uint32_t*) varDataVal(ptr) = TSDB_DATA_NCHAR_NULL; + } else { + setNull(ptr, schema[i].type, schema[i].bytes); + } + } + } + + offset += schema[i].bytes; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t fillTablesColumnsNull(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + + STableDataBlocks** p = taosHashIterate(pCmd->pTableBlockHashList, NULL); + + STableDataBlocks* pOneTableBlock = *p; + while(pOneTableBlock) { + SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; + if (pBlocks->numOfRows > 0 && pOneTableBlock->boundColumnInfo.numOfBound < pOneTableBlock->boundColumnInfo.numOfCols) { + fillColumnsNull(pOneTableBlock, pBlocks->numOfRows); + } + + p = taosHashIterate(pCmd->pTableBlockHashList, p); + if (p == NULL) { + break; + } + + pOneTableBlock = *p; + } + + return TSDB_CODE_SUCCESS; +} + + + //////////////////////////////////////////////////////////////////////////////// // functions for insertion statement preparation static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { @@ -1027,6 +1081,8 @@ static int insertStmtExecute(STscStmt* stmt) { pBlk->uid = pTableMeta->id.uid; pBlk->tid = pTableMeta->id.tid; + fillTablesColumnsNull(stmt->pSql); + int code = tscMergeTableDataBlocks(stmt->pSql, false); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1120,10 +1176,15 @@ static int insertBatchStmtExecute(STscStmt* pStmt) { pStmt->pSql->retry = pStmt->pSql->maxRetry + 1; //no retry - if (taosHashGetSize(pStmt->pSql->cmd.pTableBlockHashList) > 0) { // merge according to vgId - if ((code = tscMergeTableDataBlocks(pStmt->pSql, false)) != TSDB_CODE_SUCCESS) { - return code; - } + if (taosHashGetSize(pStmt->pSql->cmd.pTableBlockHashList) <= 0) { // merge according to vgId + tscError("0x%"PRIx64" no data block to insert", pStmt->pSql->self); + return TSDB_CODE_TSC_APP_ERROR; + } + + fillTablesColumnsNull(pStmt->pSql); + + if ((code = tscMergeTableDataBlocks(pStmt->pSql, false)) != TSDB_CODE_SUCCESS) { + return code; } code = tscHandleMultivnodeInsert(pStmt->pSql); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 8f1337486e..db48632cb7 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -28,6 +28,445 @@ unsigned long long getCurrentTime(){ + + +int stmt_scol_func1(TAOS_STMT *stmt) { + struct { + int64_t ts; + int8_t b; + int8_t v1; + int16_t v2; + int32_t v4; + int64_t v8; + float f4; + double f8; + char bin[40]; + char blob[80]; + } v = {0}; + + TAOS_BIND params[10]; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts); + params[0].buffer = &v.ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + + params[1].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[1].buffer_length = sizeof(v.v1); + params[1].buffer = &v.v1; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + + params[2].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[2].buffer_length = sizeof(v.v2); + params[2].buffer = &v.v2; + params[2].length = ¶ms[2].buffer_length; + params[2].is_null = NULL; + + params[3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[3].buffer_length = sizeof(v.f4); + params[3].buffer = &v.f4; + params[3].length = ¶ms[3].buffer_length; + params[3].is_null = NULL; + + params[4].buffer_type = TSDB_DATA_TYPE_BINARY; + params[4].buffer_length = sizeof(v.bin); + params[4].buffer = v.bin; + params[4].length = ¶ms[4].buffer_length; + params[4].is_null = NULL; + + params[5].buffer_type = TSDB_DATA_TYPE_BINARY; + params[5].buffer_length = sizeof(v.bin); + params[5].buffer = v.bin; + params[5].length = ¶ms[5].buffer_length; + params[5].is_null = NULL; + + char *sql = "insert into ? (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + for (int zz = 0; zz < 10; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + exit(1); + } + v.ts = 1591060628000 + zz * 10; + for (int i = 0; i < 10; ++i) { + v.ts += 1; + + v.b = (int8_t)(i+zz*10) % 2; + v.v1 = (int8_t)(i+zz*10); + v.v2 = (int16_t)((i+zz*10) * 2); + v.v4 = (int32_t)((i+zz*10) * 4); + v.v8 = (int64_t)((i+zz*10) * 8); + v.f4 = (float)((i+zz*10) * 40); + v.f8 = (double)((i+zz*10) * 80); + for (int j = 0; j < sizeof(v.bin) - 1; ++j) { + v.bin[j] = (char)((i)%10 + '0'); + } + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + } + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + return 0; +} + + + +int stmt_scol_func2(TAOS_STMT *stmt) { + struct { + int64_t ts; + int8_t b; + int8_t v1; + int16_t v2; + int32_t v4; + int64_t v8; + float f4; + double f8; + char bin[40]; + char blob[80]; + } v = {0}; + + TAOS_BIND params[10]; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts); + params[0].buffer = &v.ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + + params[1].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[1].buffer_length = sizeof(v.v1); + params[1].buffer = &v.v1; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + + params[2].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[2].buffer_length = sizeof(v.v2); + params[2].buffer = &v.v2; + params[2].length = ¶ms[2].buffer_length; + params[2].is_null = NULL; + + params[3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[3].buffer_length = sizeof(v.f4); + params[3].buffer = &v.f4; + params[3].length = ¶ms[3].buffer_length; + params[3].is_null = NULL; + + params[4].buffer_type = TSDB_DATA_TYPE_BINARY; + params[4].buffer_length = sizeof(v.bin); + params[4].buffer = v.bin; + params[4].length = ¶ms[4].buffer_length; + params[4].is_null = NULL; + + params[5].buffer_type = TSDB_DATA_TYPE_BINARY; + params[5].buffer_length = sizeof(v.bin); + params[5].buffer = v.bin; + params[5].length = ¶ms[5].buffer_length; + params[5].is_null = NULL; + + char *sql = "insert into m0 (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + for (int zz = 0; zz < 10; zz++) { + v.ts = 1591060628000 + zz * 10; + for (int i = 0; i < 10; ++i) { + v.ts += 1; + + v.b = (int8_t)(i+zz*10) % 2; + v.v1 = (int8_t)(i+zz*10); + v.v2 = (int16_t)((i+zz*10) * 2); + v.v4 = (int32_t)((i+zz*10) * 4); + v.v8 = (int64_t)((i+zz*10) * 8); + v.f4 = (float)((i+zz*10) * 40); + v.f8 = (double)((i+zz*10) * 80); + for (int j = 0; j < sizeof(v.bin) - 1; ++j) { + v.bin[j] = (char)((i)%10 + '0'); + } + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + } + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + return 0; +} + + + + +//300 tables 60 records +int stmt_scol_func3(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[60]; + int8_t v1[60]; + int16_t v2[60]; + int32_t v4[60]; + int64_t v8[60]; + float f4[60]; + double f8[60]; + char bin[60][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 900000 * 60); + + int *lb = malloc(60 * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = malloc(sizeof(char) * 60); + char* no_null = malloc(sizeof(char) * 60); + + for (int i = 0; i < 60; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 9000000; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[10*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 10; + + params[i+1].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.v1; + params[i+1].length = NULL; + params[i+1].is_null = no_null; + params[i+1].num = 10; + + params[i+2].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+2].buffer_length = sizeof(int16_t); + params[i+2].buffer = v.v2; + params[i+2].length = NULL; + params[i+2].is_null = no_null; + params[i+2].num = 10; + + params[i+3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+3].buffer_length = sizeof(float); + params[i+3].buffer = v.f4; + params[i+3].length = NULL; + params[i+3].is_null = no_null; + params[i+3].num = 10; + + params[i+4].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+4].buffer_length = 40; + params[i+4].buffer = v.bin; + params[i+4].length = lb; + params[i+4].is_null = no_null; + params[i+4].num = 10; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+5].buffer_length = 40; + params[i+5].buffer = v.bin; + params[i+5].length = lb; + params[i+5].is_null = no_null; + params[i+5].num = 10; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 54000000; ++i) { + v.ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int l = 0; l < 2; l++) { + for (int zz = 0; zz < 300; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + } + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + + + +//10 tables 10 records single column bind +int stmt_scol_func4(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[60]; + int8_t v1[60]; + int16_t v2[60]; + int32_t v4[60]; + int64_t v8[60]; + float f4[60]; + double f8[60]; + char bin[60][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 1000 * 60); + + int *lb = malloc(60 * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1000*10); + char* is_null = malloc(sizeof(char) * 60); + char* no_null = malloc(sizeof(char) * 60); + + for (int i = 0; i < 60; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 10000; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[10*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 2; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = no_null; + params[i+1].num = 2; + + params[i+2].buffer_type = TSDB_DATA_TYPE_INT; + params[i+2].buffer_length = sizeof(int32_t); + params[i+2].buffer = v.v4; + params[i+2].length = NULL; + params[i+2].is_null = no_null; + params[i+2].num = 2; + + params[i+3].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+3].buffer_length = sizeof(int64_t); + params[i+3].buffer = v.v8; + params[i+3].length = NULL; + params[i+3].is_null = no_null; + params[i+3].num = 2; + + params[i+4].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+4].buffer_length = sizeof(double); + params[i+4].buffer = v.f8; + params[i+4].length = NULL; + params[i+4].is_null = no_null; + params[i+4].num = 2; + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 60000; ++i) { + v.ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? (ts,b,v4,v8,f8) values(?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int l = 0; l < 10; l++) { + for (int zz = 0; zz < 10; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + + for (int col=0; col < 10; ++col) { + taos_stmt_bind_single_param_batch(stmt, params + id++, col); + } + + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + } + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + + + int stmt_func1(TAOS_STMT *stmt) { struct { int64_t ts; @@ -2201,6 +2640,90 @@ void* runcase(void *par) { (void)idx; +#if 0 + prepare(taos, 1); + + stmt = taos_stmt_init(taos); + + printf("10t+10records+specifycol start\n"); + stmt_scol_func1(stmt); + printf("10t+10records+specifycol end\n"); + printf("check result start\n"); + check_result(taos, "m0", 1, 10); + check_result(taos, "m1", 1, 10); + check_result(taos, "m2", 1, 10); + check_result(taos, "m3", 1, 10); + check_result(taos, "m4", 1, 10); + check_result(taos, "m5", 1, 10); + check_result(taos, "m6", 1, 10); + check_result(taos, "m7", 1, 10); + check_result(taos, "m8", 1, 10); + check_result(taos, "m9", 1, 10); + printf("check result end\n"); + taos_stmt_close(stmt); +#endif + + +#if 0 + prepare(taos, 1); + + stmt = taos_stmt_init(taos); + + printf("1t+100records+specifycol start\n"); + stmt_scol_func2(stmt); + printf("1t+100records+specifycol end\n"); + printf("check result start\n"); + check_result(taos, "m0", 1, 100); + printf("check result end\n"); + taos_stmt_close(stmt); +#endif + + +#if 0 + prepare(taos, 1); + + stmt = taos_stmt_init(taos); + + printf("300t+10r+bm+specifycol start\n"); + stmt_scol_func3(stmt); + printf("300t+10r+bm+specifycol end\n"); + printf("check result start\n"); + check_result(taos, "m0", 1, 20); + check_result(taos, "m1", 1, 20); + check_result(taos, "m111", 1, 20); + check_result(taos, "m223", 1, 20); + check_result(taos, "m299", 1, 20); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + +#if 1 + prepare(taos, 1); + + stmt = taos_stmt_init(taos); + + printf("10t+2r+bm+specifycol start\n"); + stmt_scol_func4(stmt); + printf("10t+2r+bm+specifycol end\n"); + printf("check result start\n"); + check_result(taos, "m0", 1, 20); + check_result(taos, "m1", 1, 20); + check_result(taos, "m2", 1, 20); + check_result(taos, "m3", 1, 20); + check_result(taos, "m4", 1, 20); + check_result(taos, "m5", 1, 20); + check_result(taos, "m6", 1, 20); + check_result(taos, "m7", 1, 20); + check_result(taos, "m8", 1, 20); + check_result(taos, "m9", 1, 20); + printf("check result end\n"); + taos_stmt_close(stmt); + + return NULL; +#endif + + #if 1 prepare(taos, 1); From 29ac33bf4707d36331cde4fab315f78cc71ff239 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Wed, 12 May 2021 15:42:29 +0800 Subject: [PATCH 040/157] fix bug --- src/client/src/tscParseInsert.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index bade9bb66a..e5d4d00dc0 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -581,7 +581,7 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SSq if (sToken.n == 0 || sToken.type != TK_RP) { tscSQLSyntaxErrMsg(pCmd->payload, ") expected", *str); code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - return -1; + return code; } (*numOfRows)++; @@ -712,6 +712,9 @@ static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, STableDataBlock int32_t numOfRows = 0; code = tsParseValues(str, dataBuf, maxNumOfRows, pCmd, &numOfRows, tmpTokenBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) { SParamInfo *param = dataBuf->params + i; From fcfad6c0d930f8fe26e7778a0f7451a670d92068 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 12 May 2021 15:43:23 +0800 Subject: [PATCH 041/157] [TD-2639] : update "stime" parameter in "taos_open_stream" func. --- documentation20/cn/08.connector/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index 59f80b0a55..15c722c180 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -345,7 +345,7 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 * taos：已经建立好的数据库连接 * sql：SQL查询语句（仅能使用查询语句） * fp：用户定义的回调函数指针，每次流式计算完成后，TDengine将查询的结果（TAOS_ROW）、查询状态（TAOS_RES）、用户定义参数（PARAM）传递给回调函数，在回调函数内，用户可以使用taos_num_fields获取结果集列数，taos_fetch_fields获取结果集每列数据的类型。 - * stime：是流式计算开始的时间，如果是0，表示从现在开始，如果不为零，表示从指定的时间开始计算（UTC时间从1970/1/1算起的毫秒数） + * stime：是流式计算开始的时间。如果是“64位整数最小值”，表示从现在开始；如果不为“64位整数最小值”，表示从指定的时间开始计算（UTC时间从1970/1/1算起的毫秒数）。 * param：是应用提供的用于回调的一个参数，回调时，提供给应用 * callback: 第二个回调函数，会在连续查询自动停止时被调用。 From a8f23214798fb1143dc4b4fe7f9523e2bded207c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 12 May 2021 16:11:36 +0800 Subject: [PATCH 042/157] [TD-4068]: taosdemo supports stmt interface. --- src/kit/taosdemo/taosdemo.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 0e468347ee..57f4b24ab5 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -120,6 +120,13 @@ enum MODE { MODE_BUT }; +enum INTERFACE { + TAOSC_INTERFACE, + REST_INTERFACE, + STMT_INTERFACE, + INTERFACE_BUT +}; + typedef enum enum_INSERT_MODE { PROGRESSIVE_INSERT_MODE, INTERLACE_INSERT_MODE, @@ -188,6 +195,7 @@ typedef struct SArguments_S { uint32_t test_mode; char * host; uint16_t port; + uint16_t interface; char * user; char * password; char * database; @@ -536,6 +544,7 @@ SArguments g_args = { 0, // test_mode "127.0.0.1", // host 6030, // port + TAOSC_INTERFACE, // interface "root", // user #ifdef _TD_POWER_ "powerdb", // password @@ -652,6 +661,8 @@ static void printHelp() { "The host to connect to TDengine. Default is localhost."); printf("%s%s%s%s\n", indent, "-p", indent, "The TCP/IP port number to use for the connection. Default is 0."); + printf("%s%s%s%s\n", indent, "-I", indent, + "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'."); printf("%s%s%s%s\n", indent, "-d", indent, "Destination database. Default is 'test'."); printf("%s%s%s%s\n", indent, "-a", indent, @@ -741,6 +752,23 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->port = atoi(argv[++i]); + } else if (strcmp(argv[i], "-I") == 0) { + if (argc == i+1) { + printHelp(); + errorPrint("%s", "\n\t-I need a valid string following!\n"); + exit(EXIT_FAILURE); + } + ++i; + if (0 == strcasecmp(argv[i], "taosc")) { + arguments->interface = TAOSC_INTERFACE; + } else if (0 == strcasecmp(argv[i], "rest")) { + arguments->interface = REST_INTERFACE; + } else if (0 == strcasecmp(argv[i], "stmt")) { + arguments->interface = STMT_INTERFACE; + } else { + errorPrint("%s", "\n\t-I need a valid string following!\n"); + exit(EXIT_FAILURE); + } } else if (strcmp(argv[i], "-u") == 0) { if (argc == i+1) { printHelp(); @@ -1141,7 +1169,8 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { free(databuf); } -static void selectAndGetResult(threadInfo *pThreadInfo, char *command, char* resultFileName) { +static void selectAndGetResult( + threadInfo *pThreadInfo, char *command, char* resultFileName) { if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) { TAOS_RES *res = taos_query(pThreadInfo->taos, command); if (res == NULL || taos_errno(res) != 0) { From 6262dffb54a82ab787c909acd7b3d85b5b8abe24 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 12 May 2021 17:38:26 +0800 Subject: [PATCH 043/157] [TD-3914]: taosdemo input argument validation. (#6097) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 8866bf2607..bb1bbf2129 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -725,7 +725,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN); - } else if (strcmp(argv[i], "-h") == 0) { if (argc == i+1) { printHelp(); @@ -849,6 +848,11 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->num_of_CPR = atoi(argv[++i]); } else if (strcmp(argv[i], "-b") == 0) { + if (argc == i+1) { + printHelp(); + errorPrint("%s", "\n\t-b need valid string following!\n"); + exit(EXIT_FAILURE); + } sptr = arguments->datatype; ++i; if (strstr(argv[i], ",") == NULL) { From 00204198aa99024b894cd00ae647d80286c105c4 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Wed, 12 May 2021 18:50:53 +0800 Subject: [PATCH 044/157] [TD-182]: bump python connector version to v2.0.10 (#6091) Also fix url error in setup.py --- src/connector/python/setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/connector/python/setup.py b/src/connector/python/setup.py index 4d083d7ddb..901e8396c0 100644 --- a/src/connector/python/setup.py +++ b/src/connector/python/setup.py @@ -5,16 +5,15 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.9", + version="2.0.10", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", long_description=long_description, long_description_content_type="text/markdown", - url="https://github.com/pypa/sampleproject", + url="https://github.com/taosdata/TDengine/tree/develop/src/connector/python", packages=setuptools.find_packages(), classifiers=[ - "Environment :: Console", "Environment :: MacOS X", "Environment :: Win32 (MS Windows)", From 15ad9bd137ed11b251ba8066193d5eb795e74323 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 12 May 2021 19:43:44 +0800 Subject: [PATCH 045/157] [TD-4130]: taosdemo subscribe stable. (#6090) * [TD-4130]: taosdemo subscribe stable. for develop branch. * [TD-4130]: taosdemo subscribe stable. fix append result to file. Co-authored-by: Shuduo Sang From 487d7e41b72452869ac98a475b7340d04336a634 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 12 May 2021 20:30:08 +0800 Subject: [PATCH 046/157] [TD-4152]: taosdemo reduce prompt and query systeminfo. (#6094) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index bb1bbf2129..9b73284f79 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3410,8 +3410,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.interlace_rows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); + if (!g_args.answer_yes) { + printf(" press Enter key to continue or Ctrl-C to stop."); + (void)getchar(); + } g_args.interlace_rows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -3930,8 +3932,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); + if (!g_args.answer_yes) { + printf(" press Enter key to continue or Ctrl-C to stop."); + (void)getchar(); + } g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -6313,7 +6317,9 @@ static int queryTestProcess() { (void)getchar(); } - printfQuerySystemInfo(taos); + if (g_args.debug_print || g_args.verbose_print) { + printfQuerySystemInfo(taos); + } if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { if (convertHostToServAddr( From 769721eb15630f1822c1ec33e64e9a56fbbdaf17 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 12 May 2021 20:32:54 +0800 Subject: [PATCH 047/157] Hotfix/sangshuduo/td 3992 taosdemo subscribe (#6100) * [TD-3902]: taosdemo subscribe. * [TD-3992]: taosdemo subscribe. refactor sync/async mode. * [TD-3992]: taosdemo subscribe stable. add missed code lines Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 43 +++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9b73284f79..49d89dc597 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -769,16 +769,16 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->sqlFile = argv[++i]; } else if (strcmp(argv[i], "-q") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) + || (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n"); exit(EXIT_FAILURE); } arguments->async_mode = atoi(argv[++i]); } else if (strcmp(argv[i], "-T") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) + || (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-T need a number following!\n"); exit(EXIT_FAILURE); @@ -793,24 +793,24 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->insert_interval = atoi(argv[++i]); } else if (strcmp(argv[i], "-qt") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) + || (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-qt need a number following!\n"); exit(EXIT_FAILURE); } arguments->query_times = atoi(argv[++i]); } else if (strcmp(argv[i], "-B") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) + || (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-B need a number following!\n"); exit(EXIT_FAILURE); } arguments->interlace_rows = atoi(argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) + || (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-r need a number following!\n"); exit(EXIT_FAILURE); @@ -1075,7 +1075,7 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { if (code != 0) { if (!quiet) { debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command); - errorPrint("Failed to run %s, reason: %s\n", command, taos_errstr(res)); + errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res)); } taos_free_result(res); //taos_close(taos); @@ -4079,7 +4079,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times"); if (gQueryTimes && gQueryTimes->type == cJSON_Number) { - if (gQueryTimes->valueint < 0) { + if (gQueryTimes->valueint <= 0) { errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__); goto PARSE_OVER; @@ -4130,9 +4130,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, "query_times"); if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { - if (specifiedQueryTimes->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", - __func__, __LINE__); + if (specifiedQueryTimes->valueint <= 0) { + errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", + __func__, __LINE__, specifiedQueryTimes->valueint); goto PARSE_OVER; } @@ -4274,9 +4274,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times"); if (superQueryTimes && superQueryTimes->type == cJSON_Number) { - if (superQueryTimes->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", - __func__, __LINE__); + if (superQueryTimes->valueint <= 0) { + errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", + __func__, __LINE__, superQueryTimes->valueint); goto PARSE_OVER; } g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint; @@ -5240,6 +5240,13 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { startTs = taosGetTimestampMs(); + if (recOfBatch == 0) { + errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, + recOfBatch); + errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n"); + goto free_of_interlace; + } int64_t affectedRows = execInsert(pThreadInfo, buffer, recOfBatch); endTs = taosGetTimestampMs(); From 7307647ddbd2a722a5f63a7084ca703827992f80 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 12 May 2021 22:01:03 +0800 Subject: [PATCH 048/157] [TD-4158]: taosdemo cmake failed with low version git. (#6106) --- src/kit/taosdemo/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index 4e38a8842e..5f75be0e19 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -10,7 +10,11 @@ IF (GIT_FOUND) COMMAND ${GIT_EXECUTABLE} log --pretty=oneline -n 1 ${CMAKE_CURRENT_LIST_DIR}/taosdemo.c RESULT_VARIABLE RESULT OUTPUT_VARIABLE TAOSDEMO_COMMIT_SHA1) - STRING(SUBSTRING "${TAOSDEMO_COMMIT_SHA1}" 0 7 TAOSDEMO_COMMIT_SHA1) + IF ("${TAOSDEMO_COMMIT_SHA1}" STREQUAL "") + MESSAGE("taosdemo's latest commit in short is:" ${TAOSDEMO_COMMIT_SHA1}) + ELSE () + STRING(SUBSTRING "${TAOSDEMO_COMMIT_SHA1}" 0 7 TAOSDEMO_COMMIT_SHA1) + ENDIF () EXECUTE_PROCESS( COMMAND ${GIT_EXECUTABLE} status -z -s ${CMAKE_CURRENT_LIST_DIR}/taosdemo.c RESULT_VARIABLE RESULT From c4ac43af993550ca505799935c81c5938cf469f7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 May 2021 00:19:13 +0800 Subject: [PATCH 049/157] [TD-4158]: taosdemo cmake failed with low version git. (#6107) --- src/kit/taosdemo/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index 4e38a8842e..5f75be0e19 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -10,7 +10,11 @@ IF (GIT_FOUND) COMMAND ${GIT_EXECUTABLE} log --pretty=oneline -n 1 ${CMAKE_CURRENT_LIST_DIR}/taosdemo.c RESULT_VARIABLE RESULT OUTPUT_VARIABLE TAOSDEMO_COMMIT_SHA1) - STRING(SUBSTRING "${TAOSDEMO_COMMIT_SHA1}" 0 7 TAOSDEMO_COMMIT_SHA1) + IF ("${TAOSDEMO_COMMIT_SHA1}" STREQUAL "") + MESSAGE("taosdemo's latest commit in short is:" ${TAOSDEMO_COMMIT_SHA1}) + ELSE () + STRING(SUBSTRING "${TAOSDEMO_COMMIT_SHA1}" 0 7 TAOSDEMO_COMMIT_SHA1) + ENDIF () EXECUTE_PROCESS( COMMAND ${GIT_EXECUTABLE} status -z -s ${CMAKE_CURRENT_LIST_DIR}/taosdemo.c RESULT_VARIABLE RESULT From d7a11fb8b847a61ece56b84873e0d5ceb12c7b24 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 May 2021 04:25:10 +0800 Subject: [PATCH 050/157] Hotfix/sangshuduo/td 3992 taosdemo subscribe for develop (#6109) * [TD-3992]: taosdemo subscribe. * [TD-3992]: taosdemo subscribe. refactor sync/async mode. * [TD-3992]: taosdemo subscribe stable. add missed code lines Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 43 +++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 8866bf2607..f584448e63 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -770,16 +770,16 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->sqlFile = argv[++i]; } else if (strcmp(argv[i], "-q") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) + || (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n"); exit(EXIT_FAILURE); } arguments->async_mode = atoi(argv[++i]); } else if (strcmp(argv[i], "-T") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) + || (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-T need a number following!\n"); exit(EXIT_FAILURE); @@ -794,24 +794,24 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->insert_interval = atoi(argv[++i]); } else if (strcmp(argv[i], "-qt") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) + || (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-qt need a number following!\n"); exit(EXIT_FAILURE); } arguments->query_times = atoi(argv[++i]); } else if (strcmp(argv[i], "-B") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) + || (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-B need a number following!\n"); exit(EXIT_FAILURE); } arguments->interlace_rows = atoi(argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) + || (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-r need a number following!\n"); exit(EXIT_FAILURE); @@ -1071,7 +1071,7 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { if (code != 0) { if (!quiet) { debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command); - errorPrint("Failed to run %s, reason: %s\n", command, taos_errstr(res)); + errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res)); } taos_free_result(res); //taos_close(taos); @@ -4071,7 +4071,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times"); if (gQueryTimes && gQueryTimes->type == cJSON_Number) { - if (gQueryTimes->valueint < 0) { + if (gQueryTimes->valueint <= 0) { errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__); goto PARSE_OVER; @@ -4122,9 +4122,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, "query_times"); if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { - if (specifiedQueryTimes->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", - __func__, __LINE__); + if (specifiedQueryTimes->valueint <= 0) { + errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", + __func__, __LINE__, specifiedQueryTimes->valueint); goto PARSE_OVER; } @@ -4266,9 +4266,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times"); if (superQueryTimes && superQueryTimes->type == cJSON_Number) { - if (superQueryTimes->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", - __func__, __LINE__); + if (superQueryTimes->valueint <= 0) { + errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", + __func__, __LINE__, superQueryTimes->valueint); goto PARSE_OVER; } g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint; @@ -5232,6 +5232,13 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { startTs = taosGetTimestampMs(); + if (recOfBatch == 0) { + errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, + recOfBatch); + errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n"); + goto free_of_interlace; + } int64_t affectedRows = execInsert(pThreadInfo, buffer, recOfBatch); endTs = taosGetTimestampMs(); From 5e2348782450bb2583427c057d8a13bba4977407 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Thu, 13 May 2021 09:40:15 +0800 Subject: [PATCH 051/157] [TD-4089]: fix systemd sent SIGKILL before taosd shutdown normally (#6101) Co-authored-by: plum-lihui --- packaging/tools/install.sh | 3 +++ packaging/tools/install_arbi.sh | 1 + packaging/tools/install_arbi_power.sh | 1 + packaging/tools/install_power.sh | 3 +++ packaging/tools/make_install.sh | 1 + packaging/tools/post.sh | 1 + src/connector/grafanaplugin | 2 +- 7 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index dca3dd2ff6..178a248cfe 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -607,6 +607,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" @@ -630,6 +631,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" @@ -655,6 +657,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo 'PIDFile=/usr/local/nginxd/logs/nginx.pid' >> ${nginx_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/local/nginxd/sbin/nginx' >> ${nginx_service_config}" ${csudo} bash -c "echo 'ExecStop=/usr/local/nginxd/sbin/nginx -s stop' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${nginx_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${nginx_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${nginx_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${nginx_service_config}" diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh index a89d2257dc..f47c3672cb 100755 --- a/packaging/tools/install_arbi.sh +++ b/packaging/tools/install_arbi.sh @@ -205,6 +205,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" diff --git a/packaging/tools/install_arbi_power.sh b/packaging/tools/install_arbi_power.sh index 4b12913760..3f27175151 100755 --- a/packaging/tools/install_arbi_power.sh +++ b/packaging/tools/install_arbi_power.sh @@ -205,6 +205,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index ba6ace4009..9f28435cb5 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -577,6 +577,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo 'Type=simple' >> ${powerd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/powerd' >> ${powerd_service_config}" ${csudo} bash -c "echo 'ExecStartPre=/usr/local/power/bin/startPre.sh' >> ${powerd_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${powerd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${powerd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${powerd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${powerd_service_config}" @@ -599,6 +600,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" @@ -624,6 +626,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo 'PIDFile=/usr/local/nginxd/logs/nginx.pid' >> ${nginx_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/local/nginxd/sbin/nginx' >> ${nginx_service_config}" ${csudo} bash -c "echo 'ExecStop=/usr/local/nginxd/sbin/nginx -s stop' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${nginx_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${nginx_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${nginx_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${nginx_service_config}" diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 1fd0e943b1..f03065d70c 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -333,6 +333,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 8665b3fec3..3aa8083175 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -405,6 +405,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index 32e2c97a4c..3530c6df09 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df +Subproject commit 3530c6df097134a410bacec6b3cd013ef38a61aa From 2cb090997df343ace459d500026aef226214697b Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Thu, 13 May 2021 10:44:10 +0800 Subject: [PATCH 052/157] fix msg issue --- src/client/src/tscParseInsert.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index e5d4d00dc0..8ce0efa27a 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -577,12 +577,13 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SSq index = 0; sToken = tStrGetToken(*str, &index, false); - *str += index; if (sToken.n == 0 || sToken.type != TK_RP) { tscSQLSyntaxErrMsg(pCmd->payload, ") expected", *str); code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; return code; } + + *str += index; (*numOfRows)++; } From 920ff4d40cfabaed0cd0b6a072548eabb77da8a0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 13 May 2021 11:37:49 +0800 Subject: [PATCH 053/157] [td-4175]: improve the error message while the binary string overflow. --- src/client/src/tscParseInsert.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 923d95c888..f09ca351ff 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -709,6 +709,9 @@ static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, STableDataBlock int32_t numOfRows = 0; code = tsParseValues(str, dataBuf, maxNumOfRows, pCmd, &numOfRows, tmpTokenBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) { SParamInfo *param = dataBuf->params + i; From b14d3ea7729701e5bfb56b66f0d46c03c3109f8e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 13 May 2021 11:49:12 +0800 Subject: [PATCH 054/157] [TD-4172] --- src/client/src/tscLocal.c | 5 ++--- src/client/src/tscSQLParser.c | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 6b55780af9..1be5e29230 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -713,13 +713,12 @@ static int32_t tscProcessShowCreateDatabase(SSqlObj *pSql) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCreateBuilder *param = (SCreateBuilder *)malloc(sizeof(SCreateBuilder)); + SCreateBuilder *param = (SCreateBuilder *)calloc(1, sizeof(SCreateBuilder)); if (param == NULL) { free(pInterSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - - strncpy(param->buf, tNameGetTableName(&pTableMetaInfo->name), TSDB_TABLE_NAME_LEN); + tNameGetDbName(&pTableMetaInfo->name, param->buf); param->pParentSql = pSql; param->pInterSql = pInterSql; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e2ce0da739..acfc1b0cf5 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -467,8 +467,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pToken->n > TSDB_DB_NAME_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - - return tscSetTableFullName(pTableMetaInfo, pToken, pSql); + return tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); } case TSDB_SQL_CFG_DNODE: { const char* msg2 = "invalid configure options or values, such as resetlog / debugFlag 135 / balance 'vnode:2-dnode:2' / monitor 1 "; From 78c24ae1356ecf5aa106489dda071940f877ff34 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 May 2021 12:24:45 +0800 Subject: [PATCH 055/157] Hotfix/sangshuduo/td 3914 taosdemo func param segfault for develop (#6110) * [TD-3914]: taosdemo check func param input more. for develop branch. * [TD-3914]: taosdemo check func param more. for develop branch. * [TD-3914]: taosdemo input argument validation. (#6097) Co-authored-by: Shuduo Sang Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f584448e63..f6b511e5fa 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -725,7 +725,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN); - } else if (strcmp(argv[i], "-h") == 0) { if (argc == i+1) { printHelp(); @@ -849,6 +848,11 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->num_of_CPR = atoi(argv[++i]); } else if (strcmp(argv[i], "-b") == 0) { + if (argc == i+1) { + printHelp(); + errorPrint("%s", "\n\t-b need valid string following!\n"); + exit(EXIT_FAILURE); + } sptr = arguments->datatype; ++i; if (strstr(argv[i], ",") == NULL) { From 6892a32fe5c2c7dc87d64355182f6c10f27a4421 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 May 2021 13:02:35 +0800 Subject: [PATCH 056/157] [TD-4173]: compile failed with gcc 4.8.5 on centos7. (#6111) --- cmake/define.inc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 4115dd0c41..e825dce024 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -57,7 +57,7 @@ IF (TD_LINUX_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_LINUX_64) MESSAGE(STATUS "linux64 is defined") - SET(COMMON_FLAGS "-Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ADD_DEFINITIONS(-DUSE_LIBICONV) ENDIF () @@ -65,7 +65,7 @@ IF (TD_LINUX_32) ADD_DEFINITIONS(-D_TD_LINUX_32) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "linux32 is defined") - SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_ARM_64) @@ -73,7 +73,7 @@ IF (TD_ARM_64) ADD_DEFINITIONS(-D_TD_ARM_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "arm64 is defined") - SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_ARM_32) @@ -81,7 +81,7 @@ IF (TD_ARM_32) ADD_DEFINITIONS(-D_TD_ARM_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "arm32 is defined") - SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ") ENDIF () IF (TD_MIPS_64) @@ -89,7 +89,7 @@ IF (TD_MIPS_64) ADD_DEFINITIONS(-D_TD_MIPS_64) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "mips64 is defined") - SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_MIPS_32) @@ -97,7 +97,7 @@ IF (TD_MIPS_32) ADD_DEFINITIONS(-D_TD_MIPS_32) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "mips32 is defined") - SET(COMMON_FLAGS "-Wall -Werror -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_APLHINE) @@ -138,7 +138,7 @@ IF (TD_DARWIN_64) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "darwin64 is defined") - SET(COMMON_FLAGS "-Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG") SET(RELEASE_FLAGS "-Og") INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) From 8b011772ac80b5d441aa160191862e8df5ea383b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 May 2021 13:44:06 +0800 Subject: [PATCH 057/157] [TD-4152]: taosdemo reduce prompt and query systeminfo. (#6108) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f6b511e5fa..49d89dc597 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3410,8 +3410,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.interlace_rows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); + if (!g_args.answer_yes) { + printf(" press Enter key to continue or Ctrl-C to stop."); + (void)getchar(); + } g_args.interlace_rows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -3930,8 +3932,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); + if (!g_args.answer_yes) { + printf(" press Enter key to continue or Ctrl-C to stop."); + (void)getchar(); + } g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -6320,7 +6324,9 @@ static int queryTestProcess() { (void)getchar(); } - printfQuerySystemInfo(taos); + if (g_args.debug_print || g_args.verbose_print) { + printfQuerySystemInfo(taos); + } if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { if (convertHostToServAddr( From 133df75cec1379ccd1603c56cf44b73e2ea8161f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 May 2021 15:15:30 +0800 Subject: [PATCH 058/157] [TD-4170]:improve skiplist rand performance --- src/util/inc/tskiplist.h | 1 + src/util/src/tskiplist.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 2c4d1a86ef..17f5940b49 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -96,6 +96,7 @@ typedef struct tSkipListState { } tSkipListState; typedef struct SSkipList { + unsigned int seed; __compar_fn_t comparFn; __sl_key_fn_t keyFn; pthread_rwlock_t *lock; diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index e3798162e8..1bf6385288 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -50,6 +50,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ pSkipList->len = keyLen; pSkipList->flags = flags; pSkipList->keyFn = fn; + pSkipList->seed = rand(); if (comparFn == NULL) { pSkipList->comparFn = getKeyComparFunc(keyType); } else { @@ -545,7 +546,7 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) { const uint32_t factor = 4; int32_t n = 1; - while ((rand() % factor) == 0 && n <= pSkipList->maxLevel) { + while ((rand_r(&(pSkipList->seed)) % factor) == 0 && n <= pSkipList->maxLevel) { n++; } From 2b70b54cd4749d198b4f6f01db91325a5303695a Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Thu, 13 May 2021 15:27:33 +0800 Subject: [PATCH 059/157] fix bug --- src/client/src/tscProfile.c | 4 ++-- src/client/src/tscUtil.c | 8 ++++---- src/query/src/qExecutor.c | 2 ++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index be304632cc..ea2a3d8228 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -54,14 +54,14 @@ void tscAddIntoSqlList(SSqlObj *pSql) { pSql->next = pObj->sqlList; if (pObj->sqlList) pObj->sqlList->prev = pSql; pObj->sqlList = pSql; - pSql->queryId = queryId++; + pSql->queryId = atomic_fetch_add_32(&queryId, 1); pthread_mutex_unlock(&pObj->mutex); pSql->stime = taosGetTimestampMs(); pSql->listed = 1; - tscDebug("0x%"PRIx64" added into sqlList", pSql->self); + tscDebug("0x%"PRIx64" added into sqlList, queryId:%u", pSql->self, pSql->queryId); } void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 547d66a0f3..3d2cf44560 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2431,15 +2431,15 @@ void tscDoQuery(SSqlObj* pSql) { return; } - if (pCmd->command == TSDB_SQL_SELECT) { - tscAddIntoSqlList(pSql); - } - if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) { tscImportDataFromFile(pSql); } else { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); uint16_t type = pQueryInfo->type; + + if ((pCmd->command == TSDB_SQL_SELECT) && (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_SUBQUERY)) && (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_STABLE_SUBQUERY))) { + tscAddIntoSqlList(pSql); + } if (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_INSERT)) { // multi-vnodes insertion tscHandleMultivnodeInsert(pSql); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 4cf6d6072e..eca2a25a35 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -124,6 +124,8 @@ int64_t genQueryId(void) { uid |= sid; + qDebug("gen qid:0x%"PRIx64, uid); + return uid; } From 68d9c2417f280ac19b70eb9040f6aa7616364081 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 13 May 2021 15:39:53 +0800 Subject: [PATCH 060/157] [TD-4114] add test case for bind batch --- tests/script/api/makefile | 2 + tests/script/api/stmtBatchTest.c | 2036 ++++++++++++++++++++++++++++++ 2 files changed, 2038 insertions(+) create mode 100644 tests/script/api/stmtBatchTest.c diff --git a/tests/script/api/makefile b/tests/script/api/makefile index c5bbde0f0b..f3a8dec061 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -12,6 +12,8 @@ all: $(TARGET) exe: gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) + gcc $(CFLAGS) ./stmtBatchTest.c -o $(ROOT)stmtBatchTest $(LFLAGS) clean: rm $(ROOT)batchprepare + rm $(ROOT)stmtBatchTest diff --git a/tests/script/api/stmtBatchTest.c b/tests/script/api/stmtBatchTest.c new file mode 100644 index 0000000000..cb1ca3346b --- /dev/null +++ b/tests/script/api/stmtBatchTest.c @@ -0,0 +1,2036 @@ +// TAOS standard API example. The same syntax as MySQL, but only a subet +// to compile: gcc -o prepare prepare.c -ltaos + +#include +#include +#include +#include "taos.h" +#include "taoserror.h" +#include +#include +#include + +#define MAX_ROWS_OF_PER_COLUMN 32770 +#define MAX_BINARY_DEF_LEN (1024*16) + +typedef struct { + int64_t *ts; + int8_t b[MAX_ROWS_OF_PER_COLUMN]; + int8_t v1[MAX_ROWS_OF_PER_COLUMN]; + int16_t v2[MAX_ROWS_OF_PER_COLUMN]; + int32_t v4[MAX_ROWS_OF_PER_COLUMN]; + int64_t v8[MAX_ROWS_OF_PER_COLUMN]; + float f4[MAX_ROWS_OF_PER_COLUMN]; + double f8[MAX_ROWS_OF_PER_COLUMN]; + //char br[MAX_ROWS_OF_PER_COLUMN][MAX_BINARY_DEF_LEN]; + //char nr[MAX_ROWS_OF_PER_COLUMN][MAX_BINARY_DEF_LEN]; + char *br; + char *nr; + int64_t ts2[MAX_ROWS_OF_PER_COLUMN]; +} sampleValue; + + +typedef struct { + TAOS *taos; + int idx; +} ThreadInfo; + +//void taosMsleep(int mseconds); + +int g_rows = 0; + + +unsigned long long getCurrentTime(){ + struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) { + perror("Failed to get current time in ms"); + exit(EXIT_FAILURE); + } + + return (uint64_t)tv.tv_sec * 1000000ULL + (uint64_t)tv.tv_usec; +} + +static int stmt_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v->v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v->v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v->v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v->v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = rowsOfPerColum; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v->f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = rowsOfPerColum; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v->f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = rowsOfPerColum; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+8].buffer = v->br; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = rowsOfPerColum; + + params[i+9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[i+9].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+9].buffer = v->nr; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = rowsOfPerColum; + + params[i+10].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+10].buffer_length = sizeof(int64_t); + params[i+10].buffer = v->ts2; + params[i+10].length = NULL; + params[i+10].is_null = is_null; + params[i+10].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + + +static int stmt_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v->v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v->v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v->v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v->v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = rowsOfPerColum; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v->f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = rowsOfPerColum; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v->f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = rowsOfPerColum; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+8].buffer = v->br; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = rowsOfPerColum; + + params[i+9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[i+9].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+9].buffer = v->nr; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = rowsOfPerColum; + + params[i+10].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+10].buffer_length = sizeof(int64_t); + params[i+10].buffer = v->ts2; + params[i+10].length = NULL; + params[i+10].is_null = is_null; + params[i+10].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + + +static int stmt_bind_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_INT; + params[i+1].buffer_length = sizeof(int32_t); + params[i+1].buffer = v->v4; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +static int stmt_bind_case_004(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum * 2; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v->v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v->v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v->v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v->v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = rowsOfPerColum; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v->f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = rowsOfPerColum; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v->f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = rowsOfPerColum; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+8].buffer = v->br; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = rowsOfPerColum; + + params[i+9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[i+9].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+9].buffer = v->nr; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = rowsOfPerColum; + + params[i+10].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+10].buffer_length = sizeof(int64_t); + params[i+10].buffer = v->ts2; + params[i+10].length = NULL; + params[i+10].is_null = is_null; + params[i+10].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum * 2; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + // ===================================start==============================================// + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + // ===================================end==============================================// + + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +static int stmt_bind_error_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum * 2; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v->v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v->v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v->v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v->v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = rowsOfPerColum; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v->f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = rowsOfPerColum; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v->f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = rowsOfPerColum; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+8].buffer = v->br; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = rowsOfPerColum; + + params[i+9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[i+9].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+9].buffer = v->nr; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = rowsOfPerColum; + + params[i+10].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+10].buffer_length = sizeof(int64_t); + params[i+10].buffer = v->ts2; + params[i+10].length = NULL; + params[i+10].is_null = is_null; + params[i+10].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum * 2; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + //------- add one batch ------// + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + //----------------------------// + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + + +static int stmt_bind_error_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum * 2; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v->v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v->v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v->v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v->v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = rowsOfPerColum; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v->f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = rowsOfPerColum; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v->f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = rowsOfPerColum; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+8].buffer = v->br; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = rowsOfPerColum; + + params[i+9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[i+9].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+9].buffer = v->nr; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = rowsOfPerColum; + + params[i+10].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+10].buffer_length = sizeof(int64_t); + params[i+10].buffer = v->ts2; + params[i+10].length = NULL; + params[i+10].is_null = is_null; + params[i+10].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum * 2; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + //code = taos_stmt_add_batch(stmt); + //if (code != 0) { + // printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + // return -1; + //} + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +static int stmt_bind_error_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum * 2; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v->v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v->v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v->v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v->v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = rowsOfPerColum; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v->f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = rowsOfPerColum; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v->f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = rowsOfPerColum; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+8].buffer = v->br; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = rowsOfPerColum; + + params[i+9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[i+9].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+9].buffer = v->nr; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = rowsOfPerColum; + + params[i+10].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+10].buffer_length = sizeof(int64_t); + params[i+10].buffer = v->ts2; + params[i+10].length = NULL; + params[i+10].is_null = is_null; + params[i+10].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum * 2; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + //==================add one=================// + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + //==========================================// + + for (int col=0; col < columnNum; ++col) { + + //==================add one=================// + if (1==col) { + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + //==========================================// + + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +static void checkResult(TAOS *taos, char *tname, int printr, int expected) { + char sql[255] = "SELECT * FROM "; + TAOS_RES *result; + + strcat(sql, tname); + + result = taos_query(taos, sql); + int code = taos_errno(result); + if (code != 0) { + printf("failed to query table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + return; + } + + TAOS_ROW row; + int rows = 0; + int num_fields = taos_num_fields(result); + TAOS_FIELD *fields = taos_fetch_fields(result); + char temp[256]; + + // fetch the records row by row + while ((row = taos_fetch_row(result))) { + rows++; + if (printr) { + memset(temp, 0, sizeof(temp)); + taos_print_row(temp, row, fields, num_fields); + printf("[%s]\n", temp); + } + } + + if (rows == expected) { + printf("%d rows are fetched as expectation\n", rows); + } else { + printf("!!!expect %d rows, but %d rows are fetched\n", expected, rows); + return; + } + + taos_free_result(result); + +} + + +static void prepareV(TAOS *taos, int schemaCase, int tableNum, int lenOfBinaryDef) { + TAOS_RES *result; + int code; + + result = taos_query(taos, "drop database if exists demo"); + taos_free_result(result); + + result = taos_query(taos, "create database demo"); + code = taos_errno(result); + if (code != 0) { + printf("failed to create database, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + result = taos_query(taos, "use demo"); + taos_free_result(result); + + // create table + for (int i = 0 ; i < tableNum; i++) { + char buf[1024]; + if (schemaCase) { + sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp)", i, lenOfBinaryDef, lenOfBinaryDef) ; + } else { + sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; + } + + result = taos_query(taos, buf); + code = taos_errno(result); + if (code != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + } + +} + +static void preparemV(TAOS *taos, int schemaCase, int idx) { + TAOS_RES *result; + int code; + char dbname[32],sql[255]; + + sprintf(dbname, "demo%d", idx); + sprintf(sql, "drop database if exists %s", dbname); + + + result = taos_query(taos, sql); + taos_free_result(result); + + sprintf(sql, "create database %s", dbname); + result = taos_query(taos, sql); + code = taos_errno(result); + if (code != 0) { + printf("failed to create database, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + sprintf(sql, "use %s", dbname); + result = taos_query(taos, sql); + taos_free_result(result); + + // create table + for (int i = 0 ; i < 300; i++) { + char buf[1024]; + if (schemaCase) { + sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40), t2 timestamp)", i) ; + } else { + sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; + } + result = taos_query(taos, buf); + code = taos_errno(result); + if (code != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + } + +} + + + +//void runcase(TAOS *taos, int idx) { +static void* runCase(void *para) { + ThreadInfo* tInfo = (ThreadInfo *)para; + TAOS *taos = tInfo->taos; + int idx = tInfo->idx; + + TAOS_STMT *stmt = NULL; + + (void)idx; + + int tableNum; + int lenOfBinaryDef; + int rowsOfPerColum; + int bingNum; + int lenOfBinaryAct; + int columnNum; + + int totalRowsPerTbl; + +//=======================================================================// +//=============================== single table ==========================// +//========== case 1: ======================// +#if 1 +{ + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 1; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 16; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 1 check result end\n\n"); +} +#endif + + //========== case 2: ======================// +#if 1 +{ + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 5; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 2 check result end\n\n"); +} +#endif + + //========== case 3: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 1; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 3 check result end\n\n"); + } +#endif + + //========== case 4: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 5; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 4 check result end\n\n"); + } +#endif + +//=======================================================================// +//=============================== multi tables ==========================// + //========== case 5: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 1; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 16; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 5 check result end\n\n"); + } +#endif + + //========== case 6: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 5; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 6 check result end\n\n"); + } +#endif + + //========== case 7: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 1; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 7 check result end\n\n"); + } +#endif + + //========== case 8: ======================// +#if 1 +{ + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 5; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 8 check result end\n\n"); +} +#endif + + //=======================================================================// + //=============================== multi-rows to single table ==========================// + //========== case 9: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 23740; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 16; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 9 check result end\n\n"); + } +#endif + + //========== case 10: ======================// +#if 1 + { + printf("====case 10 error test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 23741; // WAL size exceeds limit + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 16; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("====case 10 check result end\n\n"); + } +#endif + + + //========== case 11: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 32767; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 16; + columnNum = 2; + + prepareV(taos, 0, tableNum, lenOfBinaryDef); + stmt_bind_case_003(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 11 check result end\n\n"); + } +#endif + + //========== case 12: ======================// +#if 1 + { + printf("====case 12 error test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 32768; // invalid parameter + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 16; + columnNum = 2; + + prepareV(taos, 0, tableNum, lenOfBinaryDef); + stmt_bind_case_003(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("====case 12 check result end\n\n"); + } +#endif + + //=======================================================================// + //=============================== multi tables, multi bind one same table ==========================// + //========== case 13: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 1; + bingNum = 5; + lenOfBinaryDef = 40; + lenOfBinaryAct = 16; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 13 check result end\n\n"); + } +#endif + + //========== case 14: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 5; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 14 check result end\n\n"); + } +#endif + + + //========== case 15: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1000; + rowsOfPerColum = 10; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 8; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m111", 0, totalRowsPerTbl); + checkResult(taos, "m222", 0, totalRowsPerTbl); + checkResult(taos, "m333", 0, totalRowsPerTbl); + checkResult(taos, "m999", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 15 check result end\n\n"); + } +#endif + + //========== case 16: ======================// +#if 1 + { + printf("====case 16 error test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 10; + rowsOfPerColum = 10; + bingNum = 1; + lenOfBinaryDef = 100; + lenOfBinaryAct = 8; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_error_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m11", 0, totalRowsPerTbl); + //checkResult(taos, "m22", 0, totalRowsPerTbl); + //checkResult(taos, "m33", 0, totalRowsPerTbl); + //checkResult(taos, "m99", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("====case 16 check result end\n\n"); + } +#endif + + //========== case 17: ======================// +#if 1 + { + //printf("case 17 test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 10; + rowsOfPerColum = 10; + bingNum = 1; + lenOfBinaryDef = 100; + lenOfBinaryAct = 8; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_004(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum * 2; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m11", 0, totalRowsPerTbl); + //checkResult(taos, "m22", 0, totalRowsPerTbl); + //checkResult(taos, "m33", 0, totalRowsPerTbl); + //checkResult(taos, "m99", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 17 check result end\n\n"); + } +#endif + + //========== case 18: ======================// +#if 1 + { + printf("====case 18 error test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 10; + bingNum = 1; + lenOfBinaryDef = 100; + lenOfBinaryAct = 8; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_error_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m11", 0, totalRowsPerTbl); + //checkResult(taos, "m22", 0, totalRowsPerTbl); + //checkResult(taos, "m33", 0, totalRowsPerTbl); + //checkResult(taos, "m99", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("====case 18 check result end\n\n"); + } +#endif + + //========== case 19: ======================// +#if 1 + { + printf("====case 19 error test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 10; + bingNum = 1; + lenOfBinaryDef = 100; + lenOfBinaryAct = 8; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_error_case_003(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m11", 0, totalRowsPerTbl); + //checkResult(taos, "m22", 0, totalRowsPerTbl); + //checkResult(taos, "m33", 0, totalRowsPerTbl); + //checkResult(taos, "m99", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("====case 19 check result end\n\n"); + } +#endif + + return NULL; + +} + +int main(int argc, char *argv[]) +{ + TAOS *taos; + char host[32] = "127.0.0.1"; + char* serverIp = NULL; + int threadNum = 1; + + // connect to server + if (argc == 1) { + serverIp = host; + } else if (argc == 2) { + serverIp = argv[1]; + } else if (argc == 3) { + serverIp = argv[1]; + threadNum = atoi(argv[2]); + } else if (argc == 4) { + serverIp = argv[1]; + threadNum = atoi(argv[2]); + g_rows = atoi(argv[3]); + } + + printf("server:%s, threadNum:%d, rows:%d\n\n", serverIp, threadNum, g_rows); + + pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), (size_t)threadNum); + ThreadInfo* threadInfo = (ThreadInfo *) calloc(sizeof(ThreadInfo), (size_t)threadNum); + + ThreadInfo* tInfo = threadInfo; + for (int i = 0; i < threadNum; i++) { + taos = taos_connect(serverIp, "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to TDengine, reason:%s\n", taos_errstr(taos)); + return -1; + } + + tInfo->taos = taos; + tInfo->idx = i; + pthread_create(&(pThreadList[0]), NULL, runCase, (void *)tInfo); + tInfo++; + } + + for (int i = 0; i < threadNum; i++) { + pthread_join(pThreadList[i], NULL); + } + + free(pThreadList); + free(threadInfo); + return 0; +} + From 8178a81a34ea8d67fc814828352b8a902dec6b58 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 13 May 2021 16:07:53 +0800 Subject: [PATCH 061/157] fix compile error --- src/util/src/tskiplist.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 1bf6385288..842ded19a6 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -546,7 +546,12 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) { const uint32_t factor = 4; int32_t n = 1; + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + while ((rand() % factor) == 0 && n <= pSkipList->maxLevel) { +#else while ((rand_r(&(pSkipList->seed)) % factor) == 0 && n <= pSkipList->maxLevel) { +#endif n++; } From 57aceae597c513c72736dea7efc75d37c315092c Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Thu, 13 May 2021 16:48:37 +0800 Subject: [PATCH 062/157] fix bug --- src/mnode/src/mnodeProfile.c | 5 +++-- src/mnode/src/mnodeShow.c | 4 ---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 17a4282d05..f0ff016694 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -123,8 +123,9 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po if (/* pConn->ip != ip || */ pConn->port != port /* || strcmp(pConn->user, user) != 0 */) { mDebug("connId:%d, incoming conn user:%s ip:%s:%u, not match exist conn user:%s ip:%s:%u", connId, user, taosIpStr(ip), port, pConn->user, taosIpStr(pConn->ip), pConn->port); - taosCacheRelease(tsMnodeConnCache, (void **)&pConn, false); - return NULL; + pConn->port = port; + //taosCacheRelease(tsMnodeConnCache, (void **)&pConn, false); + //return NULL; } // mDebug("connId:%d, is incoming, user:%s ip:%s:%u", connId, pConn->user, taosIpStr(pConn->ip), pConn->port); diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index c0fa6368f3..5fe22826b7 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -253,10 +253,6 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { int32_t connId = htonl(pHBMsg->connId); SConnObj *pConn = mnodeAccquireConn(connId, connInfo.user, connInfo.clientIp, connInfo.clientPort); - if (pConn == NULL) { - pHBMsg->pid = htonl(pHBMsg->pid); - pConn = mnodeCreateConn(connInfo.user, connInfo.clientIp, connInfo.clientPort, pHBMsg->pid, pHBMsg->appName); - } if (pConn == NULL) { // do not close existing links, otherwise From 6db4fc6c000f1781961d6e56f5c680ef4b7a0356 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 13 May 2021 09:44:10 +0000 Subject: [PATCH 063/157] [TD-4179]: add test case --- .../jdbc/TSDBPreparedStatementTest.java | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java index dc6fd4c501..8804cc5da0 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -8,6 +8,8 @@ import org.junit.Test; import java.io.IOException; import java.io.Serializable; import java.sql.*; +import java.util.ArrayList; +import java.util.Random; public class TSDBPreparedStatementTest { private static final String host = "127.0.0.1"; @@ -97,6 +99,118 @@ public class TSDBPreparedStatementTest { Assert.assertEquals(1, result); } + @Test + public void executeTest() throws SQLException { + Statement stmt = conn.createStatement(); + + int numOfRows = 1000; + + for (int loop = 0; loop < 10; loop++){ + stmt.execute("drop table if exists weather_test"); + stmt.execute("create table weather_test(ts timestamp, f1 nchar(4), f2 float, f3 double, f4 timestamp, f5 int, f6 bool, f7 binary(10))"); + + TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? values(?, ?, ?, ?, ?, ?, ?, ?)"); + Random r = new Random(); + s.setTableName("weather_test"); + + ArrayList ts = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + ts.add(System.currentTimeMillis() + i); + } + s.setTimestamp(0, ts); + + int random = 10 + r.nextInt(5); + ArrayList s2 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + s2.add(null); + }else{ + s2.add("分支" + i % 4); + } + } + s.setNString(1, s2, 4); + + random = 10 + r.nextInt(5); + ArrayList s3 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + s3.add(null); + }else{ + s3.add(r.nextFloat()); + } + } + s.setFloat(2, s3); + + random = 10 + r.nextInt(5); + ArrayList s4 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + s4.add(null); + }else{ + s4.add(r.nextDouble()); + } + } + s.setDouble(3, s4); + + random = 10 + r.nextInt(5); + ArrayList ts2 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + ts2.add(null); + }else{ + ts2.add(System.currentTimeMillis() + i); + } + } + s.setTimestamp(4, ts2); + + random = 10 + r.nextInt(5); + ArrayList vals = new ArrayList<>(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + vals.add(null); + }else{ + vals.add(r.nextInt()); + } + } + s.setInt(5, vals); + + random = 10 + r.nextInt(5); + ArrayList sb = new ArrayList<>(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + sb.add(null); + }else{ + sb.add(i % 2 == 0 ? true : false); + } + } + s.setBoolean(6, sb); + + random = 10 + r.nextInt(5); + ArrayList s5 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + s5.add(null); + }else{ + s5.add("test" + i % 10); + } + } + s.setString(7, s5, 10); + + s.columnDataAddBatch(); + s.columnDataExecuteBatch(); + s.columnDataCloseBatch(); + + String sql = "select * from weather_test"; + PreparedStatement statement = conn.prepareStatement(sql); + ResultSet rs = statement.executeQuery(); + int rows = 0; + while(rs.next()) { + rows++; + } + Assert.assertEquals(numOfRows, rows); + } + } + @Test public void setBoolean() throws SQLException { pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); From ee36db4af9a5cf3fe4f81ca535403a808ae7b50c Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 13 May 2021 10:09:04 +0000 Subject: [PATCH 064/157] [TD-4167]add smoke test on centos7 in CI --- .drone.yml | 47 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/.drone.yml b/.drone.yml index e7ae6ebbda..d13aad3153 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7,41 +7,22 @@ platform: arch: amd64 steps: -- name: smoke_test - image: python:3.8 +- name: build + image: gcc commands: - apt-get update - - apt-get install -y cmake build-essential gcc - - pip3 install psutil - - pip3 install guppy3 - - pip3 install src/connector/python/linux/python3/ + - apt-get install -y cmake build-essential - mkdir debug - cd debug - cmake .. - make - - cd ../tests - - ./test-all.sh smoke + trigger: + event: + - pull_request when: branch: - develop - master - - -- name: crash_gen - image: python:3.8 - commands: - - pip3 install requests - - pip3 install src/connector/python/linux/python3/ - - pip3 install psutil - - pip3 install guppy3 - - cd tests/pytest - - ./crash_gen.sh -a -p -t 4 -s 2000 - when: - branch: - - develop - - master - - --- kind: pipeline name: test_arm64 @@ -60,6 +41,9 @@ steps: - cd debug - cmake .. -DCPUTYPE=aarch64 > /dev/null - make + trigger: + event: + - pull_request when: branch: - develop @@ -82,6 +66,9 @@ steps: - cd debug - cmake .. -DCPUTYPE=aarch32 > /dev/null - make + trigger: + event: + - pull_request when: branch: - develop @@ -106,11 +93,13 @@ steps: - cd debug - cmake .. - make + trigger: + event: + - pull_request when: branch: - develop - master - --- kind: pipeline name: build_xenial @@ -129,6 +118,9 @@ steps: - cd debug - cmake .. - make + trigger: + event: + - pull_request when: branch: - develop @@ -151,6 +143,9 @@ steps: - cd debug - cmake .. - make + trigger: + event: + - pull_request when: branch: - develop From 8517694940f73be2aa479076f7b543d0d20c2d68 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 May 2021 20:37:34 +0800 Subject: [PATCH 065/157] [TD-4068]: taosdemo support stmt interface. construct framework. --- src/kit/taosdemo/taosdemo.c | 147 +++++++++++++++++++++++------------- 1 file changed, 93 insertions(+), 54 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index e5d8556467..108acfc78a 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -121,9 +121,9 @@ enum MODE { }; enum INTERFACE { - TAOSC_INTERFACE, - REST_INTERFACE, - STMT_INTERFACE, + TAOSC_IFACE, + REST_IFACE, + STMT_IFACE, INTERFACE_BUT }; @@ -131,7 +131,7 @@ typedef enum enum_INSERT_MODE { PROGRESSIVE_INSERT_MODE, INTERLACE_INSERT_MODE, INVALID_INSERT_MODE -} INSERT_MODE; +} PROG_OR_INTERLACE_MODE; typedef enum enumQUERY_TYPE { NO_INSERT_TYPE, @@ -246,9 +246,9 @@ typedef struct SSuperTable_S { uint8_t autoCreateTable; // 0: create sub table, 1: auto create sub table char childTblPrefix[MAX_TB_NAME_SIZE]; char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample - char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest + uint16_t insertMode; // 0: taosc, 1: rest, 2: stmt int64_t childTblLimit; - uint64_t childTblOffset; + uint64_t childTblOffset; // int multiThreadWriteOneTbl; // 0: no, 1: yes uint64_t interlaceRows; // @@ -266,7 +266,7 @@ typedef struct SSuperTable_S { uint32_t columnCount; StrColumn columns[MAX_COLUMN_COUNT]; - uint32_t tagCount; + uint32_t tagCount; StrColumn tags[MAX_TAG_COUNT]; char* childTblName; @@ -291,7 +291,7 @@ typedef struct SSuperTable_S { typedef struct { char name[TSDB_DB_NAME_LEN + 1]; char create_time[32]; - int32_t ntables; + uint64_t ntables; int32_t vgroups; int16_t replica; int16_t quorum; @@ -413,6 +413,7 @@ typedef struct SQueryMetaInfo_S { typedef struct SThreadInfo_S { TAOS * taos; + TAOS_STMT *stmt; int threadID; char db_name[MAX_DB_NAME_SIZE+1]; uint32_t time_precision; @@ -544,7 +545,7 @@ SArguments g_args = { 0, // test_mode "127.0.0.1", // host 6030, // port - TAOSC_INTERFACE, // interface + TAOSC_IFACE, // interface "root", // user #ifdef _TD_POWER_ "powerdb", // password @@ -759,11 +760,11 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } ++i; if (0 == strcasecmp(argv[i], "taosc")) { - arguments->interface = TAOSC_INTERFACE; + arguments->interface = TAOSC_IFACE; } else if (0 == strcasecmp(argv[i], "rest")) { - arguments->interface = REST_INTERFACE; + arguments->interface = REST_IFACE; } else if (0 == strcasecmp(argv[i], "stmt")) { - arguments->interface = STMT_INTERFACE; + arguments->interface = STMT_IFACE; } else { errorPrint("%s", "\n\t-I need a valid string following!\n"); exit(EXIT_FAILURE); @@ -1025,7 +1026,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->port ); printf("# User: %s\n", arguments->user); printf("# Password: %s\n", arguments->password); - printf("# Use metric: %s\n", arguments->use_metric ? "true" : "false"); + printf("# Use metric: %s\n", + arguments->use_metric ? "true" : "false"); if (*(arguments->datatype)) { printf("# Specified data type: "); for (int i = 0; i < MAX_NUM_DATATYPE; i++) @@ -1319,6 +1321,8 @@ static void init_rand_data() { static int printfInsertMeta() { SHOW_PARSE_RESULT_START(); + printf("interface: \033[33m%s\033[0m\n", + (g_args.interface==TAOSC_IFACE)?"taosc":(g_args.interface==REST_IFACE)?"rest":"stmt"); printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); printf("user: \033[33m%s\033[0m\n", g_Dbs.user); printf("password: \033[33m%s\033[0m\n", g_Dbs.password); @@ -1423,7 +1427,8 @@ static int printfInsertMeta() { printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); printf(" insertMode: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].insertMode); + (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc": + (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt"); if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) { printf(" childTblLimit: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].childTblLimit); @@ -1606,7 +1611,8 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); fprintf(fp, " insertMode: %s\n", - g_Dbs.db[i].superTbls[j].insertMode); + (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc": + (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt"); fprintf(fp, " insertRows: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].insertRows); fprintf(fp, " interlace rows: %"PRIu64"\n", @@ -2916,7 +2922,7 @@ static int startMultiThreadCreateChildTable( char* db_name, SSuperTable* superTblInfo) { pthread_t *pids = malloc(threads * sizeof(pthread_t)); - threadInfo *infos = malloc(threads * sizeof(threadInfo)); + threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { printf("malloc failed\n"); @@ -3110,10 +3116,12 @@ static int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { return 0; } +#if 0 int readSampleFromJsonFileToMem(SSuperTable * superTblInfo) { // TODO return 0; } +#endif /* Read 10000 lines at most. If more than 10000 lines, continue to read after using @@ -3813,15 +3821,24 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest + cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt if (insertMode && insertMode->type == cJSON_String && insertMode->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, - insertMode->valuestring, MAX_DB_NAME_SIZE); + if (0 == strcasecmp(insertMode->valuestring, "taosc")) { + g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE; + } else if (0 == strcasecmp(insertMode->valuestring, "rest")) { + g_Dbs.db[i].superTbls[j].insertMode = REST_IFACE; + } else if (0 == strcasecmp(insertMode->valuestring, "stmt")) { + g_Dbs.db[i].superTbls[j].insertMode = STMT_IFACE; + } else { + errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n", + __func__, __LINE__, insertMode->valuestring); + goto PARSE_OVER; + } } else if (!insertMode) { - tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE); + g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE; } else { - printf("ERROR: failed to read json, insert_mode not found\n"); + errorPrint("%s", "failed to read json, insert_mode not found\n"); goto PARSE_OVER; } @@ -4751,9 +4768,9 @@ static int64_t execInsert(threadInfo *pThreadInfo, char *buffer, uint64_t k) verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID, __func__, __LINE__, buffer); if (superTblInfo) { - if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) { + if (superTblInfo->insertMode == TAOSC_IFACE) { affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false); - } else if (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest"))) { + } else if (superTblInfo->insertMode == REST_IFACE) { if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port, buffer, NULL /* not set result file */)) { affectedRows = -1; @@ -4762,8 +4779,13 @@ static int64_t execInsert(threadInfo *pThreadInfo, char *buffer, uint64_t k) } else { affectedRows = k; } + } else if (superTblInfo->insertMode == STMT_IFACE) { + // TODO: add stmt support + errorPrint("%s() LN%d, %s\n", + __func__, __LINE__, "!!! need support stmt here"); + exit(-1); } else { - errorPrint("%s() LN%d: unknown insert mode: %s\n", + errorPrint("%s() LN%d: unknown insert mode: %d\n", __func__, __LINE__, superTblInfo->insertMode); affectedRows = 0; } @@ -4800,7 +4822,7 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t table static int64_t generateDataTail( SSuperTable* superTblInfo, uint64_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows, - int64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { + uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { uint64_t len = 0; uint32_t ncols_per_record = 1; // count first col ts @@ -5114,17 +5136,14 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (interlaceRows > g_args.num_of_RPR) interlaceRows = g_args.num_of_RPR; - int insertMode; + int progOrInterlace; if (interlaceRows > 0) { - insertMode = INTERLACE_INSERT_MODE; + progOrInterlace= INTERLACE_INSERT_MODE; } else { - insertMode = PROGRESSIVE_INSERT_MODE; + progOrInterlace = PROGRESSIVE_INSERT_MODE; } - // TODO: prompt tbl count multple interlace rows and batch - // - uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; char* buffer = calloc(maxSqlLen, 1); if (NULL == buffer) { @@ -5230,7 +5249,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__, batchPerTbl, recOfBatch); - if (insertMode == INTERLACE_INSERT_MODE) { + if (progOrInterlace == INTERLACE_INSERT_MODE) { if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { // turn to first table tableSeq = pThreadInfo->start_table_from; @@ -5609,15 +5628,6 @@ static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in * static void startMultiThreadInsertData(int threads, char* db_name, char* precision,SSuperTable* superTblInfo) { - pthread_t *pids = malloc(threads * sizeof(pthread_t)); - assert(pids != NULL); - - threadInfo *infos = malloc(threads * sizeof(threadInfo)); - assert(infos != NULL); - - memset(pids, 0, threads * sizeof(pthread_t)); - memset(infos, 0, threads * sizeof(threadInfo)); - //TAOS* taos; //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { // taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); @@ -5678,17 +5688,17 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } - TAOS* taos = taos_connect( + TAOS* taos0 = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); - if (NULL == taos) { + if (NULL == taos0) { errorPrint("%s() LN%d, connect to server fail , reason: %s\n", __func__, __LINE__, taos_errstr(NULL)); exit(-1); } - int ntables = 0; - int startFrom; + uint64_t ntables = 0; + uint startFrom; if (superTblInfo) { int64_t limit; @@ -5740,13 +5750,13 @@ static void startMultiThreadInsertData(int threads, char* db_name, limit * TSDB_TABLE_NAME_LEN); if (superTblInfo->childTblName == NULL) { errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); - taos_close(taos); + taos_close(taos0); exit(-1); } uint64_t childTblCount; getChildNameOfSuperTableWithLimitAndOffset( - taos, + taos0, db_name, superTblInfo->sTblName, &superTblInfo->childTblName, &childTblCount, limit, @@ -5756,7 +5766,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, startFrom = 0; } - taos_close(taos); + taos_close(taos0); uint64_t a = ntables / threads; if (a < 1) { @@ -5770,11 +5780,21 @@ static void startMultiThreadInsertData(int threads, char* db_name, } if ((superTblInfo) - && (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest")))) { - if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) - exit(-1); + && (superTblInfo->insertMode == REST_IFACE)) { + if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) { + exit(-1); + } } + pthread_t *pids = malloc(threads * sizeof(pthread_t)); + assert(pids != NULL); + + threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); + assert(infos != NULL); + + memset(pids, 0, threads * sizeof(pthread_t)); + memset(infos, 0, threads * sizeof(threadInfo)); + for (int i = 0; i < threads; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; @@ -5786,17 +5806,32 @@ static void startMultiThreadInsertData(int threads, char* db_name, t_info->minDelay = UINT64_MAX; if ((NULL == superTblInfo) || - (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) { + (superTblInfo->insertMode != REST_IFACE)) { //t_info->taos = taos; t_info->taos = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); if (NULL == t_info->taos) { errorPrint( - "connect to server fail from insert sub thread, reason: %s\n", + "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n", + __func__, __LINE__, taos_errstr(NULL)); + free(infos); exit(-1); } + + if ((superTblInfo) && (superTblInfo->insertMode == STMT_IFACE)) { + t_info->stmt = taos_stmt_init(t_info->taos); + if (NULL == t_info->stmt) { + errorPrint( + "%s() LN%d, failed init stmt, reason: %s\n", + __func__, __LINE__, + taos_errstr(NULL)); + free(pids); + free(infos); + exit(-1); + } + } } else { t_info->taos = NULL; } @@ -5836,6 +5871,10 @@ static void startMultiThreadInsertData(int threads, char* db_name, threadInfo *t_info = infos + i; tsem_destroy(&(t_info->lock_sem)); + + if (t_info->stmt) { + taos_stmt_close(t_info->stmt); + } taos_close(t_info->taos); debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n", @@ -6908,7 +6947,7 @@ static void setParaFromArg(){ tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); - tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].insertMode = g_args.interface; tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; From 91249c2e50eab1e0cdd823a4090f087d260f5256 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 May 2021 22:54:54 +0800 Subject: [PATCH 066/157] Hotfix/sangshuduo/td 4136 taosdemo records morethan 32767 for develop (#6119) * [TD-4136]: taosdemo records per req more than 32767. for develop branch. * change taosRandom() to rand_bool() for bool data. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 49d89dc597..72044753d8 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4661,7 +4661,7 @@ static int64_t generateData(char *recBuf, char **data_type, double t = rand_double(); pstr += sprintf(pstr, ",%20.8f", t); } else if (strcasecmp(data_type[i % c], "BOOL") == 0) { - bool b = taosRandom() & 1; + bool b = rand_bool() & 1; pstr += sprintf(pstr, ",%s", b ? "true" : "false"); } else if (strcasecmp(data_type[i % c], "BINARY") == 0) { char *s = malloc(lenOfBinary); From 770bfd7b39c3c0bb105dffb291858792b3c1cd68 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 May 2021 22:55:12 +0800 Subject: [PATCH 067/157] [TD-4136]: taosdemo max records per req < 32767 (#6118) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 49d89dc597..72044753d8 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4661,7 +4661,7 @@ static int64_t generateData(char *recBuf, char **data_type, double t = rand_double(); pstr += sprintf(pstr, ",%20.8f", t); } else if (strcasecmp(data_type[i % c], "BOOL") == 0) { - bool b = taosRandom() & 1; + bool b = rand_bool() & 1; pstr += sprintf(pstr, ",%s", b ? "true" : "false"); } else if (strcasecmp(data_type[i % c], "BINARY") == 0) { char *s = malloc(lenOfBinary); From f5f714a3ce945aa0974b1c9220e7273ec82507ac Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 May 2021 23:06:35 +0800 Subject: [PATCH 068/157] merge with develop branch. --- src/kit/taosdemo/taosdemo.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 566a5918f6..83094df9b8 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -120,14 +120,14 @@ enum MODE { MODE_BUT }; -enum INTERFACE { +enum enum_TAOS_INTERFACE { TAOSC_IFACE, REST_IFACE, STMT_IFACE, INTERFACE_BUT }; -typedef enum enum_INSERT_MODE { +typedef enum enum_PROGRESSIVE_OR_INTERLACE { PROGRESSIVE_INSERT_MODE, INTERLACE_INSERT_MODE, INVALID_INSERT_MODE @@ -1964,7 +1964,7 @@ static void printfDbInfoForQueryToFile( fprintf(fp, "================ database[%d] ================\n", index); fprintf(fp, "name: %s\n", dbInfos->name); fprintf(fp, "created_time: %s\n", dbInfos->create_time); - fprintf(fp, "ntables: %d\n", dbInfos->ntables); + fprintf(fp, "ntables: %"PRIu64"\n", dbInfos->ntables); fprintf(fp, "vgroups: %d\n", dbInfos->vgroups); fprintf(fp, "replica: %d\n", dbInfos->replica); fprintf(fp, "quorum: %d\n", dbInfos->quorum); From fe4984290cb904ff6bea14883abf5fb2ca1c8582 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 May 2021 23:33:54 +0800 Subject: [PATCH 069/157] fix windows compile issue. --- src/kit/taosdemo/taosdemo.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 83094df9b8..2657869150 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -195,7 +195,7 @@ typedef struct SArguments_S { uint32_t test_mode; char * host; uint16_t port; - uint16_t interface; + uint16_t iface; char * user; char * password; char * database; @@ -545,7 +545,7 @@ SArguments g_args = { 0, // test_mode "127.0.0.1", // host 6030, // port - TAOSC_IFACE, // interface + TAOSC_IFACE, // iface "root", // user #ifdef _TD_POWER_ "powerdb", // password @@ -760,11 +760,11 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } ++i; if (0 == strcasecmp(argv[i], "taosc")) { - arguments->interface = TAOSC_IFACE; + arguments->iface = TAOSC_IFACE; } else if (0 == strcasecmp(argv[i], "rest")) { - arguments->interface = REST_IFACE; + arguments->iface = REST_IFACE; } else if (0 == strcasecmp(argv[i], "stmt")) { - arguments->interface = STMT_IFACE; + arguments->iface = STMT_IFACE; } else { errorPrint("%s", "\n\t-I need a valid string following!\n"); exit(EXIT_FAILURE); @@ -1322,7 +1322,7 @@ static int printfInsertMeta() { SHOW_PARSE_RESULT_START(); printf("interface: \033[33m%s\033[0m\n", - (g_args.interface==TAOSC_IFACE)?"taosc":(g_args.interface==REST_IFACE)?"rest":"stmt"); + (g_args.iface==TAOSC_IFACE)?"taosc":(g_args.iface==REST_IFACE)?"rest":"stmt"); printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); printf("user: \033[33m%s\033[0m\n", g_Dbs.user); printf("password: \033[33m%s\033[0m\n", g_Dbs.password); @@ -6947,7 +6947,7 @@ static void setParaFromArg(){ tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].insertMode = g_args.interface; + g_Dbs.db[0].superTbls[0].insertMode = g_args.iface; tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; From 11d3aaf2daa068db85614fb62ccf678e5be4936f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 May 2021 23:40:52 +0800 Subject: [PATCH 070/157] fix windows compile issue. again. --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 2657869150..c83b8349b7 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5698,7 +5698,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } uint64_t ntables = 0; - uint startFrom; + uint64_t startFrom; if (superTblInfo) { int64_t limit; From 65471ac1a5a4e6595ffee2b519eef0c99053c6dd Mon Sep 17 00:00:00 2001 From: hope Date: Fri, 14 May 2021 00:20:21 +0800 Subject: [PATCH 071/157] fix some type error, and add compile-instruction in order to reduce potential errors --- .gitignore | 1 + CMakeLists.txt | 7 ++++++- packaging/release.sh | 4 ++-- src/client/src/tscSubquery.c | 4 ++-- src/kit/taosdemo/taosdemo.c | 2 +- src/mnode/src/mnodeDnode.c | 6 +++--- src/plugins/http/src/httpGzip.c | 4 ++-- src/plugins/http/src/httpParser.c | 6 +++--- src/plugins/http/src/httpTgHandle.c | 2 +- src/rpc/src/rpcTcp.c | 2 +- src/sync/src/syncMain.c | 14 +++++++------- src/sync/src/syncMsg.c | 4 ++-- src/tsdb/inc/tsdbMeta.h | 8 ++++---- src/tsdb/src/tsdbFS.c | 12 ++++++------ src/tsdb/src/tsdbFile.c | 10 +++++----- src/tsdb/src/tsdbMeta.c | 10 +++++----- src/util/src/tdes.c | 14 +++++++------- src/wal/src/walWrite.c | 2 +- 18 files changed, 59 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index 1ff1108056..da47590a2f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ build/ .vscode/ .idea/ cmake-build-debug/ +cmake-build-release/ cscope.out .DS_Store debug/ diff --git a/CMakeLists.txt b/CMakeLists.txt index e0d6e82923..0a3531908c 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ IF (CMAKE_VERSION VERSION_LESS 3.0) PROJECT(TDengine CXX) SET(PROJECT_VERSION_MAJOR "${LIB_MAJOR_VERSION}") SET(PROJECT_VERSION_MINOR "${LIB_MINOR_VERSION}") - SET(PROJECT_VERSION_PATCH"${LIB_PATCH_VERSION}") + SET(PROJECT_VERSION_PATCH "${LIB_PATCH_VERSION}") SET(PROJECT_VERSION "${LIB_VERSION_STRING}") ELSE () CMAKE_POLICY(SET CMP0048 NEW) @@ -42,6 +42,11 @@ INCLUDE(cmake/env.inc) INCLUDE(cmake/version.inc) INCLUDE(cmake/install.inc) +#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pipe -Wall -Wshadow -Wextra -Werror") +#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pipe -Wall -Wshadow -Wextra -Werror") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pipe -Wall -Wshadow -Werror") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pipe -Wall -Wshadow -Werror") + ADD_SUBDIRECTORY(deps) ADD_SUBDIRECTORY(src) ADD_SUBDIRECTORY(tests) diff --git a/packaging/release.sh b/packaging/release.sh index 68f947ccab..1e54bc2872 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -1,11 +1,11 @@ #!/bin/bash # -# Generate the deb package for ubunt, or rpm package for centos, or tar.gz package for other linux os +# Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os set -e #set -x -# releash.sh -v [cluster | edge] +# release.sh -v [cluster | edge] # -c [aarch32 | aarch64 | x64 | x86 | mips64 ...] # -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] # -V [stable | beta] diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e9f87b3115..101cb6f714 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -55,9 +55,9 @@ static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) { } while (tsBufNextPos(pTSBuf)) { - STSElem el1 = tsBufGetElem(pTSBuf); + el1 = tsBufGetElem(pTSBuf); - int32_t res = tVariantCompare(el1.tag, tag1); + res = tVariantCompare(el1.tag, tag1); if (res != 0) { // it is a record with new tag return; } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 72044753d8..49d89dc597 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4661,7 +4661,7 @@ static int64_t generateData(char *recBuf, char **data_type, double t = rand_double(); pstr += sprintf(pstr, ",%20.8f", t); } else if (strcasecmp(data_type[i % c], "BOOL") == 0) { - bool b = rand_bool() & 1; + bool b = taosRandom() & 1; pstr += sprintf(pstr, ",%s", b ? "true" : "false"); } else if (strcasecmp(data_type[i % c], "BINARY") == 0) { char *s = malloc(lenOfBinary); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index b513da29f4..3525bcac18 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -522,13 +522,13 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { pStatus->lastReboot = htonl(pStatus->lastReboot); pStatus->numOfCores = htons(pStatus->numOfCores); - uint32_t version = htonl(pStatus->version); - if (version != tsVersion) { + uint32_t _version = htonl(pStatus->version); + if (_version != tsVersion) { pDnode = mnodeGetDnodeByEp(pStatus->dnodeEp); if (pDnode != NULL && pDnode->status != TAOS_DN_STATUS_READY) { pDnode->offlineReason = TAOS_DN_OFF_VERSION_NOT_MATCH; } - mError("dnode:%d, status msg version:%d not equal with cluster:%d", pStatus->dnodeId, version, tsVersion); + mError("dnode:%d, status msg version:%d not equal with cluster:%d", pStatus->dnodeId, _version, tsVersion); return TSDB_CODE_MND_INVALID_MSG_VERSION; } diff --git a/src/plugins/http/src/httpGzip.c b/src/plugins/http/src/httpGzip.c index ecda0e1fe0..6a6e995c18 100644 --- a/src/plugins/http/src/httpGzip.c +++ b/src/plugins/http/src/httpGzip.c @@ -132,10 +132,10 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { if (ret != Z_STREAM_END) continue; } - int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk); + int32_t _len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk); gzip->gzip->next_out[0] = '\0'; - gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); + gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, _len); gzip->gzip->next_out = (z_const Bytef *)gzip->chunk; gzip->gzip->avail_out = gzip->conf.chunk_size; } diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index 18cea56cfe..599991da63 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -163,9 +163,9 @@ static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target // parse decode method for (int32_t i = 0; i < tsHttpServer.methodScannerLen; i++) { - HttpDecodeMethod *method = tsHttpServer.methodScanner[i]; - if (strcmp(method->module, pParser->path[0].str) == 0) { - pContext->decodeMethod = method; + HttpDecodeMethod *_method = tsHttpServer.methodScanner[i]; + if (strcmp(_method->module, pParser->path[0].str) == 0) { + pContext->decodeMethod = _method; break; } } diff --git a/src/plugins/http/src/httpTgHandle.c b/src/plugins/http/src/httpTgHandle.c index c1d006ff5a..69ac3e19c5 100644 --- a/src/plugins/http/src/httpTgHandle.c +++ b/src/plugins/http/src/httpTgHandle.c @@ -209,7 +209,7 @@ void tgParseSchemaMetric(cJSON *metric) { parsedOk = false; goto ParseEnd; } - int32_t nameLen = (int32_t)strlen(field->valuestring); + nameLen = (int32_t)strlen(field->valuestring); if (nameLen == 0 || nameLen >= TSDB_TABLE_NAME_LEN) { parsedOk = false; goto ParseEnd; diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 09857610d2..029629eff0 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -576,7 +576,7 @@ static void *taosProcessTcpData(void *param) { } while (pThreadObj->pHead) { - SFdObj *pFdObj = pThreadObj->pHead; + pFdObj = pThreadObj->pHead; pThreadObj->pHead = pFdObj->next; taosReportBrokenLink(pFdObj); } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index e5f2d94c4a..3d8f54ad9b 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -389,17 +389,17 @@ int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype, return code; } -void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force) { +void syncConfirmForward(int64_t rid, uint64_t _version, int32_t code, bool force) { SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; SSyncPeer *pPeer = pNode->pMaster; if (pPeer && (pNode->quorum > 1 || force)) { SFwdRsp rsp; - syncBuildSyncFwdRsp(&rsp, pNode->vgId, version, code); + syncBuildSyncFwdRsp(&rsp, pNode->vgId, _version, code); if (taosWriteMsg(pPeer->peerFd, &rsp, sizeof(SFwdRsp)) == sizeof(SFwdRsp)) { - sTrace("%s, forward-rsp is sent, code:0x%x hver:%" PRIu64, pPeer->id, code, version); + sTrace("%s, forward-rsp is sent, code:0x%x hver:%" PRIu64, pPeer->id, code, _version); } else { sDebug("%s, failed to send forward-rsp, restart", pPeer->id); syncRestartConnection(pPeer); @@ -1302,14 +1302,14 @@ static void syncProcessBrokenLink(int64_t rid) { syncReleasePeer(pPeer); } -static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { +static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t _version, void *mhandle) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; int64_t time = taosGetTimestampMs(); if (pSyncFwds->fwds >= SYNC_MAX_FWDS) { // pSyncFwds->first = (pSyncFwds->first + 1) % SYNC_MAX_FWDS; // pSyncFwds->fwds--; - sError("vgId:%d, failed to save fwd info, hver:%" PRIu64 " fwds:%d", pNode->vgId, version, pSyncFwds->fwds); + sError("vgId:%d, failed to save fwd info, hver:%" PRIu64 " fwds:%d", pNode->vgId, _version, pSyncFwds->fwds); return TSDB_CODE_SYN_TOO_MANY_FWDINFO; } @@ -1319,12 +1319,12 @@ static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->last; memset(pFwdInfo, 0, sizeof(SFwdInfo)); - pFwdInfo->version = version; + pFwdInfo->version = _version; pFwdInfo->mhandle = mhandle; pFwdInfo->time = time; pSyncFwds->fwds++; - sTrace("vgId:%d, fwd info is saved, hver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds); + sTrace("vgId:%d, fwd info is saved, hver:%" PRIu64 " fwds:%d ", pNode->vgId, _version, pSyncFwds->fwds); return 0; } diff --git a/src/sync/src/syncMsg.c b/src/sync/src/syncMsg.c index 3348f1ec33..64d4e72fac 100644 --- a/src/sync/src/syncMsg.c +++ b/src/sync/src/syncMsg.c @@ -61,13 +61,13 @@ void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len) { syncBuildHead(pHead); } -void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code) { +void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t _version, int32_t code) { pMsg->head.type = TAOS_SMSG_SYNC_FWD_RSP; pMsg->head.vgId = vgId; pMsg->head.len = sizeof(SFwdRsp) - sizeof(SSyncHead); syncBuildHead(&pMsg->head); - pMsg->version = version; + pMsg->version = _version; pMsg->code = code; } diff --git a/src/tsdb/inc/tsdbMeta.h b/src/tsdb/inc/tsdbMeta.h index cc916fa689..7484071ce3 100644 --- a/src/tsdb/inc/tsdbMeta.h +++ b/src/tsdb/inc/tsdbMeta.h @@ -69,7 +69,7 @@ void tsdbFreeMeta(STsdbMeta* pMeta); int tsdbOpenMeta(STsdbRepo* pRepo); int tsdbCloseMeta(STsdbRepo* pRepo); STable* tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid); -STSchema* tsdbGetTableSchemaByVersion(STable* pTable, int16_t version); +STSchema* tsdbGetTableSchemaByVersion(STable* pTable, int16_t _version); int tsdbWLockRepoMeta(STsdbRepo* pRepo); int tsdbRLockRepoMeta(STsdbRepo* pRepo); int tsdbUnlockRepoMeta(STsdbRepo* pRepo); @@ -89,16 +89,16 @@ static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *k } } -static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t version) { +static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t _version) { STable* pDTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable; STSchema* pSchema = NULL; STSchema* pTSchema = NULL; if (lock) TSDB_RLOCK_TABLE(pDTable); - if (version < 0) { // get the latest version of schema + if (_version < 0) { // get the latest version of schema pTSchema = pDTable->schema[pDTable->numOfSchemas - 1]; } else { // get the schema with version - void* ptr = taosbsearch(&version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*), + void* ptr = taosbsearch(&_version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*), tsdbCompareSchemaVersion, TD_EQ); if (ptr == NULL) { terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index f6e721e3d3..fd9b5e77e3 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -957,10 +957,10 @@ static int tsdbRestoreMeta(STsdbRepo *pRepo) { regfree(®ex); return -1; } else { - uint32_t version = 0; + uint32_t _version = 0; if (strcmp(bname, "meta") != 0) { - sscanf(bname, "meta-ver%" PRIu32, &version); - pfs->cstatus->meta.version = version; + sscanf(bname, "meta-ver%" PRIu32, &_version); + pfs->cstatus->meta.version = _version; } pfs->cstatus->pmf = &(pfs->cstatus->mf); @@ -1103,10 +1103,10 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { int tvid, tfid; TSDB_FILE_T ttype; uint32_t tversion; - char bname[TSDB_FILENAME_LEN]; + char _bname[TSDB_FILENAME_LEN]; - tfsbasename(pf, bname); - tsdbParseDFilename(bname, &tvid, &tfid, &ttype, &tversion); + tfsbasename(pf, _bname); + tsdbParseDFilename(_bname, &tvid, &tfid, &ttype, &tversion); ASSERT(tvid == REPO_ID(pRepo)); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 5db993e463..50fa393e9f 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -410,7 +410,7 @@ int tsdbUpdateDFileHeader(SDFile *pDFile) { int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) { char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - uint32_t version; + uint32_t _version; ASSERT(TSDB_FILE_OPENED(pDFile)); @@ -428,7 +428,7 @@ int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) { } void *pBuf = buf; - pBuf = taosDecodeFixedU32(pBuf, &version); + pBuf = taosDecodeFixedU32(pBuf, &_version); pBuf = tsdbDecodeDFInfo(pBuf, pInfo); return 0; } @@ -660,12 +660,12 @@ int tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet *pSet) { return 0; } -int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version) { +int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *_version) { char *p = NULL; - *version = 0; + *_version = 0; *ftype = TSDB_FILE_MAX; - sscanf(fname, "v%df%d.%m[a-z]-ver%" PRIu32, vid, fid, &p, version); + sscanf(fname, "v%df%d.%m[a-z]-ver%" PRIu32, vid, fid, &p, _version); for (TSDB_FILE_T i = 0; i < TSDB_FILE_MAX; i++) { if (strcmp(p, TSDB_FNAME_SUFFIX[i]) == 0) { *ftype = i; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 3e6263b9d3..e6cbc4da9e 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -531,8 +531,8 @@ STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid) { return *(STable **)ptr; } -STSchema *tsdbGetTableSchemaByVersion(STable *pTable, int16_t version) { - return tsdbGetTableSchemaImpl(pTable, true, false, version); +STSchema *tsdbGetTableSchemaByVersion(STable *pTable, int16_t _version) { + return tsdbGetTableSchemaImpl(pTable, true, false, _version); } int tsdbWLockRepoMeta(STsdbRepo *pRepo) { @@ -891,9 +891,9 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro maxCols = 0; maxRowBytes = 0; for (int i = 0; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable != NULL) { - pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); + STable *_pTable = pMeta->tables[i]; + if (_pTable != NULL) { + pSchema = tsdbGetTableSchemaImpl(_pTable, false, false, -1); maxCols = MAX(maxCols, schemaNCols(pSchema)); maxRowBytes = MAX(maxRowBytes, schemaTLen(pSchema)); } diff --git a/src/util/src/tdes.c b/src/util/src/tdes.c index 6e003756a3..f72ddcaa3b 100644 --- a/src/util/src/tdes.c +++ b/src/util/src/tdes.c @@ -155,26 +155,26 @@ void generate_key(unsigned char* key) { } } -void print_key_set(key_set key_set) { +void print_key_set(key_set _key_set) { int i; printf("K: \n"); for (i = 0; i < 8; i++) { - printf("%02X : ", key_set.k[i]); - print_char_as_binary(key_set.k[i]); + printf("%02X : ", _key_set.k[i]); + print_char_as_binary(_key_set.k[i]); printf("\n"); } printf("\nC: \n"); for (i = 0; i < 4; i++) { - printf("%02X : ", key_set.c[i]); - print_char_as_binary(key_set.c[i]); + printf("%02X : ", _key_set.c[i]); + print_char_as_binary(_key_set.c[i]); printf("\n"); } printf("\nD: \n"); for (i = 0; i < 4; i++) { - printf("%02X : ", key_set.d[i]); - print_char_as_binary(key_set.d[i]); + printf("%02X : ", _key_set.d[i]); + print_char_as_binary(_key_set.d[i]); printf("\n"); } printf("\n"); diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 4368ddd7d3..0981d7a996 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -199,7 +199,7 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { snprintf(walName, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); wInfo("vgId:%d, file:%s, will be restored", pWal->vgId, walName); - int32_t code = walRestoreWalFile(pWal, pVnode, writeFp, walName, fileId); + code = walRestoreWalFile(pWal, pVnode, writeFp, walName, fileId); if (code != TSDB_CODE_SUCCESS) { wError("vgId:%d, file:%s, failed to restore since %s", pWal->vgId, walName, tstrerror(code)); continue; From e08fe44fdfebe0cfef0e52b062d88734ed6acbc0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 14 May 2021 09:38:04 +0800 Subject: [PATCH 072/157] DB/create: fix create database when default quorum > replica --- src/mnode/src/mnodeDb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8af20aa862..7cc5d41ae1 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -389,7 +389,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->compression < 0) pCfg->compression = tsCompression; if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL; if (pCfg->replications < 0) pCfg->replications = tsReplications; - if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; + if (pCfg->quorum < 0) pCfg->quorum = MIN(tsQuorum, pCfg->replications); if (pCfg->update < 0) pCfg->update = tsUpdate; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow; if (pCfg->dbType < 0) pCfg->dbType = 0; From be6a583034ef9dfe42768ea5d65bc2043f71b198 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 14 May 2021 09:47:01 +0800 Subject: [PATCH 073/157] [TD-4114] add test case for bind batch --- tests/script/api/stmtBatchTest.c | 300 +++++++++++++++++++++++++++++-- 1 file changed, 285 insertions(+), 15 deletions(-) diff --git a/tests/script/api/stmtBatchTest.c b/tests/script/api/stmtBatchTest.c index cb1ca3346b..1f50e579ae 100644 --- a/tests/script/api/stmtBatchTest.c +++ b/tests/script/api/stmtBatchTest.c @@ -1409,20 +1409,14 @@ static void prepareV(TAOS *taos, int schemaCase, int tableNum, int lenOfBina } -static void preparemV(TAOS *taos, int schemaCase, int idx) { +static void prepareV_long(TAOS *taos, int schemaCase, int tableNum, int lenOfBinaryDef) { TAOS_RES *result; int code; - char dbname[32],sql[255]; - sprintf(dbname, "demo%d", idx); - sprintf(sql, "drop database if exists %s", dbname); - - - result = taos_query(taos, sql); + result = taos_query(taos, "drop database if exists demol"); taos_free_result(result); - sprintf(sql, "create database %s", dbname); - result = taos_query(taos, sql); + result = taos_query(taos, "create database demol"); code = taos_errno(result); if (code != 0) { printf("failed to create database, reason:%s\n", taos_errstr(result)); @@ -1431,18 +1425,18 @@ static void preparemV(TAOS *taos, int schemaCase, int idx) { } taos_free_result(result); - sprintf(sql, "use %s", dbname); - result = taos_query(taos, sql); + result = taos_query(taos, "use demol"); taos_free_result(result); // create table - for (int i = 0 ; i < 300; i++) { + for (int i = 0 ; i < tableNum; i++) { char buf[1024]; if (schemaCase) { - sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40), t2 timestamp)", i) ; + sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp)", i, lenOfBinaryDef, lenOfBinaryDef) ; } else { sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; } + result = taos_query(taos, buf); code = taos_errno(result); if (code != 0) { @@ -1985,12 +1979,284 @@ static void* runCase(void *para) { } + +static int stmt_bind_case_001_long(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum, int64_t* startTs) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = *startTs; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v->v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v->v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v->v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v->v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = rowsOfPerColum; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v->f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = rowsOfPerColum; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v->f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = rowsOfPerColum; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+8].buffer = v->br; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = rowsOfPerColum; + + params[i+9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[i+9].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+9].buffer = v->nr; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = rowsOfPerColum; + + params[i+10].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+10].buffer_length = sizeof(int64_t); + params[i+10].buffer = v->ts2; + params[i+10].length = NULL; + params[i+10].is_null = is_null; + params[i+10].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + *startTs = tts + totalRowsPerTbl * tableNum; // return to next + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +//void runcase(TAOS *taos, int idx) { +static void* runCase_long(void *para) { + ThreadInfo* tInfo = (ThreadInfo *)para; + TAOS *taos = tInfo->taos; + int idx = tInfo->idx; + + TAOS_STMT *stmt = NULL; + + (void)idx; + + int tableNum; + int lenOfBinaryDef; + int rowsOfPerColum; + int bingNum; + int lenOfBinaryAct; + int columnNum; + + int totalRowsPerTbl; + +//=======================================================================// + //========== long case 14: ======================// +#if 0 + { + stmt = taos_stmt_init(taos); + + tableNum = 1000; + rowsOfPerColum = 10; + bingNum = 5000000; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 11; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("long case 14 check result end\n\n"); + } +#endif + + + //========== case 15: ======================// +#if 1 + { + printf("====long case 15 test start\n\n"); + + tableNum = 200; + rowsOfPerColum = 110; + bingNum = 100; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 8; + columnNum = 11; + + int64_t startTs = 1591060628000; + prepareV_long(taos, 1, tableNum, lenOfBinaryDef); + + totalRowsPerTbl = 0; + for (int i = 0; i < 30000; i++) { + stmt = taos_stmt_init(taos); + stmt_bind_case_001_long(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum, &startTs); + + totalRowsPerTbl += rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m11", 0, totalRowsPerTbl); + checkResult(taos, "m22", 0, totalRowsPerTbl); + checkResult(taos, "m133", 0, totalRowsPerTbl); + checkResult(taos, "m199", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + } + + printf("====long case 15 check result end\n\n"); + } +#endif + + return NULL; + +} + + int main(int argc, char *argv[]) { TAOS *taos; char host[32] = "127.0.0.1"; char* serverIp = NULL; - int threadNum = 1; + int threadNum = 2; // connect to server if (argc == 1) { @@ -2021,7 +2287,11 @@ int main(int argc, char *argv[]) tInfo->taos = taos; tInfo->idx = i; - pthread_create(&(pThreadList[0]), NULL, runCase, (void *)tInfo); + if (0 == i) { + pthread_create(&(pThreadList[0]), NULL, runCase, (void *)tInfo); + } else if (1 == i){ + pthread_create(&(pThreadList[0]), NULL, runCase_long, (void *)tInfo); + } tInfo++; } From db6c390289bdf2196ef2243896b3fe15f7cd25f5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 14 May 2021 09:52:07 +0800 Subject: [PATCH 074/157] TD-4176 --- src/wal/src/walWrite.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 4368ddd7d3..ca19a39f2b 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -426,8 +426,8 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch #endif offset = offset + sizeof(SWalHead) + pHead->len; - wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, - fileId, pHead->version, pWal->version, pHead->len); + wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d offset:%" PRId64, + pWal->vgId, fileId, pHead->version, pWal->version, pHead->len, offset); pWal->version = pHead->version; (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL); From 1873cc9c5b459d9bc057fc1533e04dd7411fc2ad Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 14 May 2021 09:59:43 +0800 Subject: [PATCH 075/157] [TD-4114] add test case for bind batch --- tests/script/api/makefile | 2 +- tests/script/api/stmtBatchTest.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/script/api/makefile b/tests/script/api/makefile index f3a8dec061..5eeb134288 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -11,7 +11,7 @@ CFLAGS = -O0 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ all: $(TARGET) exe: - gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) + gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) gcc $(CFLAGS) ./stmtBatchTest.c -o $(ROOT)stmtBatchTest $(LFLAGS) clean: diff --git a/tests/script/api/stmtBatchTest.c b/tests/script/api/stmtBatchTest.c index 1f50e579ae..3ce2fefb0d 100644 --- a/tests/script/api/stmtBatchTest.c +++ b/tests/script/api/stmtBatchTest.c @@ -2166,7 +2166,6 @@ static int stmt_bind_case_001_long(TAOS_STMT *stmt, int tableNum, int rowsOfPerC return 0; } -//void runcase(TAOS *taos, int idx) { static void* runCase_long(void *para) { ThreadInfo* tInfo = (ThreadInfo *)para; TAOS *taos = tInfo->taos; From 5d4b05ecd1dbf46254e77d3b4776db45925f8e2a Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Fri, 14 May 2021 10:00:05 +0800 Subject: [PATCH 076/157] Fix/td 4148 (#6122) * [TD-4148]: JDBC-Restful Compatible version before 2.0.18.0 * [TD-4174]: add test case for TD-4174 * change * change * change * change * change --- .../java/com/taosdata/jdbc/TSDBConstants.java | 1 + .../taosdata/jdbc/rs/RestfulResultSet.java | 152 ++++++++++++++---- .../taosdata/jdbc/utils/UtcTimestampUtil.java | 12 -- .../java/com/taosdata/jdbc/utils/Utils.java | 52 ++++++ .../InsertSpecialCharacterRestfulTest.java | 1 - .../com/taosdata/jdbc/cases/TD4174Test.java | 62 +++++++ ...woTypeTimestampPercisionInRestfulTest.java | 6 +- .../jdbc/cases/UnsignedNumberJniTest.java | 92 +++++------ .../jdbc/cases/UnsignedNumberRestfulTest.java | 105 ++++++------ .../jdbc/rs/RestfulPreparedStatementTest.java | 2 - .../jdbc/rs/RestfulResultSetTest.java | 6 +- 11 files changed, 335 insertions(+), 156 deletions(-) delete mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/UtcTimestampUtil.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4174Test.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java index 37073e243f..f38555ce8a 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java @@ -30,6 +30,7 @@ public abstract class TSDBConstants { public static final int JNI_FETCH_END = -6; public static final int JNI_OUT_OF_MEMORY = -7; // TSDB Data Types + public static final int TSDB_DATA_TYPE_NULL = 0; public static final int TSDB_DATA_TYPE_BOOL = 1; public static final int TSDB_DATA_TYPE_TINYINT = 2; public static final int TSDB_DATA_TYPE_SMALLINT = 3; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index db635f5f79..530b433d42 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -6,11 +6,13 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.primitives.Shorts; import com.taosdata.jdbc.*; +import com.taosdata.jdbc.utils.Utils; import java.math.BigDecimal; import java.sql.*; import java.time.Instant; import java.time.ZoneOffset; +import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Calendar; @@ -18,14 +20,13 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { private volatile boolean isClosed; private int pos = -1; - private final String database; private final Statement statement; // data - private final ArrayList> resultSet; + private final ArrayList> resultSet = new ArrayList<>(); // meta - private ArrayList columnNames; - private ArrayList columns; + private ArrayList columnNames = new ArrayList<>(); + private ArrayList columns = new ArrayList<>(); private RestfulResultSetMetaData metaData; /** @@ -37,10 +38,46 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { this.database = database; this.statement = statement; - // column metadata + // get column metadata JSONArray columnMeta = resultJson.getJSONArray("column_meta"); - columnNames = new ArrayList<>(); - columns = new ArrayList<>(); + // get row data + JSONArray data = resultJson.getJSONArray("data"); + if (data == null || data.isEmpty()) { + columnNames.clear(); + columns.clear(); + this.resultSet.clear(); + return; + } + // get head + JSONArray head = resultJson.getJSONArray("head"); + // get rows + Integer rows = resultJson.getInteger("rows"); + // parse column_meta + if (columnMeta != null) { + parseColumnMeta_new(columnMeta); + } else { + parseColumnMeta_old(head, data, rows); + } + this.metaData = new RestfulResultSetMetaData(this.database, columns, this); + // parse row data + resultSet.clear(); + for (int rowIndex = 0; rowIndex < data.size(); rowIndex++) { + ArrayList row = new ArrayList(); + JSONArray jsonRow = data.getJSONArray(rowIndex); + for (int colIndex = 0; colIndex < this.metaData.getColumnCount(); colIndex++) { + row.add(parseColumnData(jsonRow, colIndex, columns.get(colIndex).taos_type)); + } + resultSet.add(row); + } + } + + /*** + * use this method after TDengine-2.0.18.0 to parse column meta, restful add column_meta in resultSet + * @Param columnMeta + */ + private void parseColumnMeta_new(JSONArray columnMeta) throws SQLException { + columnNames.clear(); + columns.clear(); for (int colIndex = 0; colIndex < columnMeta.size(); colIndex++) { JSONArray col = columnMeta.getJSONArray(colIndex); String col_name = col.getString(0); @@ -50,23 +87,55 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { columnNames.add(col_name); columns.add(new Field(col_name, col_type, col_length, "", taos_type)); } - this.metaData = new RestfulResultSetMetaData(this.database, columns, this); + } - // row data - JSONArray data = resultJson.getJSONArray("data"); - resultSet = new ArrayList<>(); - for (int rowIndex = 0; rowIndex < data.size(); rowIndex++) { - ArrayList row = new ArrayList(); - JSONArray jsonRow = data.getJSONArray(rowIndex); - for (int colIndex = 0; colIndex < jsonRow.size(); colIndex++) { - row.add(parseColumnData(jsonRow, colIndex, columns.get(colIndex).taos_type)); + /** + * use this method before TDengine-2.0.18.0 to parse column meta + */ + private void parseColumnMeta_old(JSONArray head, JSONArray data, int rows) { + columnNames.clear(); + columns.clear(); + for (int colIndex = 0; colIndex < head.size(); colIndex++) { + String col_name = head.getString(colIndex); + columnNames.add(col_name); + + int col_type = Types.NULL; + int col_length = 0; + int taos_type = TSDBConstants.TSDB_DATA_TYPE_NULL; + + JSONArray row0Json = data.getJSONArray(0); + if (colIndex < row0Json.size()) { + Object value = row0Json.get(colIndex); + if (value instanceof Boolean) { + col_type = Types.BOOLEAN; + col_length = 1; + taos_type = TSDBConstants.TSDB_DATA_TYPE_BOOL; + } + if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long) { + col_type = Types.BIGINT; + col_length = 8; + taos_type = TSDBConstants.TSDB_DATA_TYPE_BIGINT; + } + if (value instanceof Float || value instanceof Double || value instanceof BigDecimal) { + col_type = Types.DOUBLE; + col_length = 8; + taos_type = TSDBConstants.TSDB_DATA_TYPE_DOUBLE; + } + if (value instanceof String) { + col_type = Types.NCHAR; + col_length = ((String) value).length(); + taos_type = TSDBConstants.TSDB_DATA_TYPE_NCHAR; + } } - resultSet.add(row); + columns.add(new Field(col_name, col_type, col_length, "", taos_type)); } } + private Object parseColumnData(JSONArray row, int colIndex, int taosType) throws SQLException { switch (taosType) { + case TSDBConstants.TSDB_DATA_TYPE_NULL: + return null; case TSDBConstants.TSDB_DATA_TYPE_BOOL: return row.getBoolean(colIndex); case TSDBConstants.TSDB_DATA_TYPE_TINYINT: @@ -290,8 +359,10 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return 0; } wasNull = false; - if (value instanceof Float || value instanceof Double) + if (value instanceof Float) return (float) value; + if (value instanceof Double) + return new Float((Double) value); return Float.parseFloat(value.toString()); } @@ -329,6 +400,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return Shorts.toByteArray((short) value); if (value instanceof Byte) return new byte[]{(byte) value}; + if (value instanceof Timestamp) { + return Utils.formatTimestamp((Timestamp) value).getBytes(); + } return value.toString().getBytes(); } @@ -342,7 +416,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return null; if (value instanceof Timestamp) return new Date(((Timestamp) value).getTime()); - return Date.valueOf(value.toString()); + Date date = null; + date = Utils.parseDate(value.toString()); + return date; } @Override @@ -354,7 +430,13 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return null; if (value instanceof Timestamp) return new Time(((Timestamp) value).getTime()); - return Time.valueOf(value.toString()); + Time time = null; + try { + time = Utils.parseTime(value.toString()); + } catch (DateTimeParseException e) { + time = null; + } + return time; } @Override @@ -366,14 +448,20 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return null; if (value instanceof Timestamp) return (Timestamp) value; -// if (value instanceof Long) { -// if (1_0000_0000_0000_0L > (long) value) -// return Timestamp.from(Instant.ofEpochMilli((long) value)); -// long epochSec = (long) value / 1000_000L; -// long nanoAdjustment = (long) ((long) value % 1000_000L * 1000); -// return Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); -// } - return Timestamp.valueOf(value.toString()); + if (value instanceof Long) { + if (1_0000_0000_0000_0L > (long) value) + return Timestamp.from(Instant.ofEpochMilli((long) value)); + long epochSec = (long) value / 1000_000L; + long nanoAdjustment = (long) value % 1000_000L * 1000; + return Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + } + Timestamp ret; + try { + ret = Utils.parseTimestamp(value.toString()); + } catch (Exception e) { + ret = null; + } + return ret; } @Override @@ -415,7 +503,13 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return new BigDecimal(Double.valueOf(value.toString())); if (value instanceof Timestamp) return new BigDecimal(((Timestamp) value).getTime()); - return new BigDecimal(value.toString()); + BigDecimal ret; + try { + ret = new BigDecimal(value.toString()); + } catch (Exception e) { + ret = null; + } + return ret; } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/UtcTimestampUtil.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/UtcTimestampUtil.java deleted file mode 100644 index 04a11a2beb..0000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/UtcTimestampUtil.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.taosdata.jdbc.utils; - -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; - -public class UtcTimestampUtil { - public static final DateTimeFormatter formatter = new DateTimeFormatterBuilder() - .appendPattern("yyyy-MM-ddTHH:mm:ss.SSS+") -// .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) - .toFormatter(); - -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java index 0ce4bd3dc1..082cec1e24 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java @@ -5,7 +5,15 @@ import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; import java.nio.charset.Charset; +import java.sql.Date; +import java.sql.Time; import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.format.DateTimeParseException; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -17,6 +25,41 @@ public class Utils { private static Pattern ptn = Pattern.compile(".*?'"); + private static final DateTimeFormatter formatter = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd HH:mm:ss.SSS").toFormatter(); + private static final DateTimeFormatter formatter2 = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd HH:mm:ss.SSSSSS").toFormatter(); + + public static Time parseTime(String timestampStr) throws DateTimeParseException { + LocalTime time; + try { + time = LocalTime.parse(timestampStr, formatter); + } catch (DateTimeParseException e) { + time = LocalTime.parse(timestampStr, formatter2); + } + return Time.valueOf(time); + } + + public static Date parseDate(String timestampStr) throws DateTimeParseException { + LocalDate date; + try { + date = LocalDate.parse(timestampStr, formatter); + } catch (DateTimeParseException e) { + date = LocalDate.parse(timestampStr, formatter2); + } + return Date.valueOf(date); + } + + public static Timestamp parseTimestamp(String timeStampStr) { + LocalDateTime dateTime; + try { + dateTime = LocalDateTime.parse(timeStampStr, formatter); + } catch (DateTimeParseException e) { + dateTime = LocalDateTime.parse(timeStampStr, formatter2); + } + return Timestamp.valueOf(dateTime); + } + public static String escapeSingleQuota(String origin) { Matcher m = ptn.matcher(origin); StringBuffer sb = new StringBuffer(); @@ -132,4 +175,13 @@ public class Utils { }).collect(Collectors.joining()); } + + public static String formatTimestamp(Timestamp timestamp) { + int nanos = timestamp.getNanos(); + if (nanos % 1000000l != 0) + return timestamp.toLocalDateTime().format(formatter2); + return timestamp.toLocalDateTime().format(formatter); + } + + } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java index ea0d1aec41..fa6cbd22b5 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java @@ -7,7 +7,6 @@ import java.sql.*; public class InsertSpecialCharacterRestfulTest { private static final String host = "127.0.0.1"; - // private static final String host = "master"; private static Connection conn; private static String dbName = "spec_char_test"; private static String tbname1 = "test"; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4174Test.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4174Test.java new file mode 100644 index 0000000000..2704d4cfa5 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4174Test.java @@ -0,0 +1,62 @@ +package com.taosdata.jdbc.cases; + +import com.alibaba.fastjson.JSONObject; +import com.taosdata.jdbc.TSDBDriver; +import org.junit.*; + +import java.sql.*; +import java.util.Properties; + +public class TD4174Test { + private Connection conn; + private static final String host = "127.0.0.1"; + + @Test + public void test() { + long ts = System.currentTimeMillis(); + try (PreparedStatement pstmt = conn.prepareStatement("insert into weather values(" + ts + ", ?)")) { + JSONObject value = new JSONObject(); + value.put("name", "John Smith"); + value.put("age", 20); + Assert.assertEquals("{\"name\":\"John Smith\",\"age\":20}",value.toJSONString()); + pstmt.setString(1, value.toJSONString()); + + int ret = pstmt.executeUpdate(); + Assert.assertEquals(1, ret); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + public static void main(String[] args) { + JSONObject value = new JSONObject(); + value.put("name", "John Smith"); + value.put("age", 20); + System.out.println(value.toJSONString()); + } + + @Before + public void before() throws SQLException { + String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + conn = DriverManager.getConnection(url, properties); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists td4174"); + stmt.execute("create database if not exists td4174"); + stmt.execute("use td4174"); + stmt.execute("create table weather(ts timestamp, text binary(64))"); + } + } + + @After + public void after() throws SQLException { + if (conn != null) + conn.close(); + + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java index ed4f979ef3..5c83b5a9da 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java @@ -13,6 +13,7 @@ import java.util.Properties; public class TwoTypeTimestampPercisionInRestfulTest { private static final String host = "127.0.0.1"; + private static final String ms_timestamp_db = "ms_precision_test"; private static final String us_timestamp_db = "us_precision_test"; private static final long timestamp1 = System.currentTimeMillis(); @@ -94,7 +95,8 @@ public class TwoTypeTimestampPercisionInRestfulTest { try (Statement stmt = conn3.createStatement()) { ResultSet rs = stmt.executeQuery("select last_row(ts) from " + ms_timestamp_db + ".weather"); rs.next(); - long ts = rs.getTimestamp(1).getTime(); + Timestamp actual = rs.getTimestamp(1); + long ts = actual == null ? 0 : actual.getTime(); Assert.assertEquals(timestamp1, ts); ts = rs.getLong(1); Assert.assertEquals(timestamp1, ts); @@ -110,7 +112,7 @@ public class TwoTypeTimestampPercisionInRestfulTest { rs.next(); Timestamp timestamp = rs.getTimestamp(1); - long ts = timestamp.getTime(); + long ts = timestamp == null ? 0 : timestamp.getTime(); Assert.assertEquals(timestamp1, ts); int nanos = timestamp.getNanos(); Assert.assertEquals(timestamp2 % 1000_000l * 1000, nanos); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java index d1816a3e7c..fb23c0e64a 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java @@ -9,19 +9,19 @@ import java.util.Properties; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class UnsignedNumberJniTest { + private static final String host = "127.0.0.1"; private static Connection conn; + private static long ts; @Test public void testCase001() { try (Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery("select * from us_table"); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); + Assert.assertEquals(ts, rs.getTimestamp(1).getTime()); Assert.assertEquals("127", rs.getString(2)); Assert.assertEquals("32767", rs.getString(3)); Assert.assertEquals("2147483647", rs.getString(4)); @@ -37,13 +37,10 @@ public class UnsignedNumberJniTest { try (Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery("select * from us_table"); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); + while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(ts, rs.getTimestamp(1).getTime()); Assert.assertEquals(127, rs.getByte(2)); Assert.assertEquals(32767, rs.getShort(3)); Assert.assertEquals(2147483647, rs.getInt(4)); @@ -61,16 +58,14 @@ public class UnsignedNumberJniTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 32767,2147483647, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); Assert.assertEquals(127, rs.getByte(2)); Assert.assertEquals(32767, rs.getShort(3)); Assert.assertEquals(2147483647, rs.getInt(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getLong(5); } } } @@ -82,15 +77,15 @@ public class UnsignedNumberJniTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 32767,4294967294, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); + while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); Assert.assertEquals(127, rs.getByte(2)); Assert.assertEquals(32767, rs.getShort(3)); + Assert.assertEquals("4294967294", rs.getString(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getInt(4); } } } @@ -102,15 +97,15 @@ public class UnsignedNumberJniTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 65534,4294967294, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); - while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + assertResultSetMetaData(meta); + while (rs.next()) { + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals("65534", rs.getString(3)); + Assert.assertEquals("4294967294", rs.getString(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getShort(3); } } } @@ -122,37 +117,27 @@ public class UnsignedNumberJniTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 254, 65534,4294967294, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); - while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); - } - } - } + assertResultSetMetaData(meta); - @Test - public void testCase007() throws SQLException { - try (Statement stmt = conn.createStatement()) { - long now = System.currentTimeMillis(); - stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 254, 65534,4294967294, 18446744073709551614)"); - ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); - ResultSetMetaData meta = rs.getMetaData(); while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); Assert.assertEquals("254", rs.getString(2)); Assert.assertEquals("65534", rs.getString(3)); Assert.assertEquals("4294967294", rs.getString(4)); Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getByte(2); } } } + private void assertResultSetMetaData(ResultSetMetaData meta) throws SQLException { + Assert.assertEquals(5, meta.getColumnCount()); + Assert.assertEquals("ts", meta.getColumnLabel(1)); + Assert.assertEquals("f1", meta.getColumnLabel(2)); + Assert.assertEquals("f2", meta.getColumnLabel(3)); + Assert.assertEquals("f3", meta.getColumnLabel(4)); + Assert.assertEquals("f4", meta.getColumnLabel(5)); + } @BeforeClass public static void beforeClass() { @@ -160,20 +145,19 @@ public class UnsignedNumberJniTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + ts = System.currentTimeMillis(); try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; conn = DriverManager.getConnection(url, properties); - Statement stmt = conn.createStatement(); stmt.execute("drop database if exists unsign_jni"); stmt.execute("create database if not exists unsign_jni"); stmt.execute("use unsign_jni"); stmt.execute("create table us_table(ts timestamp, f1 tinyint unsigned, f2 smallint unsigned, f3 int unsigned, f4 bigint unsigned)"); - stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(now, 127, 32767,2147483647, 9223372036854775807)"); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + ts + ", 127, 32767,2147483647, 9223372036854775807)"); stmt.close(); - } catch (ClassNotFoundException | SQLException e) { + } catch (SQLException e) { e.printStackTrace(); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java index 4ae2f36fe9..a659a490cb 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java @@ -13,17 +13,20 @@ public class UnsignedNumberRestfulTest { private static final String host = "127.0.0.1"; private static Connection conn; + private static long ts; @Test public void testCase001() { try (Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery("select * from us_table"); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); + Assert.assertEquals(ts, rs.getTimestamp(1).getTime()); + Assert.assertEquals("127", rs.getString(2)); + Assert.assertEquals("32767", rs.getString(3)); + Assert.assertEquals("2147483647", rs.getString(4)); + Assert.assertEquals("9223372036854775807", rs.getString(5)); } } catch (SQLException e) { e.printStackTrace(); @@ -35,13 +38,14 @@ public class UnsignedNumberRestfulTest { try (Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery("select * from us_table"); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); + while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(ts, rs.getTimestamp(1).getTime()); + Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals(32767, rs.getShort(3)); + Assert.assertEquals(2147483647, rs.getInt(4)); + Assert.assertEquals(9223372036854775807l, rs.getLong(5)); } } catch (SQLException e) { e.printStackTrace(); @@ -55,13 +59,14 @@ public class UnsignedNumberRestfulTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 32767,2147483647, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); + Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals(32767, rs.getShort(3)); + Assert.assertEquals(2147483647, rs.getInt(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getLong(5); } } } @@ -73,13 +78,15 @@ public class UnsignedNumberRestfulTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 32767,4294967294, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); + while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); + Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals(32767, rs.getShort(3)); + Assert.assertEquals("4294967294", rs.getString(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getInt(4); } } } @@ -91,13 +98,15 @@ public class UnsignedNumberRestfulTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 65534,4294967294, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); + while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); + Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals("65534", rs.getString(3)); + Assert.assertEquals("4294967294", rs.getString(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getShort(3); } } } @@ -109,57 +118,47 @@ public class UnsignedNumberRestfulTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 254, 65534,4294967294, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); - while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); - } - } - } + assertResultSetMetaData(meta); - @Test - public void testCase007() throws SQLException { - try (Statement stmt = conn.createStatement()) { - long now = System.currentTimeMillis(); - stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 254, 65534,4294967294, 18446744073709551614)"); - ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); - ResultSetMetaData meta = rs.getMetaData(); while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); Assert.assertEquals("254", rs.getString(2)); Assert.assertEquals("65534", rs.getString(3)); Assert.assertEquals("4294967294", rs.getString(4)); Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getByte(2); } } } + private void assertResultSetMetaData(ResultSetMetaData meta) throws SQLException { + Assert.assertEquals(5, meta.getColumnCount()); + Assert.assertEquals("ts", meta.getColumnLabel(1)); + Assert.assertEquals("f1", meta.getColumnLabel(2)); + Assert.assertEquals("f2", meta.getColumnLabel(3)); + Assert.assertEquals("f3", meta.getColumnLabel(4)); + Assert.assertEquals("f4", meta.getColumnLabel(5)); + } + @BeforeClass public static void beforeClass() { Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + ts = System.currentTimeMillis(); try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; conn = DriverManager.getConnection(url, properties); - Statement stmt = conn.createStatement(); stmt.execute("drop database if exists unsign_restful"); stmt.execute("create database if not exists unsign_restful"); stmt.execute("use unsign_restful"); stmt.execute("create table us_table(ts timestamp, f1 tinyint unsigned, f2 smallint unsigned, f3 int unsigned, f4 bigint unsigned)"); - stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(now, 127, 32767,2147483647, 9223372036854775807)"); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + ts + ", 127, 32767,2147483647, 9223372036854775807)"); stmt.close(); - } catch (ClassNotFoundException | SQLException e) { + } catch (SQLException e) { e.printStackTrace(); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java index e4dd6384f9..ee457ff412 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java @@ -10,7 +10,6 @@ import java.sql.*; public class RestfulPreparedStatementTest { private static final String host = "127.0.0.1"; - // private static final String host = "master"; private static Connection conn; private static final String sql_insert = "insert into t1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static PreparedStatement pstmt_insert; @@ -371,7 +370,6 @@ public class RestfulPreparedStatementTest { pstmt_insert.setSQLXML(1, null); } - @BeforeClass public static void beforeClass() { try { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java index 9bfe9a04ff..81e762c5ca 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java @@ -18,7 +18,6 @@ import java.text.SimpleDateFormat; public class RestfulResultSetTest { private static final String host = "127.0.0.1"; - private static Connection conn; private static Statement stmt; private static ResultSet rs; @@ -95,7 +94,8 @@ public class RestfulResultSetTest { @Test public void getBigDecimal() throws SQLException { BigDecimal f1 = rs.getBigDecimal("f1"); - Assert.assertEquals(1609430400000l, f1.longValue()); + long actual = (f1 == null) ? 0 : f1.longValue(); + Assert.assertEquals(1609430400000l, actual); BigDecimal f2 = rs.getBigDecimal("f2"); Assert.assertEquals(1, f2.intValue()); @@ -119,7 +119,7 @@ public class RestfulResultSetTest { @Test public void getBytes() throws SQLException { byte[] f1 = rs.getBytes("f1"); - Assert.assertEquals("2021-01-01 00:00:00.0", new String(f1)); + Assert.assertEquals("2021-01-01 00:00:00.000", new String(f1)); byte[] f2 = rs.getBytes("f2"); Assert.assertEquals(1, Ints.fromByteArray(f2)); From 4681a7bf7c87d64ebd136c86747aff5072e214b0 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 14 May 2021 02:00:12 +0000 Subject: [PATCH 077/157] add build test in centos --- .drone.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.drone.yml b/.drone.yml index d13aad3153..dd90a0440e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -150,6 +150,29 @@ steps: branch: - develop - master +--- +kind: pipeline +name: build_centos7 +platform: + os: linux + arch: amd64 + +steps: +- name: build + image: ansible/centos7-ansible + commands: + - yum install gcc cmake build-essential -y + - mkdir debug + - cd debug + - cmake .. + - make + trigger: + event: + - pull_request + when: + branch: + - develop + - master --- kind: pipeline From 8c06d5952a70589804562409eef0e2a3171545b5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 14 May 2021 10:07:35 +0800 Subject: [PATCH 078/157] [TD-4081]: fix vnode dropping --- src/vnode/src/vnodeMain.c | 2 -- src/vnode/src/vnodeMgmt.c | 2 ++ src/vnode/src/vnodeRead.c | 8 ++++++-- src/vnode/src/vnodeWrite.c | 2 ++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 8ec66316e3..21d4480a73 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -124,8 +124,6 @@ int32_t vnodeDrop(int32_t vgId) { vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); pVnode->dropped = 1; - // remove from hash, so new messages wont be consumed - vnodeRemoveFromHash(pVnode); vnodeRelease(pVnode); vnodeCleanupInMWorker(pVnode); diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index 32f9532138..62eb4dadcc 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -118,6 +118,8 @@ void vnodeRelease(void *vparam) { tsem_post(&pVnode->sem); } } else { + vnodeRemoveFromHash(pVnode); + vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode); vnodeDestroyInMWorker(pVnode); int32_t count = taosHashGetSize(tsVnodesHash); diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index b60fc5a8cd..2f8da0012b 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -117,14 +117,18 @@ static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t con } int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { + SVnodeObj *pVnode = vparam; + + if (pVnode->dropped) { + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } + SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam); if (pRead == NULL) { assert(terrno != 0); return terrno; } - SVnodeObj *pVnode = vparam; - int32_t code = vnodeCheckRead(pVnode); if (code != TSDB_CODE_SUCCESS) { taosFreeQitem(pRead); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 36516d81df..1b6e2e37ca 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -386,4 +386,6 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { vTrace("vgId:%d, queued wmsg num:%d", pVnode->vgId, pVnode->queuedWMsg); taosMsleep(10); } + + taosMsleep(1000 * 3); } From cb1064c1a714b116793cce87c0d0304a178c6c49 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 14 May 2021 10:10:54 +0800 Subject: [PATCH 079/157] [TD-4179]: add test case --- .../jdbc/TSDBPreparedStatementTest.java | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java index 8804cc5da0..52858e7f88 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -207,10 +207,69 @@ public class TSDBPreparedStatementTest { while(rs.next()) { rows++; } - Assert.assertEquals(numOfRows, rows); + Assert.assertEquals(numOfRows, rows); } } + @Test + public void bindDataSelectColumnTest() throws SQLException { + Statement stmt = conn.createStatement(); + + int numOfRows = 1000; + + for (int loop = 0; loop < 10; loop++){ + stmt.execute("drop table if exists weather_test"); + stmt.execute("create table weather_test(ts timestamp, f1 nchar(4), f2 float, f3 double, f4 timestamp, f5 int, f6 bool, f7 binary(10))"); + + TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? (ts, f1, f7) values(?, ?, ?)"); + Random r = new Random(); + s.setTableName("weather_test"); + + ArrayList ts = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + ts.add(System.currentTimeMillis() + i); + } + s.setTimestamp(0, ts); + + int random = 10 + r.nextInt(5); + ArrayList s2 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + s2.add(null); + }else{ + s2.add("分支" + i % 4); + } + } + s.setNString(1, s2, 4); + + random = 10 + r.nextInt(5); + ArrayList s5 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + s5.add(null); + }else{ + s5.add("test" + i % 10); + } + } + s.setString(2, s5, 10); + + s.columnDataAddBatch(); + s.columnDataExecuteBatch(); + s.columnDataCloseBatch(); + + String sql = "select * from weather_test"; + PreparedStatement statement = conn.prepareStatement(sql); + ResultSet rs = statement.executeQuery(); + int rows = 0; + while(rs.next()) { + rows++; + } + Assert.assertEquals(numOfRows, rows); + } + } + + + @Test public void setBoolean() throws SQLException { pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); From 88107acde7933dd8e08342d4c0e7b0ed843cf039 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Fri, 14 May 2021 10:27:13 +0800 Subject: [PATCH 080/157] [TD-4160]: remove python connector soft links, fix tests and documents --- Jenkinsfile | 2 +- documentation20/cn/08.connector/docs.md | 17 +++++-------- packaging/tools/makeclient_power.sh | 15 +++-------- packaging/tools/makepkg_power.sh | 15 +++-------- src/connector/python/linux/python2 | 1 - .../python3/taos/__init__.py~origin_master | 25 ------------------- src/connector/python/osx/python3 | 1 - src/connector/python/windows/python2 | 1 - src/connector/python/windows/python3 | 1 - ...o-Run-Test-And-How-To-Add-New-Test-Case.md | 4 +-- tests/Jenkinsfile | 2 +- tests/pytest/concurrent_inquiry.sh | 2 +- tests/pytest/crash_gen.sh | 2 +- tests/pytest/hivemq-extension-test.py | 2 +- tests/pytest/perf_gen.sh | 2 +- tests/pytest/simpletest_no_sudo.sh | 2 +- tests/pytest/test.py | 2 +- tests/pytest/test.sh | 2 +- tests/pytest/testCompress.py | 2 +- tests/pytest/testMinTablesPerVnode.py | 2 +- tests/pytest/testNoCompress.py | 2 +- 21 files changed, 26 insertions(+), 78 deletions(-) delete mode 120000 src/connector/python/linux/python2 delete mode 100644 src/connector/python/linux/python3/taos/__init__.py~origin_master delete mode 120000 src/connector/python/osx/python3 delete mode 120000 src/connector/python/windows/python2 delete mode 120000 src/connector/python/windows/python3 diff --git a/Jenkinsfile b/Jenkinsfile index 33ce784bce..b48dca0241 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -94,7 +94,7 @@ def pre_test(){ make > /dev/null make install > /dev/null cd ${WKC}/tests - pip3 install ${WKC}/src/connector/python/linux/python3/ + pip3 install ${WKC}/src/connector/python ''' return 1 } diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index 15c722c180..9edeb78c68 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -400,27 +400,22 @@ Python连接器的使用参见[视频教程](https://www.taosdata.com/blog/2020/ #### Linux -用户可以在源代码的src/connector/python（或者tar.gz的/connector/python）文件夹下找到python2和python3的connector安装包。用户可以通过pip命令安装： +用户可以在源代码的src/connector/python（或者tar.gz的/connector/python）文件夹下找到connector安装包。用户可以通过pip命令安装： -​ `pip install src/connector/python/linux/python2/` +​ `pip install src/connector/python/` 或 -​ `pip3 install src/connector/python/linux/python3/` +​ `pip3 install src/connector/python/` #### Windows 在已安装Windows TDengine 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos cmd 命令行界面 ```cmd -cd C:\TDengine\connector\python\windows -python -m pip install python2\ -``` -或 -```cmd -cd C:\TDengine\connector\python\windows -python -m pip install python3\ +cd C:\TDengine\connector\python +python -m pip install . ``` -* 如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。 +* 如果机器上没有pip命令，用户可将src/connector/python下的taos文件夹拷贝到应用程序的目录使用。 对于windows 客户端，安装TDengine windows 客户端后，将C:\TDengine\driver\taos.dll拷贝到C:\windows\system32目录下即可。 ### 使用 diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index 181536b7f1..23ee300313 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -148,20 +148,11 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${connector_dir}/python ${install_dir}/connector/ cp -r ${connector_dir}/go ${install_dir}/connector - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python2/taos/cinterface.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python3/taos/cinterface.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/windows/python2/taos/cinterface.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/windows/python3/taos/cinterface.py + sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/cinterface.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python2/taos/subscription.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python3/taos/subscription.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/windows/python2/taos/subscription.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/windows/python3/taos/subscription.py + sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/subscription.py - sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python2/taos/connection.py - sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python3/taos/connection.py - sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/windows/python2/taos/connection.py - sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/windows/python3/taos/connection.py + sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/connection.py fi # Copy release note # cp ${script_dir}/release_note ${install_dir} diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index 554e7884b1..359c5dba46 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -170,20 +170,11 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${connector_dir}/python ${install_dir}/connector/ cp -r ${connector_dir}/go ${install_dir}/connector - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python2/taos/cinterface.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python3/taos/cinterface.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/windows/python2/taos/cinterface.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/windows/python3/taos/cinterface.py + sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/cinterface.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python2/taos/subscription.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python3/taos/subscription.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/windows/python2/taos/subscription.py - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/windows/python3/taos/subscription.py + sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/subscription.py - sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python2/taos/connection.py - sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python3/taos/connection.py - sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/windows/python2/taos/connection.py - sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/windows/python3/taos/connection.py + sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/connection.py fi # Copy release note # cp ${script_dir}/release_note ${install_dir} diff --git a/src/connector/python/linux/python2 b/src/connector/python/linux/python2 deleted file mode 120000 index b870225aa0..0000000000 --- a/src/connector/python/linux/python2 +++ /dev/null @@ -1 +0,0 @@ -../ \ No newline at end of file diff --git a/src/connector/python/linux/python3/taos/__init__.py~origin_master b/src/connector/python/linux/python3/taos/__init__.py~origin_master deleted file mode 100644 index 1b086f36ec..0000000000 --- a/src/connector/python/linux/python3/taos/__init__.py~origin_master +++ /dev/null @@ -1,25 +0,0 @@ - -from .connection import TDengineConnection -from .cursor import TDengineCursor -from .error import Error - -# Globals -threadsafety = 0 -paramstyle = 'pyformat' - -__all__ = ['connection', 'cursor'] - - -def connect(*args, **kwargs): - """ Function to return a TDengine connector object - - Current supporting keyword parameters: - @dsn: Data source name as string - @user: Username as string(optional) - @password: Password as string(optional) - @host: Hostname(optional) - @database: Database name(optional) - - @rtype: TDengineConnector - """ - return TDengineConnection(*args, **kwargs) diff --git a/src/connector/python/osx/python3 b/src/connector/python/osx/python3 deleted file mode 120000 index b870225aa0..0000000000 --- a/src/connector/python/osx/python3 +++ /dev/null @@ -1 +0,0 @@ -../ \ No newline at end of file diff --git a/src/connector/python/windows/python2 b/src/connector/python/windows/python2 deleted file mode 120000 index b870225aa0..0000000000 --- a/src/connector/python/windows/python2 +++ /dev/null @@ -1 +0,0 @@ -../ \ No newline at end of file diff --git a/src/connector/python/windows/python3 b/src/connector/python/windows/python3 deleted file mode 120000 index b870225aa0..0000000000 --- a/src/connector/python/windows/python3 +++ /dev/null @@ -1 +0,0 @@ -../ \ No newline at end of file diff --git a/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md b/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md index b476c118a7..6845d091b5 100644 --- a/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md +++ b/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md @@ -8,8 +8,8 @@ 3. mkdir debug; cd debug; cmake ..; make ; sudo make install -4. pip install ../src/connector/python/linux/python2 ; pip3 install - ../src/connector/python/linux/python3 +4. pip install ../src/connector/python ; pip3 install + ../src/connector/python 5. pip install numpy; pip3 install numpy (numpy is required only if you need to run querySort.py) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 93db09ad6c..709b39d077 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -21,7 +21,7 @@ def pre_test(){ cmake .. > /dev/null make > /dev/null make install > /dev/null - pip3 install ${WKC}/src/connector/python/linux/python3/ + pip3 install ${WKC}/src/connector/python ''' return 1 } diff --git a/tests/pytest/concurrent_inquiry.sh b/tests/pytest/concurrent_inquiry.sh index e5918792f4..6ac15fb46f 100755 --- a/tests/pytest/concurrent_inquiry.sh +++ b/tests/pytest/concurrent_inquiry.sh @@ -48,7 +48,7 @@ fi PYTHON_EXEC=python3.8 # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. -export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3:$(pwd) +export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd) # Then let us set up the library path so that our compiled SO file can be loaded by Python export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index 0af09634df..127e13c5be 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -48,7 +48,7 @@ fi PYTHON_EXEC=python3.8 # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. -export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3:$(pwd) +export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd) # Then let us set up the library path so that our compiled SO file can be loaded by Python export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py index 3d0b1ef83f..9d293ea5ed 100644 --- a/tests/pytest/hivemq-extension-test.py +++ b/tests/pytest/hivemq-extension-test.py @@ -10,7 +10,7 @@ # ################################################################### # install pip -# pip install src/connector/python/linux/python2/ +# pip install src/connector/python/ import sys import os import os.path diff --git a/tests/pytest/perf_gen.sh b/tests/pytest/perf_gen.sh index fcedd2d407..d28b5422f8 100755 --- a/tests/pytest/perf_gen.sh +++ b/tests/pytest/perf_gen.sh @@ -48,7 +48,7 @@ fi PYTHON_EXEC=python3.8 # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. -export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3:$(pwd) +export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd) # Then let us set up the library path so that our compiled SO file can be loaded by Python export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR diff --git a/tests/pytest/simpletest_no_sudo.sh b/tests/pytest/simpletest_no_sudo.sh index 61faf3df52..36edfc027f 100755 --- a/tests/pytest/simpletest_no_sudo.sh +++ b/tests/pytest/simpletest_no_sudo.sh @@ -4,7 +4,7 @@ # 2. No files are needed outside the development tree, everything is done in the local source code directory # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. -export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3 +export PYTHONPATH=$(pwd)/../../src/connector/python # Then let us set up the library path so that our compiled SO file can be loaded by Python export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/../../build/build/lib diff --git a/tests/pytest/test.py b/tests/pytest/test.py index c7781f2087..65abd3ef93 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -10,7 +10,7 @@ # ################################################################### # install pip -# pip install src/connector/python/linux/python2/ +# pip install src/connector/python/ # -*- coding: utf-8 -*- import sys diff --git a/tests/pytest/test.sh b/tests/pytest/test.sh index fbb9ba9879..4e74341f70 100755 --- a/tests/pytest/test.sh +++ b/tests/pytest/test.sh @@ -13,7 +13,7 @@ else fi TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1` LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib -export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3 +export PYTHONPATH=$(pwd)/../../src/connector/python export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR if [[ "$1" == *"test.py"* ]]; then diff --git a/tests/pytest/testCompress.py b/tests/pytest/testCompress.py index 0f5d9ef3b1..1ad032f05f 100644 --- a/tests/pytest/testCompress.py +++ b/tests/pytest/testCompress.py @@ -10,7 +10,7 @@ # ################################################################### # install pip -# pip install src/connector/python/linux/python2/ +# pip install src/connector/python/ # -*- coding: utf-8 -*- import sys diff --git a/tests/pytest/testMinTablesPerVnode.py b/tests/pytest/testMinTablesPerVnode.py index 91cea833e7..a111113c07 100644 --- a/tests/pytest/testMinTablesPerVnode.py +++ b/tests/pytest/testMinTablesPerVnode.py @@ -10,7 +10,7 @@ # ################################################################### # install pip -# pip install src/connector/python/linux/python2/ +# pip install src/connector/python/ # -*- coding: utf-8 -*- import sys diff --git a/tests/pytest/testNoCompress.py b/tests/pytest/testNoCompress.py index e3b40b4426..d41055c755 100644 --- a/tests/pytest/testNoCompress.py +++ b/tests/pytest/testNoCompress.py @@ -10,7 +10,7 @@ # ################################################################### # install pip -# pip install src/connector/python/linux/python2/ +# pip install src/connector/python/ # -*- coding: utf-8 -*- import sys From 70213e65261e6ad7290d497383c7580efe93d16a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 14 May 2021 11:55:49 +0800 Subject: [PATCH 081/157] test/sim: fix null to 0 in show dnodes; --- tests/script/unique/dnode/remove2.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index 1d707bc4a3..11f6762129 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -162,7 +162,7 @@ print dnode3 openVnodes $data2_3 if $data2_1 != 1 then goto show4 endi -if $data2_2 != null then +if $data2_2 != 0 then goto show4 endi if $data2_3 != 3 then From cf93ca70d1d79d5d3ddfc78bfa7b50e14c886342 Mon Sep 17 00:00:00 2001 From: ZhongChunHe Date: Fri, 14 May 2021 11:56:37 +0800 Subject: [PATCH 082/157] fix declaration shadows previous local/global errors --- src/connector/odbc/examples/c/main.c | 6 +-- src/connector/odbc/src/base/tsdb_impl.c | 4 +- src/kit/shell/src/shellCheck.c | 24 +++++------ src/kit/shell/src/shellEngine.c | 42 ++++++++++---------- src/kit/shell/src/shellImport.c | 22 +++++----- src/kit/shell/src/shellLinux.c | 2 +- src/kit/taosdemo/taosdemo.c | 4 +- src/kit/taosdump/taosdump.c | 2 +- src/tsdb/src/tsdbCommit.c | 6 +-- tests/comparisonTest/tdengine/tdengineTest.c | 8 ++-- 10 files changed, 60 insertions(+), 60 deletions(-) diff --git a/src/connector/odbc/examples/c/main.c b/src/connector/odbc/examples/c/main.c index e36c75688e..3ad32dbe53 100644 --- a/src/connector/odbc/examples/c/main.c +++ b/src/connector/odbc/examples/c/main.c @@ -18,8 +18,8 @@ #define CHK_TEST(statement) \ do { \ D("testing: %s", #statement); \ - int r = (statement); \ - if (r) { \ + int _r = (statement); \ + if (_r) { \ D("testing failed: %s", #statement); \ return 1; \ } \ @@ -181,7 +181,7 @@ static int do_statement(SQLHSTMT stmt, const char *statement) { r = traverse_cols(stmt, cols); char buf[4096]; while (1) { - SQLRETURN r = SQLFetch(stmt); + r = SQLFetch(stmt); if (r==SQL_NO_DATA) break; CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); for (size_t i=0; itsdb_params = tsdb_params; for (int i=0; ithreadNum, sizeof(ShellThreadObj)); - for (int t = 0; t < args->threadNum; ++t) { + ShellThreadObj *threadObj = (ShellThreadObj *)calloc(_args->threadNum, sizeof(ShellThreadObj)); + for (int t = 0; t < _args->threadNum; ++t) { ShellThreadObj *pThread = threadObj + t; pThread->threadIndex = t; - pThread->totalThreads = args->threadNum; + pThread->totalThreads = _args->threadNum; pThread->taos = con; - pThread->db = args->database; + pThread->db = _args->database; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -167,31 +167,31 @@ static void shellRunCheckThreads(TAOS *con, SShellArguments *args) { } } - for (int t = 0; t < args->threadNum; ++t) { + for (int t = 0; t < _args->threadNum; ++t) { pthread_join(threadObj[t].threadID, NULL); } - for (int t = 0; t < args->threadNum; ++t) { + for (int t = 0; t < _args->threadNum; ++t) { taos_close(threadObj[t].taos); } free(threadObj); } -void shellCheck(TAOS *con, SShellArguments *args) { +void shellCheck(TAOS *con, SShellArguments *_args) { int64_t start = taosGetTimestampMs(); - if (shellUseDb(con, args->database) != 0) { + if (shellUseDb(con, _args->database) != 0) { shellFreeTbnames(); return; } - if (shellShowTables(con, args->database) != 0) { + if (shellShowTables(con, _args->database) != 0) { shellFreeTbnames(); return; } - fprintf(stdout, "total %d tables will be checked by %d threads\n", tbNum, args->threadNum); - shellRunCheckThreads(con, args); + fprintf(stdout, "total %d tables will be checked by %d threads\n", tbNum, _args->threadNum); + shellRunCheckThreads(con, _args); int64_t end = taosGetTimestampMs(); fprintf(stdout, "total %d tables checked, failed:%d, time spent %.2f seconds\n", checkedNum, errorNum, diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 0eb1248fad..d4176fca91 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -56,24 +56,24 @@ extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *aut /* * FUNCTION: Initialize the shell. */ -TAOS *shellInit(SShellArguments *args) { +TAOS *shellInit(SShellArguments *_args) { printf("\n"); printf(CLIENT_VERSION, tsOsName, taos_get_client_info()); fflush(stdout); // set options before initializing - if (args->timezone != NULL) { - taos_options(TSDB_OPTION_TIMEZONE, args->timezone); + if (_args->timezone != NULL) { + taos_options(TSDB_OPTION_TIMEZONE, _args->timezone); } - if (args->is_use_passwd) { - if (args->password == NULL) args->password = getpass("Enter password: "); + if (_args->is_use_passwd) { + if (_args->password == NULL) _args->password = getpass("Enter password: "); } else { - args->password = TSDB_DEFAULT_PASS; + _args->password = TSDB_DEFAULT_PASS; } - if (args->user == NULL) { - args->user = TSDB_DEFAULT_USER; + if (_args->user == NULL) { + _args->user = TSDB_DEFAULT_USER; } if (taos_init()) { @@ -84,10 +84,10 @@ TAOS *shellInit(SShellArguments *args) { // Connect to the database. TAOS *con = NULL; - if (args->auth == NULL) { - con = taos_connect(args->host, args->user, args->password, args->database, args->port); + if (_args->auth == NULL) { + con = taos_connect(_args->host, _args->user, _args->password, _args->database, _args->port); } else { - con = taos_connect_auth(args->host, args->user, args->auth, args->database, args->port); + con = taos_connect_auth(_args->host, _args->user, _args->auth, _args->database, _args->port); } if (con == NULL) { @@ -100,14 +100,14 @@ TAOS *shellInit(SShellArguments *args) { read_history(); // Check if it is temperory run - if (args->commands != NULL || args->file[0] != 0) { - if (args->commands != NULL) { - printf("%s%s\n", PROMPT_HEADER, args->commands); - shellRunCommand(con, args->commands); + if (_args->commands != NULL || _args->file[0] != 0) { + if (_args->commands != NULL) { + printf("%s%s\n", PROMPT_HEADER, _args->commands); + shellRunCommand(con, _args->commands); } - if (args->file[0] != 0) { - source_file(con, args->file); + if (_args->file[0] != 0) { + source_file(con, _args->file); } taos_close(con); @@ -116,14 +116,14 @@ TAOS *shellInit(SShellArguments *args) { } #ifndef WINDOWS - if (args->dir[0] != 0) { - source_dir(con, args); + if (_args->dir[0] != 0) { + source_dir(con, _args); taos_close(con); exit(EXIT_SUCCESS); } - if (args->check != 0) { - shellCheck(con, args); + if (_args->check != 0) { + shellCheck(con, _args); taos_close(con); exit(EXIT_SUCCESS); } diff --git a/src/kit/shell/src/shellImport.c b/src/kit/shell/src/shellImport.c index af61995c61..5de50a3aaf 100644 --- a/src/kit/shell/src/shellImport.c +++ b/src/kit/shell/src/shellImport.c @@ -233,15 +233,15 @@ void* shellImportThreadFp(void *arg) return NULL; } -static void shellRunImportThreads(SShellArguments* args) +static void shellRunImportThreads(SShellArguments* _args) { pthread_attr_t thattr; - ShellThreadObj *threadObj = (ShellThreadObj *)calloc(args->threadNum, sizeof(ShellThreadObj)); - for (int t = 0; t < args->threadNum; ++t) { + ShellThreadObj *threadObj = (ShellThreadObj *)calloc(_args->threadNum, sizeof(ShellThreadObj)); + for (int t = 0; t < _args->threadNum; ++t) { ShellThreadObj *pThread = threadObj + t; pThread->threadIndex = t; - pThread->totalThreads = args->threadNum; - pThread->taos = taos_connect(args->host, args->user, args->password, args->database, tsDnodeShellPort); + pThread->totalThreads = _args->threadNum; + pThread->taos = taos_connect(_args->host, _args->user, _args->password, _args->database, tsDnodeShellPort); if (pThread->taos == NULL) { fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, "null taos"/*taos_errstr(pThread->taos)*/); exit(0); @@ -256,18 +256,18 @@ static void shellRunImportThreads(SShellArguments* args) } } - for (int t = 0; t < args->threadNum; ++t) { + for (int t = 0; t < _args->threadNum; ++t) { pthread_join(threadObj[t].threadID, NULL); } - for (int t = 0; t < args->threadNum; ++t) { + for (int t = 0; t < _args->threadNum; ++t) { taos_close(threadObj[t].taos); } free(threadObj); } -void source_dir(TAOS* con, SShellArguments* args) { - shellGetDirectoryFileList(args->dir); +void source_dir(TAOS* con, SShellArguments* _args) { + shellGetDirectoryFileList(_args->dir); int64_t start = taosGetTimestampMs(); if (shellTablesSQLFile[0] != 0) { @@ -276,7 +276,7 @@ void source_dir(TAOS* con, SShellArguments* args) { fprintf(stdout, "import %s finished, time spent %.2f seconds\n", shellTablesSQLFile, (end - start) / 1000.0); } - shellRunImportThreads(args); + shellRunImportThreads(_args); int64_t end = taosGetTimestampMs(); - fprintf(stdout, "import %s finished, time spent %.2f seconds\n", args->dir, (end - start) / 1000.0); + fprintf(stdout, "import %s finished, time spent %.2f seconds\n", _args->dir, (end - start) / 1000.0); } diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 37050c416c..4eead252fd 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -415,7 +415,7 @@ void set_terminal_mode() { } } -void get_history_path(char *history) { snprintf(history, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), HISTORY_FILE); } +void get_history_path(char *_history) { snprintf(_history, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), HISTORY_FILE); } void clearScreen(int ecmd_pos, int cursor_pos) { struct winsize w; diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 49d89dc597..3afb4d5876 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3884,7 +3884,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); + maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); if (maxSqlLen && maxSqlLen->type == cJSON_Number) { int32_t len = maxSqlLen->valueint; if (len > TSDB_MAX_ALLOWED_SQL_LEN) { @@ -3918,7 +3918,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } */ - cJSON* interlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); + interlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); if (interlaceRows && interlaceRows->type == cJSON_Number) { if (interlaceRows->valueint < 0) { errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n", diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index f80ac069a0..15db83297c 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -1017,7 +1017,7 @@ int taosDumpOut(struct arguments *arguments) { sprintf(command, "use %s", dbInfos[0]->name); result = taos_query(taos, command); - int32_t code = taos_errno(result); + code = taos_errno(result); if (code != 0) { fprintf(stderr, "invalid database %s\n", dbInfos[0]->name); goto _exit_failure; diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 4351cce51c..24f1c11628 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -958,11 +958,11 @@ static int tsdbWriteBlockInfo(SCommitH *pCommih) { } static int tsdbWriteBlockIdx(SCommitH *pCommih) { - SBlockIdx *pBlkIdx; + SBlockIdx *pBlkIdx = NULL; SDFile * pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); size_t nidx = taosArrayGetSize(pCommih->aBlkIdx); - int tlen = 0, size; - int64_t offset; + int tlen = 0, size = 0; + int64_t offset = 0; if (nidx <= 0) { // All data are deleted diff --git a/tests/comparisonTest/tdengine/tdengineTest.c b/tests/comparisonTest/tdengine/tdengineTest.c index 1298aa8323..d1cf3a1f98 100644 --- a/tests/comparisonTest/tdengine/tdengineTest.c +++ b/tests/comparisonTest/tdengine/tdengineTest.c @@ -189,8 +189,8 @@ void writeDataImp(void *param) { counter++; if (counter >= arguments.rowsPerRequest) { - TAOS_RES *result = taos_query(taos, sql); - int32_t code = taos_errno(result); + result = taos_query(taos, sql); + code = taos_errno(result); if (code != 0) { printf("thread:%d error:%d reason:%s\n", pThread->threadId, code, taos_errstr(taos)); } @@ -207,8 +207,8 @@ void writeDataImp(void *param) { } if (counter > 0) { - TAOS_RES *result = taos_query(taos, sql); - int32_t code = taos_errno(result); + result = taos_query(taos, sql); + code = taos_errno(result); if (code != 0) { printf("thread:%d error:%d reason:%s\n", pThread->threadId, code, taos_errstr(taos)); } From 64b462a40e131896e4bce94da9ebce6fce5eba02 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 14 May 2021 11:57:35 +0800 Subject: [PATCH 083/157] refactor --- src/kit/taosdemo/taosdemo.c | 132 +++++++++++++++++++----------------- 1 file changed, 68 insertions(+), 64 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index c83b8349b7..3593fb6193 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -291,7 +291,7 @@ typedef struct SSuperTable_S { typedef struct { char name[TSDB_DB_NAME_LEN + 1]; char create_time[32]; - uint64_t ntables; + int64_t ntables; int32_t vgroups; int16_t replica; int16_t quorum; @@ -421,12 +421,13 @@ typedef struct SThreadInfo_S { char tb_prefix[MAX_TB_NAME_SIZE]; uint64_t start_table_from; uint64_t end_table_to; - uint64_t ntables; + int64_t ntables; uint64_t data_of_rate; int64_t start_time; char* cols; bool use_metric; SSuperTable* superTblInfo; + char *buffer; // sql cmd buffer // for async insert tsem_t lock_sem; @@ -591,7 +592,7 @@ SArguments g_args = { static SDbs g_Dbs; -static int g_totalChildTables = 0; +static uint64_t g_totalChildTables = 0; static SQueryMetaInfo g_queryInfo; static FILE * g_fpOfInsertResult = NULL; @@ -1089,7 +1090,7 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { TAOS_RES *res = NULL; int32_t code = -1; - for (i = 0; i < 5; i++) { + for (i = 0; i < 5 /* retry */; i++) { if (NULL != res) { taos_free_result(res); res = NULL; @@ -1135,7 +1136,6 @@ static void appendResultBufToFile(char *resultBuf, char *resultFile) } } - fprintf(fp, "%s", resultBuf); tmfclose(fp); } @@ -1583,8 +1583,8 @@ static void printfInsertMetaToFile(FILE* fp) { } fprintf(fp, " super table count: %"PRIu64"\n", g_Dbs.db[i].superTblCount); - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - fprintf(fp, " super table[%d]:\n", j); + for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { + fprintf(fp, " super table[%"PRIu64"]:\n", j); fprintf(fp, " stbName: %s\n", g_Dbs.db[i].superTbls[j].sTblName); @@ -1913,7 +1913,7 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { formatTimestamp(dbInfos[count]->create_time, *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX], TSDB_TIME_PRECISION_MILLI); - dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); + dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); @@ -1964,7 +1964,7 @@ static void printfDbInfoForQueryToFile( fprintf(fp, "================ database[%d] ================\n", index); fprintf(fp, "name: %s\n", dbInfos->name); fprintf(fp, "created_time: %s\n", dbInfos->create_time); - fprintf(fp, "ntables: %"PRIu64"\n", dbInfos->ntables); + fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables); fprintf(fp, "vgroups: %d\n", dbInfos->vgroups); fprintf(fp, "replica: %d\n", dbInfos->replica); fprintf(fp, "quorum: %d\n", dbInfos->quorum); @@ -2783,7 +2783,7 @@ static int createDatabasesAndStables() { int validStbCount = 0; - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command); @@ -2795,7 +2795,7 @@ static int createDatabasesAndStables() { &g_Dbs.db[i].superTbls[j]); if (0 != ret) { - errorPrint("create super table %d failed!\n\n", j); + errorPrint("create super table %"PRIu64" failed!\n\n", j); continue; } } @@ -2823,7 +2823,7 @@ static void* createTable(void *sarg) threadInfo *pThreadInfo = (threadInfo *)sarg; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - int64_t lastPrintTime = taosGetTimestampMs(); + uint64_t lastPrintTime = taosGetTimestampMs(); int buff_len; buff_len = BUFFER_SIZE / 8; @@ -2898,7 +2898,7 @@ static void* createTable(void *sarg) return NULL; } - int64_t currentPrintTime = taosGetTimestampMs(); + uint64_t currentPrintTime = taosGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30*1000) { printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n", pThreadInfo->threadID, pThreadInfo->start_table_from, i); @@ -2918,7 +2918,7 @@ static void* createTable(void *sarg) } static int startMultiThreadCreateChildTable( - char* cols, int threads, uint64_t startFrom, uint64_t ntables, + char* cols, int threads, uint64_t startFrom, int64_t ntables, char* db_name, SSuperTable* superTblInfo) { pthread_t *pids = malloc(threads * sizeof(pthread_t)); @@ -2933,16 +2933,16 @@ static int startMultiThreadCreateChildTable( threads = 1; } - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; b = ntables % threads; - for (int64_t i = 0; i < threads; i++) { + for (int i = 0; i < threads; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); @@ -2995,7 +2995,7 @@ static void createChildTables() { if (g_Dbs.use_metric) { if (g_Dbs.db[i].superTblCount > 0) { // with super table - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { continue; @@ -3006,7 +3006,7 @@ static void createChildTables() { int startFrom = 0; g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; - verbosePrint("%s() LN%d: create %d child tables from %d\n", + verbosePrint("%s() LN%d: create %"PRIu64" child tables from %d\n", __func__, __LINE__, g_totalChildTables, startFrom); startMultiThreadCreateChildTable( g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, @@ -4555,7 +4555,7 @@ static void prepareSampleData() { static void postFreeResource() { tmfclose(g_fpOfInsertResult); for (int i = 0; i < g_Dbs.dbCount; i++) { - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) { free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL; @@ -4760,19 +4760,21 @@ static int prepareSampleDataForSTable(SSuperTable *superTblInfo) { return 0; } -static int64_t execInsert(threadInfo *pThreadInfo, char *buffer, uint64_t k) +static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k) { int affectedRows; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID, - __func__, __LINE__, buffer); + __func__, __LINE__, pThreadInfo->buffer); if (superTblInfo) { if (superTblInfo->insertMode == TAOSC_IFACE) { - affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false); + affectedRows = queryDbExec( + pThreadInfo->taos, + pThreadInfo->buffer, INSERT_TYPE, false); } else if (superTblInfo->insertMode == REST_IFACE) { if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port, - buffer, NULL /* not set result file */)) { + pThreadInfo->buffer, NULL /* not set result file */)) { affectedRows = -1; printf("========restful return fail, threadID[%d]\n", pThreadInfo->threadID); @@ -4780,17 +4782,19 @@ static int64_t execInsert(threadInfo *pThreadInfo, char *buffer, uint64_t k) affectedRows = k; } } else if (superTblInfo->insertMode == STMT_IFACE) { - // TODO: add stmt support - errorPrint("%s() LN%d, %s\n", - __func__, __LINE__, "!!! need support stmt here"); - exit(-1); + debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt); + if (0 != taos_stmt_execute(pThreadInfo->stmt)) { + errorPrint("%s() LN%d, failied to execute insert statement\n", + __func__, __LINE__); + exit(-1); + } } else { errorPrint("%s() LN%d: unknown insert mode: %d\n", __func__, __LINE__, superTblInfo->insertMode); affectedRows = 0; } } else { - affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false); + affectedRows = queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, INSERT_TYPE, false); } return affectedRows; @@ -5145,8 +5149,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; - char* buffer = calloc(maxSqlLen, 1); - if (NULL == buffer) { + pThreadInfo->buffer = calloc(maxSqlLen, 1); + if (NULL == pThreadInfo->buffer) { errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", __func__, __LINE__, maxSqlLen, strerror(errno)); return NULL; @@ -5170,7 +5174,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { uint64_t tableSeq = pThreadInfo->start_table_from; - debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRIu64" insertRows=%"PRIu64"\n", + debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, pThreadInfo->ntables, insertRows); @@ -5201,10 +5205,10 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { flagSleep = false; } // generate data - memset(buffer, 0, maxSqlLen); + memset(pThreadInfo->buffer, 0, maxSqlLen); uint64_t remainderBufLen = maxSqlLen; - char *pstr = buffer; + char *pstr = pThreadInfo->buffer; int len = snprintf(pstr, nInsertBufLen + 1, "%s", strInsertInto); pstr += len; @@ -5217,7 +5221,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (0 == strlen(tableName)) { errorPrint("[%d] %s() LN%d, getTableName return null\n", pThreadInfo->threadID, __func__, __LINE__); - free(buffer); + free(pThreadInfo->buffer); return NULL; } @@ -5283,7 +5287,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__, recOfBatch, pThreadInfo->totalInsertRows); verbosePrint("[%d] %s() LN%d, buffer=%s\n", - pThreadInfo->threadID, __func__, __LINE__, buffer); + pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); startTs = taosGetTimestampMs(); @@ -5294,7 +5298,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n"); goto free_of_interlace; } - int64_t affectedRows = execInsert(pThreadInfo, buffer, recOfBatch); + int64_t affectedRows = execInsert(pThreadInfo, recOfBatch); endTs = taosGetTimestampMs(); uint64_t delay = endTs - startTs; @@ -5312,7 +5316,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (recOfBatch != affectedRows) { errorPrint("[%d] %s() LN%d execInsert insert %"PRIu64", affected rows: %"PRId64"\n%s\n", pThreadInfo->threadID, __func__, __LINE__, - recOfBatch, affectedRows, buffer); + recOfBatch, affectedRows, pThreadInfo->buffer); goto free_of_interlace; } @@ -5341,7 +5345,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } free_of_interlace: - tmfree(buffer); + tmfree(pThreadInfo->buffer); printStatPerThread(pThreadInfo); return NULL; } @@ -5360,8 +5364,8 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { SSuperTable* superTblInfo = pThreadInfo->superTblInfo; uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; - char* buffer = calloc(maxSqlLen, 1); - if (NULL == buffer) { + pThreadInfo->buffer = calloc(maxSqlLen, 1); + if (NULL == pThreadInfo->buffer) { errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n", maxSqlLen, strerror(errno)); @@ -5407,7 +5411,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->threadID, tableSeq, tableName); int64_t remainderBufLen = maxSqlLen; - char *pstr = buffer; + char *pstr = pThreadInfo->buffer; int nInsertBufLen = strlen("insert into "); int len = snprintf(pstr, nInsertBufLen + 1, "%s", "insert into "); @@ -5430,7 +5434,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { startTs = taosGetTimestampMs(); - int64_t affectedRows = execInsert(pThreadInfo, buffer, generated); + int64_t affectedRows = execInsert(pThreadInfo, generated); endTs = taosGetTimestampMs(); uint64_t delay = endTs - startTs; @@ -5489,7 +5493,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { } // tableSeq free_of_progressive: - tmfree(buffer); + tmfree(pThreadInfo->buffer); printStatPerThread(pThreadInfo); return NULL; } @@ -5697,7 +5701,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(-1); } - uint64_t ntables = 0; + int64_t ntables = 0; uint64_t startFrom; if (superTblInfo) { @@ -5768,13 +5772,13 @@ static void startMultiThreadInsertData(int threads, char* db_name, taos_close(taos0); - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; if (threads != 0) { b = ntables % threads; } @@ -5967,15 +5971,15 @@ static void *readTable(void *sarg) { num_of_DPT = g_args.num_of_DPT; // } - int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; - int totalData = num_of_DPT * num_of_tables; + int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; if (!do_aggreFunc) { printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); } - printf("%d records:\n", totalData); + printf("%"PRId64" records:\n", totalData); fprintf(fp, "| QFunctions | QRecords | QSpeed(R/s) | QLatency(ms) |\n"); for (uint64_t j = 0; j < n; j++) { @@ -6007,7 +6011,7 @@ static void *readTable(void *sarg) { taos_free_result(pSql); } - fprintf(fp, "|%10s | %10d | %12.2f | %10.2f |\n", + fprintf(fp, "|%10s | %"PRIu64" | %12.2f | %10.2f |\n", aggreFunc[j][0] == '*' ? " * " : aggreFunc[j], totalData, (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000); printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000); @@ -6030,16 +6034,16 @@ static void *readMetric(void *sarg) { } int num_of_DPT = rinfo->superTblInfo->insertRows; - int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; - int totalData = num_of_DPT * num_of_tables; + int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; if (!do_aggreFunc) { printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); } - printf("%d records:\n", totalData); - fprintf(fp, "Querying On %d records:\n", totalData); + printf("%"PRId64" records:\n", totalData); + fprintf(fp, "Querying On %"PRId64" records:\n", totalData); for (int j = 0; j < n; j++) { char condition[COND_BUF_LEN] = "\0"; @@ -6137,11 +6141,11 @@ static int insertTestProcess() { end = taosGetTimestampMs(); if (g_totalChildTables > 0) { - fprintf(stderr, "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + fprintf(stderr, "Spent %.4f seconds to create %"PRIu64" tables with %d thread(s)\n\n", (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); if (g_fpOfInsertResult) { fprintf(g_fpOfInsertResult, - "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + "Spent %.4f seconds to create %"PRIu64" tables with %d thread(s)\n\n", (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); } } @@ -6152,7 +6156,7 @@ static int insertTestProcess() { for (int i = 0; i < g_Dbs.dbCount; i++) { if (g_Dbs.use_metric) { if (g_Dbs.db[i].superTblCount > 0) { - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; @@ -6467,16 +6471,16 @@ static int queryTestProcess() { ERROR_EXIT("memory allocation failed for create threads\n"); } - uint64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; int threads = g_queryInfo.superQueryInfo.threadCnt; - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; if (threads != 0) { b = ntables % threads; } @@ -6815,16 +6819,16 @@ static int subscribeTestProcess() { exit(-1); } - uint64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; int threads = g_queryInfo.superQueryInfo.threadCnt; - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; if (threads != 0) { b = ntables % threads; } From 6a7a0d3254e872e421906738effe8c0e42a26f2c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 14 May 2021 12:20:11 +0800 Subject: [PATCH 084/157] improve more. --- src/kit/taosdemo/taosdemo.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3593fb6193..5a5b205d52 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3003,10 +3003,10 @@ static void createChildTables() { verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); - int startFrom = 0; + uint64_t startFrom = 0; g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; - verbosePrint("%s() LN%d: create %"PRIu64" child tables from %d\n", + verbosePrint("%s() LN%d: create %"PRIu64" child tables from %"PRIu64"\n", __func__, __LINE__, g_totalChildTables, startFrom); startMultiThreadCreateChildTable( g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, @@ -5335,8 +5335,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { et = taosGetTimestampMs(); if (insert_interval > (et - st) ) { - int sleepTime = insert_interval - (et -st); - performancePrint("%s() LN%d sleep: %d ms for insert interval\n", + uint64_t sleepTime = insert_interval - (et -st); + performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n", __func__, __LINE__, sleepTime); taosMsleep(sleepTime); // ms sleepTimeTotal += insert_interval; @@ -5963,7 +5963,7 @@ static void *readTable(void *sarg) { return NULL; } - int num_of_DPT; + uint64_t num_of_DPT; /* if (rinfo->superTblInfo) { num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; } else { @@ -6033,7 +6033,7 @@ static void *readMetric(void *sarg) { return NULL; } - int num_of_DPT = rinfo->superTblInfo->insertRows; + uint64_t num_of_DPT = rinfo->superTblInfo->insertRows; int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; From 27ac73a41bdc806f48f9da81f8f1d6e94034940f Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Fri, 14 May 2021 12:47:27 +0800 Subject: [PATCH 085/157] : fix function name typo --- src/client/src/tscSubquery.c | 2 +- src/common/inc/texpr.h | 2 +- src/connector/go | 2 +- src/connector/grafanaplugin | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 48c1880ebc..f759cf8eb4 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2861,7 +2861,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR tscDebug("0x%"PRIx64" sub:%p retrieve numOfRows:%d totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", pParentSql->self, pSql, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); - if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0) && !(tscGetQueryInfoDetail(&pParentSql->cmd, 0)->distinctTag)) { + if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0) && !(tscGetQueryInfo(&pParentSql->cmd, 0)->distinctTag)) { tscError("0x%"PRIx64" sub:0x%"PRIx64" num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, pParentSql->self, pSql->self, tsMaxNumOfOrderedResults, num); tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_SORTED_RES_TOO_MANY); diff --git a/src/common/inc/texpr.h b/src/common/inc/texpr.h index 3376a7c251..275dd12fd7 100644 --- a/src/common/inc/texpr.h +++ b/src/common/inc/texpr.h @@ -89,7 +89,7 @@ tExprNode* exprdup(tExprNode* pTree); void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); -bool exprTreeApplayFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param); +bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param); void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, const char*, int32_t)); diff --git a/src/connector/go b/src/connector/go index 050667e5b4..7a26c432f8 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f +Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5 diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index 3530c6df09..32e2c97a4c 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit 3530c6df097134a410bacec6b3cd013ef38a61aa +Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df From a07f6137bfeec37b24a179fa016457b027b472c8 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 14 May 2021 05:15:34 +0000 Subject: [PATCH 086/157] fix error --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index dd90a0440e..f7ee4e976f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -161,7 +161,7 @@ steps: - name: build image: ansible/centos7-ansible commands: - - yum install gcc cmake build-essential -y + - yum install -y gcc gcc-c++ make cmake - mkdir debug - cd debug - cmake .. From d33e0f597f1c42522e005e6e628e93bd6b793d18 Mon Sep 17 00:00:00 2001 From: ZhongChunHe Date: Fri, 14 May 2021 14:01:45 +0800 Subject: [PATCH 087/157] limit compile-instruction to linux platform --- CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a3531908c..6f50aca079 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,10 +42,12 @@ INCLUDE(cmake/env.inc) INCLUDE(cmake/version.inc) INCLUDE(cmake/install.inc) -#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pipe -Wall -Wshadow -Wextra -Werror") -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pipe -Wall -Wshadow -Wextra -Werror") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pipe -Wall -Wshadow -Werror") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pipe -Wall -Wshadow -Werror") +IF (CMAKE_SYSTEM_NAME MATCHES "Linux") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pipe -Wall -Wshadow -Werror") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pipe -Wall -Wshadow -Werror") +ENDIF () +MESSAGE(STATUS "CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") +MESSAGE(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}") ADD_SUBDIRECTORY(deps) ADD_SUBDIRECTORY(src) From c091f2733c5ceff98613e58bdd46aea632be6a9d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 14 May 2021 14:48:14 +0800 Subject: [PATCH 088/157] fix mac clang compile error. --- src/kit/taosdemo/taosdemo.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 5a5b205d52..44474b9e20 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4788,6 +4788,8 @@ static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k) __func__, __LINE__); exit(-1); } + + affectedRows = k; } else { errorPrint("%s() LN%d: unknown insert mode: %d\n", __func__, __LINE__, superTblInfo->insertMode); From 226f86c8bf4d155f1ecc06ba196b5af95e3eb9e7 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 14 May 2021 15:09:49 +0800 Subject: [PATCH 089/157] [TD-4081]: [v3] fix vnode closing --- src/vnode/src/vnodeMain.c | 6 ++++++ src/vnode/src/vnodeMgmt.c | 2 -- src/vnode/src/vnodeStatus.c | 3 +++ src/vnode/src/vnodeSync.c | 11 +++++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 21d4480a73..58baefefd2 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -120,6 +120,10 @@ int32_t vnodeDrop(int32_t vgId) { vDebug("vgId:%d, failed to drop, vnode not find", vgId); return TSDB_CODE_VND_INVALID_VGROUP_ID; } + if (pVnode->dropped) { + vnodeRelease(pVnode); + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); pVnode->dropped = 1; @@ -473,6 +477,8 @@ void vnodeCleanUp(SVnodeObj *pVnode) { vnodeSetClosingStatus(pVnode); + vnodeRemoveFromHash(pVnode); + // stop replication module if (pVnode->sync > 0) { int64_t sync = pVnode->sync; diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index 62eb4dadcc..32f9532138 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -118,8 +118,6 @@ void vnodeRelease(void *vparam) { tsem_post(&pVnode->sem); } } else { - vnodeRemoveFromHash(pVnode); - vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode); vnodeDestroyInMWorker(pVnode); int32_t count = taosHashGetSize(tsVnodesHash); diff --git a/src/vnode/src/vnodeStatus.c b/src/vnode/src/vnodeStatus.c index c482d1fd1a..1eaddc3d25 100644 --- a/src/vnode/src/vnodeStatus.c +++ b/src/vnode/src/vnodeStatus.c @@ -66,6 +66,9 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) { } bool vnodeSetClosingStatus(SVnodeObj* pVnode) { + if (pVnode->status == TAOS_VN_STATUS_CLOSING) + return true; + while (!vnodeSetClosingStatusImp(pVnode)) { taosMsleep(1); } diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index 05af34a34f..e5a1964915 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -55,6 +55,11 @@ void vnodeNotifyRole(int32_t vgId, int8_t role) { vTrace("vgId:%d, vnode not found while notify role", vgId); return; } + if (pVnode->dropped) { + vTrace("vgId:%d, vnode dropped while notify role", vgId); + vnodeRelease(pVnode); + return; + } vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); pVnode->role = role; @@ -75,6 +80,11 @@ void vnodeCtrlFlow(int32_t vgId, int32_t level) { vTrace("vgId:%d, vnode not found while flow ctrl", vgId); return; } + if (pVnode->dropped) { + vTrace("vgId:%d, vnode dropped while flow ctrl", vgId); + vnodeRelease(pVnode); + return; + } if (pVnode->flowctrlLevel != level) { vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level); @@ -129,6 +139,7 @@ int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rpara SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while write to cache", vgId); + vnodeRelease(pVnode); return TSDB_CODE_VND_INVALID_VGROUP_ID; } From 39f54b7cb8153a7b15a8ab845404e48113a90370 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 14 May 2021 15:17:48 +0800 Subject: [PATCH 090/157] [TD-4092] : describe "SHOW CREATE" series of commands. --- documentation20/cn/12.taos-sql/docs.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 112ad99391..fbb82ee140 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -135,6 +135,14 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM SHOW DATABASES; ``` +- **显示一个数据库的创建语句** + + ```mysql + SHOW CREATE DATABASE db_name; + ``` + 常用于数据库迁移。对一个已经存在的数据库，返回其创建语句；在另一个集群中执行该语句，就能得到一个设置完全相同的 Database。 + + ## 表管理 - **创建数据表** @@ -200,6 +208,13 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM 通配符匹配：1）’%’ (百分号)匹配0到任意个字符；2）’\_’下划线匹配一个字符。 +- **显示一个数据表的创建语句** + + ```mysql + SHOW CREATE TABLE tb_name; + ``` + 常用于数据库迁移。对一个已经存在的数据表，返回其创建语句；在另一个集群中执行该语句，就能得到一个结构完全相同的数据表。 + - **在线修改显示字符宽度** ```mysql @@ -265,6 +280,13 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM ``` 查看数据库内全部 STable，及其相关信息，包括 STable 的名称、创建时间、列数量、标签（TAG）数量、通过该 STable 建表的数量。 +- **显示一个超级表的创建语句** + + ```mysql + SHOW CREATE STABLE stb_name; + ``` + 常用于数据库迁移。对一个已经存在的超级表，返回其创建语句；在另一个集群中执行该语句，就能得到一个结构完全相同的超级表。 + - **获取超级表的结构信息** ```mysql From 61da9bbe84ec017a8c2768e06add74713d02594c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 14 May 2021 15:18:05 +0800 Subject: [PATCH 091/157] TD-4100 --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 2ec508f050..75312a0b03 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1471,7 +1471,7 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { * only the compressed size is less than the value of contLen - overhead, the compression is applied * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message */ - if (compLen < contLen - overhead) { + if (compLen > 0 && compLen < contLen - overhead) { SRpcComp *pComp = (SRpcComp *)pCont; pComp->reserved = 0; pComp->contLen = htonl(contLen); From 1d24ab6e0a6548c9bcc27ba7e584b36a26f6611e Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 14 May 2021 16:39:20 +0800 Subject: [PATCH 092/157] [TD-4202] : fix Grafana plugin installation directory. --- documentation20/cn/09.connections/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/09.connections/docs.md b/documentation20/cn/09.connections/docs.md index 79380f3bbd..6a2ead3766 100644 --- a/documentation20/cn/09.connections/docs.md +++ b/documentation20/cn/09.connections/docs.md @@ -16,7 +16,7 @@ TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin 以CentOS 7.2操作系统为例，将grafanaplugin目录拷贝到/var/lib/grafana/plugins目录下，重新启动grafana即可。 ```bash -sudo cp -rf /usr/local/taos/connector/grafanaplugin /var/lib/grafana/tdengine +sudo cp -rf /usr/local/taos/connector/grafanaplugin /var/lib/grafana/plugins/tdengine ``` ### 使用 Grafana From cf902d175023733ab02cbe2d3668cdac446fc378 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Sat, 15 May 2021 10:31:30 +0800 Subject: [PATCH 093/157] [TD-4205]: fix drone ci error --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index e7ae6ebbda..eed8b6306d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -14,7 +14,7 @@ steps: - apt-get install -y cmake build-essential gcc - pip3 install psutil - pip3 install guppy3 - - pip3 install src/connector/python/linux/python3/ + - pip3 install src/connector/python/ - mkdir debug - cd debug - cmake .. @@ -31,7 +31,7 @@ steps: image: python:3.8 commands: - pip3 install requests - - pip3 install src/connector/python/linux/python3/ + - pip3 install src/connector/python/ - pip3 install psutil - pip3 install guppy3 - cd tests/pytest From c933f78964b03598f7047091ccdea67100868367 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Sat, 15 May 2021 02:50:36 +0000 Subject: [PATCH 094/157] [TD-4167]add smoke test on centos7 in CI --- .drone.yml | 70 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/.drone.yml b/.drone.yml index e7ae6ebbda..f7ee4e976f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7,41 +7,22 @@ platform: arch: amd64 steps: -- name: smoke_test - image: python:3.8 +- name: build + image: gcc commands: - apt-get update - - apt-get install -y cmake build-essential gcc - - pip3 install psutil - - pip3 install guppy3 - - pip3 install src/connector/python/linux/python3/ + - apt-get install -y cmake build-essential - mkdir debug - cd debug - cmake .. - make - - cd ../tests - - ./test-all.sh smoke + trigger: + event: + - pull_request when: branch: - develop - master - - -- name: crash_gen - image: python:3.8 - commands: - - pip3 install requests - - pip3 install src/connector/python/linux/python3/ - - pip3 install psutil - - pip3 install guppy3 - - cd tests/pytest - - ./crash_gen.sh -a -p -t 4 -s 2000 - when: - branch: - - develop - - master - - --- kind: pipeline name: test_arm64 @@ -60,6 +41,9 @@ steps: - cd debug - cmake .. -DCPUTYPE=aarch64 > /dev/null - make + trigger: + event: + - pull_request when: branch: - develop @@ -82,6 +66,9 @@ steps: - cd debug - cmake .. -DCPUTYPE=aarch32 > /dev/null - make + trigger: + event: + - pull_request when: branch: - develop @@ -106,11 +93,13 @@ steps: - cd debug - cmake .. - make + trigger: + event: + - pull_request when: branch: - develop - master - --- kind: pipeline name: build_xenial @@ -129,6 +118,9 @@ steps: - cd debug - cmake .. - make + trigger: + event: + - pull_request when: branch: - develop @@ -151,6 +143,32 @@ steps: - cd debug - cmake .. - make + trigger: + event: + - pull_request + when: + branch: + - develop + - master +--- +kind: pipeline +name: build_centos7 +platform: + os: linux + arch: amd64 + +steps: +- name: build + image: ansible/centos7-ansible + commands: + - yum install -y gcc gcc-c++ make cmake + - mkdir debug + - cd debug + - cmake .. + - make + trigger: + event: + - pull_request when: branch: - develop From 4fd37ea2a1beae14ebfc636fdcdf53171d3181bc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 12:43:08 +0800 Subject: [PATCH 095/157] [TD-4081]: vnode not close if dropped --- src/vnode/src/vnodeMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 58baefefd2..f02a5bedb0 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -392,6 +392,10 @@ int32_t vnodeOpen(int32_t vgId) { int32_t vnodeClose(int32_t vgId) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) return 0; + if (pVnode->dropped) { + vnodeRelease(pVnode); + return 0; + } vDebug("vgId:%d, vnode will be closed, pVnode:%p", pVnode->vgId, pVnode); vnodeRemoveFromHash(pVnode); From 198485c6f43667a8d14b4dfe7dfeeaf93d12307d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 15:32:31 +0800 Subject: [PATCH 096/157] Revert "test/sim: fix null to 0 in show dnodes;" This reverts commit 70213e65261e6ad7290d497383c7580efe93d16a. --- tests/script/unique/dnode/remove2.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index 11f6762129..1d707bc4a3 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -162,7 +162,7 @@ print dnode3 openVnodes $data2_3 if $data2_1 != 1 then goto show4 endi -if $data2_2 != 0 then +if $data2_2 != null then goto show4 endi if $data2_3 != 3 then From 7c93c4217a84a951dde190053abfa4364821b15f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 15:33:19 +0800 Subject: [PATCH 097/157] vnode/write: make last write msg to be written or confirmed --- src/vnode/src/vnodeWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 1b6e2e37ca..3d16ae1faf 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -387,5 +387,5 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { taosMsleep(10); } - taosMsleep(1000 * 3); + taosMsleep(1000); } From cd931e106a021221fed0f22a6575337dffb7c76a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 15 May 2021 16:29:09 +0800 Subject: [PATCH 098/157] Hotfix/sangshuduo/td 3985 taosdemo timestamp step overflow (#6131) * [TD-3985]: taosdemo timestamp step overflow. * fix specified subscribe test. * replace potential overflow of int32 to int64 * replace potential int32 overflow variables to int64. * replace potential int32 overflow variables to int64. * change max of int16 to int64 after type changed. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 70 ++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 72044753d8..7935546719 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -283,7 +283,7 @@ typedef struct SSuperTable_S { typedef struct { char name[TSDB_DB_NAME_LEN + 1]; char create_time[32]; - int32_t ntables; + int64_t ntables; int32_t vgroups; int16_t replica; int16_t quorum; @@ -412,7 +412,7 @@ typedef struct SThreadInfo_S { char tb_prefix[MAX_TB_NAME_SIZE]; uint64_t start_table_from; uint64_t end_table_to; - uint64_t ntables; + int64_t ntables; uint64_t data_of_rate; int64_t start_time; char* cols; @@ -581,7 +581,7 @@ SArguments g_args = { static SDbs g_Dbs; -static int g_totalChildTables = 0; +static uint64_t g_totalChildTables = 0; static SQueryMetaInfo g_queryInfo; static FILE * g_fpOfInsertResult = NULL; @@ -1879,7 +1879,7 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { formatTimestamp(dbInfos[count]->create_time, *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX], TSDB_TIME_PRECISION_MILLI); - dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); + dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); @@ -1930,7 +1930,7 @@ static void printfDbInfoForQueryToFile( fprintf(fp, "================ database[%d] ================\n", index); fprintf(fp, "name: %s\n", dbInfos->name); fprintf(fp, "created_time: %s\n", dbInfos->create_time); - fprintf(fp, "ntables: %d\n", dbInfos->ntables); + fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables); fprintf(fp, "vgroups: %d\n", dbInfos->vgroups); fprintf(fp, "replica: %d\n", dbInfos->replica); fprintf(fp, "quorum: %d\n", dbInfos->quorum); @@ -2884,7 +2884,7 @@ static void* createTable(void *sarg) } static int startMultiThreadCreateChildTable( - char* cols, int threads, uint64_t startFrom, uint64_t ntables, + char* cols, int threads, uint64_t startFrom, int64_t ntables, char* db_name, SSuperTable* superTblInfo) { pthread_t *pids = malloc(threads * sizeof(pthread_t)); @@ -2899,13 +2899,13 @@ static int startMultiThreadCreateChildTable( threads = 1; } - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; b = ntables % threads; for (int64_t i = 0; i < threads; i++) { @@ -2969,10 +2969,10 @@ static void createChildTables() { verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); - int startFrom = 0; + uint64_t startFrom = 0; g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; - verbosePrint("%s() LN%d: create %d child tables from %d\n", + verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n", __func__, __LINE__, g_totalChildTables, startFrom); startMultiThreadCreateChildTable( g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, @@ -4756,7 +4756,7 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t table (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN); } else { - verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRIu64" seq=%"PRIu64"\n", + verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, pThreadInfo->ntables, tableSeq); @@ -5123,14 +5123,12 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { uint64_t tableSeq = pThreadInfo->start_table_from; - debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRIu64" insertRows=%"PRIu64"\n", + debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, pThreadInfo->ntables, insertRows); int64_t startTime = pThreadInfo->start_time; - assert(pThreadInfo->ntables > 0); - uint64_t batchPerTbl = interlaceRows; uint64_t batchPerTblTimes; @@ -5659,8 +5657,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(-1); } - int ntables = 0; - int startFrom; + int64_t ntables = 0; + int64_t startFrom; if (superTblInfo) { int64_t limit; @@ -5730,13 +5728,13 @@ static void startMultiThreadInsertData(int threads, char* db_name, taos_close(taos); - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; if (threads != 0) { b = ntables % threads; } @@ -5892,7 +5890,7 @@ static void *readTable(void *sarg) { return NULL; } - int num_of_DPT; + uint64_t num_of_DPT; /* if (rinfo->superTblInfo) { num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; } else { @@ -5900,15 +5898,15 @@ static void *readTable(void *sarg) { num_of_DPT = g_args.num_of_DPT; // } - int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; - int totalData = num_of_DPT * num_of_tables; + int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; if (!do_aggreFunc) { printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); } - printf("%d records:\n", totalData); + printf("%"PRId64" records:\n", totalData); fprintf(fp, "| QFunctions | QRecords | QSpeed(R/s) | QLatency(ms) |\n"); for (uint64_t j = 0; j < n; j++) { @@ -5940,7 +5938,7 @@ static void *readTable(void *sarg) { taos_free_result(pSql); } - fprintf(fp, "|%10s | %10d | %12.2f | %10.2f |\n", + fprintf(fp, "|%10s | %"PRId64" | %12.2f | %10.2f |\n", aggreFunc[j][0] == '*' ? " * " : aggreFunc[j], totalData, (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000); printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000); @@ -5962,17 +5960,17 @@ static void *readMetric(void *sarg) { return NULL; } - int num_of_DPT = rinfo->superTblInfo->insertRows; - int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; - int totalData = num_of_DPT * num_of_tables; + int64_t num_of_DPT = rinfo->superTblInfo->insertRows; + int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; if (!do_aggreFunc) { printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); } - printf("%d records:\n", totalData); - fprintf(fp, "Querying On %d records:\n", totalData); + printf("%"PRId64" records:\n", totalData); + fprintf(fp, "Querying On %"PRId64" records:\n", totalData); for (int j = 0; j < n; j++) { char condition[COND_BUF_LEN] = "\0"; @@ -6070,11 +6068,11 @@ static int insertTestProcess() { end = taosGetTimestampMs(); if (g_totalChildTables > 0) { - fprintf(stderr, "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); if (g_fpOfInsertResult) { fprintf(g_fpOfInsertResult, - "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); } } @@ -6400,16 +6398,16 @@ static int queryTestProcess() { ERROR_EXIT("memory allocation failed for create threads\n"); } - uint64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; int threads = g_queryInfo.superQueryInfo.threadCnt; - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; if (threads != 0) { b = ntables % threads; } @@ -6748,16 +6746,16 @@ static int subscribeTestProcess() { exit(-1); } - uint64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; int threads = g_queryInfo.superQueryInfo.threadCnt; - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; if (threads != 0) { b = ntables % threads; } From 2fb7c16b8757f5a1cf13bd8b5e008ead24aeafb4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 16:46:43 +0800 Subject: [PATCH 099/157] vnode/drop: ingore invalid vgroup id if already dropped --- src/vnode/src/vnodeMain.c | 2 +- src/vnode/src/vnodeWrite.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index f02a5bedb0..ee28be3d2f 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -122,7 +122,7 @@ int32_t vnodeDrop(int32_t vgId) { } if (pVnode->dropped) { vnodeRelease(pVnode); - return TSDB_CODE_VND_INVALID_VGROUP_ID; + return TSDB_CODE_SUCCESS; } vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 3d16ae1faf..7c67865a2b 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -387,5 +387,5 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { taosMsleep(10); } - taosMsleep(1000); + taosMsleep(2100); } From 6f6dab4d01f5bf59ac5007e7fa6c0996f9d45d62 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 15 May 2021 20:21:16 +0800 Subject: [PATCH 100/157] [TD-4198]: support AddressSanitizer on windows. (#6132) --- cmake/define.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/define.inc b/cmake/define.inc index e825dce024..57351e5478 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -157,7 +157,7 @@ IF (TD_WINDOWS) IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") ENDIF () - SET(DEBUG_FLAGS "/Zi /W3 /GL") + SET(DEBUG_FLAGS "/fsanitize=thread /fsanitize=leak /fsanitize=memory /fsanitize=undefined /fsanitize=hwaddress /Zi /W3 /GL") SET(RELEASE_FLAGS "/W0 /O3 /GL") ENDIF () From dae9262562cedbb6c6c61448f39342fb5f7e2b2e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 20:42:48 +0800 Subject: [PATCH 101/157] vnode/read: use app not ready instead of invalid vgroup id to make crash_gen happy --- src/vnode/src/vnodeRead.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 2f8da0012b..5d344ebf11 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -118,9 +118,8 @@ static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t con int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { SVnodeObj *pVnode = vparam; - if (pVnode->dropped) { - return TSDB_CODE_VND_INVALID_VGROUP_ID; + return TSDB_CODE_APP_NOT_READY; } SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam); From 8b4a46de899acae5ecc89e204bfaf111bf074d90 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 15 May 2021 21:24:31 +0800 Subject: [PATCH 102/157] Hotfix/sangshuduo/td 3985 taosdemo timestamp step overflow (#6139) * [TD-3985]: taosdemo timestamp step overflow. * fix specified subscribe test. * replace potential overflow of int32 to int64 * replace potential int32 overflow variables to int64. * replace potential int32 overflow variables to int64. * change max of int16 to int64 after type changed. * improve more and more. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 68 ++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 7935546719..12262f7a32 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -212,8 +212,8 @@ typedef struct SArguments_S { uint64_t interlace_rows; uint64_t num_of_RPR; // num_of_records_per_req uint64_t max_sql_len; - uint64_t num_of_tables; - uint64_t num_of_DPT; + int64_t num_of_tables; + int64_t num_of_DPT; int abort; int disorderRatio; // 0: no disorder, >0: x% int disorderRange; // ms or us by database precision @@ -240,7 +240,7 @@ typedef struct SSuperTable_S { char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest int64_t childTblLimit; - uint64_t childTblOffset; + uint64_t childTblOffset; // int multiThreadWriteOneTbl; // 0: no, 1: yes uint64_t interlaceRows; // @@ -249,7 +249,7 @@ typedef struct SSuperTable_S { uint64_t maxSqlLen; // uint64_t insertInterval; // insert interval, will override global insert interval - uint64_t insertRows; + int64_t insertRows; int64_t timeStampStep; char startTimestamp[MAX_TB_NAME_SIZE]; char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json @@ -258,7 +258,7 @@ typedef struct SSuperTable_S { uint32_t columnCount; StrColumn columns[MAX_COLUMN_COUNT]; - uint32_t tagCount; + uint32_t tagCount; StrColumn tags[MAX_TAG_COUNT]; char* childTblName; @@ -377,7 +377,7 @@ typedef struct SuperQueryInfo_S { int subscribeRestart; int subscribeKeepProgress; uint64_t queryTimes; - uint64_t childTblCount; + int64_t childTblCount; char childTblPrefix[MAX_TB_NAME_SIZE]; uint64_t sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; @@ -581,7 +581,7 @@ SArguments g_args = { static SDbs g_Dbs; -static uint64_t g_totalChildTables = 0; +static int64_t g_totalChildTables = 0; static SQueryMetaInfo g_queryInfo; static FILE * g_fpOfInsertResult = NULL; @@ -1015,9 +1015,9 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->max_sql_len); printf("# Length of Binary: %d\n", arguments->len_of_binary); printf("# Number of Threads: %d\n", arguments->num_of_threads); - printf("# Number of Tables: %"PRIu64"\n", + printf("# Number of Tables: %"PRId64"\n", arguments->num_of_tables); - printf("# Number of Data per Table: %"PRIu64"\n", + printf("# Number of Data per Table: %"PRId64"\n", arguments->num_of_DPT); printf("# Database name: %s\n", arguments->database); printf("# Table prefix: %s\n", arguments->tb_prefix); @@ -1388,7 +1388,7 @@ static int printfInsertMeta() { printf(" childTblExists: \033[33m%s\033[0m\n", "error"); } - printf(" childTblCount: \033[33m%"PRIu64"\033[0m\n", + printf(" childTblCount: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].childTblCount); printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); @@ -1404,7 +1404,7 @@ static int printfInsertMeta() { printf(" childTblOffset: \033[33m%"PRIu64"\033[0m\n", g_Dbs.db[i].superTbls[j].childTblOffset); } - printf(" insertRows: \033[33m%"PRIu64"\033[0m\n", + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); /* if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { @@ -1571,7 +1571,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " childTblExists: %s\n", "error"); } - fprintf(fp, " childTblCount: %"PRIu64"\n", + fprintf(fp, " childTblCount: %"PRId64"\n", g_Dbs.db[i].superTbls[j].childTblCount); fprintf(fp, " childTblPrefix: %s\n", g_Dbs.db[i].superTbls[j].childTblPrefix); @@ -1579,7 +1579,7 @@ static void printfInsertMetaToFile(FILE* fp) { g_Dbs.db[i].superTbls[j].dataSource); fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); - fprintf(fp, " insertRows: %"PRIu64"\n", + fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); fprintf(fp, " interlace rows: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].interlaceRows); @@ -1697,7 +1697,7 @@ static void printfQueryMeta() { g_queryInfo.superQueryInfo.queryInterval); printf("threadCnt: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.threadCnt); - printf("childTblCount: \033[33m%"PRIu64"\033[0m\n", + printf("childTblCount: \033[33m%"PRId64"\033[0m\n", g_queryInfo.superQueryInfo.childTblCount); printf("stable name: \033[33m%s\033[0m\n", g_queryInfo.superQueryInfo.sTblName); @@ -2327,7 +2327,7 @@ static int calcRowLen(SSuperTable* superTbls) { static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, char* dbName, char* sTblName, char** childTblNameOfSuperTbl, - uint64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) { + int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) { char command[BUFFER_SIZE] = "\0"; char limitBuf[100] = "\0"; @@ -2356,8 +2356,8 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, exit(-1); } - int childTblCount = (limit < 0)?10000:limit; - int count = 0; + int64_t childTblCount = (limit < 0)?10000:limit; + int64_t count = 0; if (childTblName == NULL) { childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); if (NULL == childTblName) { @@ -2404,7 +2404,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, char* sTblName, char** childTblNameOfSuperTbl, - uint64_t* childTblCountOfSuperTbl) { + int64_t* childTblCountOfSuperTbl) { return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName, childTblNameOfSuperTbl, childTblCountOfSuperTbl, @@ -3000,7 +3000,7 @@ static void createChildTables() { snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); - verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRIu64" schema: %s\n", + verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n", __func__, __LINE__, g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf); startMultiThreadCreateChildTable( @@ -4772,7 +4772,7 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t table static int64_t generateDataTail( SSuperTable* superTblInfo, uint64_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows, - int64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { + uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { uint64_t len = 0; uint32_t ncols_per_record = 1; // count first col ts @@ -4948,7 +4948,7 @@ static int64_t generateInterlaceDataBuffer( char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes, uint64_t tableSeq, threadInfo *pThreadInfo, char *buffer, - uint64_t insertRows, + int64_t insertRows, int64_t startTime, uint64_t *pRemainderBufLen) { @@ -5008,7 +5008,7 @@ static int64_t generateProgressiveDataBuffer( int64_t tableSeq, threadInfo *pThreadInfo, char *buffer, int64_t insertRows, - int64_t startFrom, int64_t startTime, int64_t *pSamplePos, + uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *pRemainderBufLen) { SSuperTable* superTblInfo = pThreadInfo->superTblInfo; @@ -5061,7 +5061,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { debugPrint("[%d] %s() LN%d: ### interlace write\n", pThreadInfo->threadID, __func__, __LINE__); - uint64_t insertRows; + int64_t insertRows; uint64_t interlaceRows; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; @@ -5341,7 +5341,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { tableSeq ++) { int64_t start_time = pThreadInfo->start_time; - uint64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); for (uint64_t i = 0; i < insertRows;) { @@ -5658,7 +5658,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } int64_t ntables = 0; - int64_t startFrom; + uint64_t startFrom; if (superTblInfo) { int64_t limit; @@ -5714,7 +5714,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(-1); } - uint64_t childTblCount; + int64_t childTblCount; getChildNameOfSuperTableWithLimitAndOffset( taos, db_name, superTblInfo->sTblName, @@ -5890,7 +5890,7 @@ static void *readTable(void *sarg) { return NULL; } - uint64_t num_of_DPT; + int64_t num_of_DPT; /* if (rinfo->superTblInfo) { num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; } else { @@ -5909,11 +5909,11 @@ static void *readTable(void *sarg) { printf("%"PRId64" records:\n", totalData); fprintf(fp, "| QFunctions | QRecords | QSpeed(R/s) | QLatency(ms) |\n"); - for (uint64_t j = 0; j < n; j++) { + for (int j = 0; j < n; j++) { double totalT = 0; uint64_t count = 0; - for (uint64_t i = 0; i < num_of_tables; i++) { - sprintf(command, "select %s from %s%"PRIu64" where ts>= %" PRIu64, + for (int64_t i = 0; i < num_of_tables; i++) { + sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64, aggreFunc[j], tb_prefix, i, sTime); double t = taosGetTimestampMs(); @@ -5976,13 +5976,13 @@ static void *readMetric(void *sarg) { char condition[COND_BUF_LEN] = "\0"; char tempS[64] = "\0"; - int m = 10 < num_of_tables ? 10 : num_of_tables; + int64_t m = 10 < num_of_tables ? 10 : num_of_tables; - for (int i = 1; i <= m; i++) { + for (int64_t i = 1; i <= m; i++) { if (i == 1) { - sprintf(tempS, "t1 = %d", i); + sprintf(tempS, "t1 = %"PRId64"", i); } else { - sprintf(tempS, " or t1 = %d ", i); + sprintf(tempS, " or t1 = %"PRId64" ", i); } strncat(condition, tempS, COND_BUF_LEN - 1); From a0f7116ce580317973e9ba532d5e43f9f5f1d0df Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 15 May 2021 21:24:44 +0800 Subject: [PATCH 103/157] Hotfix/sangshuduo/td 3985 for develop (#6138) * [TD-3985]: taosdemo timestamp step overflow. * fix specified subscribe test. * replace potential overflow of int32 to int64 * replace potential int32 overflow variables to int64. * replace potential int32 overflow variables to int64. * change max of int16 to int64 after type changed. * improve more. * improve more and more. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 132 ++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 67 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 72044753d8..12262f7a32 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -212,8 +212,8 @@ typedef struct SArguments_S { uint64_t interlace_rows; uint64_t num_of_RPR; // num_of_records_per_req uint64_t max_sql_len; - uint64_t num_of_tables; - uint64_t num_of_DPT; + int64_t num_of_tables; + int64_t num_of_DPT; int abort; int disorderRatio; // 0: no disorder, >0: x% int disorderRange; // ms or us by database precision @@ -240,7 +240,7 @@ typedef struct SSuperTable_S { char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest int64_t childTblLimit; - uint64_t childTblOffset; + uint64_t childTblOffset; // int multiThreadWriteOneTbl; // 0: no, 1: yes uint64_t interlaceRows; // @@ -249,7 +249,7 @@ typedef struct SSuperTable_S { uint64_t maxSqlLen; // uint64_t insertInterval; // insert interval, will override global insert interval - uint64_t insertRows; + int64_t insertRows; int64_t timeStampStep; char startTimestamp[MAX_TB_NAME_SIZE]; char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json @@ -258,7 +258,7 @@ typedef struct SSuperTable_S { uint32_t columnCount; StrColumn columns[MAX_COLUMN_COUNT]; - uint32_t tagCount; + uint32_t tagCount; StrColumn tags[MAX_TAG_COUNT]; char* childTblName; @@ -283,7 +283,7 @@ typedef struct SSuperTable_S { typedef struct { char name[TSDB_DB_NAME_LEN + 1]; char create_time[32]; - int32_t ntables; + int64_t ntables; int32_t vgroups; int16_t replica; int16_t quorum; @@ -377,7 +377,7 @@ typedef struct SuperQueryInfo_S { int subscribeRestart; int subscribeKeepProgress; uint64_t queryTimes; - uint64_t childTblCount; + int64_t childTblCount; char childTblPrefix[MAX_TB_NAME_SIZE]; uint64_t sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; @@ -412,7 +412,7 @@ typedef struct SThreadInfo_S { char tb_prefix[MAX_TB_NAME_SIZE]; uint64_t start_table_from; uint64_t end_table_to; - uint64_t ntables; + int64_t ntables; uint64_t data_of_rate; int64_t start_time; char* cols; @@ -581,7 +581,7 @@ SArguments g_args = { static SDbs g_Dbs; -static int g_totalChildTables = 0; +static int64_t g_totalChildTables = 0; static SQueryMetaInfo g_queryInfo; static FILE * g_fpOfInsertResult = NULL; @@ -1015,9 +1015,9 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->max_sql_len); printf("# Length of Binary: %d\n", arguments->len_of_binary); printf("# Number of Threads: %d\n", arguments->num_of_threads); - printf("# Number of Tables: %"PRIu64"\n", + printf("# Number of Tables: %"PRId64"\n", arguments->num_of_tables); - printf("# Number of Data per Table: %"PRIu64"\n", + printf("# Number of Data per Table: %"PRId64"\n", arguments->num_of_DPT); printf("# Database name: %s\n", arguments->database); printf("# Table prefix: %s\n", arguments->tb_prefix); @@ -1388,7 +1388,7 @@ static int printfInsertMeta() { printf(" childTblExists: \033[33m%s\033[0m\n", "error"); } - printf(" childTblCount: \033[33m%"PRIu64"\033[0m\n", + printf(" childTblCount: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].childTblCount); printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); @@ -1404,7 +1404,7 @@ static int printfInsertMeta() { printf(" childTblOffset: \033[33m%"PRIu64"\033[0m\n", g_Dbs.db[i].superTbls[j].childTblOffset); } - printf(" insertRows: \033[33m%"PRIu64"\033[0m\n", + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); /* if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { @@ -1571,7 +1571,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " childTblExists: %s\n", "error"); } - fprintf(fp, " childTblCount: %"PRIu64"\n", + fprintf(fp, " childTblCount: %"PRId64"\n", g_Dbs.db[i].superTbls[j].childTblCount); fprintf(fp, " childTblPrefix: %s\n", g_Dbs.db[i].superTbls[j].childTblPrefix); @@ -1579,7 +1579,7 @@ static void printfInsertMetaToFile(FILE* fp) { g_Dbs.db[i].superTbls[j].dataSource); fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); - fprintf(fp, " insertRows: %"PRIu64"\n", + fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); fprintf(fp, " interlace rows: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].interlaceRows); @@ -1697,7 +1697,7 @@ static void printfQueryMeta() { g_queryInfo.superQueryInfo.queryInterval); printf("threadCnt: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.threadCnt); - printf("childTblCount: \033[33m%"PRIu64"\033[0m\n", + printf("childTblCount: \033[33m%"PRId64"\033[0m\n", g_queryInfo.superQueryInfo.childTblCount); printf("stable name: \033[33m%s\033[0m\n", g_queryInfo.superQueryInfo.sTblName); @@ -1879,7 +1879,7 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { formatTimestamp(dbInfos[count]->create_time, *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX], TSDB_TIME_PRECISION_MILLI); - dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); + dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); @@ -1930,7 +1930,7 @@ static void printfDbInfoForQueryToFile( fprintf(fp, "================ database[%d] ================\n", index); fprintf(fp, "name: %s\n", dbInfos->name); fprintf(fp, "created_time: %s\n", dbInfos->create_time); - fprintf(fp, "ntables: %d\n", dbInfos->ntables); + fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables); fprintf(fp, "vgroups: %d\n", dbInfos->vgroups); fprintf(fp, "replica: %d\n", dbInfos->replica); fprintf(fp, "quorum: %d\n", dbInfos->quorum); @@ -2327,7 +2327,7 @@ static int calcRowLen(SSuperTable* superTbls) { static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, char* dbName, char* sTblName, char** childTblNameOfSuperTbl, - uint64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) { + int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) { char command[BUFFER_SIZE] = "\0"; char limitBuf[100] = "\0"; @@ -2356,8 +2356,8 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, exit(-1); } - int childTblCount = (limit < 0)?10000:limit; - int count = 0; + int64_t childTblCount = (limit < 0)?10000:limit; + int64_t count = 0; if (childTblName == NULL) { childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); if (NULL == childTblName) { @@ -2404,7 +2404,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, char* sTblName, char** childTblNameOfSuperTbl, - uint64_t* childTblCountOfSuperTbl) { + int64_t* childTblCountOfSuperTbl) { return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName, childTblNameOfSuperTbl, childTblCountOfSuperTbl, @@ -2884,7 +2884,7 @@ static void* createTable(void *sarg) } static int startMultiThreadCreateChildTable( - char* cols, int threads, uint64_t startFrom, uint64_t ntables, + char* cols, int threads, uint64_t startFrom, int64_t ntables, char* db_name, SSuperTable* superTblInfo) { pthread_t *pids = malloc(threads * sizeof(pthread_t)); @@ -2899,13 +2899,13 @@ static int startMultiThreadCreateChildTable( threads = 1; } - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; b = ntables % threads; for (int64_t i = 0; i < threads; i++) { @@ -2969,10 +2969,10 @@ static void createChildTables() { verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); - int startFrom = 0; + uint64_t startFrom = 0; g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; - verbosePrint("%s() LN%d: create %d child tables from %d\n", + verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n", __func__, __LINE__, g_totalChildTables, startFrom); startMultiThreadCreateChildTable( g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, @@ -3000,7 +3000,7 @@ static void createChildTables() { snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); - verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRIu64" schema: %s\n", + verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n", __func__, __LINE__, g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf); startMultiThreadCreateChildTable( @@ -4756,7 +4756,7 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t table (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN); } else { - verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRIu64" seq=%"PRIu64"\n", + verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, pThreadInfo->ntables, tableSeq); @@ -4772,7 +4772,7 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t table static int64_t generateDataTail( SSuperTable* superTblInfo, uint64_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows, - int64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { + uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { uint64_t len = 0; uint32_t ncols_per_record = 1; // count first col ts @@ -4948,7 +4948,7 @@ static int64_t generateInterlaceDataBuffer( char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes, uint64_t tableSeq, threadInfo *pThreadInfo, char *buffer, - uint64_t insertRows, + int64_t insertRows, int64_t startTime, uint64_t *pRemainderBufLen) { @@ -5008,7 +5008,7 @@ static int64_t generateProgressiveDataBuffer( int64_t tableSeq, threadInfo *pThreadInfo, char *buffer, int64_t insertRows, - int64_t startFrom, int64_t startTime, int64_t *pSamplePos, + uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *pRemainderBufLen) { SSuperTable* superTblInfo = pThreadInfo->superTblInfo; @@ -5061,7 +5061,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { debugPrint("[%d] %s() LN%d: ### interlace write\n", pThreadInfo->threadID, __func__, __LINE__); - uint64_t insertRows; + int64_t insertRows; uint64_t interlaceRows; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; @@ -5123,14 +5123,12 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { uint64_t tableSeq = pThreadInfo->start_table_from; - debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRIu64" insertRows=%"PRIu64"\n", + debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, pThreadInfo->ntables, insertRows); int64_t startTime = pThreadInfo->start_time; - assert(pThreadInfo->ntables > 0); - uint64_t batchPerTbl = interlaceRows; uint64_t batchPerTblTimes; @@ -5343,7 +5341,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { tableSeq ++) { int64_t start_time = pThreadInfo->start_time; - uint64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); for (uint64_t i = 0; i < insertRows;) { @@ -5659,8 +5657,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(-1); } - int ntables = 0; - int startFrom; + int64_t ntables = 0; + uint64_t startFrom; if (superTblInfo) { int64_t limit; @@ -5716,7 +5714,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(-1); } - uint64_t childTblCount; + int64_t childTblCount; getChildNameOfSuperTableWithLimitAndOffset( taos, db_name, superTblInfo->sTblName, @@ -5730,13 +5728,13 @@ static void startMultiThreadInsertData(int threads, char* db_name, taos_close(taos); - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; if (threads != 0) { b = ntables % threads; } @@ -5892,7 +5890,7 @@ static void *readTable(void *sarg) { return NULL; } - int num_of_DPT; + int64_t num_of_DPT; /* if (rinfo->superTblInfo) { num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; } else { @@ -5900,22 +5898,22 @@ static void *readTable(void *sarg) { num_of_DPT = g_args.num_of_DPT; // } - int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; - int totalData = num_of_DPT * num_of_tables; + int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; if (!do_aggreFunc) { printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); } - printf("%d records:\n", totalData); + printf("%"PRId64" records:\n", totalData); fprintf(fp, "| QFunctions | QRecords | QSpeed(R/s) | QLatency(ms) |\n"); - for (uint64_t j = 0; j < n; j++) { + for (int j = 0; j < n; j++) { double totalT = 0; uint64_t count = 0; - for (uint64_t i = 0; i < num_of_tables; i++) { - sprintf(command, "select %s from %s%"PRIu64" where ts>= %" PRIu64, + for (int64_t i = 0; i < num_of_tables; i++) { + sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64, aggreFunc[j], tb_prefix, i, sTime); double t = taosGetTimestampMs(); @@ -5940,7 +5938,7 @@ static void *readTable(void *sarg) { taos_free_result(pSql); } - fprintf(fp, "|%10s | %10d | %12.2f | %10.2f |\n", + fprintf(fp, "|%10s | %"PRId64" | %12.2f | %10.2f |\n", aggreFunc[j][0] == '*' ? " * " : aggreFunc[j], totalData, (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000); printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000); @@ -5962,29 +5960,29 @@ static void *readMetric(void *sarg) { return NULL; } - int num_of_DPT = rinfo->superTblInfo->insertRows; - int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; - int totalData = num_of_DPT * num_of_tables; + int64_t num_of_DPT = rinfo->superTblInfo->insertRows; + int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; if (!do_aggreFunc) { printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); } - printf("%d records:\n", totalData); - fprintf(fp, "Querying On %d records:\n", totalData); + printf("%"PRId64" records:\n", totalData); + fprintf(fp, "Querying On %"PRId64" records:\n", totalData); for (int j = 0; j < n; j++) { char condition[COND_BUF_LEN] = "\0"; char tempS[64] = "\0"; - int m = 10 < num_of_tables ? 10 : num_of_tables; + int64_t m = 10 < num_of_tables ? 10 : num_of_tables; - for (int i = 1; i <= m; i++) { + for (int64_t i = 1; i <= m; i++) { if (i == 1) { - sprintf(tempS, "t1 = %d", i); + sprintf(tempS, "t1 = %"PRId64"", i); } else { - sprintf(tempS, " or t1 = %d ", i); + sprintf(tempS, " or t1 = %"PRId64" ", i); } strncat(condition, tempS, COND_BUF_LEN - 1); @@ -6070,11 +6068,11 @@ static int insertTestProcess() { end = taosGetTimestampMs(); if (g_totalChildTables > 0) { - fprintf(stderr, "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); if (g_fpOfInsertResult) { fprintf(g_fpOfInsertResult, - "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); } } @@ -6400,16 +6398,16 @@ static int queryTestProcess() { ERROR_EXIT("memory allocation failed for create threads\n"); } - uint64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; int threads = g_queryInfo.superQueryInfo.threadCnt; - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; if (threads != 0) { b = ntables % threads; } @@ -6748,16 +6746,16 @@ static int subscribeTestProcess() { exit(-1); } - uint64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; int threads = g_queryInfo.superQueryInfo.threadCnt; - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; if (threads != 0) { b = ntables % threads; } From 610603478155d87fa71f5c2d2fa97bb6b41e0f96 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sun, 16 May 2021 11:24:35 +0800 Subject: [PATCH 104/157] vnode/close: wait write 900ms --- src/vnode/src/vnodeWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 7c67865a2b..56ea32ccc0 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -387,5 +387,5 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { taosMsleep(10); } - taosMsleep(2100); + taosMsleep(900); } From 292732315e0707d003fc5f6c4f9bc7be8846e5ed Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 16 May 2021 19:53:01 +0800 Subject: [PATCH 105/157] merge with develop branch. (#6143) change query/tests/CMakeLists.txt to allow unused function and variable. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 18 +++++++++--------- src/query/tests/CMakeLists.txt | 7 +++++++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3f1a63eed2..b4689a262d 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3929,9 +3929,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); - if (maxSqlLen && maxSqlLen->type == cJSON_Number) { - int32_t len = maxSqlLen->valueint; + cJSON* stbMaxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); + if (stbMaxSqlLen && stbMaxSqlLen->type == cJSON_Number) { + int32_t len = stbMaxSqlLen->valueint; if (len > TSDB_MAX_ALLOWED_SQL_LEN) { len = TSDB_MAX_ALLOWED_SQL_LEN; } else if (len < 5) { @@ -3941,7 +3941,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!maxSqlLen) { g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len; } else { - errorPrint("%s() LN%d, failed to read json, maxSqlLen input mistake\n", + errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n", __func__, __LINE__); goto PARSE_OVER; } @@ -3963,14 +3963,14 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } */ - interlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); - if (interlaceRows && interlaceRows->type == cJSON_Number) { - if (interlaceRows->valueint < 0) { + cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); + if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) { + if (stbInterlaceRows->valueint < 0) { errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n", __func__, __LINE__); goto PARSE_OVER; } - g_Dbs.db[i].superTbls[j].interlaceRows = interlaceRows->valueint; + g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint; // rows per table need be less than insert batch if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) { printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n", @@ -3983,7 +3983,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; } - } else if (!interlaceRows) { + } else if (!stbInterlaceRows) { g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req } else { errorPrint( diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt index 1856223391..f8b6daaa90 100644 --- a/src/query/tests/CMakeLists.txt +++ b/src/query/tests/CMakeLists.txt @@ -13,3 +13,10 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread gcov) ENDIF() + +SET_SOURCE_FILES_PROPERTIES(./astTest.cpp PROPERTIES COMPILE_FLAGS -w) +SET_SOURCE_FILES_PROPERTIES(./histogramTest.cpp PROPERTIES COMPILE_FLAGS -w) +SET_SOURCE_FILES_PROPERTIES(./percentileTest.cpp PROPERTIES COMPILE_FLAGS -w) +SET_SOURCE_FILES_PROPERTIES(./resultBufferTest.cpp PROPERTIES COMPILE_FLAGS -w) +SET_SOURCE_FILES_PROPERTIES(./tsBufTest.cpp PROPERTIES COMPILE_FLAGS -w) +SET_SOURCE_FILES_PROPERTIES(./unitTest.cpp PROPERTIES COMPILE_FLAGS -w) From e620e2e83a26c8e8784d203010ef914172607a95 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 May 2021 12:53:35 +0800 Subject: [PATCH 106/157] [td-4209]: add the uid check for super table metadata that is already cached in local buffer. --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 6 ++++-- src/client/src/tscSchemaUtil.c | 1 + src/client/src/tscServer.c | 28 +++++++++++++--------------- src/client/src/tscUtil.c | 21 ++++++++++----------- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index b80dcfb577..47b2865313 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -298,7 +298,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild); uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); uint32_t tscGetTableMetaMaxSize(); -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf); +int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf); STableMeta* tscTableMetaDup(STableMeta* pTableMeta); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 6d94d270a7..4bfd3bc88f 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -67,14 +67,16 @@ typedef struct CChildTableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; //super table name, not full name + char sTableName[TSDB_TABLE_FNAME_LEN]; // TODO: refactor super table name, not full name + uint64_t suid; // super table id } CChildTableMeta; typedef struct STableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; + char sTableName[TSDB_TABLE_FNAME_LEN]; // super table name + uint64_t suid; // super table id int16_t sversion; int16_t tversion; STableComInfo tableInfo; diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 2ea382132b..114fc8ee73 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -94,6 +94,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { pTableMeta->tableType = pTableMetaMsg->tableType; pTableMeta->vgId = pTableMetaMsg->vgroup.vgId; + pTableMeta->suid = pTableMetaMsg->suid; pTableMeta->tableInfo = (STableComInfo) { .numOfTags = pTableMetaMsg->numOfTags, diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 33325c7fd7..664f2989a8 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1920,13 +1920,13 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscProcessTableMetaRsp(SSqlObj *pSql) { STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - pMetaMsg->tid = htonl(pMetaMsg->tid); - pMetaMsg->sversion = htons(pMetaMsg->sversion); - pMetaMsg->tversion = htons(pMetaMsg->tversion); + pMetaMsg->tid = htonl(pMetaMsg->tid); + pMetaMsg->sversion = htons(pMetaMsg->sversion); + pMetaMsg->tversion = htons(pMetaMsg->tversion); pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); - - pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->contLen = htons(pMetaMsg->contLen); + pMetaMsg->uid = htobe64(pMetaMsg->uid); + pMetaMsg->suid = htobe64(pMetaMsg->suid); + pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && @@ -2537,19 +2537,16 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { pTableMetaInfo->pTableMeta = calloc(1, size); pTableMetaInfo->tableMetaSize = size; } else if (pTableMetaInfo->tableMetaSize < size) { - char *tmp = realloc(pTableMetaInfo->pTableMeta, size); - if (tmp == NULL) { + char *tmp = realloc(pTableMetaInfo->pTableMeta, size); + if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } pTableMetaInfo->pTableMeta = (STableMeta *)tmp; - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; - } else { - //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; } + memset(pTableMetaInfo->pTableMeta, 0, size); + pTableMetaInfo->tableMetaSize = size; + pTableMetaInfo->pTableMeta->tableType = -1; pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; @@ -2565,8 +2562,9 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { STableMeta* pMeta = pTableMetaInfo->pTableMeta; if (pMeta->id.uid > 0) { + // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, name, buf); + int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, buf); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 3d2cf44560..89749202ef 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2941,22 +2941,25 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { assert(pTableMeta != NULL); CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); + cMeta->tableType = TSDB_CHILD_TABLE; - cMeta->vgId = pTableMeta->vgId; - cMeta->id = pTableMeta->id; + cMeta->vgId = pTableMeta->vgId; + cMeta->id = pTableMeta->id; + cMeta->suid = pTableMeta->suid; tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); return cMeta; } -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf) { +int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf) { assert(pChild != NULL && buf != NULL); -// uint32_t size = tscGetTableMetaMaxSize(); - STableMeta* p = buf;//calloc(1, size); - + STableMeta* p = buf; taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); - if (p->id.uid > 0) { // tableMeta exists, build child table meta and return + + // tableMeta exists, build child table meta according to the super table meta + // the uid need to be checked in addition to the general name of the super table. + if (p->id.uid > 0 && pChild->suid == p->id.uid) { pChild->sversion = p->sversion; pChild->tversion = p->tversion; @@ -2964,13 +2967,9 @@ int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, v int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); - -// tfree(p); return TSDB_CODE_SUCCESS; } else { // super table has been removed, current tableMeta is also expired. remove it here taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - -// tfree(p); return -1; } } From 07aed4fa708386e96b441ea1becfb89af80bea50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 May 2021 15:23:44 +0800 Subject: [PATCH 107/157] [td-4209] --- src/client/src/tscServer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 664f2989a8..0e16369cad 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1925,7 +1925,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->tversion = htons(pMetaMsg->tversion); pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->suid = htobe64(pMetaMsg->suid); + pMetaMsg->suid = pMetaMsg->suid; pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); From c824330bb5548f963f27255799c7f73d10765dcb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 May 2021 15:51:49 +0800 Subject: [PATCH 108/157] [td-225] refactor a sim script. --- tests/script/general/parser/commit.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/general/parser/commit.sim b/tests/script/general/parser/commit.sim index dfe521b92b..7c4c883fb1 100644 --- a/tests/script/general/parser/commit.sim +++ b/tests/script/general/parser/commit.sim @@ -68,7 +68,7 @@ while $loop <= $loops while $i < 10 sql select count(*) from $stb where t1 = $i if $data00 != $rowNum then - print expect $rowNum, actual: $data00 + print expect $rowNum , actual: $data00 return -1 endi $i = $i + 1 From e4f135b4bd9235ce0218a076be67973b84b51353 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Mon, 17 May 2021 21:04:02 +0800 Subject: [PATCH 109/157] [TD-4224]: reduce grafana module size in packaging (#6146) --- packaging/deb/makedeb.sh | 2 +- packaging/rpm/tdengine.spec | 2 +- packaging/tools/make_install.sh | 2 +- packaging/tools/makeclient.sh | 8 ++++---- packaging/tools/makeclient_power.sh | 2 +- packaging/tools/makepkg.sh | 2 +- packaging/tools/makepkg_power.sh | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 36870b2ebe..a593c7ca16 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -58,7 +58,7 @@ cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_pat cp ${compile_dir}/../src/inc/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../src/inc/taoserror.h ${pkg_dir}${install_home_path}/include cp -r ${top_dir}/tests/examples/* ${pkg_dir}${install_home_path}/examples -cp -r ${top_dir}/src/connector/grafanaplugin ${pkg_dir}${install_home_path}/connector +cp -r ${top_dir}/src/connector/grafanaplugin/dist ${pkg_dir}${install_home_path}/connector/grafanaplugin cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 92c917cb3d..73efe2e1ab 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -66,7 +66,7 @@ cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include -cp -r %{_compiledir}/../src/connector/grafanaplugin %{buildroot}%{homepath}/connector +cp -r %{_compiledir}/../src/connector/grafanaplugin/dist %{buildroot}%{homepath}/connector/grafanaplugin cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index f03065d70c..5f1739d6e0 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -243,7 +243,7 @@ function install_data() { } function install_connector() { - ${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin ${install_main_dir}/connector + ${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin/dist ${install_main_dir}/connector/grafanaplugin ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector ${csudo} cp -rf ${source_dir}/src/connector/go ${install_main_dir}/connector diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 30e9fa51a7..7418bcc312 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -117,10 +117,10 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi - cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ - cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector - cp -r ${connector_dir}/nodejs ${install_dir}/connector + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + cp -r ${connector_dir}/python ${install_dir}/connector/ + cp -r ${connector_dir}/go ${install_dir}/connector + cp -r ${connector_dir}/nodejs ${install_dir}/connector fi # Copy release note # cp ${script_dir}/release_note ${install_dir} diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index 23ee300313..911cc574a5 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -144,7 +144,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi - cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin cp -r ${connector_dir}/python ${install_dir}/connector/ cp -r ${connector_dir}/go ${install_dir}/connector diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 36b1fe5bd8..343e2a4224 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -131,7 +131,7 @@ connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: - cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin cp -r ${connector_dir}/python ${install_dir}/connector/ cp -r ${connector_dir}/go ${install_dir}/connector cp -r ${connector_dir}/nodejs ${install_dir}/connector diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index 359c5dba46..8ea7f363c8 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -166,7 +166,7 @@ connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: - cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin cp -r ${connector_dir}/python ${install_dir}/connector/ cp -r ${connector_dir}/go ${install_dir}/connector From 4cb5bedfaeaeeec84a79e7a72e1a279f4f9111f6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 18 May 2021 13:48:45 +0800 Subject: [PATCH 110/157] [TD-4232]: fix vnode queueWMsg & queuedWMsgSize counting under flow ctrl --- src/vnode/src/vnodeWrite.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 56ea32ccc0..16089c8e91 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -340,8 +340,11 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { if (pWrite->processedCount >= 100) { vError("vgId:%d, msg:%p, failed to process since %s, retry:%d", pVnode->vgId, pWrite, tstrerror(code), pWrite->processedCount); - pWrite->processedCount = 1; - dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + void *handle = pWrite->rpcMsg.handle; + taosFreeQitem(pWrite); + vnodeRelease(pVnode); + SRpcMsg rpcRsp = {.handle = handle, .code = code}; + rpcSendResponse(&rpcRsp); } else { code = vnodePerformFlowCtrl(pWrite); if (code == 0) { From 6d448dcac1e522e3496c133d7d6d2a72dc2b7d5f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 18 May 2021 16:48:17 +0800 Subject: [PATCH 111/157] Hotfix/sangshuduo/td 4136 taosdemo records morethan 32767 (#6147) * [TD-4136]: taosdemo max records per req < 32767 * [TD-4136]: taosdemo check insert rows not more than 32767. check insert rows for progressive. * fix with answer_yes. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 48 ++++++++++++---------- tests/pytest/tools/taosdemoTestWithJson.py | 22 +++++----- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 12262f7a32..f6ac4a75ba 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -516,6 +516,8 @@ static int taosRandom() #endif // ifdef Windows +static void prompt(); +static void prompt2(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); @@ -1031,10 +1033,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { printf("# Print debug info: %d\n", arguments->debug_print); printf("# Print verbose info: %d\n", arguments->verbose_print); printf("###################################################################\n"); - if (!arguments->answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + + prompt(); } } @@ -3410,10 +3410,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.interlace_rows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } + prompt2(); g_args.interlace_rows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -3470,9 +3467,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.answer_yes = false; } } else if (!answerPrompt) { - g_args.answer_yes = false; + g_args.answer_yes = true; // default is no, mean answer_yes. } else { - printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); + errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n"); goto PARSE_OVER; } @@ -5342,6 +5339,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int64_t start_time = pThreadInfo->start_time; int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); for (uint64_t i = 0; i < insertRows;) { @@ -6022,6 +6020,21 @@ static void *readMetric(void *sarg) { return NULL; } +static void prompt() +{ + if (!g_args.answer_yes) { + printf("Press enter key to continue\n\n"); + (void)getchar(); + } +} + +static void prompt2() +{ + if (!g_args.answer_yes) { + printf(" press Enter key to continue or Ctrl-C to stop."); + (void)getchar(); + } +} static int insertTestProcess() { @@ -6042,10 +6055,7 @@ static int insertTestProcess() { if (g_fpOfInsertResult) printfInsertMetaToFile(g_fpOfInsertResult); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); init_rand_data(); @@ -6317,10 +6327,7 @@ static int queryTestProcess() { &g_queryInfo.superQueryInfo.childTblCount); } - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); if (g_args.debug_print || g_args.verbose_print) { printfQuerySystemInfo(taos); @@ -6673,10 +6680,7 @@ static int subscribeTestProcess() { printfQueryMeta(); resetAfterAnsiEscape(); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + prompt(); TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, diff --git a/tests/pytest/tools/taosdemoTestWithJson.py b/tests/pytest/tools/taosdemoTestWithJson.py index f57af9ce5c..b2ecd54976 100644 --- a/tests/pytest/tools/taosdemoTestWithJson.py +++ b/tests/pytest/tools/taosdemoTestWithJson.py @@ -23,32 +23,32 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] else: - projPath = selfPath[:selfPath.find("tests")] + projPath = selfPath[: selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if "taosd" in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + if "packaging" not in rootRealPath: + buildPath = root[: len(root) - len("/build/bin")] break return buildPath - + def run(self): tdSql.prepare() buildPath = self.getBuildPath() - if (buildPath == ""): + if buildPath == "": tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - os.system("yes | %staosdemo -f tools/insert.json" % binPath) + binPath = buildPath + "/build/bin/" + os.system("%staosdemo -f tools/insert.json -y" % binPath) tdSql.execute("use db01") tdSql.query("select count(*) from stb01") From e8d2162cce72044cc42ac8baa505a0e5c9880615 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 18 May 2021 21:34:10 +0800 Subject: [PATCH 112/157] Hotfix/sangshuduo/td 4240 taosdemo subscribe morethan 100 (#6155) * DB/create: fix create database when default quorum > replica * [TD-4081]: fix vnode dropping * test/sim: fix null to 0 in show dnodes; * [TD-4081]: [v3] fix vnode closing * [TD-4081]: vnode not close if dropped * Revert "test/sim: fix null to 0 in show dnodes;" This reverts commit 70213e65261e6ad7290d497383c7580efe93d16a. * vnode/write: make last write msg to be written or confirmed * vnode/drop: ingore invalid vgroup id if already dropped * vnode/read: use app not ready instead of invalid vgroup id to make crash_gen happy * vnode/close: wait write 900ms * [td-4209]: add the uid check for super table metadata that is already cached in local buffer. * [td-4209] * [td-225] refactor a sim script. * Hotfix/sangshuduo/td 4136 taosdemo records morethan 32767 (#6147) * [TD-4136]: taosdemo max records per req < 32767 * [TD-4136]: taosdemo check insert rows not more than 32767. check insert rows for progressive. * fix with answer_yes. Co-authored-by: Shuduo Sang * [TD-4240]: taosdemo subscribe more than max query sql count. Co-authored-by: Minglei Jin Co-authored-by: Minglei Jin Co-authored-by: Shengliang Guan Co-authored-by: Haojun Liao Co-authored-by: haojun Liao Co-authored-by: Shuduo Sang --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 6 ++- src/client/src/tscSchemaUtil.c | 1 + src/client/src/tscServer.c | 28 +++++------ src/client/src/tscUtil.c | 21 ++++---- src/kit/taosdemo/taosdemo.c | 57 +++++++++++++--------- src/mnode/src/mnodeDb.c | 2 +- src/vnode/src/vnodeMain.c | 12 ++++- src/vnode/src/vnodeRead.c | 7 ++- src/vnode/src/vnodeStatus.c | 3 ++ src/vnode/src/vnodeSync.c | 11 +++++ src/vnode/src/vnodeWrite.c | 2 + tests/pytest/tools/taosdemoTestWithJson.py | 22 ++++----- tests/script/general/parser/commit.sim | 2 +- 14 files changed, 108 insertions(+), 68 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 56d595ff1f..0348606d6b 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -307,7 +307,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild); uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); uint32_t tscGetTableMetaMaxSize(); -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf); +int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf); STableMeta* tscTableMetaDup(STableMeta* pTableMeta); int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index bf41449e13..81443360f4 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -68,14 +68,16 @@ typedef struct CChildTableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; //super table name, not full name + char sTableName[TSDB_TABLE_FNAME_LEN]; // TODO: refactor super table name, not full name + uint64_t suid; // super table id } CChildTableMeta; typedef struct STableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; + char sTableName[TSDB_TABLE_FNAME_LEN]; // super table name + uint64_t suid; // super table id int16_t sversion; int16_t tversion; STableComInfo tableInfo; diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 2ea382132b..114fc8ee73 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -94,6 +94,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { pTableMeta->tableType = pTableMetaMsg->tableType; pTableMeta->vgId = pTableMetaMsg->vgroup.vgId; + pTableMeta->suid = pTableMetaMsg->suid; pTableMeta->tableInfo = (STableComInfo) { .numOfTags = pTableMetaMsg->numOfTags, diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 6587a55152..9b7d5c0c7f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1828,13 +1828,13 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscProcessTableMetaRsp(SSqlObj *pSql) { STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - pMetaMsg->tid = htonl(pMetaMsg->tid); - pMetaMsg->sversion = htons(pMetaMsg->sversion); - pMetaMsg->tversion = htons(pMetaMsg->tversion); + pMetaMsg->tid = htonl(pMetaMsg->tid); + pMetaMsg->sversion = htons(pMetaMsg->sversion); + pMetaMsg->tversion = htons(pMetaMsg->tversion); pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); - - pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->contLen = htons(pMetaMsg->contLen); + pMetaMsg->uid = htobe64(pMetaMsg->uid); + pMetaMsg->suid = pMetaMsg->suid; + pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && @@ -2448,19 +2448,16 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { pTableMetaInfo->pTableMeta = calloc(1, size); pTableMetaInfo->tableMetaSize = size; } else if (pTableMetaInfo->tableMetaSize < size) { - char *tmp = realloc(pTableMetaInfo->pTableMeta, size); - if (tmp == NULL) { + char *tmp = realloc(pTableMetaInfo->pTableMeta, size); + if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } pTableMetaInfo->pTableMeta = (STableMeta *)tmp; - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; - } else { - //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; } + memset(pTableMetaInfo->pTableMeta, 0, size); + pTableMetaInfo->tableMetaSize = size; + pTableMetaInfo->pTableMeta->tableType = -1; pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; @@ -2476,8 +2473,9 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { STableMeta* pMeta = pTableMetaInfo->pTableMeta; if (pMeta->id.uid > 0) { + // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, name, buf); + int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, buf); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index a698e239df..37e836dc1c 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3370,22 +3370,25 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { assert(pTableMeta != NULL); CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); + cMeta->tableType = TSDB_CHILD_TABLE; - cMeta->vgId = pTableMeta->vgId; - cMeta->id = pTableMeta->id; + cMeta->vgId = pTableMeta->vgId; + cMeta->id = pTableMeta->id; + cMeta->suid = pTableMeta->suid; tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); return cMeta; } -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf) { +int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf) { assert(pChild != NULL && buf != NULL); -// uint32_t size = tscGetTableMetaMaxSize(); - STableMeta* p = buf;//calloc(1, size); - + STableMeta* p = buf; taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); - if (p->id.uid > 0) { // tableMeta exists, build child table meta and return + + // tableMeta exists, build child table meta according to the super table meta + // the uid need to be checked in addition to the general name of the super table. + if (p->id.uid > 0 && pChild->suid == p->id.uid) { pChild->sversion = p->sversion; pChild->tversion = p->tversion; @@ -3393,13 +3396,9 @@ int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, v int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); - -// tfree(p); return TSDB_CODE_SUCCESS; } else { // super table has been removed, current tableMeta is also expired. remove it here taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - -// tfree(p); return -1; } } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index b4689a262d..4b4c44e519 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -526,6 +526,8 @@ static int taosRandom() #endif // ifdef Windows +static void prompt(); +static void prompt2(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); @@ -1062,10 +1064,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { printf("# Print debug info: %d\n", arguments->debug_print); printf("# Print verbose info: %d\n", arguments->verbose_print); printf("###################################################################\n"); - if (!arguments->answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + + prompt(); } } @@ -3446,10 +3446,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.interlace_rows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } + prompt2(); g_args.interlace_rows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -3506,9 +3503,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.answer_yes = false; } } else if (!answerPrompt) { - g_args.answer_yes = false; + g_args.answer_yes = true; // default is no, mean answer_yes. } else { - printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); + errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n"); goto PARSE_OVER; } @@ -5395,6 +5392,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int64_t start_time = pThreadInfo->start_time; int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); for (uint64_t i = 0; i < insertRows;) { @@ -6095,6 +6093,21 @@ static void *readMetric(void *sarg) { return NULL; } +static void prompt() +{ + if (!g_args.answer_yes) { + printf("Press enter key to continue\n\n"); + (void)getchar(); + } +} + +static void prompt2() +{ + if (!g_args.answer_yes) { + printf(" press Enter key to continue or Ctrl-C to stop."); + (void)getchar(); + } +} static int insertTestProcess() { @@ -6115,10 +6128,7 @@ static int insertTestProcess() { if (g_fpOfInsertResult) printfInsertMetaToFile(g_fpOfInsertResult); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); init_rand_data(); @@ -6390,10 +6400,7 @@ static int queryTestProcess() { &g_queryInfo.superQueryInfo.childTblCount); } - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); if (g_args.debug_print || g_args.verbose_print) { printfQuerySystemInfo(taos); @@ -6576,6 +6583,15 @@ static void *superSubscribe(void *sarg) { if (g_queryInfo.superQueryInfo.sqlCount == 0) return NULL; + if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { + errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n", + g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables, + g_queryInfo.superQueryInfo.sqlCount, + pThreadInfo->ntables, + MAX_QUERY_SQL_COUNT); + exit(-1); + } + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, @@ -6746,10 +6762,7 @@ static int subscribeTestProcess() { printfQueryMeta(); resetAfterAnsiEscape(); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + prompt(); TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8af20aa862..7cc5d41ae1 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -389,7 +389,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->compression < 0) pCfg->compression = tsCompression; if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL; if (pCfg->replications < 0) pCfg->replications = tsReplications; - if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; + if (pCfg->quorum < 0) pCfg->quorum = MIN(tsQuorum, pCfg->replications); if (pCfg->update < 0) pCfg->update = tsUpdate; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow; if (pCfg->dbType < 0) pCfg->dbType = 0; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index ca3746c598..f133101cca 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -120,12 +120,14 @@ int32_t vnodeDrop(int32_t vgId) { vDebug("vgId:%d, failed to drop, vnode not find", vgId); return TSDB_CODE_VND_INVALID_VGROUP_ID; } + if (pVnode->dropped) { + vnodeRelease(pVnode); + return TSDB_CODE_SUCCESS; + } vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); pVnode->dropped = 1; - // remove from hash, so new messages wont be consumed - vnodeRemoveFromHash(pVnode); vnodeRelease(pVnode); vnodeCleanupInMWorker(pVnode); @@ -425,6 +427,10 @@ int32_t vnodeOpen(int32_t vgId) { int32_t vnodeClose(int32_t vgId) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) return 0; + if (pVnode->dropped) { + vnodeRelease(pVnode); + return 0; + } vDebug("vgId:%d, vnode will be closed, pVnode:%p", pVnode->vgId, pVnode); vnodeRemoveFromHash(pVnode); @@ -510,6 +516,8 @@ void vnodeCleanUp(SVnodeObj *pVnode) { vnodeSetClosingStatus(pVnode); + vnodeRemoveFromHash(pVnode); + // stop replication module if (pVnode->sync > 0) { int64_t sync = pVnode->sync; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index b28bdbf130..2831090267 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -117,14 +117,17 @@ static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t con } int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { + SVnodeObj *pVnode = vparam; + if (pVnode->dropped) { + return TSDB_CODE_APP_NOT_READY; + } + SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam); if (pRead == NULL) { assert(terrno != 0); return terrno; } - SVnodeObj *pVnode = vparam; - int32_t code = vnodeCheckRead(pVnode); if (code != TSDB_CODE_SUCCESS) { taosFreeQitem(pRead); diff --git a/src/vnode/src/vnodeStatus.c b/src/vnode/src/vnodeStatus.c index c482d1fd1a..1eaddc3d25 100644 --- a/src/vnode/src/vnodeStatus.c +++ b/src/vnode/src/vnodeStatus.c @@ -66,6 +66,9 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) { } bool vnodeSetClosingStatus(SVnodeObj* pVnode) { + if (pVnode->status == TAOS_VN_STATUS_CLOSING) + return true; + while (!vnodeSetClosingStatusImp(pVnode)) { taosMsleep(1); } diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index 05af34a34f..e5a1964915 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -55,6 +55,11 @@ void vnodeNotifyRole(int32_t vgId, int8_t role) { vTrace("vgId:%d, vnode not found while notify role", vgId); return; } + if (pVnode->dropped) { + vTrace("vgId:%d, vnode dropped while notify role", vgId); + vnodeRelease(pVnode); + return; + } vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); pVnode->role = role; @@ -75,6 +80,11 @@ void vnodeCtrlFlow(int32_t vgId, int32_t level) { vTrace("vgId:%d, vnode not found while flow ctrl", vgId); return; } + if (pVnode->dropped) { + vTrace("vgId:%d, vnode dropped while flow ctrl", vgId); + vnodeRelease(pVnode); + return; + } if (pVnode->flowctrlLevel != level) { vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level); @@ -129,6 +139,7 @@ int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rpara SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while write to cache", vgId); + vnodeRelease(pVnode); return TSDB_CODE_VND_INVALID_VGROUP_ID; } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 36516d81df..56ea32ccc0 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -386,4 +386,6 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { vTrace("vgId:%d, queued wmsg num:%d", pVnode->vgId, pVnode->queuedWMsg); taosMsleep(10); } + + taosMsleep(900); } diff --git a/tests/pytest/tools/taosdemoTestWithJson.py b/tests/pytest/tools/taosdemoTestWithJson.py index f57af9ce5c..b2ecd54976 100644 --- a/tests/pytest/tools/taosdemoTestWithJson.py +++ b/tests/pytest/tools/taosdemoTestWithJson.py @@ -23,32 +23,32 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] else: - projPath = selfPath[:selfPath.find("tests")] + projPath = selfPath[: selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if "taosd" in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + if "packaging" not in rootRealPath: + buildPath = root[: len(root) - len("/build/bin")] break return buildPath - + def run(self): tdSql.prepare() buildPath = self.getBuildPath() - if (buildPath == ""): + if buildPath == "": tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - os.system("yes | %staosdemo -f tools/insert.json" % binPath) + binPath = buildPath + "/build/bin/" + os.system("%staosdemo -f tools/insert.json -y" % binPath) tdSql.execute("use db01") tdSql.query("select count(*) from stb01") diff --git a/tests/script/general/parser/commit.sim b/tests/script/general/parser/commit.sim index dfe521b92b..7c4c883fb1 100644 --- a/tests/script/general/parser/commit.sim +++ b/tests/script/general/parser/commit.sim @@ -68,7 +68,7 @@ while $loop <= $loops while $i < 10 sql select count(*) from $stb where t1 = $i if $data00 != $rowNum then - print expect $rowNum, actual: $data00 + print expect $rowNum , actual: $data00 return -1 endi $i = $i + 1 From 42771b3145a58b164d4b7745e533e3116fb36307 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 18 May 2021 21:35:40 +0800 Subject: [PATCH 113/157] Hotfix/sangshuduo/td 4136 taosdemo records morethan 32767 for develop (#6148) * [TD-4136]: taosdemo records per req more than 32767. for develop branch. * change taosRandom() to rand_bool() for bool data. * [TD-4136]: taosdemo check insert rows not more than 32767. check insert rows for progressive. * fix with answer_yes. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 4b4c44e519..658093a385 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4703,7 +4703,7 @@ static int64_t generateData(char *recBuf, char **data_type, double t = rand_double(); pstr += sprintf(pstr, ",%20.8f", t); } else if (strcasecmp(data_type[i % c], "BOOL") == 0) { - bool b = taosRandom() & 1; + bool b = rand_bool() & 1; pstr += sprintf(pstr, ",%s", b ? "true" : "false"); } else if (strcasecmp(data_type[i % c], "BINARY") == 0) { char *s = malloc(lenOfBinary); From 4412c52f27bca02a0b645f24858d94010d1d1039 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 18 May 2021 21:36:07 +0800 Subject: [PATCH 114/157] Hotfix/sangshuduo/td 4187 taosdemo keepprogress (#6149) * [TD-4187]: taosdemo keepProgress. * add consumed. * [TD-4187]: taosdemo keepProgress. add resubAfterConsume process. * print resub msg for test. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/subscribe.json | 40 +++-- src/kit/taosdemo/taosdemo.c | 281 +++++++++++++++++++++++--------- 2 files changed, 234 insertions(+), 87 deletions(-) diff --git a/src/kit/taosdemo/subscribe.json b/src/kit/taosdemo/subscribe.json index fd33a2e2e2..9faf03a03d 100644 --- a/src/kit/taosdemo/subscribe.json +++ b/src/kit/taosdemo/subscribe.json @@ -1,17 +1,37 @@ { - "filetype":"subscribe", + "filetype": "subscribe", "cfgdir": "/etc/taos", "host": "127.0.0.1", "port": 6030, "user": "root", "password": "taosdata", - "databases": "dbx", - "specified_table_query": - {"concurrent":1, "mode":"sync", "interval":5000, "restart":"yes", "keepProgress":"yes", - "sqls": [{"sql": "select avg(col1) from stb01 where col1 > 1;", "result": "./subscribe_res0.txt"}] - }, - "super_table_query": - {"stblname": "stb", "threads":1, "mode":"sync", "interval":10000, "restart":"yes", "keepProgress":"yes", - "sqls": [{"sql": "select col1 from xxxx where col1 > 10;", "result": "./subscribe_res1.txt"}] - } + "databases": "test", + "specified_table_query": { + "concurrent": 1, + "mode": "sync", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "resubAfterConsume": 10, + "sqls": [ + { + "sql": "select avg(col1) from meters where col1 > 1;", + "result": "./subscribe_res0.txt" + } + ] + }, + "super_table_query": { + "stblname": "meters", + "threads": 1, + "mode": "sync", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "sqls": [ + { + "sql": "select col1 from xxxx where col1 > 10;", + "result": "./subscribe_res1.txt" + } + ] + } } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f6ac4a75ba..08c1142070 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -227,7 +227,7 @@ typedef struct SColumn_S { char field[TSDB_COL_NAME_LEN + 1]; char dataType[MAX_TB_NAME_SIZE]; uint32_t dataLen; - char note[128]; + char note[128]; } StrColumn; typedef struct SSuperTable_S { @@ -360,10 +360,11 @@ typedef struct SpecifiedQueryInfo_S { uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms uint64_t queryTimes; - int subscribeRestart; + bool subscribeRestart; int subscribeKeepProgress; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + int resubAfterConsume[MAX_QUERY_SQL_COUNT]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; uint64_t totalQueried; } SpecifiedQueryInfo; @@ -374,7 +375,7 @@ typedef struct SuperQueryInfo_S { uint32_t threadCnt; uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms - int subscribeRestart; + bool subscribeRestart; int subscribeKeepProgress; uint64_t queryTimes; int64_t childTblCount; @@ -382,6 +383,7 @@ typedef struct SuperQueryInfo_S { uint64_t sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + int resubAfterConsume[MAX_QUERY_SQL_COUNT]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; char* childTblName; @@ -399,7 +401,7 @@ typedef struct SQueryMetaInfo_S { char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest SpecifiedQueryInfo specifiedQueryInfo; - SuperQueryInfo superQueryInfo; + SuperQueryInfo superQueryInfo; uint64_t totalQueried; } SQueryMetaInfo; @@ -521,8 +523,8 @@ static void prompt2(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); -static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, - char* sqlstr, char *resultFile); +static int postProceSql(char *host, struct sockaddr_in *pServAddr, + uint16_t port, char* sqlstr, char *resultFile); /* ************ Global variables ************ */ @@ -1141,12 +1143,14 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { totalLen += len; } - verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile); + verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", + __func__, __LINE__, databuf, resultFile); appendResultBufToFile(databuf, resultFile); free(databuf); } -static void selectAndGetResult(threadInfo *pThreadInfo, char *command, char* resultFile) +static void selectAndGetResult( + threadInfo *pThreadInfo, char *command, char* resultFile) { if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) { TAOS_RES *res = taos_query(pThreadInfo->taos, command); @@ -1291,13 +1295,15 @@ static void init_rand_data() { static int printfInsertMeta() { SHOW_PARSE_RESULT_START(); - printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); + printf("host: \033[33m%s:%u\033[0m\n", + g_Dbs.host, g_Dbs.port); printf("user: \033[33m%s\033[0m\n", g_Dbs.user); printf("password: \033[33m%s\033[0m\n", g_Dbs.password); printf("configDir: \033[33m%s\033[0m\n", configDir); printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); - printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl); + printf("thread num of create table: \033[33m%d\033[0m\n", + g_Dbs.threadCountByCreateTbl); printf("top insert interval: \033[33m%"PRIu64"\033[0m\n", g_args.insert_interval); printf("number of records per req: \033[33m%"PRIu64"\033[0m\n", @@ -1309,7 +1315,8 @@ static int printfInsertMeta() { for (int i = 0; i < g_Dbs.dbCount; i++) { printf("database[\033[33m%d\033[0m]:\n", i); - printf(" database[%d] name: \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName); + printf(" database[%d] name: \033[33m%s\033[0m\n", + i, g_Dbs.db[i].dbName); if (0 == g_Dbs.db[i].drop) { printf(" drop: \033[33mno\033[0m\n"); } else { @@ -1317,40 +1324,51 @@ static int printfInsertMeta() { } if (g_Dbs.db[i].dbCfg.blocks > 0) { - printf(" blocks: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks); + printf(" blocks: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.blocks); } if (g_Dbs.db[i].dbCfg.cache > 0) { - printf(" cache: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache); + printf(" cache: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.cache); } if (g_Dbs.db[i].dbCfg.days > 0) { - printf(" days: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days); + printf(" days: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.days); } if (g_Dbs.db[i].dbCfg.keep > 0) { - printf(" keep: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep); + printf(" keep: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.keep); } if (g_Dbs.db[i].dbCfg.replica > 0) { - printf(" replica: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica); + printf(" replica: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.replica); } if (g_Dbs.db[i].dbCfg.update > 0) { - printf(" update: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update); + printf(" update: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.update); } if (g_Dbs.db[i].dbCfg.minRows > 0) { - printf(" minRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows); + printf(" minRows: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.minRows); } if (g_Dbs.db[i].dbCfg.maxRows > 0) { - printf(" maxRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows); + printf(" maxRows: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.maxRows); } if (g_Dbs.db[i].dbCfg.comp > 0) { printf(" comp: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp); } if (g_Dbs.db[i].dbCfg.walLevel > 0) { - printf(" walLevel: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel); + printf(" walLevel: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.walLevel); } if (g_Dbs.db[i].dbCfg.fsync > 0) { - printf(" fsync: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync); + printf(" fsync: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.fsync); } if (g_Dbs.db[i].dbCfg.quorum > 0) { - printf(" quorum: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum); + printf(" quorum: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.quorum); } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) @@ -1543,21 +1561,26 @@ static void printfInsertMetaToFile(FILE* fp) { if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { - fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); + fprintf(fp, " precision: %s\n", + g_Dbs.db[i].dbCfg.precision); } else { - fprintf(fp, " precision error: %s\n", g_Dbs.db[i].dbCfg.precision); + fprintf(fp, " precision error: %s\n", + g_Dbs.db[i].dbCfg.precision); } } - fprintf(fp, " super table count: %"PRIu64"\n", g_Dbs.db[i].superTblCount); + fprintf(fp, " super table count: %"PRIu64"\n", + g_Dbs.db[i].superTblCount); for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { fprintf(fp, " super table[%d]:\n", j); - fprintf(fp, " stbName: %s\n", g_Dbs.db[i].superTbls[j].sTblName); + fprintf(fp, " stbName: %s\n", + g_Dbs.db[i].superTbls[j].sTblName); if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { fprintf(fp, " autoCreateTable: %s\n", "no"); - } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + } else if (AUTO_CREATE_SUBTBL + == g_Dbs.db[i].superTbls[j].autoCreateTable) { fprintf(fp, " autoCreateTable: %s\n", "yes"); } else { fprintf(fp, " autoCreateTable: %s\n", "error"); @@ -1565,7 +1588,8 @@ static void printfInsertMetaToFile(FILE* fp) { if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { fprintf(fp, " childTblExists: %s\n", "no"); - } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + } else if (TBL_ALREADY_EXISTS + == g_Dbs.db[i].superTbls[j].childTblExists) { fprintf(fp, " childTblExists: %s\n", "yes"); } else { fprintf(fp, " childTblExists: %s\n", "error"); @@ -1596,8 +1620,10 @@ static void printfInsertMetaToFile(FILE* fp) { */ fprintf(fp, " interlaceRows: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].interlaceRows); - fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); - fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); + fprintf(fp, " disorderRange: %d\n", + g_Dbs.db[i].superTbls[j].disorderRange); + fprintf(fp, " disorderRatio: %d\n", + g_Dbs.db[i].superTbls[j].disorderRatio); fprintf(fp, " maxSqlLen: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].maxSqlLen); @@ -1605,23 +1631,29 @@ static void printfInsertMetaToFile(FILE* fp) { g_Dbs.db[i].superTbls[j].timeStampStep); fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); - fprintf(fp, " sampleFormat: %s\n", g_Dbs.db[i].superTbls[j].sampleFormat); - fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); - fprintf(fp, " tagsFile: %s\n", g_Dbs.db[i].superTbls[j].tagsFile); + fprintf(fp, " sampleFormat: %s\n", + g_Dbs.db[i].superTbls[j].sampleFormat); + fprintf(fp, " sampleFile: %s\n", + g_Dbs.db[i].superTbls[j].sampleFile); + fprintf(fp, " tagsFile: %s\n", + g_Dbs.db[i].superTbls[j].tagsFile); - fprintf(fp, " columnCount: %d\n ", g_Dbs.db[i].superTbls[j].columnCount); + fprintf(fp, " columnCount: %d\n ", + g_Dbs.db[i].superTbls[j].columnCount); for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); if ((0 == strncasecmp( g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", strlen("binary"))) - || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + || (0 == strncasecmp( + g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", strlen("nchar")))) { fprintf(fp, "column[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); } else { - fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); + fprintf(fp, "column[%d]:%s ", + k, g_Dbs.db[i].superTbls[j].columns[k].dataType); } } fprintf(fp, "\n"); @@ -1634,7 +1666,8 @@ static void printfInsertMetaToFile(FILE* fp) { "binary", strlen("binary"))) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", strlen("nchar")))) { - fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, + fprintf(fp, "tag[%d]:%s(%d) ", + k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); } else { fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); @@ -4128,7 +4161,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { "query_times"); if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { if (specifiedQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", + errorPrint( + "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", __func__, __LINE__, specifiedQueryTimes->valueint); goto PARSE_OVER; @@ -4145,7 +4179,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent"); if (concurrent && concurrent->type == cJSON_Number) { if (concurrent->valueint <= 0) { - errorPrint("%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n", + errorPrint( + "%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n", __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount, g_queryInfo.specifiedQueryInfo.concurrent); @@ -4184,15 +4219,15 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart"); if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { if (0 == strcmp("yes", restart->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeRestart = 1; + g_queryInfo.specifiedQueryInfo.subscribeRestart = true; } else if (0 == strcmp("no", restart->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeRestart = 0; + g_queryInfo.specifiedQueryInfo.subscribeRestart = false; } else { printf("ERROR: failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { - g_queryInfo.specifiedQueryInfo.subscribeRestart = 1; + g_queryInfo.specifiedQueryInfo.subscribeRestart = true; } cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress"); @@ -4237,13 +4272,29 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { printf("ERROR: failed to read json, sql not found\n"); goto PARSE_OVER; } - tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], + sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + + cJSON* resubAfterConsume = + cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume"); + if (resubAfterConsume + && resubAfterConsume->type == cJSON_Number) { + g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] + = resubAfterConsume->valueint; + } else if (!resubAfterConsume) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = 1; + } cJSON *result = cJSON_GetObjectItem(sql, "result"); - if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) { - tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + if ((NULL != result) && (result->type == cJSON_String) + && (result->valuestring != NULL)) { + tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], + result->valuestring, MAX_FILE_NAME_LEN); } else if (NULL == result) { - memset(g_queryInfo.specifiedQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + memset(g_queryInfo.specifiedQueryInfo.result[j], + 0, MAX_FILE_NAME_LEN); } else { printf("ERROR: failed to read json, super query result file not found\n"); goto PARSE_OVER; @@ -4350,27 +4401,27 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) { if (0 == strcmp("yes", subrestart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = 1; + g_queryInfo.superQueryInfo.subscribeRestart = true; } else if (0 == strcmp("no", subrestart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = 0; + g_queryInfo.superQueryInfo.subscribeRestart = false; } else { printf("ERROR: failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { - g_queryInfo.superQueryInfo.subscribeRestart = 1; + g_queryInfo.superQueryInfo.subscribeRestart = true; } - cJSON* subkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); - if (subkeepProgress && - subkeepProgress->type == cJSON_String - && subkeepProgress->valuestring != NULL) { - if (0 == strcmp("yes", subkeepProgress->valuestring)) { + cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); + if (superkeepProgress && + superkeepProgress->type == cJSON_String + && superkeepProgress->valuestring != NULL) { + if (0 == strcmp("yes", superkeepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; - } else if (0 == strcmp("no", subkeepProgress->valuestring)) { + } else if (0 == strcmp("no", superkeepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } else { - printf("ERROR: failed to read json, subscribe keepProgress error\n"); + printf("ERROR: failed to read json, subscribe super table keepProgress error\n"); goto PARSE_OVER; } } else { @@ -4408,6 +4459,18 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + cJSON* superResubAfterConsume = + cJSON_GetObjectItem(sql, "resubAfterConsume"); + if (superResubAfterConsume + && superResubAfterConsume->type == cJSON_Number) { + g_queryInfo.superQueryInfo.resubAfterConsume[j] = + superResubAfterConsume->valueint; + } else if (!superResubAfterConsume) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.superQueryInfo.resubAfterConsume[j] = 1; + } + cJSON *result = cJSON_GetObjectItem(sql, "result"); if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ @@ -5506,7 +5569,8 @@ static void callBack(void *param, TAOS_RES *res, int code) { int rand_num = taosRandom() % 100; if (0 != pThreadInfo->superTblInfo->disorderRatio && rand_num < pThreadInfo->superTblInfo->disorderRatio) { - int64_t d = pThreadInfo->lastTs - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); + int64_t d = pThreadInfo->lastTs + - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); generateRowData(data, d, pThreadInfo->superTblInfo); } else { generateRowData(data, pThreadInfo->lastTs += 1000, pThreadInfo->superTblInfo); @@ -6480,17 +6544,18 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c } static TAOS_SUB* subscribeImpl( - TAOS *taos, char *sql, char* topic, char* resultFileName) { + TAOS *taos, char *sql, char* topic, bool restart, + char* resultFileName) { TAOS_SUB* tsub = NULL; if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { tsub = taos_subscribe(taos, - g_queryInfo.specifiedQueryInfo.subscribeRestart, + restart, topic, sql, subscribe_callback, (void*)resultFileName, g_queryInfo.specifiedQueryInfo.subscribeInterval); } else { tsub = taos_subscribe(taos, - g_queryInfo.specifiedQueryInfo.subscribeRestart, + restart, topic, sql, NULL, NULL, 0); } @@ -6551,7 +6616,9 @@ static void *superSubscribe(void *sarg) { uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", __func__, __LINE__, subSeq, subSqlstr); - tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile); + tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart, + tmpFile); if (NULL == tsub[subSeq]) { taos_close(pThreadInfo->taos); return NULL; @@ -6559,17 +6626,22 @@ static void *superSubscribe(void *sarg) { } } + int consumed[MAX_QUERY_SQL_COUNT]; + for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) + consumed[i] = 0; + // start loop to consume result TAOS_RES* res = NULL; while(1) { - for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to; i++) { for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { continue; } uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - taosMsleep(100); // ms + taosMsleep(g_queryInfo.superQueryInfo.subscribeInterval); // ms res = taos_consume(tsub[subSeq]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; @@ -6579,6 +6651,28 @@ static void *superSubscribe(void *sarg) { pThreadInfo->threadID); appendResultToFile(res, tmpFile); } + consumed[j] ++; + + if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) + && (consumed[j] >= + g_queryInfo.superQueryInfo.resubAfterConsume[j])) { + printf("keepProgress:%d, resub super table query: %d\n", + g_queryInfo.superQueryInfo.subscribeKeepProgress, j); + taos_unsubscribe(tsub[subSeq], + g_queryInfo.superQueryInfo.subscribeKeepProgress); + consumed[j]= 0; + uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", + __func__, __LINE__, subSeq, subSqlstr); + tsub[subSeq] = subscribeImpl( + pThreadInfo->taos, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart, + tmpFile); + if (NULL == tsub[subSeq]) { + taos_close(pThreadInfo->taos); + return NULL; + } + } } } } @@ -6589,8 +6683,7 @@ static void *superSubscribe(void *sarg) { i <= pThreadInfo->end_table_to; i++) { for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - taos_unsubscribe(tsub[subSeq], - g_queryInfo.superQueryInfo.subscribeKeepProgress); + taos_unsubscribe(tsub[subSeq], 0); } } @@ -6635,40 +6728,68 @@ static void *specifiedSubscribe(void *sarg) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); + g_queryInfo.specifiedQueryInfo.result[i], + pThreadInfo->threadID); } tsub[i] = subscribeImpl(pThreadInfo->taos, - g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile); + g_queryInfo.specifiedQueryInfo.sql[i], topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart, + tmpFile); if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); return NULL; } } + // start loop to consume result TAOS_RES* res = NULL; + + int consumed[MAX_QUERY_SQL_COUNT]; + for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) + consumed[i] = 0; + while(1) { for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { continue; } - taosMsleep(1000); // ms + taosMsleep(g_queryInfo.specifiedQueryInfo.subscribeInterval); // ms res = taos_consume(tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); + g_queryInfo.specifiedQueryInfo.result[i], + pThreadInfo->threadID); appendResultToFile(res, tmpFile); } + consumed[i] ++; + + if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) + && (consumed[i] >= + g_queryInfo.specifiedQueryInfo.resubAfterConsume[i])) { + printf("keepProgress:%d, resub specified query: %d\n", + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, i); + consumed[i] = 0; + taos_unsubscribe(tsub[i], + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + tsub[i] = subscribeImpl(pThreadInfo->taos, + g_queryInfo.specifiedQueryInfo.sql[i], topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart, + tmpFile); + if (NULL == tsub[i]) { + taos_close(pThreadInfo->taos); + return NULL; + } + } } } } taos_free_result(res); for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - taos_unsubscribe(tsub[i], - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + taos_unsubscribe(tsub[i], 0); } taos_close(pThreadInfo->taos); @@ -6719,8 +6840,10 @@ static int subscribeTestProcess() { exit(-1); } - pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); - infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); + pids = malloc( + g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc( + g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); exit(-1); @@ -6905,17 +7028,21 @@ static void setParaFromArg(){ if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; } else { - for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { - tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE); + for (int i = g_Dbs.db[0].superTbls[0].columnCount; + i < g_args.num_of_CPR; i++) { + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + "INT", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; g_Dbs.db[0].superTbls[0].columnCount++; } } - tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, + "INT", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; - tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, + "BINARY", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { From 8e5681f1b004cb8b5d3982fb36f0ffd4d712e2ca Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 18 May 2021 21:36:35 +0800 Subject: [PATCH 115/157] [TD-4240]: taosdemo subscribe more than max query sql count. (#6154) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 08c1142070..92b5f6abff 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6575,6 +6575,15 @@ static void *superSubscribe(void *sarg) { if (g_queryInfo.superQueryInfo.sqlCount == 0) return NULL; + if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { + errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n", + g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables, + g_queryInfo.superQueryInfo.sqlCount, + pThreadInfo->ntables, + MAX_QUERY_SQL_COUNT); + exit(-1); + } + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, From 4c1e13b60e3f50d1d0c38bb6f774ae4981b462b5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 18 May 2021 21:43:28 +0800 Subject: [PATCH 116/157] Revert "Hotfix/sangshuduo/td 4240 taosdemo subscribe morethan 100 (#6155)" (#6156) This reverts commit e8d2162cce72044cc42ac8baa505a0e5c9880615. --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 6 +-- src/client/src/tscSchemaUtil.c | 1 - src/client/src/tscServer.c | 28 ++++++----- src/client/src/tscUtil.c | 21 ++++---- src/kit/taosdemo/taosdemo.c | 57 +++++++++------------- src/mnode/src/mnodeDb.c | 2 +- src/vnode/src/vnodeMain.c | 12 +---- src/vnode/src/vnodeRead.c | 7 +-- src/vnode/src/vnodeStatus.c | 3 -- src/vnode/src/vnodeSync.c | 11 ----- src/vnode/src/vnodeWrite.c | 2 - tests/pytest/tools/taosdemoTestWithJson.py | 22 ++++----- tests/script/general/parser/commit.sim | 2 +- 14 files changed, 68 insertions(+), 108 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 0348606d6b..56d595ff1f 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -307,7 +307,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild); uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); uint32_t tscGetTableMetaMaxSize(); -int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf); +int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf); STableMeta* tscTableMetaDup(STableMeta* pTableMeta); int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 81443360f4..bf41449e13 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -68,16 +68,14 @@ typedef struct CChildTableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; // TODO: refactor super table name, not full name - uint64_t suid; // super table id + char sTableName[TSDB_TABLE_FNAME_LEN]; //super table name, not full name } CChildTableMeta; typedef struct STableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; // super table name - uint64_t suid; // super table id + char sTableName[TSDB_TABLE_FNAME_LEN]; int16_t sversion; int16_t tversion; STableComInfo tableInfo; diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 114fc8ee73..2ea382132b 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -94,7 +94,6 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { pTableMeta->tableType = pTableMetaMsg->tableType; pTableMeta->vgId = pTableMetaMsg->vgroup.vgId; - pTableMeta->suid = pTableMetaMsg->suid; pTableMeta->tableInfo = (STableComInfo) { .numOfTags = pTableMetaMsg->numOfTags, diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9b7d5c0c7f..6587a55152 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1828,13 +1828,13 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscProcessTableMetaRsp(SSqlObj *pSql) { STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - pMetaMsg->tid = htonl(pMetaMsg->tid); - pMetaMsg->sversion = htons(pMetaMsg->sversion); - pMetaMsg->tversion = htons(pMetaMsg->tversion); + pMetaMsg->tid = htonl(pMetaMsg->tid); + pMetaMsg->sversion = htons(pMetaMsg->sversion); + pMetaMsg->tversion = htons(pMetaMsg->tversion); pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); - pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->suid = pMetaMsg->suid; - pMetaMsg->contLen = htons(pMetaMsg->contLen); + + pMetaMsg->uid = htobe64(pMetaMsg->uid); + pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && @@ -2448,16 +2448,19 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { pTableMetaInfo->pTableMeta = calloc(1, size); pTableMetaInfo->tableMetaSize = size; } else if (pTableMetaInfo->tableMetaSize < size) { - char *tmp = realloc(pTableMetaInfo->pTableMeta, size); - if (tmp == NULL) { + char *tmp = realloc(pTableMetaInfo->pTableMeta, size); + if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } pTableMetaInfo->pTableMeta = (STableMeta *)tmp; + memset(pTableMetaInfo->pTableMeta, 0, size); + pTableMetaInfo->tableMetaSize = size; + } else { + //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); + memset(pTableMetaInfo->pTableMeta, 0, size); + pTableMetaInfo->tableMetaSize = size; } - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; - pTableMetaInfo->pTableMeta->tableType = -1; pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; @@ -2473,9 +2476,8 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { STableMeta* pMeta = pTableMetaInfo->pTableMeta; if (pMeta->id.uid > 0) { - // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, buf); + int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, name, buf); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 37e836dc1c..a698e239df 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3370,25 +3370,22 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { assert(pTableMeta != NULL); CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); - cMeta->tableType = TSDB_CHILD_TABLE; - cMeta->vgId = pTableMeta->vgId; - cMeta->id = pTableMeta->id; - cMeta->suid = pTableMeta->suid; + cMeta->vgId = pTableMeta->vgId; + cMeta->id = pTableMeta->id; tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); return cMeta; } -int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf) { +int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf) { assert(pChild != NULL && buf != NULL); - STableMeta* p = buf; - taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); +// uint32_t size = tscGetTableMetaMaxSize(); + STableMeta* p = buf;//calloc(1, size); - // tableMeta exists, build child table meta according to the super table meta - // the uid need to be checked in addition to the general name of the super table. - if (p->id.uid > 0 && pChild->suid == p->id.uid) { + taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); + if (p->id.uid > 0) { // tableMeta exists, build child table meta and return pChild->sversion = p->sversion; pChild->tversion = p->tversion; @@ -3396,9 +3393,13 @@ int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, v int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); + +// tfree(p); return TSDB_CODE_SUCCESS; } else { // super table has been removed, current tableMeta is also expired. remove it here taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + +// tfree(p); return -1; } } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 658093a385..0d5d370bf5 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -526,8 +526,6 @@ static int taosRandom() #endif // ifdef Windows -static void prompt(); -static void prompt2(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); @@ -1064,8 +1062,10 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { printf("# Print debug info: %d\n", arguments->debug_print); printf("# Print verbose info: %d\n", arguments->verbose_print); printf("###################################################################\n"); - - prompt(); + if (!arguments->answer_yes) { + printf("Press enter key to continue\n\n"); + (void) getchar(); + } } } @@ -3446,7 +3446,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.interlace_rows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - prompt2(); + if (!g_args.answer_yes) { + printf(" press Enter key to continue or Ctrl-C to stop."); + (void)getchar(); + } g_args.interlace_rows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -3503,9 +3506,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.answer_yes = false; } } else if (!answerPrompt) { - g_args.answer_yes = true; // default is no, mean answer_yes. + g_args.answer_yes = false; } else { - errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n"); + printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); goto PARSE_OVER; } @@ -5392,7 +5395,6 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int64_t start_time = pThreadInfo->start_time; int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; - verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); for (uint64_t i = 0; i < insertRows;) { @@ -6093,21 +6095,6 @@ static void *readMetric(void *sarg) { return NULL; } -static void prompt() -{ - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } -} - -static void prompt2() -{ - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } -} static int insertTestProcess() { @@ -6128,7 +6115,10 @@ static int insertTestProcess() { if (g_fpOfInsertResult) printfInsertMetaToFile(g_fpOfInsertResult); - prompt(); + if (!g_args.answer_yes) { + printf("Press enter key to continue\n\n"); + (void)getchar(); + } init_rand_data(); @@ -6400,7 +6390,10 @@ static int queryTestProcess() { &g_queryInfo.superQueryInfo.childTblCount); } - prompt(); + if (!g_args.answer_yes) { + printf("Press enter key to continue\n\n"); + (void)getchar(); + } if (g_args.debug_print || g_args.verbose_print) { printfQuerySystemInfo(taos); @@ -6583,15 +6576,6 @@ static void *superSubscribe(void *sarg) { if (g_queryInfo.superQueryInfo.sqlCount == 0) return NULL; - if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { - errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n", - g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables, - g_queryInfo.superQueryInfo.sqlCount, - pThreadInfo->ntables, - MAX_QUERY_SQL_COUNT); - exit(-1); - } - if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, @@ -6762,7 +6746,10 @@ static int subscribeTestProcess() { printfQueryMeta(); resetAfterAnsiEscape(); - prompt(); + if (!g_args.answer_yes) { + printf("Press enter key to continue\n\n"); + (void) getchar(); + } TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 7cc5d41ae1..8af20aa862 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -389,7 +389,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->compression < 0) pCfg->compression = tsCompression; if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL; if (pCfg->replications < 0) pCfg->replications = tsReplications; - if (pCfg->quorum < 0) pCfg->quorum = MIN(tsQuorum, pCfg->replications); + if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; if (pCfg->update < 0) pCfg->update = tsUpdate; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow; if (pCfg->dbType < 0) pCfg->dbType = 0; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index f133101cca..ca3746c598 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -120,14 +120,12 @@ int32_t vnodeDrop(int32_t vgId) { vDebug("vgId:%d, failed to drop, vnode not find", vgId); return TSDB_CODE_VND_INVALID_VGROUP_ID; } - if (pVnode->dropped) { - vnodeRelease(pVnode); - return TSDB_CODE_SUCCESS; - } vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); pVnode->dropped = 1; + // remove from hash, so new messages wont be consumed + vnodeRemoveFromHash(pVnode); vnodeRelease(pVnode); vnodeCleanupInMWorker(pVnode); @@ -427,10 +425,6 @@ int32_t vnodeOpen(int32_t vgId) { int32_t vnodeClose(int32_t vgId) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) return 0; - if (pVnode->dropped) { - vnodeRelease(pVnode); - return 0; - } vDebug("vgId:%d, vnode will be closed, pVnode:%p", pVnode->vgId, pVnode); vnodeRemoveFromHash(pVnode); @@ -516,8 +510,6 @@ void vnodeCleanUp(SVnodeObj *pVnode) { vnodeSetClosingStatus(pVnode); - vnodeRemoveFromHash(pVnode); - // stop replication module if (pVnode->sync > 0) { int64_t sync = pVnode->sync; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 2831090267..b28bdbf130 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -117,17 +117,14 @@ static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t con } int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { - SVnodeObj *pVnode = vparam; - if (pVnode->dropped) { - return TSDB_CODE_APP_NOT_READY; - } - SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam); if (pRead == NULL) { assert(terrno != 0); return terrno; } + SVnodeObj *pVnode = vparam; + int32_t code = vnodeCheckRead(pVnode); if (code != TSDB_CODE_SUCCESS) { taosFreeQitem(pRead); diff --git a/src/vnode/src/vnodeStatus.c b/src/vnode/src/vnodeStatus.c index 1eaddc3d25..c482d1fd1a 100644 --- a/src/vnode/src/vnodeStatus.c +++ b/src/vnode/src/vnodeStatus.c @@ -66,9 +66,6 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) { } bool vnodeSetClosingStatus(SVnodeObj* pVnode) { - if (pVnode->status == TAOS_VN_STATUS_CLOSING) - return true; - while (!vnodeSetClosingStatusImp(pVnode)) { taosMsleep(1); } diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index e5a1964915..05af34a34f 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -55,11 +55,6 @@ void vnodeNotifyRole(int32_t vgId, int8_t role) { vTrace("vgId:%d, vnode not found while notify role", vgId); return; } - if (pVnode->dropped) { - vTrace("vgId:%d, vnode dropped while notify role", vgId); - vnodeRelease(pVnode); - return; - } vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); pVnode->role = role; @@ -80,11 +75,6 @@ void vnodeCtrlFlow(int32_t vgId, int32_t level) { vTrace("vgId:%d, vnode not found while flow ctrl", vgId); return; } - if (pVnode->dropped) { - vTrace("vgId:%d, vnode dropped while flow ctrl", vgId); - vnodeRelease(pVnode); - return; - } if (pVnode->flowctrlLevel != level) { vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level); @@ -139,7 +129,6 @@ int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rpara SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while write to cache", vgId); - vnodeRelease(pVnode); return TSDB_CODE_VND_INVALID_VGROUP_ID; } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 56ea32ccc0..36516d81df 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -386,6 +386,4 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { vTrace("vgId:%d, queued wmsg num:%d", pVnode->vgId, pVnode->queuedWMsg); taosMsleep(10); } - - taosMsleep(900); } diff --git a/tests/pytest/tools/taosdemoTestWithJson.py b/tests/pytest/tools/taosdemoTestWithJson.py index b2ecd54976..f57af9ce5c 100644 --- a/tests/pytest/tools/taosdemoTestWithJson.py +++ b/tests/pytest/tools/taosdemoTestWithJson.py @@ -23,32 +23,32 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) - if "community" in selfPath: - projPath = selfPath[: selfPath.find("community")] + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] else: - projPath = selfPath[: selfPath.find("tests")] + projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if "taosd" in files: + if ("taosd" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) - if "packaging" not in rootRealPath: - buildPath = root[: len(root) - len("/build/bin")] + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] break return buildPath - + def run(self): tdSql.prepare() buildPath = self.getBuildPath() - if buildPath == "": + if (buildPath == ""): tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath + "/build/bin/" - os.system("%staosdemo -f tools/insert.json -y" % binPath) + binPath = buildPath+ "/build/bin/" + os.system("yes | %staosdemo -f tools/insert.json" % binPath) tdSql.execute("use db01") tdSql.query("select count(*) from stb01") diff --git a/tests/script/general/parser/commit.sim b/tests/script/general/parser/commit.sim index 7c4c883fb1..dfe521b92b 100644 --- a/tests/script/general/parser/commit.sim +++ b/tests/script/general/parser/commit.sim @@ -68,7 +68,7 @@ while $loop <= $loops while $i < 10 sql select count(*) from $stb where t1 = $i if $data00 != $rowNum then - print expect $rowNum , actual: $data00 + print expect $rowNum, actual: $data00 return -1 endi $i = $i + 1 From 4cc471bb41c09d16e69b39a737bdbd2b594564e0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 19 May 2021 09:39:25 +0800 Subject: [PATCH 117/157] [TD-4240]: taosdemo subscribe more than max query sql count. (#6157) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 0d5d370bf5..abdbb9a316 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6576,6 +6576,15 @@ static void *superSubscribe(void *sarg) { if (g_queryInfo.superQueryInfo.sqlCount == 0) return NULL; + if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { + errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n", + g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables, + g_queryInfo.superQueryInfo.sqlCount, + pThreadInfo->ntables, + MAX_QUERY_SQL_COUNT); + exit(-1); + } + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, From a06be90f6f305122bef6101c8dd39b8d63d9f1bf Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Wed, 19 May 2021 11:17:44 +0800 Subject: [PATCH 118/157] [TD-4228]: fix install error when grafana submodule not ready (#6151) * [TD-4228]: fix install error when grafana submodule not ready * [TD-4228]: error or warn while lost grafanaplugin or go submodule --- packaging/deb/makedeb.sh | 7 ++++++- packaging/rpm/tdengine.spec | 7 ++++++- packaging/tools/make_install.sh | 12 ++++++++++-- packaging/tools/makeclient.sh | 14 +++++++++++--- packaging/tools/makeclient_power.sh | 14 +++++++++++--- packaging/tools/makepkg.sh | 14 +++++++++++--- packaging/tools/makepkg_power.sh | 13 +++++++++++-- 7 files changed, 66 insertions(+), 15 deletions(-) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index a593c7ca16..28be037e6c 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -58,7 +58,12 @@ cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_pat cp ${compile_dir}/../src/inc/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../src/inc/taoserror.h ${pkg_dir}${install_home_path}/include cp -r ${top_dir}/tests/examples/* ${pkg_dir}${install_home_path}/examples -cp -r ${top_dir}/src/connector/grafanaplugin/dist ${pkg_dir}${install_home_path}/connector/grafanaplugin +if [ -d "${top_dir}/src/connector/grafanaplugin/dist" ]; then + cp -r ${top_dir}/src/connector/grafanaplugin/dist ${pkg_dir}${install_home_path}/connector/grafanaplugin +else + echo "grafanaplugin bundled directory not found!" + exit 1 +fi cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 73efe2e1ab..9910e20bfe 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -66,7 +66,12 @@ cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include -cp -r %{_compiledir}/../src/connector/grafanaplugin/dist %{buildroot}%{homepath}/connector/grafanaplugin +if [ -d %{_compiledir}/../src/connector/grafanaplugin/dist ]; then + cp -r %{_compiledir}/../src/connector/grafanaplugin/dist %{buildroot}%{homepath}/connector/grafanaplugin +else + echo grafanaplugin bundled directory not found! + exit 1 +fi cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 5f1739d6e0..d6ace0a063 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -243,9 +243,17 @@ function install_data() { } function install_connector() { - ${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin/dist ${install_main_dir}/connector/grafanaplugin + if [ -d "${source_dir}/src/connector/grafanaplugin/dist" ]; then + ${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin/dist ${install_main_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" + fi + if find ${source_dir}/src/connector/go -mindepth 1 -maxdepth 1 | read; then + ${csudo} cp -r ${source_dir}/src/connector/go ${install_main_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector - ${csudo} cp -rf ${source_dir}/src/connector/go ${install_main_dir}/connector ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null } diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 7418bcc312..d0eeffc86a 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -117,9 +117,17 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector cp -r ${connector_dir}/nodejs ${install_dir}/connector fi # Copy release note diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index 911cc574a5..8241319e4f 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -144,9 +144,17 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bunlded dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/cinterface.py diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 343e2a4224..d114d5eef8 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -131,9 +131,17 @@ connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if you want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector cp -r ${connector_dir}/nodejs ${install_dir}/connector fi # Copy release note diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index 8ea7f363c8..633a135c14 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -166,9 +166,18 @@ connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/cinterface.py From 6f1b157b178f432f570d795c08cbec5c34eedbfb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 19 May 2021 13:48:05 +0800 Subject: [PATCH 119/157] Hotfix/sangshuduo/td 4187 taosdemo keepprogress for develop (#6158) * [TD-4240]: taosdemo subscribe more than max query sql count. * add consumed. * [TD-4187]: taosdemo keepProgress. add resubAfterConsume process. * [TD-4187]: taosdemo support keepProgress. * [TD-4187]: taosdemo support keepProgress. for develop branch. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/subscribe.json | 38 +++++++++--- src/kit/taosdemo/taosdemo.c | 107 ++++++++++++++++++++++++++++---- 2 files changed, 123 insertions(+), 22 deletions(-) diff --git a/src/kit/taosdemo/subscribe.json b/src/kit/taosdemo/subscribe.json index fd33a2e2e2..18846c8116 100644 --- a/src/kit/taosdemo/subscribe.json +++ b/src/kit/taosdemo/subscribe.json @@ -5,13 +5,33 @@ "port": 6030, "user": "root", "password": "taosdata", - "databases": "dbx", - "specified_table_query": - {"concurrent":1, "mode":"sync", "interval":5000, "restart":"yes", "keepProgress":"yes", - "sqls": [{"sql": "select avg(col1) from stb01 where col1 > 1;", "result": "./subscribe_res0.txt"}] - }, - "super_table_query": - {"stblname": "stb", "threads":1, "mode":"sync", "interval":10000, "restart":"yes", "keepProgress":"yes", - "sqls": [{"sql": "select col1 from xxxx where col1 > 10;", "result": "./subscribe_res1.txt"}] - } + "databases": "test", + "specified_table_query": { + "concurrent": 1, + "mode": "sync", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "resubAfterConsume": 10, + "sqls": [ + { + "sql": "select avg(col1) from meters where col1 > 1;", + "result": "./subscribe_res0.txt" + } + ] + }, + "super_table_query": { + "stblname": "meters", + "threads": 1, + "mode": "sync", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "sqls": [ + { + "sql": "select col1 from xxxx where col1 > 10;", + "result": "./subscribe_res1.txt" + } + ] + } } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index abdbb9a316..95fb2f7bd4 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -372,6 +372,7 @@ typedef struct SpecifiedQueryInfo_S { int subscribeKeepProgress; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + int resubAfterConsume[MAX_QUERY_SQL_COUNT]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; uint64_t totalQueried; } SpecifiedQueryInfo; @@ -390,6 +391,7 @@ typedef struct SuperQueryInfo_S { uint64_t sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + int resubAfterConsume[MAX_QUERY_SQL_COUNT]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; char* childTblName; @@ -407,7 +409,7 @@ typedef struct SQueryMetaInfo_S { char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest SpecifiedQueryInfo specifiedQueryInfo; - SuperQueryInfo superQueryInfo; + SuperQueryInfo superQueryInfo; uint64_t totalQueried; } SQueryMetaInfo; @@ -4287,6 +4289,18 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + cJSON* resubAfterConsume = + cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume"); + if (resubAfterConsume + && resubAfterConsume->type == cJSON_Number) { + g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] + = resubAfterConsume->valueint; + } else if (!resubAfterConsume) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = 1; + } + cJSON *result = cJSON_GetObjectItem(sql, "result"); if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) { tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); @@ -4456,6 +4470,18 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + cJSON* superResubAfterConsume = + cJSON_GetObjectItem(sql, "resubAfterConsume"); + if (superResubAfterConsume + && superResubAfterConsume->type == cJSON_Number) { + g_queryInfo.superQueryInfo.resubAfterConsume[j] = + superResubAfterConsume->valueint; + } else if (!superResubAfterConsume) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.superQueryInfo.resubAfterConsume[j] = 1; + } + cJSON *result = cJSON_GetObjectItem(sql, "result"); if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ @@ -6546,17 +6572,18 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c } static TAOS_SUB* subscribeImpl( - TAOS *taos, char *sql, char* topic, char* resultFileName) { + TAOS *taos, char *sql, char* topic, bool restart, + char* resultFileName) { TAOS_SUB* tsub = NULL; if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { tsub = taos_subscribe(taos, - g_queryInfo.specifiedQueryInfo.subscribeRestart, + restart, topic, sql, subscribe_callback, (void*)resultFileName, g_queryInfo.specifiedQueryInfo.subscribeInterval); } else { tsub = taos_subscribe(taos, - g_queryInfo.specifiedQueryInfo.subscribeRestart, + restart, topic, sql, NULL, NULL, 0); } @@ -6610,6 +6637,7 @@ static void *superSubscribe(void *sarg) { return NULL; } + uint64_t subSeq; char topic[32] = {0}; for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { @@ -6623,10 +6651,12 @@ static void *superSubscribe(void *sarg) { g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", __func__, __LINE__, subSeq, subSqlstr); - tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile); + tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart, + tmpFile); if (NULL == tsub[subSeq]) { taos_close(pThreadInfo->taos); return NULL; @@ -6634,6 +6664,10 @@ static void *superSubscribe(void *sarg) { } } + int consumed[MAX_QUERY_SQL_COUNT]; + for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) + consumed[i] = 0; + // start loop to consume result TAOS_RES* res = NULL; while(1) { @@ -6643,7 +6677,7 @@ static void *superSubscribe(void *sarg) { continue; } - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; taosMsleep(100); // ms res = taos_consume(tsub[subSeq]); if (res) { @@ -6654,6 +6688,28 @@ static void *superSubscribe(void *sarg) { pThreadInfo->threadID); appendResultToFile(res, tmpFile); } + consumed[j] ++; + + if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) + && (consumed[j] >= + g_queryInfo.superQueryInfo.resubAfterConsume[j])) { + printf("keepProgress:%d, resub super table query: %d\n", + g_queryInfo.superQueryInfo.subscribeKeepProgress, j); + taos_unsubscribe(tsub[subSeq], + g_queryInfo.superQueryInfo.subscribeKeepProgress); + consumed[j]= 0; + subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", + __func__, __LINE__, subSeq, subSqlstr); + tsub[subSeq] = subscribeImpl( + pThreadInfo->taos, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart, + tmpFile); + if (NULL == tsub[subSeq]) { + taos_close(pThreadInfo->taos); + return NULL; + } + } } } } @@ -6663,9 +6719,8 @@ static void *superSubscribe(void *sarg) { for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - taos_unsubscribe(tsub[subSeq], - g_queryInfo.superQueryInfo.subscribeKeepProgress); + subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + taos_unsubscribe(tsub[subSeq], 0); } } @@ -6713,14 +6768,22 @@ static void *specifiedSubscribe(void *sarg) { g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); } tsub[i] = subscribeImpl(pThreadInfo->taos, - g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile); + g_queryInfo.specifiedQueryInfo.sql[i], topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart, + tmpFile); if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); return NULL; } } + // start loop to consume result TAOS_RES* res = NULL; + + int consumed[MAX_QUERY_SQL_COUNT]; + for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) + consumed[i] = 0; + while(1) { for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { @@ -6736,14 +6799,32 @@ static void *specifiedSubscribe(void *sarg) { g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); appendResultToFile(res, tmpFile); } + consumed[i] ++; + + if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) + && (consumed[i] >= + g_queryInfo.specifiedQueryInfo.resubAfterConsume[i])) { + printf("keepProgress:%d, resub specified query: %d\n", + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, i); + consumed[i] = 0; + taos_unsubscribe(tsub[i], + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + tsub[i] = subscribeImpl(pThreadInfo->taos, + g_queryInfo.specifiedQueryInfo.sql[i], topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart, + tmpFile); + if (NULL == tsub[i]) { + taos_close(pThreadInfo->taos); + return NULL; + } + } } } } taos_free_result(res); for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - taos_unsubscribe(tsub[i], - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + taos_unsubscribe(tsub[i], 0); } taos_close(pThreadInfo->taos); From 5c03dc9ae0f3f2a8efac6d4978878cd69eecd26b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 19 May 2021 16:20:38 +0800 Subject: [PATCH 120/157] Hotfix/sangshuduo/td 4238 taosdemo async subscribe (#6161) * [TD-4238]: taosdemo async subscribe. * [TD-4238]: taosdemo async subscribe subsribe sql command do not use aggregation functions. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/async-sub.json | 41 +++ src/kit/taosdemo/taosdemo.c | 430 +++++++++++++++++--------------- 2 files changed, 267 insertions(+), 204 deletions(-) create mode 100644 src/kit/taosdemo/async-sub.json diff --git a/src/kit/taosdemo/async-sub.json b/src/kit/taosdemo/async-sub.json new file mode 100644 index 0000000000..a30a1be45c --- /dev/null +++ b/src/kit/taosdemo/async-sub.json @@ -0,0 +1,41 @@ +{ + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "test", + "specified_table_query": { + "concurrent": 1, + "mode": "async", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "resubAfterConsume": 10, + "sqls": [ + { + "sql": "select col1 from meters where col1 > 1;", + "result": "./subscribe_res0.txt" + }, + { + "sql": "select col2 from meters where col2 > 1;", + "result": "./subscribe_res2.txt" + } + ] + }, + "super_table_query": { + "stblname": "meters", + "threads": 1, + "mode": "sync", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "sqls": [ + { + "sql": "select col1 from xxxx where col1 > 10;", + "result": "./subscribe_res1.txt" + } + ] + } +} diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 92b5f6abff..0831555d22 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -441,8 +441,9 @@ typedef struct SThreadInfo_S { uint64_t maxDelay; uint64_t minDelay; - // query + // seq of query or subscribe uint64_t querySeq; // sequence number of sql command + } threadInfo; #ifdef WINDOWS @@ -1107,7 +1108,6 @@ static void appendResultBufToFile(char *resultBuf, char *resultFile) } } - fprintf(fp, "%s", resultBuf); tmfclose(fp); } @@ -1150,7 +1150,7 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { } static void selectAndGetResult( - threadInfo *pThreadInfo, char *command, char* resultFile) + threadInfo *pThreadInfo, char *command) { if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) { TAOS_RES *res = taos_query(pThreadInfo->taos, command); @@ -1161,8 +1161,8 @@ static void selectAndGetResult( return; } - if ((resultFile) && (strlen(resultFile))) { - appendResultToFile(res, resultFile); + if ((strlen(pThreadInfo->fp))) { + appendResultToFile(res, pThreadInfo->fp); } taos_free_result(res); @@ -1170,7 +1170,7 @@ static void selectAndGetResult( int retCode = postProceSql( g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port, command, - resultFile); + pThreadInfo->fp); if (0 != retCode) { printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID); } @@ -6230,23 +6230,22 @@ static void *specifiedTableQuery(void *sarg) { uint64_t lastPrintTime = taosGetTimestampMs(); uint64_t startTs = taosGetTimestampMs(); + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + } + while(queryTimes --) { if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) { taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms } - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - } - st = taosGetTimestampMs(); selectAndGetResult(pThreadInfo, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], tmpFile); + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]); et = taosGetTimestampMs(); printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n", @@ -6332,13 +6331,12 @@ static void *superTableQuery(void *sarg) { for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { memset(sqlstr,0,sizeof(sqlstr)); replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(tmpFile, "%s-%d", + sprintf(pThreadInfo->fp, "%s-%d", g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } - selectAndGetResult(pThreadInfo, sqlstr, tmpFile); + selectAndGetResult(pThreadInfo, sqlstr); totalQueried++; g_queryInfo.superQueryInfo.totalQueried ++; @@ -6407,7 +6405,7 @@ static int queryTestProcess() { threadInfo *infos = NULL; //==== create sub threads for query from specify table int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; - int nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount; + uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount; uint64_t startTs = taosGetTimestampMs(); @@ -6421,32 +6419,33 @@ static int queryTestProcess() { ERROR_EXIT("memory allocation failed for create threads\n"); } - for (int i = 0; i < nConcurrent; i++) { - for (int j = 0; j < nSqlCount; j++) { - threadInfo *t_info = infos + i * nSqlCount + j; - t_info->threadID = i * nSqlCount + j; - t_info->querySeq = j; + for (uint64_t i = 0; i < nSqlCount; i++) { + for (int j = 0; j < nConcurrent; j++) { + uint64_t seq = i * nConcurrent + j; + threadInfo *t_info = infos + seq; + t_info->threadID = seq; + t_info->querySeq = i; - if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); - verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); - if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { - taos_close(taos); - free(infos); - free(pids); - errorPrint( "use database %s failed!\n\n", - g_queryInfo.dbName); - return -1; - } + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); + if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(taos); + free(infos); + free(pids); + errorPrint( "use database %s failed!\n\n", + g_queryInfo.dbName); + return -1; + } + } + + t_info->taos = NULL;// TODO: workaround to use separate taos connection; + + pthread_create(pids + seq, NULL, specifiedTableQuery, + t_info); } - - t_info->taos = NULL;// TODO: workaround to use separate taos connection; - - pthread_create(pids + i * nSqlCount + j, NULL, specifiedTableQuery, - t_info); - } } } else { g_queryInfo.specifiedQueryInfo.concurrent = 0; @@ -6531,7 +6530,8 @@ static int queryTestProcess() { return 0; } -static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { +static void stable_sub_callback( + TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { if (res == NULL || taos_errno(res) != 0) { errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", __func__, __LINE__, code, taos_errstr(res)); @@ -6539,22 +6539,44 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c } if (param) - appendResultToFile(res, (char*)param); + appendResultToFile(res, ((threadInfo *)param)->fp); + // tao_unscribe() will free result. +} + +static void specified_sub_callback( + TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { + if (res == NULL || taos_errno(res) != 0) { + errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", + __func__, __LINE__, code, taos_errstr(res)); + return; + } + + if (param) + appendResultToFile(res, ((threadInfo *)param)->fp); // tao_unscribe() will free result. } static TAOS_SUB* subscribeImpl( - TAOS *taos, char *sql, char* topic, bool restart, - char* resultFileName) { + threadInfo *pThreadInfo, + char *sql, char* topic, bool restart) +{ TAOS_SUB* tsub = NULL; if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { - tsub = taos_subscribe(taos, + tsub = taos_subscribe( + pThreadInfo->taos, restart, - topic, sql, subscribe_callback, (void*)resultFileName, + topic, sql, specified_sub_callback, (void*)pThreadInfo, g_queryInfo.specifiedQueryInfo.subscribeInterval); + } else if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { + tsub = taos_subscribe( + pThreadInfo->taos, + restart, + topic, sql, stable_sub_callback, (void*)pThreadInfo, + g_queryInfo.superQueryInfo.subscribeInterval); } else { - tsub = taos_subscribe(taos, + tsub = taos_subscribe( + pThreadInfo->taos, restart, topic, sql, NULL, NULL, 0); } @@ -6572,13 +6594,8 @@ static void *superSubscribe(void *sarg) { char subSqlstr[MAX_QUERY_SQL_LENGTH]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; - if (g_queryInfo.superQueryInfo.sqlCount == 0) - return NULL; - - if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { - errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n", - g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables, - g_queryInfo.superQueryInfo.sqlCount, + if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { + errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n", pThreadInfo->ntables, MAX_QUERY_SQL_COUNT); exit(-1); @@ -6612,88 +6629,80 @@ static void *superSubscribe(void *sarg) { char topic[32] = {0}; for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { - sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%d", i, j); + sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"", + i, pThreadInfo->querySeq); memset(subSqlstr,0,sizeof(subSqlstr)); - replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], subSqlstr, i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); + replaceChildTblName( + g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq], + subSqlstr, i); + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); } - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", - __func__, __LINE__, subSeq, subSqlstr); - tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, - g_queryInfo.superQueryInfo.subscribeRestart, - tmpFile); - if (NULL == tsub[subSeq]) { + debugPrint("%s() LN%d, [%d] subSqlstr: %s\n", + __func__, __LINE__, pThreadInfo->threadID, subSqlstr); + tsub[i] = subscribeImpl( + pThreadInfo, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart); + if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); return NULL; } - } } - int consumed[MAX_QUERY_SQL_COUNT]; - for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) - consumed[i] = 0; - // start loop to consume result + int consumed[MAX_QUERY_SQL_COUNT]; + for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) { + consumed[i] = 0; + } TAOS_RES* res = NULL; + while(1) { for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { continue; } - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; taosMsleep(g_queryInfo.superQueryInfo.subscribeInterval); // ms - res = taos_consume(tsub[subSeq]); + res = taos_consume(tsub[i]); if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[j], + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - appendResultToFile(res, tmpFile); + appendResultToFile(res, pThreadInfo->fp); } - consumed[j] ++; + consumed[i] ++; if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) - && (consumed[j] >= - g_queryInfo.superQueryInfo.resubAfterConsume[j])) { - printf("keepProgress:%d, resub super table query: %d\n", - g_queryInfo.superQueryInfo.subscribeKeepProgress, j); - taos_unsubscribe(tsub[subSeq], + && (consumed[i] >= + g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + printf("keepProgress:%d, resub super table query: %"PRIu64"\n", + g_queryInfo.superQueryInfo.subscribeKeepProgress, + pThreadInfo->querySeq); + taos_unsubscribe(tsub, g_queryInfo.superQueryInfo.subscribeKeepProgress); - consumed[j]= 0; - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", - __func__, __LINE__, subSeq, subSqlstr); - tsub[subSeq] = subscribeImpl( - pThreadInfo->taos, subSqlstr, topic, - g_queryInfo.superQueryInfo.subscribeRestart, - tmpFile); - if (NULL == tsub[subSeq]) { + consumed[i]= 0; + tsub[i] = subscribeImpl( + pThreadInfo, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart + ); + if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); return NULL; } } } - } } } taos_free_result(res); for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - taos_unsubscribe(tsub[subSeq], 0); - } + taos_unsubscribe(tsub[i], 0); } taos_close(pThreadInfo->taos); @@ -6702,10 +6711,7 @@ static void *superSubscribe(void *sarg) { static void *specifiedSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; - TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; - - if (g_queryInfo.specifiedQueryInfo.sqlCount == 0) - return NULL; + TAOS_SUB* tsub = NULL; if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; @@ -6732,76 +6738,61 @@ static void *specifiedSubscribe(void *sarg) { } char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - sprintf(topic, "taosdemo-subscribe-%d", i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], + sprintf(topic, "taosdemo-subscribe-%"PRIu64"", pThreadInfo->querySeq); + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - } - tsub[i] = subscribeImpl(pThreadInfo->taos, - g_queryInfo.specifiedQueryInfo.sql[i], topic, - g_queryInfo.specifiedQueryInfo.subscribeRestart, - tmpFile); - if (NULL == tsub[i]) { - taos_close(pThreadInfo->taos); - return NULL; - } + } + tsub = subscribeImpl(pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], + topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart); + if (NULL == tsub) { + taos_close(pThreadInfo->taos); + return NULL; } // start loop to consume result TAOS_RES* res = NULL; - int consumed[MAX_QUERY_SQL_COUNT]; - for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) - consumed[i] = 0; + int consumed; while(1) { - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { continue; } taosMsleep(g_queryInfo.specifiedQueryInfo.subscribeInterval); // ms - res = taos_consume(tsub[i]); + res = taos_consume(tsub); if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], - pThreadInfo->threadID); - appendResultToFile(res, tmpFile); - } - consumed[i] ++; - - if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) - && (consumed[i] >= - g_queryInfo.specifiedQueryInfo.resubAfterConsume[i])) { - printf("keepProgress:%d, resub specified query: %d\n", - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, i); - consumed[i] = 0; - taos_unsubscribe(tsub[i], - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); - tsub[i] = subscribeImpl(pThreadInfo->taos, - g_queryInfo.specifiedQueryInfo.sql[i], topic, - g_queryInfo.specifiedQueryInfo.subscribeRestart, - tmpFile); - if (NULL == tsub[i]) { + consumed ++; + if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) + && (consumed >= + g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + printf("keepProgress:%d, resub specified query: %"PRIu64"\n", + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, + pThreadInfo->querySeq); + consumed = 0; + taos_unsubscribe(tsub, + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + tsub = subscribeImpl( + pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], + topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart + ); + if (NULL == tsub) { taos_close(pThreadInfo->taos); return NULL; - } - } + } + } } - } } taos_free_result(res); - - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - taos_unsubscribe(tsub[i], 0); - } - + taos_unsubscribe(tsub, 0); taos_close(pThreadInfo->taos); + return NULL; } @@ -6836,7 +6827,11 @@ static int subscribeTestProcess() { pthread_t *pids = NULL; threadInfo *infos = NULL; - //==== create sub threads for query for specified table + + pthread_t *pidsOfStable = NULL; + threadInfo *infosOfStable = NULL; + + //==== create threads for query for specified table if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", __func__, __LINE__, @@ -6850,81 +6845,108 @@ static int subscribeTestProcess() { } pids = malloc( - g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); + g_queryInfo.specifiedQueryInfo.sqlCount * + g_queryInfo.specifiedQueryInfo.concurrent * + sizeof(pthread_t)); infos = malloc( - g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); + g_queryInfo.specifiedQueryInfo.sqlCount * + g_queryInfo.specifiedQueryInfo.concurrent * + sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); exit(-1); } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pids + i, NULL, specifiedSubscribe, t_info); + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { + uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; + threadInfo *t_info = infos + seq; + t_info->threadID = seq; + t_info->querySeq = i; + t_info->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pids + seq, NULL, specifiedSubscribe, t_info); + } } } - //==== create sub threads for super table query - pthread_t *pidsOfSub = NULL; - threadInfo *infosOfSub = NULL; - if ((g_queryInfo.superQueryInfo.sqlCount > 0) + //==== create threads for super table query + if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { + printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", + __func__, __LINE__, + g_queryInfo.specifiedQueryInfo.sqlCount); + } else { + if ((g_queryInfo.superQueryInfo.sqlCount > 0) && (g_queryInfo.superQueryInfo.threadCnt > 0)) { - pidsOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * + pidsOfStable = malloc( + g_queryInfo.superQueryInfo.sqlCount * + g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * + infosOfStable = malloc( + g_queryInfo.superQueryInfo.sqlCount * + g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); - if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { - errorPrint("%s() LN%d, malloc failed for create threads\n", + if ((NULL == pidsOfStable) || (NULL == infosOfStable)) { + errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); - // taos_close(taos); - exit(-1); - } + // taos_close(taos); + exit(-1); + } - int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; - int threads = g_queryInfo.superQueryInfo.threadCnt; + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int threads = g_queryInfo.superQueryInfo.threadCnt; - int64_t a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } + int64_t a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } - int64_t b = 0; - if (threads != 0) { - b = ntables % threads; - } + int64_t b = 0; + if (threads != 0) { + b = ntables % threads; + } - uint64_t startFrom = 0; - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infosOfSub + i; - t_info->threadID = i; + uint64_t startFrom = 0; + for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + for (int j = 0; j < threads; j++) { + uint64_t seq = i * threads + j; + threadInfo *t_info = infosOfStable + seq; + t_info->threadID = seq; + t_info->querySeq = i; - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pidsOfSub + i, NULL, superSubscribe, t_info); - } + t_info->start_table_from = startFrom; + t_info->ntables = jend_table_to = jend_table_to + 1; + t_info->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pidsOfStable + seq, + NULL, superSubscribe, t_info); + } + } - g_queryInfo.superQueryInfo.threadCnt = threads; + g_queryInfo.superQueryInfo.threadCnt = threads; - for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { - pthread_join(pidsOfSub[i], NULL); + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + for (int j = 0; j < threads; j++) { + uint64_t seq = i * threads + j; + pthread_join(pidsOfStable[seq], NULL); + } + } } } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { - pthread_join(pids[i], NULL); + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { + uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; + pthread_join(pids[seq], NULL); + } } tmfree((char*)pids); tmfree((char*)infos); - tmfree((char*)pidsOfSub); - tmfree((char*)infosOfSub); + tmfree((char*)pidsOfStable); + tmfree((char*)infosOfStable); // taos_close(taos); return 0; } From 78b8473e03670ec434fe7fe30fbeaf7730beace2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 19 May 2021 20:12:19 +0800 Subject: [PATCH 121/157] Hotfix/sangshuduo/td 4238 taosdemo async subscribe (#6162) * [TD-4238]: taosdemo async subscribe. * [TD-4238]: taosdemo async subscribe subsribe sql command do not use aggregation functions. * [TD-4238]: taosdemo async subscribe. interval. * [TD-4238]: taosdemo async subscribe. fix super table sub result file. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 61 +++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 0831555d22..f8d3f83d47 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -114,12 +114,18 @@ typedef enum TALBE_EXISTS_EN { TBL_EXISTS_BUTT } TALBE_EXISTS_EN; -enum MODE { +enum enumSYNC_MODE { SYNC_MODE, ASYNC_MODE, MODE_BUT }; +typedef enum enumQUERY_CLASS { + SPECIFIED_CLASS, + STABLE_CLASS, + CLASS_BUT +} QUERY_CLASS; + typedef enum enum_INSERT_MODE { PROGRESSIVE_INSERT_MODE, INTERLACE_INSERT_MODE, @@ -6557,18 +6563,21 @@ static void specified_sub_callback( } static TAOS_SUB* subscribeImpl( + QUERY_CLASS class, threadInfo *pThreadInfo, - char *sql, char* topic, bool restart) + char *sql, char* topic, bool restart, uint64_t interval) { TAOS_SUB* tsub = NULL; - if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { + if ((SPECIFIED_CLASS == class) + && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) { tsub = taos_subscribe( pThreadInfo->taos, restart, topic, sql, specified_sub_callback, (void*)pThreadInfo, g_queryInfo.specifiedQueryInfo.subscribeInterval); - } else if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { + } else if ((STABLE_CLASS == class) + && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) { tsub = taos_subscribe( pThreadInfo->taos, restart, @@ -6578,7 +6587,7 @@ static TAOS_SUB* subscribeImpl( tsub = taos_subscribe( pThreadInfo->taos, restart, - topic, sql, NULL, NULL, 0); + topic, sql, NULL, NULL, interval); } if (tsub == NULL) { @@ -6629,9 +6638,14 @@ static void *superSubscribe(void *sarg) { char topic[32] = {0}; for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { + verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n", + __func__, __LINE__, + pThreadInfo->threadID, + pThreadInfo->start_table_from, + pThreadInfo->end_table_to, i); sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"", i, pThreadInfo->querySeq); - memset(subSqlstr,0,sizeof(subSqlstr)); + memset(subSqlstr, 0, sizeof(subSqlstr)); replaceChildTblName( g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq], subSqlstr, i); @@ -6644,8 +6658,10 @@ static void *superSubscribe(void *sarg) { debugPrint("%s() LN%d, [%d] subSqlstr: %s\n", __func__, __LINE__, pThreadInfo->threadID, subSqlstr); tsub[i] = subscribeImpl( + STABLE_CLASS, pThreadInfo, subSqlstr, topic, - g_queryInfo.superQueryInfo.subscribeRestart); + g_queryInfo.superQueryInfo.subscribeRestart, + g_queryInfo.superQueryInfo.subscribeInterval); if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); return NULL; @@ -6687,8 +6703,10 @@ static void *superSubscribe(void *sarg) { g_queryInfo.superQueryInfo.subscribeKeepProgress); consumed[i]= 0; tsub[i] = subscribeImpl( + STABLE_CLASS, pThreadInfo, subSqlstr, topic, - g_queryInfo.superQueryInfo.subscribeRestart + g_queryInfo.superQueryInfo.subscribeRestart, + g_queryInfo.superQueryInfo.subscribeInterval ); if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); @@ -6744,10 +6762,12 @@ static void *specifiedSubscribe(void *sarg) { g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); } - tsub = subscribeImpl(pThreadInfo, + tsub = subscribeImpl( + SPECIFIED_CLASS, pThreadInfo, g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], topic, - g_queryInfo.specifiedQueryInfo.subscribeRestart); + g_queryInfo.specifiedQueryInfo.subscribeRestart, + g_queryInfo.specifiedQueryInfo.subscribeInterval); if (NULL == tsub) { taos_close(pThreadInfo->taos); return NULL; @@ -6777,11 +6797,12 @@ static void *specifiedSubscribe(void *sarg) { taos_unsubscribe(tsub, g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); tsub = subscribeImpl( - pThreadInfo, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], - topic, - g_queryInfo.specifiedQueryInfo.subscribeRestart - ); + SPECIFIED_CLASS, + pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], + topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart, + g_queryInfo.specifiedQueryInfo.subscribeInterval); if (NULL == tsub) { taos_close(pThreadInfo->taos); return NULL; @@ -6833,7 +6854,7 @@ static int subscribeTestProcess() { //==== create threads for query for specified table if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { - printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", + debugPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount); } else { @@ -6870,10 +6891,10 @@ static int subscribeTestProcess() { } //==== create threads for super table query - if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { - printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", + if (g_queryInfo.superQueryInfo.sqlCount <= 0) { + printf("%s() LN%d, super table query sqlCount %"PRIu64".\n", __func__, __LINE__, - g_queryInfo.specifiedQueryInfo.sqlCount); + g_queryInfo.superQueryInfo.sqlCount); } else { if ((g_queryInfo.superQueryInfo.sqlCount > 0) && (g_queryInfo.superQueryInfo.threadCnt > 0)) { @@ -6906,8 +6927,8 @@ static int subscribeTestProcess() { b = ntables % threads; } - uint64_t startFrom = 0; for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + uint64_t startFrom = 0; for (int j = 0; j < threads; j++) { uint64_t seq = i * threads + j; threadInfo *t_info = infosOfStable + seq; From 4e32f1cc72bf39ee432541935abbeea8adab8475 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 20 May 2021 09:23:10 +0800 Subject: [PATCH 122/157] Hotfix/sangshuduo/td 4238 taosdemo async for develop (#6164) * [TD-4238]: taosdemo async sub. for develop branch. * [TD-4238]: taosdemo async subscribe subsribe sql command do not use aggregation functions. * cherry pick f5e4cdd1f7cffc4ee04fc7d043e4841bd1dd9961 * [TD-4238]: taosdemo async subscribe. fix super table sub result file. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/async-sub.json | 41 +++ src/kit/taosdemo/taosdemo.c | 471 ++++++++++++++++++-------------- 2 files changed, 301 insertions(+), 211 deletions(-) create mode 100644 src/kit/taosdemo/async-sub.json diff --git a/src/kit/taosdemo/async-sub.json b/src/kit/taosdemo/async-sub.json new file mode 100644 index 0000000000..a30a1be45c --- /dev/null +++ b/src/kit/taosdemo/async-sub.json @@ -0,0 +1,41 @@ +{ + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "test", + "specified_table_query": { + "concurrent": 1, + "mode": "async", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "resubAfterConsume": 10, + "sqls": [ + { + "sql": "select col1 from meters where col1 > 1;", + "result": "./subscribe_res0.txt" + }, + { + "sql": "select col2 from meters where col2 > 1;", + "result": "./subscribe_res2.txt" + } + ] + }, + "super_table_query": { + "stblname": "meters", + "threads": 1, + "mode": "sync", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "sqls": [ + { + "sql": "select col1 from xxxx where col1 > 10;", + "result": "./subscribe_res1.txt" + } + ] + } +} diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 95fb2f7bd4..068aa52283 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -114,7 +114,7 @@ typedef enum TALBE_EXISTS_EN { TBL_EXISTS_BUTT } TALBE_EXISTS_EN; -enum MODE { +enum enumSYNC_MODE { SYNC_MODE, ASYNC_MODE, MODE_BUT @@ -127,6 +127,12 @@ enum enum_TAOS_INTERFACE { INTERFACE_BUT }; +typedef enum enumQUERY_CLASS { + SPECIFIED_CLASS, + STABLE_CLASS, + CLASS_BUT +} QUERY_CLASS; + typedef enum enum_PROGRESSIVE_OR_INTERLACE { PROGRESSIVE_INSERT_MODE, INTERLACE_INSERT_MODE, @@ -451,8 +457,9 @@ typedef struct SThreadInfo_S { uint64_t maxDelay; uint64_t minDelay; - // query + // seq of query or subscribe uint64_t querySeq; // sequence number of sql command + } threadInfo; #ifdef WINDOWS @@ -1179,7 +1186,8 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { } static void selectAndGetResult( - threadInfo *pThreadInfo, char *command, char* resultFile) { + threadInfo *pThreadInfo, char *command) +{ if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) { TAOS_RES *res = taos_query(pThreadInfo->taos, command); if (res == NULL || taos_errno(res) != 0) { @@ -1189,8 +1197,8 @@ static void selectAndGetResult( return; } - if ((resultFile) && (strlen(resultFile))) { - appendResultToFile(res, resultFile); + if ((strlen(pThreadInfo->fp))) { + appendResultToFile(res, pThreadInfo->fp); } taos_free_result(res); @@ -1198,7 +1206,7 @@ static void selectAndGetResult( int retCode = postProceSql( g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port, command, - resultFile); + pThreadInfo->fp); if (0 != retCode) { printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID); } @@ -6255,23 +6263,22 @@ static void *specifiedTableQuery(void *sarg) { uint64_t lastPrintTime = taosGetTimestampMs(); uint64_t startTs = taosGetTimestampMs(); + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + } + while(queryTimes --) { if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) { taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms } - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - } - st = taosGetTimestampMs(); selectAndGetResult(pThreadInfo, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], tmpFile); + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]); et = taosGetTimestampMs(); printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n", @@ -6357,13 +6364,12 @@ static void *superTableQuery(void *sarg) { for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { memset(sqlstr,0,sizeof(sqlstr)); replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(tmpFile, "%s-%d", + sprintf(pThreadInfo->fp, "%s-%d", g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } - selectAndGetResult(pThreadInfo, sqlstr, tmpFile); + selectAndGetResult(pThreadInfo, sqlstr); totalQueried++; g_queryInfo.superQueryInfo.totalQueried ++; @@ -6435,7 +6441,7 @@ static int queryTestProcess() { threadInfo *infos = NULL; //==== create sub threads for query from specify table int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; - int nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount; + uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount; uint64_t startTs = taosGetTimestampMs(); @@ -6449,32 +6455,33 @@ static int queryTestProcess() { ERROR_EXIT("memory allocation failed for create threads\n"); } - for (int i = 0; i < nConcurrent; i++) { - for (int j = 0; j < nSqlCount; j++) { - threadInfo *t_info = infos + i * nSqlCount + j; - t_info->threadID = i * nSqlCount + j; - t_info->querySeq = j; + for (uint64_t i = 0; i < nSqlCount; i++) { + for (int j = 0; j < nConcurrent; j++) { + uint64_t seq = i * nConcurrent + j; + threadInfo *t_info = infos + seq; + t_info->threadID = seq; + t_info->querySeq = i; - if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); - verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); - if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { - taos_close(taos); - free(infos); - free(pids); - errorPrint( "use database %s failed!\n\n", - g_queryInfo.dbName); - return -1; - } + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); + if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(taos); + free(infos); + free(pids); + errorPrint( "use database %s failed!\n\n", + g_queryInfo.dbName); + return -1; + } + } + + t_info->taos = NULL;// TODO: workaround to use separate taos connection; + + pthread_create(pids + seq, NULL, specifiedTableQuery, + t_info); } - - t_info->taos = NULL;// TODO: workaround to use separate taos connection; - - pthread_create(pids + i * nSqlCount + j, NULL, specifiedTableQuery, - t_info); - } } } else { g_queryInfo.specifiedQueryInfo.concurrent = 0; @@ -6559,7 +6566,8 @@ static int queryTestProcess() { return 0; } -static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { +static void stable_sub_callback( + TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { if (res == NULL || taos_errno(res) != 0) { errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", __func__, __LINE__, code, taos_errstr(res)); @@ -6567,24 +6575,49 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c } if (param) - appendResultToFile(res, (char*)param); + appendResultToFile(res, ((threadInfo *)param)->fp); + // tao_unscribe() will free result. +} + +static void specified_sub_callback( + TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { + if (res == NULL || taos_errno(res) != 0) { + errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", + __func__, __LINE__, code, taos_errstr(res)); + return; + } + + if (param) + appendResultToFile(res, ((threadInfo *)param)->fp); // tao_unscribe() will free result. } static TAOS_SUB* subscribeImpl( - TAOS *taos, char *sql, char* topic, bool restart, - char* resultFileName) { + QUERY_CLASS class, + threadInfo *pThreadInfo, + char *sql, char* topic, bool restart, uint64_t interval) +{ TAOS_SUB* tsub = NULL; - if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { - tsub = taos_subscribe(taos, + if ((SPECIFIED_CLASS == class) + && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) { + tsub = taos_subscribe( + pThreadInfo->taos, restart, - topic, sql, subscribe_callback, (void*)resultFileName, + topic, sql, specified_sub_callback, (void*)pThreadInfo, g_queryInfo.specifiedQueryInfo.subscribeInterval); - } else { - tsub = taos_subscribe(taos, + } else if ((STABLE_CLASS == class) + && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) { + tsub = taos_subscribe( + pThreadInfo->taos, restart, - topic, sql, NULL, NULL, 0); + topic, sql, stable_sub_callback, (void*)pThreadInfo, + g_queryInfo.superQueryInfo.subscribeInterval); + } else { + tsub = taos_subscribe( + pThreadInfo->taos, + restart, + topic, sql, NULL, NULL, interval); } if (tsub == NULL) { @@ -6600,13 +6633,8 @@ static void *superSubscribe(void *sarg) { char subSqlstr[MAX_QUERY_SQL_LENGTH]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; - if (g_queryInfo.superQueryInfo.sqlCount == 0) - return NULL; - - if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { - errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n", - g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables, - g_queryInfo.superQueryInfo.sqlCount, + if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { + errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n", pThreadInfo->ntables, MAX_QUERY_SQL_COUNT); exit(-1); @@ -6637,91 +6665,92 @@ static void *superSubscribe(void *sarg) { return NULL; } - uint64_t subSeq; char topic[32] = {0}; for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { - sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%d", i, j); - memset(subSqlstr,0,sizeof(subSqlstr)); - replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], subSqlstr, i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); + verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n", + __func__, __LINE__, + pThreadInfo->threadID, + pThreadInfo->start_table_from, + pThreadInfo->end_table_to, i); + sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"", + i, pThreadInfo->querySeq); + memset(subSqlstr, 0, sizeof(subSqlstr)); + replaceChildTblName( + g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq], + subSqlstr, i); + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); } - subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", - __func__, __LINE__, subSeq, subSqlstr); - tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, - g_queryInfo.superQueryInfo.subscribeRestart, - tmpFile); - if (NULL == tsub[subSeq]) { + debugPrint("%s() LN%d, [%d] subSqlstr: %s\n", + __func__, __LINE__, pThreadInfo->threadID, subSqlstr); + tsub[i] = subscribeImpl( + STABLE_CLASS, + pThreadInfo, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart, + g_queryInfo.superQueryInfo.subscribeInterval); + if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); return NULL; } - } } - int consumed[MAX_QUERY_SQL_COUNT]; - for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) - consumed[i] = 0; - // start loop to consume result + int consumed[MAX_QUERY_SQL_COUNT]; + for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) { + consumed[i] = 0; + } TAOS_RES* res = NULL; + while(1) { - for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to; i++) { if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { continue; } - subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - taosMsleep(100); // ms - res = taos_consume(tsub[subSeq]); + taosMsleep(g_queryInfo.superQueryInfo.subscribeInterval); // ms + res = taos_consume(tsub[i]); if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[j], + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - appendResultToFile(res, tmpFile); + appendResultToFile(res, pThreadInfo->fp); } - consumed[j] ++; + consumed[i] ++; if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) - && (consumed[j] >= - g_queryInfo.superQueryInfo.resubAfterConsume[j])) { - printf("keepProgress:%d, resub super table query: %d\n", - g_queryInfo.superQueryInfo.subscribeKeepProgress, j); - taos_unsubscribe(tsub[subSeq], + && (consumed[i] >= + g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + printf("keepProgress:%d, resub super table query: %"PRIu64"\n", + g_queryInfo.superQueryInfo.subscribeKeepProgress, + pThreadInfo->querySeq); + taos_unsubscribe(tsub, g_queryInfo.superQueryInfo.subscribeKeepProgress); - consumed[j]= 0; - subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", - __func__, __LINE__, subSeq, subSqlstr); - tsub[subSeq] = subscribeImpl( - pThreadInfo->taos, subSqlstr, topic, - g_queryInfo.superQueryInfo.subscribeRestart, - tmpFile); - if (NULL == tsub[subSeq]) { + consumed[i]= 0; + tsub[i] = subscribeImpl( + STABLE_CLASS, + pThreadInfo, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart, + g_queryInfo.superQueryInfo.subscribeInterval + ); + if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); return NULL; } } } - } } } taos_free_result(res); for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { - subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - taos_unsubscribe(tsub[subSeq], 0); - } + taos_unsubscribe(tsub[i], 0); } taos_close(pThreadInfo->taos); @@ -6730,10 +6759,7 @@ static void *superSubscribe(void *sarg) { static void *specifiedSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; - TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; - - if (g_queryInfo.specifiedQueryInfo.sqlCount == 0) - return NULL; + TAOS_SUB* tsub = NULL; if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; @@ -6760,74 +6786,64 @@ static void *specifiedSubscribe(void *sarg) { } char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - sprintf(topic, "taosdemo-subscribe-%d", i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); - } - tsub[i] = subscribeImpl(pThreadInfo->taos, - g_queryInfo.specifiedQueryInfo.sql[i], topic, + sprintf(topic, "taosdemo-subscribe-%"PRIu64"", pThreadInfo->querySeq); + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + } + tsub = subscribeImpl( + SPECIFIED_CLASS, pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], + topic, g_queryInfo.specifiedQueryInfo.subscribeRestart, - tmpFile); - if (NULL == tsub[i]) { - taos_close(pThreadInfo->taos); - return NULL; - } + g_queryInfo.specifiedQueryInfo.subscribeInterval); + if (NULL == tsub) { + taos_close(pThreadInfo->taos); + return NULL; } // start loop to consume result TAOS_RES* res = NULL; - int consumed[MAX_QUERY_SQL_COUNT]; - for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) - consumed[i] = 0; + int consumed; while(1) { - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { continue; } - taosMsleep(1000); // ms - res = taos_consume(tsub[i]); + taosMsleep(g_queryInfo.specifiedQueryInfo.subscribeInterval); // ms + res = taos_consume(tsub); if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); - appendResultToFile(res, tmpFile); - } - consumed[i] ++; - - if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) - && (consumed[i] >= - g_queryInfo.specifiedQueryInfo.resubAfterConsume[i])) { - printf("keepProgress:%d, resub specified query: %d\n", - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, i); - consumed[i] = 0; - taos_unsubscribe(tsub[i], - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); - tsub[i] = subscribeImpl(pThreadInfo->taos, - g_queryInfo.specifiedQueryInfo.sql[i], topic, - g_queryInfo.specifiedQueryInfo.subscribeRestart, - tmpFile); - if (NULL == tsub[i]) { + consumed ++; + if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) + && (consumed >= + g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + printf("keepProgress:%d, resub specified query: %"PRIu64"\n", + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, + pThreadInfo->querySeq); + consumed = 0; + taos_unsubscribe(tsub, + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + tsub = subscribeImpl( + SPECIFIED_CLASS, + pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], + topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart, + g_queryInfo.specifiedQueryInfo.subscribeInterval); + if (NULL == tsub) { taos_close(pThreadInfo->taos); return NULL; - } - } + } + } } - } } taos_free_result(res); - - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - taos_unsubscribe(tsub[i], 0); - } - + taos_unsubscribe(tsub, 0); taos_close(pThreadInfo->taos); + return NULL; } @@ -6865,9 +6881,13 @@ static int subscribeTestProcess() { pthread_t *pids = NULL; threadInfo *infos = NULL; - //==== create sub threads for query for specified table + + pthread_t *pidsOfStable = NULL; + threadInfo *infosOfStable = NULL; + + //==== create threads for query for specified table if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { - printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", + debugPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount); } else { @@ -6878,80 +6898,109 @@ static int subscribeTestProcess() { exit(-1); } - pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); - infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); + pids = malloc( + g_queryInfo.specifiedQueryInfo.sqlCount * + g_queryInfo.specifiedQueryInfo.concurrent * + sizeof(pthread_t)); + infos = malloc( + g_queryInfo.specifiedQueryInfo.sqlCount * + g_queryInfo.specifiedQueryInfo.concurrent * + sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); exit(-1); } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pids + i, NULL, specifiedSubscribe, t_info); + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { + uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; + threadInfo *t_info = infos + seq; + t_info->threadID = seq; + t_info->querySeq = i; + t_info->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pids + seq, NULL, specifiedSubscribe, t_info); + } } } - //==== create sub threads for super table query - pthread_t *pidsOfSub = NULL; - threadInfo *infosOfSub = NULL; - if ((g_queryInfo.superQueryInfo.sqlCount > 0) + //==== create threads for super table query + if (g_queryInfo.superQueryInfo.sqlCount <= 0) { + printf("%s() LN%d, super table query sqlCount %"PRIu64".\n", + __func__, __LINE__, + g_queryInfo.superQueryInfo.sqlCount); + } else { + if ((g_queryInfo.superQueryInfo.sqlCount > 0) && (g_queryInfo.superQueryInfo.threadCnt > 0)) { - pidsOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * + pidsOfStable = malloc( + g_queryInfo.superQueryInfo.sqlCount * + g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * + infosOfStable = malloc( + g_queryInfo.superQueryInfo.sqlCount * + g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); - if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { - errorPrint("%s() LN%d, malloc failed for create threads\n", + if ((NULL == pidsOfStable) || (NULL == infosOfStable)) { + errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); - // taos_close(taos); - exit(-1); - } + // taos_close(taos); + exit(-1); + } - int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; - int threads = g_queryInfo.superQueryInfo.threadCnt; + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int threads = g_queryInfo.superQueryInfo.threadCnt; - int64_t a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } + int64_t a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } - int64_t b = 0; - if (threads != 0) { - b = ntables % threads; - } + int64_t b = 0; + if (threads != 0) { + b = ntables % threads; + } - uint64_t startFrom = 0; - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infosOfSub + i; - t_info->threadID = i; + for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + uint64_t startFrom = 0; + for (int j = 0; j < threads; j++) { + uint64_t seq = i * threads + j; + threadInfo *t_info = infosOfStable + seq; + t_info->threadID = seq; + t_info->querySeq = i; - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pidsOfSub + i, NULL, superSubscribe, t_info); - } + t_info->start_table_from = startFrom; + t_info->ntables = jend_table_to = jend_table_to + 1; + t_info->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pidsOfStable + seq, + NULL, superSubscribe, t_info); + } + } - g_queryInfo.superQueryInfo.threadCnt = threads; + g_queryInfo.superQueryInfo.threadCnt = threads; - for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { - pthread_join(pidsOfSub[i], NULL); + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + for (int j = 0; j < threads; j++) { + uint64_t seq = i * threads + j; + pthread_join(pidsOfStable[seq], NULL); + } + } } } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { - pthread_join(pids[i], NULL); + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { + uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; + pthread_join(pids[seq], NULL); + } } tmfree((char*)pids); tmfree((char*)infos); - tmfree((char*)pidsOfSub); - tmfree((char*)infosOfSub); + tmfree((char*)pidsOfStable); + tmfree((char*)infosOfStable); // taos_close(taos); return 0; } From b05e9dc640af7aa1b151c0f23d252039cd89fa66 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Thu, 20 May 2021 10:08:11 +0800 Subject: [PATCH 123/157] [TD-4258]: speedup nodejs fetchall() with taos_fetch_block (#6163) --- src/connector/nodejs/nodetaos/cinterface.js | 254 ++++++++--------- src/connector/nodejs/nodetaos/cursor.js | 121 ++++----- src/connector/nodejs/package-lock.json | 285 -------------------- src/connector/nodejs/package.json | 2 +- 4 files changed, 188 insertions(+), 474 deletions(-) delete mode 100644 src/connector/nodejs/package-lock.json diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 43a08a800a..f3961e3787 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -9,7 +9,7 @@ const ffi = require('ffi-napi'); const ArrayType = require('ref-array-napi'); const Struct = require('ref-struct-napi'); const FieldTypes = require('./constants'); -const errors = require ('./error'); +const errors = require('./error'); const TaosObjects = require('./taosobjects'); const { NULL_POINTER } = require('ref-napi'); @@ -22,7 +22,7 @@ function convertMicrosecondsToDatetime(time) { return new TaosObjects.TaosTimestamp(time * 0.001, true); } -function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { timestampConverter = convertMillisecondsToDatetime; if (micro == true) { timestampConverter = convertMicrosecondsToDatetime; @@ -44,14 +44,14 @@ function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro=false } return res; } -function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = new Array(data.length); for (let i = 0; i < data.length; i++) { if (data[i] == 0) { res[i] = false; } - else if (data[i] == 1){ + else if (data[i] == 1) { res[i] = true; } else if (data[i] == FieldTypes.C_BOOL_NULL) { @@ -60,29 +60,29 @@ function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; while (currOffset < data.length) { - let d = data.readIntLE(currOffset,1); + let d = data.readIntLE(currOffset, 1); res.push(d == FieldTypes.C_TINYINT_NULL ? null : d); currOffset += nbytes; } return res; } -function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; while (currOffset < data.length) { - let d = data.readIntLE(currOffset,2); + let d = data.readIntLE(currOffset, 2); res.push(d == FieldTypes.C_SMALLINT_NULL ? null : d); currOffset += nbytes; } return res; } -function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -93,7 +93,7 @@ function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -104,7 +104,7 @@ function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -115,7 +115,7 @@ function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -126,7 +126,7 @@ function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -142,7 +142,7 @@ function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let dataEntry = data.slice(0, nbytes); //one entry in a row under a column; @@ -153,23 +153,23 @@ function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { // Object with all the relevant converters from pblock data to javascript readable data let convertFunctions = { - [FieldTypes.C_BOOL] : convertBool, - [FieldTypes.C_TINYINT] : convertTinyint, - [FieldTypes.C_SMALLINT] : convertSmallint, - [FieldTypes.C_INT] : convertInt, - [FieldTypes.C_BIGINT] : convertBigint, - [FieldTypes.C_FLOAT] : convertFloat, - [FieldTypes.C_DOUBLE] : convertDouble, - [FieldTypes.C_BINARY] : convertBinary, - [FieldTypes.C_TIMESTAMP] : convertTimestamp, - [FieldTypes.C_NCHAR] : convertNchar + [FieldTypes.C_BOOL]: convertBool, + [FieldTypes.C_TINYINT]: convertTinyint, + [FieldTypes.C_SMALLINT]: convertSmallint, + [FieldTypes.C_INT]: convertInt, + [FieldTypes.C_BIGINT]: convertBigint, + [FieldTypes.C_FLOAT]: convertFloat, + [FieldTypes.C_DOUBLE]: convertDouble, + [FieldTypes.C_BINARY]: convertBinary, + [FieldTypes.C_TIMESTAMP]: convertTimestamp, + [FieldTypes.C_NCHAR]: convertNchar } // Define TaosField structure var char_arr = ArrayType(ref.types.char); var TaosField = Struct({ - 'name': char_arr, - }); + 'name': char_arr, +}); TaosField.fields.name.type.size = 65; TaosField.defineProperty('type', ref.types.char); TaosField.defineProperty('bytes', ref.types.short); @@ -183,7 +183,7 @@ TaosField.defineProperty('bytes', ref.types.short); * @classdesc The CTaosInterface is the interface through which Node.JS communicates data back and forth with TDengine. It is not advised to * access this class directly and use it unless you understand what these functions do. */ -function CTaosInterface (config = null, pass = false) { +function CTaosInterface(config = null, pass = false) { ref.types.char_ptr = ref.refType(ref.types.char); ref.types.void_ptr = ref.refType(ref.types.void); ref.types.void_ptr2 = ref.refType(ref.types.void_ptr); @@ -196,64 +196,65 @@ function CTaosInterface (config = null, pass = false) { taoslibname = 'libtaos'; } this.libtaos = ffi.Library(taoslibname, { - 'taos_options': [ ref.types.int, [ ref.types.int , ref.types.void_ptr ] ], - 'taos_init': [ ref.types.void, [ ] ], + 'taos_options': [ref.types.int, [ref.types.int, ref.types.void_ptr]], + 'taos_init': [ref.types.void, []], //TAOS *taos_connect(char *ip, char *user, char *pass, char *db, int port) - 'taos_connect': [ ref.types.void_ptr, [ ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int ] ], + 'taos_connect': [ref.types.void_ptr, [ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int]], //void taos_close(TAOS *taos) - 'taos_close': [ ref.types.void, [ ref.types.void_ptr ] ], - //int *taos_fetch_lengths(TAOS_RES *taos); - 'taos_fetch_lengths': [ ref.types.void_ptr, [ ref.types.void_ptr ] ], + 'taos_close': [ref.types.void, [ref.types.void_ptr]], + //int *taos_fetch_lengths(TAOS_RES *res); + 'taos_fetch_lengths': [ref.types.void_ptr, [ref.types.void_ptr]], //int taos_query(TAOS *taos, char *sqlstr) - 'taos_query': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.char_ptr ] ], - //int taos_affected_rows(TAOS *taos) - 'taos_affected_rows': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_query': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.char_ptr]], + //int taos_affected_rows(TAOS_RES *res) + 'taos_affected_rows': [ref.types.int, [ref.types.void_ptr]], //int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) - 'taos_fetch_block': [ ref.types.int, [ ref.types.void_ptr, ref.types.void_ptr] ], + 'taos_fetch_block': [ref.types.int, [ref.types.void_ptr, ref.types.void_ptr]], //int taos_num_fields(TAOS_RES *res); - 'taos_num_fields': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_num_fields': [ref.types.int, [ref.types.void_ptr]], //TAOS_ROW taos_fetch_row(TAOS_RES *res) //TAOS_ROW is void **, but we set the return type as a reference instead to get the row - 'taos_fetch_row': [ ref.refType(ref.types.void_ptr2), [ ref.types.void_ptr ] ], + 'taos_fetch_row': [ref.refType(ref.types.void_ptr2), [ref.types.void_ptr]], + 'taos_print_row': [ref.types.int, [ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int]], //int taos_result_precision(TAOS_RES *res) - 'taos_result_precision': [ ref.types.int, [ ref.types.void_ptr ] ], + 'taos_result_precision': [ref.types.int, [ref.types.void_ptr]], //void taos_free_result(TAOS_RES *res) - 'taos_free_result': [ ref.types.void, [ ref.types.void_ptr] ], + 'taos_free_result': [ref.types.void, [ref.types.void_ptr]], //int taos_field_count(TAOS *taos) - 'taos_field_count': [ ref.types.int, [ ref.types.void_ptr ] ], + 'taos_field_count': [ref.types.int, [ref.types.void_ptr]], //TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) - 'taos_fetch_fields': [ ref.refType(TaosField), [ ref.types.void_ptr ] ], + 'taos_fetch_fields': [ref.refType(TaosField), [ref.types.void_ptr]], //int taos_errno(TAOS *taos) - 'taos_errno': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_errno': [ref.types.int, [ref.types.void_ptr]], //char *taos_errstr(TAOS *taos) - 'taos_errstr': [ ref.types.char_ptr, [ ref.types.void_ptr] ], + 'taos_errstr': [ref.types.char_ptr, [ref.types.void_ptr]], //void taos_stop_query(TAOS_RES *res); - 'taos_stop_query': [ ref.types.void, [ ref.types.void_ptr] ], + 'taos_stop_query': [ref.types.void, [ref.types.void_ptr]], //char *taos_get_server_info(TAOS *taos); - 'taos_get_server_info': [ ref.types.char_ptr, [ ref.types.void_ptr ] ], + 'taos_get_server_info': [ref.types.char_ptr, [ref.types.void_ptr]], //char *taos_get_client_info(); - 'taos_get_client_info': [ ref.types.char_ptr, [ ] ], + 'taos_get_client_info': [ref.types.char_ptr, []], // ASYNC // void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *, TAOS_RES *, int), void *param) - 'taos_query_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr ] ], + 'taos_query_a': [ref.types.void, [ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr]], // void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); - 'taos_fetch_rows_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr ]], + 'taos_fetch_rows_a': [ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr]], // Subscription //TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) - 'taos_subscribe': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int] ], + 'taos_subscribe': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int]], // TAOS_RES *taos_consume(TAOS_SUB *tsub) - 'taos_consume': [ ref.types.void_ptr, [ref.types.void_ptr] ], + 'taos_consume': [ref.types.void_ptr, [ref.types.void_ptr]], //void taos_unsubscribe(TAOS_SUB *tsub); - 'taos_unsubscribe': [ ref.types.void, [ ref.types.void_ptr ] ], + 'taos_unsubscribe': [ref.types.void, [ref.types.void_ptr]], // Continuous Query //TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), // int64_t stime, void *param, void (*callback)(void *)); - 'taos_open_stream': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.int64, ref.types.void_ptr, ref.types.void_ptr ] ], + 'taos_open_stream': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.int64, ref.types.void_ptr, ref.types.void_ptr]], //void taos_close_stream(TAOS_STREAM *tstr); - 'taos_close_stream': [ ref.types.void, [ ref.types.void_ptr ] ] + 'taos_close_stream': [ref.types.void, [ref.types.void_ptr]] }); if (pass == false) { @@ -264,7 +265,7 @@ function CTaosInterface (config = null, pass = false) { try { this._config = ref.allocCString(config); } - catch(err){ + catch (err) { throw "Attribute Error: config is expected as a str"; } } @@ -276,38 +277,38 @@ function CTaosInterface (config = null, pass = false) { return this; } CTaosInterface.prototype.config = function config() { - return this._config; - } -CTaosInterface.prototype.connect = function connect(host=null, user="root", password="taosdata", db=null, port=0) { - let _host,_user,_password,_db,_port; - try { + return this._config; +} +CTaosInterface.prototype.connect = function connect(host = null, user = "root", password = "taosdata", db = null, port = 0) { + let _host, _user, _password, _db, _port; + try { _host = host != null ? ref.allocCString(host) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: host is expected as a str"; } try { _user = ref.allocCString(user) } - catch(err) { + catch (err) { throw "Attribute Error: user is expected as a str"; } try { _password = ref.allocCString(password); } - catch(err) { + catch (err) { throw "Attribute Error: password is expected as a str"; } try { _db = db != null ? ref.allocCString(db) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: db is expected as a str"; } try { _port = ref.alloc(ref.types.int, port); } - catch(err) { + catch (err) { throw TypeError("port is expected as an int") } let connection = this.libtaos.taos_connect(_host, _user, _password, _db, _port); @@ -324,10 +325,10 @@ CTaosInterface.prototype.close = function close(connection) { console.log("Connection is closed"); } CTaosInterface.prototype.query = function query(connection, sql) { - return this.libtaos.taos_query(connection, ref.allocCString(sql)); + return this.libtaos.taos_query(connection, ref.allocCString(sql)); } -CTaosInterface.prototype.affectedRows = function affectedRows(connection) { - return this.libtaos.taos_affected_rows(connection); +CTaosInterface.prototype.affectedRows = function affectedRows(result) { + return this.libtaos.taos_affected_rows(result); } CTaosInterface.prototype.useResult = function useResult(result) { @@ -337,8 +338,8 @@ CTaosInterface.prototype.useResult = function useResult(result) { pfields = ref.reinterpret(pfields, this.fieldsCount(result) * 68, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,65,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 65, i)), type: pfields[i + 65], bytes: pfields[i + 66] }) @@ -347,11 +348,10 @@ CTaosInterface.prototype.useResult = function useResult(result) { return fields; } CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { - //let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data - let pblock = this.libtaos.taos_fetch_row(result); - let num_of_rows = 1; - if (ref.isNull(pblock) == true) { - return {block:null, num_of_rows:0}; + let pblock = ref.NULL_POINTER; + let num_of_rows = this.libtaos.taos_fetch_block(result, pblock); + if (ref.isNull(pblock.deref()) == true) { + return { block: null, num_of_rows: 0 }; } var fieldL = this.libtaos.taos_fetch_lengths(result); @@ -359,10 +359,10 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO); var fieldlens = []; - + if (ref.isNull(fieldL) == false) { - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 4, i*4); + for (let i = 0; i < fields.length; i++) { + let plen = ref.reinterpret(fieldL, 4, i * 4); let len = plen.readInt32LE(0); fieldlens.push(len); } @@ -370,21 +370,23 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { let blocks = new Array(fields.length); blocks.fill(null); - //num_of_rows = Math.abs(num_of_rows); + num_of_rows = Math.abs(num_of_rows); let offset = 0; + let ptr = pblock.deref(); + for (let i = 0; i < fields.length; i++) { - pdata = ref.reinterpret(pblock,8,i*8); - if(ref.isNull(pdata.readPointer())){ - blocks[i] = new Array(); - }else{ - pdata = ref.ref(pdata.readPointer()); - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); - } + pdata = ref.reinterpret(ptr, 8, i * 8); + if (ref.isNull(pdata.readPointer())) { + blocks[i] = new Array(); + } else { + pdata = ref.ref(pdata.readPointer()); + if (!convertFunctions[fields[i]['type']]) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + blocks[i] = convertFunctions[fields[i]['type']](pdata, num_of_rows, fieldlens[i], offset, isMicro); + } } - return {blocks: blocks, num_of_rows:Math.abs(num_of_rows)} + return { blocks: blocks, num_of_rows } } CTaosInterface.prototype.fetchRow = function fetchRow(result, fields) { let row = this.libtaos.taos_fetch_row(result); @@ -414,7 +416,7 @@ CTaosInterface.prototype.errStr = function errStr(result) { // Async CTaosInterface.prototype.query_a = function query_a(connection, sql, callback, param = ref.ref(ref.NULL)) { // void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, int), void *param) - callback = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.int ], callback); + callback = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.int], callback); this.libtaos.taos_query_a(connection, ref.allocCString(sql), callback, param); return param; } @@ -439,46 +441,46 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback, var fieldL = cti.libtaos.taos_fetch_lengths(result); var fieldlens = []; if (ref.isNull(fieldL) == false) { - - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 8, i*8); - let len = ref.get(plen,0,ref.types.int32); + + for (let i = 0; i < fields.length; i++) { + let plen = ref.reinterpret(fieldL, 8, i * 8); + let len = ref.get(plen, 0, ref.types.int32); fieldlens.push(len); } } - if (numOfRows2 > 0){ + if (numOfRows2 > 0) { for (let i = 0; i < fields.length; i++) { - if(ref.isNull(pdata.readPointer())){ - blocks[i] = new Array(); - }else{ - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - let prow = ref.reinterpret(row,8,i*8); - prow = prow.readPointer(); - prow = ref.ref(prow); - blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); - //offset += fields[i]['bytes'] * numOfRows2; - } + if (ref.isNull(pdata.readPointer())) { + blocks[i] = new Array(); + } else { + if (!convertFunctions[fields[i]['type']]) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + let prow = ref.reinterpret(row, 8, i * 8); + prow = prow.readPointer(); + prow = ref.ref(prow); + blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); + //offset += fields[i]['bytes'] * numOfRows2; + } } } callback(param2, result2, numOfRows2, blocks); } - asyncCallbackWrapper = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.int ], asyncCallbackWrapper); + asyncCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.int], asyncCallbackWrapper); this.libtaos.taos_fetch_rows_a(result, asyncCallbackWrapper, param); return param; } // Fetch field meta data by result handle -CTaosInterface.prototype.fetchFields_a = function fetchFields_a (result) { +CTaosInterface.prototype.fetchFields_a = function fetchFields_a(result) { let pfields = this.fetchFields(result); let pfieldscount = this.numFields(result); let fields = []; if (ref.isNull(pfields) == false) { - pfields = ref.reinterpret(pfields, 68 * pfieldscount , 0); + pfields = ref.reinterpret(pfields, 68 * pfieldscount, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 64 = name //65 = type, 66 - 67 = bytes - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,65,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 65, i)), type: pfields[i + 65], bytes: pfields[i + 66] }) @@ -488,7 +490,7 @@ CTaosInterface.prototype.fetchFields_a = function fetchFields_a (result) { } // Stop a query by result handle CTaosInterface.prototype.stopQuery = function stopQuery(result) { - if (result != null){ + if (result != null) { this.libtaos.taos_stop_query(result); } else { @@ -509,13 +511,13 @@ CTaosInterface.prototype.subscribe = function subscribe(connection, restart, top try { sql = sql != null ? ref.allocCString(sql) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: sql is expected as a str"; } try { topic = topic != null ? ref.allocCString(topic) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw TypeError("topic is expected as a str"); } @@ -539,8 +541,8 @@ CTaosInterface.prototype.consume = function consume(subscription) { pfields = ref.reinterpret(pfields, this.numFields(result) * 68, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,64,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 64, i)), bytes: pfields[i + 64], type: pfields[i + 66] }) @@ -548,7 +550,7 @@ CTaosInterface.prototype.consume = function consume(subscription) { } let data = []; - while(true) { + while (true) { let { blocks, num_of_rows } = this.fetchBlock(result, fields); if (num_of_rows == 0) { break; @@ -559,7 +561,7 @@ CTaosInterface.prototype.consume = function consume(subscription) { for (let j = 0; j < fields.length; j++) { rowBlock[j] = blocks[j][i]; } - data[data.length-1] = (rowBlock); + data[data.length - 1] = (rowBlock); } } return { data: data, fields: fields, result: result }; @@ -570,11 +572,11 @@ CTaosInterface.prototype.unsubscribe = function unsubscribe(subscription) { } // Continuous Query -CTaosInterface.prototype.openStream = function openStream(connection, sql, callback, stime,stoppingCallback, param = ref.ref(ref.NULL)) { +CTaosInterface.prototype.openStream = function openStream(connection, sql, callback, stime, stoppingCallback, param = ref.ref(ref.NULL)) { try { sql = ref.allocCString(sql); } - catch(err) { + catch (err) { throw "Attribute Error: sql string is expected as a str"; } var cti = this; @@ -587,7 +589,7 @@ CTaosInterface.prototype.openStream = function openStream(connection, sql, callb let offset = 0; if (numOfRows2 > 0) { for (let i = 0; i < fields.length; i++) { - if (!convertFunctions[fields[i]['type']] ) { + if (!convertFunctions[fields[i]['type']]) { throw new errors.DatabaseError("Invalid data type returned from database"); } blocks[i] = convertFunctions[fields[i]['type']](row, numOfRows2, fields[i]['bytes'], offset, isMicro); @@ -596,8 +598,8 @@ CTaosInterface.prototype.openStream = function openStream(connection, sql, callb } callback(param2, result2, blocks, fields); } - asyncCallbackWrapper = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.refType(ref.types.void_ptr2) ], asyncCallbackWrapper); - asyncStoppingCallbackWrapper = ffi.Callback( ref.types.void, [ ref.types.void_ptr ], stoppingCallback); + asyncCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.refType(ref.types.void_ptr2)], asyncCallbackWrapper); + asyncStoppingCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr], stoppingCallback); let streamHandle = this.libtaos.taos_open_stream(connection, sql, asyncCallbackWrapper, stime, param, asyncStoppingCallbackWrapper); if (ref.isNull(streamHandle)) { throw new errors.TDError('Failed to open a stream with TDengine'); diff --git a/src/connector/nodejs/nodetaos/cursor.js b/src/connector/nodejs/nodetaos/cursor.js index e18e6c2500..f879d89d48 100644 --- a/src/connector/nodejs/nodetaos/cursor.js +++ b/src/connector/nodejs/nodetaos/cursor.js @@ -1,7 +1,7 @@ const ref = require('ref-napi'); require('./globalfunc.js') const CTaosInterface = require('./cinterface') -const errors = require ('./error') +const errors = require('./error') const TaosQuery = require('./taosquery') const { PerformanceObserver, performance } = require('perf_hooks'); module.exports = TDengineCursor; @@ -22,7 +22,7 @@ module.exports = TDengineCursor; * @property {fields} - Array of the field objects in order from left to right of the latest data retrieved * @since 1.0.0 */ -function TDengineCursor(connection=null) { +function TDengineCursor(connection = null) { //All parameters are store for sync queries only. this._rowcount = -1; this._connection = null; @@ -91,7 +91,7 @@ TDengineCursor.prototype.execute = function execute(operation, options, callback return null; } - if (typeof options == 'function') { + if (typeof options == 'function') { callback = options; } if (typeof options != 'object') options = {} @@ -144,10 +144,10 @@ TDengineCursor.prototype.execute = function execute(operation, options, callback } TDengineCursor.prototype._createAffectedResponse = function (num, time) { - return "Query OK, " + num + " row(s) affected (" + (time * 0.001).toFixed(8) + "s)"; + return "Query OK, " + num + " row(s) affected (" + (time * 0.001).toFixed(8) + "s)"; } TDengineCursor.prototype._createSetResponse = function (num, time) { - return "Query OK, " + num + " row(s) in set (" + (time * 0.001).toFixed(8) + "s)"; + return "Query OK, " + num + " row(s) in set (" + (time * 0.001).toFixed(8) + "s)"; } TDengineCursor.prototype.executemany = function executemany() { @@ -176,27 +176,22 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { throw new errors.OperationalError("Invalid use of fetchall, either result or fields from query are null. First execute a query first"); } - let data = []; + let num_of_rows = this._chandle.affectedRows(this._result); + let data = new Array(num_of_rows); + this._rowcount = 0; - //let nodetime = 0; + let time = 0; const obs = new PerformanceObserver((items) => { time += items.getEntries()[0].duration; performance.clearMarks(); }); - /* - const obs2 = new PerformanceObserver((items) => { - nodetime += items.getEntries()[0].duration; - performance.clearMarks(); - }); - obs2.observe({ entryTypes: ['measure'] }); - performance.mark('nodea'); - */ obs.observe({ entryTypes: ['measure'] }); performance.mark('A'); - while(true) { - + while (true) { let blockAndRows = this._chandle.fetchBlock(this._result, this._fields); + // console.log(blockAndRows); + // break; let block = blockAndRows.blocks; let num_of_rows = blockAndRows.num_of_rows; if (num_of_rows == 0) { @@ -205,22 +200,24 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { this._rowcount += num_of_rows; let numoffields = this._fields.length; for (let i = 0; i < num_of_rows; i++) { - data.push([]); - + // data.push([]); + let rowBlock = new Array(numoffields); for (let j = 0; j < numoffields; j++) { rowBlock[j] = block[j][i]; } - data[data.length-1] = (rowBlock); + data[this._rowcount - num_of_rows + i] = (rowBlock); + // data.push(rowBlock); } } + performance.mark('B'); performance.measure('query', 'A', 'B'); let response = this._createSetResponse(this._rowcount, time) console.log(response); - // this._connection._clearResultSet(); + // this._connection._clearResultSet(); let fields = this.fields; this._reset_result(); this.data = data; @@ -239,12 +236,12 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { * @return {number | Buffer} Number of affected rows or a Buffer that points to the results of the query * @since 1.0.0 */ -TDengineCursor.prototype.execute_a = function execute_a (operation, options, callback, param) { +TDengineCursor.prototype.execute_a = function execute_a(operation, options, callback, param) { if (operation == undefined) { throw new errors.ProgrammingError('No operation passed as argument'); return null; } - if (typeof options == 'function') { + if (typeof options == 'function') { //we expect the parameter after callback to be param param = callback; callback = options; @@ -265,14 +262,14 @@ TDengineCursor.prototype.execute_a = function execute_a (operation, options, cal } if (resCode >= 0) { -// let fieldCount = cr._chandle.numFields(res2); -// if (fieldCount == 0) { -// //cr._chandle.freeResult(res2); -// return res2; -// } -// else { -// return res2; -// } + // let fieldCount = cr._chandle.numFields(res2); + // if (fieldCount == 0) { + // //cr._chandle.freeResult(res2); + // return res2; + // } + // else { + // return res2; + // } return res2; } @@ -317,7 +314,7 @@ TDengineCursor.prototype.execute_a = function execute_a (operation, options, cal * }) */ TDengineCursor.prototype.fetchall_a = function fetchall_a(result, options, callback, param = {}) { - if (typeof options == 'function') { + if (typeof options == 'function') { //we expect the parameter after callback to be param param = callback; callback = options; @@ -360,17 +357,17 @@ TDengineCursor.prototype.fetchall_a = function fetchall_a(result, options, callb for (let k = 0; k < fields.length; k++) { rowBlock[k] = block[k][j]; } - data[data.length-1] = rowBlock; + data[data.length - 1] = rowBlock; } } cr._chandle.freeResult(result2); // free result, avoid seg faults and mem leaks! - callback(param2, result2, numOfRows2, {data:data,fields:fields}); + callback(param2, result2, numOfRows2, { data: data, fields: fields }); } } ref.writeObject(buf, 0, param); param = this._chandle.fetch_rows_a(result, asyncCallbackWrapper, buf); //returned param - return {param:param,result:result}; + return { param: param, result: result }; } /** * Stop a query given the result handle. @@ -428,7 +425,7 @@ TDengineCursor.prototype.subscribe = function subscribe(config) { */ TDengineCursor.prototype.consumeData = async function consumeData(subscription, callback) { while (true) { - let { data, fields, result} = this._chandle.consume(subscription); + let { data, fields, result } = this._chandle.consume(subscription); callback(data, fields, result); } } @@ -450,30 +447,30 @@ TDengineCursor.prototype.unsubscribe = function unsubscribe(subscription) { * @return {Buffer} A buffer pointing to the stream handle * @since 1.3.0 */ - TDengineCursor.prototype.openStream = function openStream(sql, callback, stime = 0, stoppingCallback, param = {}) { - let buf = ref.alloc('Object'); - ref.writeObject(buf, 0, param); +TDengineCursor.prototype.openStream = function openStream(sql, callback, stime = 0, stoppingCallback, param = {}) { + let buf = ref.alloc('Object'); + ref.writeObject(buf, 0, param); - let asyncCallbackWrapper = function (param2, result2, blocks, fields) { - let data = []; - let num_of_rows = blocks[0].length; - for (let j = 0; j < num_of_rows; j++) { - data.push([]); - let rowBlock = new Array(fields.length); - for (let k = 0; k < fields.length; k++) { - rowBlock[k] = blocks[k][j]; - } - data[data.length-1] = rowBlock; - } - callback(param2, result2, blocks, fields); - } - return this._chandle.openStream(this._connection._conn, sql, asyncCallbackWrapper, stime, stoppingCallback, buf); - } - /** - * Close a stream - * @param {Buffer} - A buffer pointing to the handle of the stream to be closed - * @since 1.3.0 - */ - TDengineCursor.prototype.closeStream = function closeStream(stream) { - this._chandle.closeStream(stream); - } + let asyncCallbackWrapper = function (param2, result2, blocks, fields) { + let data = []; + let num_of_rows = blocks[0].length; + for (let j = 0; j < num_of_rows; j++) { + data.push([]); + let rowBlock = new Array(fields.length); + for (let k = 0; k < fields.length; k++) { + rowBlock[k] = blocks[k][j]; + } + data[data.length - 1] = rowBlock; + } + callback(param2, result2, blocks, fields); + } + return this._chandle.openStream(this._connection._conn, sql, asyncCallbackWrapper, stime, stoppingCallback, buf); +} +/** + * Close a stream + * @param {Buffer} - A buffer pointing to the handle of the stream to be closed + * @since 1.3.0 + */ +TDengineCursor.prototype.closeStream = function closeStream(stream) { + this._chandle.closeStream(stream); +} diff --git a/src/connector/nodejs/package-lock.json b/src/connector/nodejs/package-lock.json deleted file mode 100644 index 9ca174ccd1..0000000000 --- a/src/connector/nodejs/package-lock.json +++ /dev/null @@ -1,285 +0,0 @@ -{ - "name": "td2.0-connector", - "version": "2.0.6", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "array-index": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-index/-/array-index-1.0.0.tgz", - "integrity": "sha1-7FanSe4QPk4Ix5C5w1PfFgVbl/k=", - "requires": { - "debug": "^2.2.0", - "es6-symbol": "^3.0.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } - } - }, - "d": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", - "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", - "requires": { - "es5-ext": "^0.10.50", - "type": "^1.0.1" - } - }, - "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", - "requires": { - "ms": "2.1.2" - } - }, - "es5-ext": { - "version": "0.10.53", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", - "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", - "requires": { - "es6-iterator": "~2.0.3", - "es6-symbol": "~3.1.3", - "next-tick": "~1.0.0" - } - }, - "es6-iterator": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", - "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=", - "requires": { - "d": "1", - "es5-ext": "^0.10.35", - "es6-symbol": "^3.1.1" - } - }, - "es6-symbol": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", - "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", - "requires": { - "d": "^1.0.1", - "ext": "^1.1.2" - } - }, - "ext": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", - "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", - "requires": { - "type": "^2.0.0" - }, - "dependencies": { - "type": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/type/-/type-2.1.0.tgz", - "integrity": "sha512-G9absDWvhAWCV2gmF1zKud3OyC61nZDwWvBL2DApaVFogI07CprggiQAOOjvp2NRjYWFzPyu7vwtDrQFq8jeSA==" - } - } - }, - "ffi-napi": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ffi-napi/-/ffi-napi-3.1.0.tgz", - "integrity": "sha512-EsHO+sP2p/nUC/3l/l8m9niee1BLm4asUFDzkkBGR4kYVgp2KqdAYUomZhkKtzim4Fq7mcYHjpUaIHsMqs+E1g==", - "requires": { - "debug": "^4.1.1", - "get-uv-event-loop-napi-h": "^1.0.5", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1", - "ref-napi": "^2.0.1", - "ref-struct-di": "^1.1.0" - }, - "dependencies": { - "ref-napi": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-2.1.2.tgz", - "integrity": "sha512-aFl+vrIuLWUXMUTQGAwGAuSNLX3Ub5W3iVP8b7KyFFZUdn4+i4U1TXXTop0kCTUfGNu8glBGVz4lowkwMcPVVA==", - "requires": { - "debug": "^4.1.1", - "get-symbol-from-current-process-h": "^1.0.2", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - } - } - } - }, - "get-symbol-from-current-process-h": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz", - "integrity": "sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw==" - }, - "get-uv-event-loop-napi-h": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz", - "integrity": "sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg==", - "requires": { - "get-symbol-from-current-process-h": "^1.0.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "next-tick": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", - "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" - }, - "node-addon-api": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", - "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" - }, - "node-gyp-build": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.2.3.tgz", - "integrity": "sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg==" - }, - "ref-array-napi": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ref-array-napi/-/ref-array-napi-1.2.1.tgz", - "integrity": "sha512-jQp2WWSucmxkqVfoNfm7yDlDeGu3liAbzqfwjNybL80ooLOCnCZpAK2woDInY+lxNOK/VlIVSqeDEYb4gVPuNQ==", - "requires": { - "array-index": "1", - "debug": "2", - "ref-napi": "^1.4.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "ref-napi": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", - "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", - "requires": { - "debug": "^3.1.0", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - } - } - }, - "ref-napi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.1.tgz", - "integrity": "sha512-W3rcb0E+tlO9u9ySFnX5vifInwwPGToOfFgTZUHJBNiOBsW0NNvgHz2zJN7ctABo/2yIlgdPQUvuqqfORIF4LA==", - "requires": { - "debug": "^4.1.1", - "get-symbol-from-current-process-h": "^1.0.2", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - } - }, - "ref-struct-di": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz", - "integrity": "sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g==", - "requires": { - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - } - } - }, - "ref-struct-napi": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ref-struct-napi/-/ref-struct-napi-1.1.1.tgz", - "integrity": "sha512-YgS5/d7+kT5zgtySYI5ieH0hREdv+DabgDvoczxsui0f9VLm0rrDcWEj4DHKehsH+tJnVMsLwuyctWgvdEcVRw==", - "requires": { - "debug": "2", - "ref-napi": "^1.4.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "ref-napi": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", - "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", - "requires": { - "debug": "^3.1.0", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - } - } - }, - "type": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", - "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" - } - } -} diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index b39ce2c17d..d21b62108b 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "td2.0-connector", - "version": "2.0.6", + "version": "2.0.7", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", "directories": { From 7316fb89796120fc274700f109612eb1eade2cbd Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 11:06:28 +0800 Subject: [PATCH 124/157] [TD-1920]add mnode compact wal functions --- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 1 + src/dnode/src/dnodeMain.c | 1 + src/dnode/src/dnodeSystem.c | 2 ++ src/inc/mnode.h | 2 ++ src/mnode/inc/mnodeSdb.h | 1 + src/mnode/src/mnodeMain.c | 6 ++++++ src/mnode/src/mnodeSdb.c | 12 ++++++++++++ 8 files changed, 26 insertions(+) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 2f4aa4c2b2..abb4d39ee1 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -142,6 +142,7 @@ extern int32_t tsMonitorInterval; extern int8_t tsEnableStream; // internal +extern int8_t tsCompactMnodeWal; extern int8_t tsPrintAuth; extern int8_t tscEmbedded; extern char configDir[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index db97c3a5af..ab036b021c 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -176,6 +176,7 @@ int32_t tsMonitorInterval = 30; // seconds int8_t tsEnableStream = 1; // internal +int8_t tsCompactMnodeWal = 0; int8_t tsPrintAuth = 0; int8_t tscEmbedded = 0; char configDir[TSDB_FILENAME_LEN] = {0}; diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 410e6bb188..48680e20d9 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -40,6 +40,7 @@ #include "dnodeShell.h" #include "dnodeTelemetry.h" #include "module.h" +#include "mnode.h" #if !defined(_MODULE) || !defined(_TD_LINUX) int32_t moduleStart() { return 0; } diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index e49b3eba99..ee37ffdcbb 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -42,6 +42,8 @@ int32_t main(int32_t argc, char *argv[]) { } } else if (strcmp(argv[i], "-C") == 0) { dump_config = 1; + } else if (strcmp(argv[i], "--compact-mnode-wal") == 0) { + tsCompactMnodeWal = 1; } else if (strcmp(argv[i], "-V") == 0) { #ifdef _ACCT char *versionStr = "enterprise"; diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 2495a42ba2..c66dc60a1b 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -73,6 +73,8 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg); void mnodeProcessPeerRsp(SRpcMsg *pMsg); int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); +int32_t mnodeCompactWal(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index e4e4a7a054..13dd8bd06b 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -106,6 +106,7 @@ int32_t sdbGetId(void *pTable); uint64_t sdbGetVersion(); bool sdbCheckRowDeleted(void *pTable, void *pRow); +int32_t mnodeCompactWal(); #ifdef __cplusplus } #endif diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 7ef0488c42..76eebe0d01 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -57,6 +57,12 @@ static SStep tsMnodeSteps[] = { {"show", mnodeInitShow, mnodeCleanUpShow} }; +/* +static SStep tsMnodeCompactSteps[] = { + {"cluster", mnodeInitCluster, NULL}, +}; +*/ + static void mnodeInitTimer(); static void mnodeCleanupTimer(); static bool mnodeNeedStart() ; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 505d3c519c..64c7713164 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -450,6 +450,12 @@ int32_t sdbInit() { } tsSdbMgmt.status = SDB_STATUS_SERVING; + + if (tsCompactMnodeWal) { + mnodeCompactWal(); + exit(EXIT_SUCCESS); + } + return TSDB_CODE_SUCCESS; } @@ -1138,3 +1144,9 @@ static void *sdbWorkerFp(void *pWorker) { int32_t sdbGetReplicaNum() { return tsSdbMgmt.cfg.replica; } + +int32_t mnodeCompactWal() { + sdbInfo("vgId:1, start compact mnode wal..."); + + return 0; +} \ No newline at end of file From 0a07ea10ad520318e44a4e90b5327c716e0bd150 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 20 May 2021 13:11:48 +0800 Subject: [PATCH 125/157] Feature/sangshuduo/td 4068 taosdemo stmt (#6166) * merge with develop branch. change query/tests/CMakeLists.txt to allow unused function and variable. * refactor data generating. * refactor. * refactor. * refactor. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 383 ++++++++++++++++++++---------------- 1 file changed, 213 insertions(+), 170 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 068aa52283..7a444c839d 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -70,6 +70,8 @@ enum TEST_MODE { #define MAX_RECORDS_PER_REQ 32766 +#define HEAD_BUFF_LEN 1024*24 // 16*1024 + (192+32)*2 + insert into .. + #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) #define COND_BUF_LEN BUFFER_SIZE - 30 @@ -1079,8 +1081,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } static bool getInfoFromJsonFile(char* file); -//static int generateOneRowDataForStb(SSuperTable* stbInfo); -//static int getDataIntoMemForStb(SSuperTable* stbInfo); static void init_rand_data(); static void tmfclose(FILE *fp) { if (NULL != fp) { @@ -1180,7 +1180,8 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { totalLen += len; } - verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile); + verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", + __func__, __LINE__, databuf, resultFile); appendResultBufToFile(databuf, resultFile); free(databuf); } @@ -1640,8 +1641,10 @@ static void printfInsertMetaToFile(FILE* fp) { */ fprintf(fp, " interlaceRows: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].interlaceRows); - fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); - fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); + fprintf(fp, " disorderRange: %d\n", + g_Dbs.db[i].superTbls[j].disorderRange); + fprintf(fp, " disorderRatio: %d\n", + g_Dbs.db[i].superTbls[j].disorderRatio); fprintf(fp, " maxSqlLen: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].maxSqlLen); @@ -1649,11 +1652,15 @@ static void printfInsertMetaToFile(FILE* fp) { g_Dbs.db[i].superTbls[j].timeStampStep); fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); - fprintf(fp, " sampleFormat: %s\n", g_Dbs.db[i].superTbls[j].sampleFormat); - fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); - fprintf(fp, " tagsFile: %s\n", g_Dbs.db[i].superTbls[j].tagsFile); + fprintf(fp, " sampleFormat: %s\n", + g_Dbs.db[i].superTbls[j].sampleFormat); + fprintf(fp, " sampleFile: %s\n", + g_Dbs.db[i].superTbls[j].sampleFile); + fprintf(fp, " tagsFile: %s\n", + g_Dbs.db[i].superTbls[j].tagsFile); - fprintf(fp, " columnCount: %d\n ", g_Dbs.db[i].superTbls[j].columnCount); + fprintf(fp, " columnCount: %d\n ", + g_Dbs.db[i].superTbls[j].columnCount); for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); if ((0 == strncasecmp( @@ -1665,7 +1672,8 @@ static void printfInsertMetaToFile(FILE* fp) { g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); } else { - fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); + fprintf(fp, "column[%d]:%s ", + k, g_Dbs.db[i].superTbls[j].columns[k].dataType); } } fprintf(fp, "\n"); @@ -4637,16 +4645,22 @@ static int getRowDataFromSample( return dataLen; } -static int64_t generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) { +static int64_t generateStbRowData( + SSuperTable* stbInfo, + char* recBuf, int64_t timestamp + ) { int64_t dataLen = 0; char *pstr = recBuf; int64_t maxLen = MAX_DATA_SIZE; - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ",", timestamp); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, + "(%" PRId64 ",", timestamp); for (int i = 0; i < stbInfo->columnCount; i++) { - if ((0 == strncasecmp(stbInfo->columns[i].dataType, "BINARY", strlen("BINARY"))) - || (0 == strncasecmp(stbInfo->columns[i].dataType, "NCHAR", strlen("NCHAR")))) { + if ((0 == strncasecmp(stbInfo->columns[i].dataType, + "BINARY", strlen("BINARY"))) + || (0 == strncasecmp(stbInfo->columns[i].dataType, + "NCHAR", strlen("NCHAR")))) { if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) { errorPrint( "binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); @@ -4708,7 +4722,7 @@ static int64_t generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stb } static int64_t generateData(char *recBuf, char **data_type, - int num_of_cols, int64_t timestamp, int lenOfBinary) { + int64_t timestamp, int lenOfBinary) { memset(recBuf, 0, MAX_DATA_SIZE); char *pstr = recBuf; pstr += sprintf(pstr, "(%" PRId64, timestamp); @@ -4859,99 +4873,52 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t table } } -static int64_t generateDataTail( - SSuperTable* superTblInfo, - uint64_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows, - uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { +static int64_t generateDataTailWithoutStb( + uint64_t batch, char* buffer, + int64_t remainderBufLen, int64_t insertRows, + uint64_t startFrom, int64_t startTime, + /* int64_t *pSamplePos, */int64_t *dataLen) { + uint64_t len = 0; - uint32_t ncols_per_record = 1; // count first col ts - char *pstr = buffer; - if (superTblInfo == NULL) { - uint32_t datatypeSeq = 0; - while(g_args.datatype[datatypeSeq]) { - datatypeSeq ++; - ncols_per_record ++; - } - } - verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch); - uint64_t k = 0; + int64_t k = 0; for (k = 0; k < batch;) { char data[MAX_DATA_SIZE]; memset(data, 0, MAX_DATA_SIZE); int64_t retLen = 0; - if (superTblInfo) { - if (0 == strncasecmp(superTblInfo->dataSource, - "sample", strlen("sample"))) { - retLen = getRowDataFromSample( - data, - remainderBufLen, - startTime + superTblInfo->timeStampStep * k, - superTblInfo, - pSamplePos); - } else if (0 == strncasecmp(superTblInfo->dataSource, - "rand", strlen("rand"))) { + char **data_type = g_args.datatype; + int lenOfBinary = g_args.len_of_binary; - int64_t randTail = superTblInfo->timeStampStep * k; - if (superTblInfo->disorderRatio > 0) { - int rand_num = taosRandom() % 100; - if(rand_num < superTblInfo->disorderRatio) { - randTail = (randTail + (taosRandom() % superTblInfo->disorderRange + 1)) * (-1); - debugPrint("rand data generated, back %"PRId64"\n", randTail); - } - } + int64_t randTail = DEFAULT_TIMESTAMP_STEP * k; - int64_t d = startTime - + randTail; - retLen = generateRowData( - data, - d, - superTblInfo); - } - - if (retLen > remainderBufLen) { - break; - } - - pstr += snprintf(pstr , retLen + 1, "%s", data); - k++; - len += retLen; - remainderBufLen -= retLen; - } else { - char **data_type = g_args.datatype; - int lenOfBinary = g_args.len_of_binary; - - int64_t randTail = DEFAULT_TIMESTAMP_STEP * k; - - if (g_args.disorderRatio != 0) { + if (g_args.disorderRatio != 0) { int rand_num = taosRandom() % 100; if (rand_num < g_args.disorderRatio) { - randTail = (randTail + (taosRandom() % g_args.disorderRange + 1)) * (-1); + randTail = (randTail + + (taosRandom() % g_args.disorderRange + 1)) * (-1); debugPrint("rand data generated, back %"PRId64"\n", randTail); } - } else { + } else { randTail = DEFAULT_TIMESTAMP_STEP * k; - } + } - retLen = generateData(data, data_type, - ncols_per_record, - startTime + randTail, - lenOfBinary); + retLen = generateData(data, data_type, + startTime + randTail, + lenOfBinary); - if (len > remainderBufLen) + if (len > remainderBufLen) break; - pstr += sprintf(pstr, "%s", data); - k++; - len += retLen; - remainderBufLen -= retLen; - } + pstr += sprintf(pstr, "%s", data); + k++; + len += retLen; + remainderBufLen -= retLen; verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n", __func__, __LINE__, len, k, buffer); @@ -4967,17 +4934,107 @@ static int64_t generateDataTail( return k; } -static int generateSQLHead(char *tableName, int32_t tableSeq, - threadInfo* pThreadInfo, SSuperTable* superTblInfo, +static int64_t generateStbDataTail( + SSuperTable* superTblInfo, + uint64_t batch, char* buffer, + int64_t remainderBufLen, int64_t insertRows, + uint64_t startFrom, int64_t startTime, + int64_t *pSamplePos, int64_t *dataLen) { + uint64_t len = 0; + + char *pstr = buffer; + + verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch); + + int64_t k = 0; + for (k = 0; k < batch;) { + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); + + int64_t retLen = 0; + + if (0 == strncasecmp(superTblInfo->dataSource, + "sample", strlen("sample"))) { + retLen = getRowDataFromSample( + data, + remainderBufLen, + startTime + superTblInfo->timeStampStep * k, + superTblInfo, + pSamplePos); + } else if (0 == strncasecmp(superTblInfo->dataSource, + "rand", strlen("rand"))) { + int64_t randTail = superTblInfo->timeStampStep * k; + if (superTblInfo->disorderRatio > 0) { + int rand_num = taosRandom() % 100; + if(rand_num < superTblInfo->disorderRatio) { + randTail = (randTail + + (taosRandom() % superTblInfo->disorderRange + 1)) * (-1); + debugPrint("rand data generated, back %"PRId64"\n", randTail); + } + } + + int64_t d = startTime + randTail; + retLen = generateStbRowData(superTblInfo, data, d); + } + + if (retLen > remainderBufLen) { + break; + } + + pstr += snprintf(pstr , retLen + 1, "%s", data); + k++; + len += retLen; + remainderBufLen -= retLen; + + verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n", + __func__, __LINE__, len, k, buffer); + + startFrom ++; + + if (startFrom >= insertRows) { + break; + } + } + + *dataLen = len; + return k; +} + + +static int generateSQLHeadWithoutStb(char *tableName, + char *dbName, char *buffer, int remainderBufLen) { int len; -#define HEAD_BUFF_LEN 1024*24 // 16*1024 + (192+32)*2 + insert into .. char headBuf[HEAD_BUFF_LEN]; - if (superTblInfo) { - if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + len = snprintf( + headBuf, + HEAD_BUFF_LEN, + "%s.%s values", + dbName, + tableName); + + if (len > remainderBufLen) + return -1; + + tstrncpy(buffer, headBuf, len + 1); + + return len; +} + +static int generateStbSQLHead( + SSuperTable* superTblInfo, + char *tableName, int32_t tableSeq, + char *dbName, + char *buffer, int remainderBufLen) +{ + int len; + + char headBuf[HEAD_BUFF_LEN]; + + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { char* tagsValBuf = NULL; if (0 == superTblInfo->tagSource) { tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq); @@ -4996,9 +5053,9 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, headBuf, HEAD_BUFF_LEN, "%s.%s using %s.%s tags %s values", - pThreadInfo->db_name, + dbName, tableName, - pThreadInfo->db_name, + dbName, superTblInfo->sTblName, tagsValBuf); tmfree(tagsValBuf); @@ -5007,22 +5064,14 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, headBuf, HEAD_BUFF_LEN, "%s.%s values", - pThreadInfo->db_name, + dbName, tableName); } else { len = snprintf( headBuf, HEAD_BUFF_LEN, "%s.%s values", - pThreadInfo->db_name, - tableName); - } - } else { - len = snprintf( - headBuf, - HEAD_BUFF_LEN, - "%s.%s values", - pThreadInfo->db_name, + dbName, tableName); } @@ -5046,8 +5095,10 @@ static int64_t generateInterlaceDataBuffer( char *pstr = buffer; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, - superTblInfo, pstr, *pRemainderBufLen); + int headLen = generateStbSQLHead( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, + pstr, *pRemainderBufLen); if (headLen <= 0) { return 0; @@ -5065,19 +5116,24 @@ static int64_t generateInterlaceDataBuffer( pThreadInfo->threadID, __func__, __LINE__, i, batchPerTblTimes, batchPerTbl); + int64_t k; if (superTblInfo) { if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { startTime = taosGetTimestamp(pThreadInfo->time_precision); } - } else { - startTime = 1500000000000; - } - int64_t k = generateDataTail( - superTblInfo, - batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, - startTime, - &(pThreadInfo->samplePos), &dataLen); + k = generateStbDataTail( + superTblInfo, + batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, + startTime, + &(pThreadInfo->samplePos), &dataLen); + } else { + startTime = 1500000000000; + k = generateDataTailWithoutStb( + batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, + startTime, + /* &(pThreadInfo->samplePos), */&dataLen); + } if (k == batchPerTbl) { pstr += dataLen; @@ -5103,25 +5159,23 @@ static int64_t generateProgressiveDataBuffer( { SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - int ncols_per_record = 1; // count first col ts - - if (superTblInfo == NULL) { - int datatypeSeq = 0; - while(g_args.datatype[datatypeSeq]) { - datatypeSeq ++; - ncols_per_record ++; - } - } - assert(buffer != NULL); char *pstr = buffer; - int64_t k = 0; - memset(buffer, 0, *pRemainderBufLen); - int64_t headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, superTblInfo, + int64_t headLen; + + if (superTblInfo) { + headLen = generateStbSQLHead( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, buffer, *pRemainderBufLen); + } else { + headLen = generateSQLHeadWithoutStb( + tableName, pThreadInfo->db_name, + buffer, *pRemainderBufLen); + } if (headLen <= 0) { return 0; @@ -5130,10 +5184,20 @@ static int64_t generateProgressiveDataBuffer( *pRemainderBufLen -= headLen; int64_t dataLen; - k = generateDataTail(superTblInfo, - g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom, + int64_t k; + + if (superTblInfo) { + k = generateStbDataTail(superTblInfo, + g_args.num_of_RPR, pstr, *pRemainderBufLen, + insertRows, startFrom, startTime, pSamplePos, &dataLen); + } else { + k = generateDataTailWithoutStb( + g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom, + startTime, + /*pSamplePos, */&dataLen); + } return k; } @@ -5147,6 +5211,7 @@ static void printStatPerThread(threadInfo *pThreadInfo) (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0))); } +// sync write interlace data static void* syncWriteInterlace(threadInfo *pThreadInfo) { debugPrint("[%d] %s() LN%d: ### interlace write\n", pThreadInfo->threadID, __func__, __LINE__); @@ -5197,7 +5262,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->totalInsertRows = 0; pThreadInfo->totalAffectedRows = 0; - int64_t nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; + int64_t nTimeStampStep = + superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; uint64_t insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; @@ -5211,7 +5277,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { uint64_t tableSeq = pThreadInfo->start_table_from; debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", - pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, + pThreadInfo->threadID, __func__, __LINE__, + pThreadInfo->start_table_from, pThreadInfo->ntables, insertRows); int64_t startTime = pThreadInfo->start_time; @@ -5384,19 +5451,18 @@ free_of_interlace: return NULL; } -// sync insertion -/* - 1 thread: 100 tables * 2000 rows/s - 1 thread: 10 tables * 20000 rows/s - 6 thread: 300 tables * 2000 rows/s - - 2 taosinsertdata , 1 thread: 10 tables * 20000 rows/s -*/ +// sync insertion progressive data static void* syncWriteProgressive(threadInfo *pThreadInfo) { debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__); SSuperTable* superTblInfo = pThreadInfo->superTblInfo; uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; + int64_t timeStampStep = + superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; + int64_t insertRows = + (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + verbosePrint("%s() LN%d insertRows=%"PRId64"\n", + __func__, __LINE__, insertRows); pThreadInfo->buffer = calloc(maxSqlLen, 1); if (NULL == pThreadInfo->buffer) { @@ -5410,8 +5476,6 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { uint64_t startTs = taosGetTimestampMs(); uint64_t endTs; - int64_t timeStampStep = - superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; /* int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; uint64_t st = 0; @@ -5423,21 +5487,12 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->samplePos = 0; - for (uint64_t tableSeq = - pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to; - tableSeq ++) { + for (uint64_t tableSeq = pThreadInfo->start_table_from; + tableSeq <= pThreadInfo->end_table_to; + tableSeq ++) { int64_t start_time = pThreadInfo->start_time; - int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; - verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); - for (uint64_t i = 0; i < insertRows;) { - /* - if (insert_interval) { - st = taosGetTimestampMs(); - } - */ - char tableName[TSDB_TABLE_NAME_LEN]; getTableName(tableName, pThreadInfo, tableSeq); verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n", @@ -5502,27 +5557,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { if (i >= insertRows) break; -/* - if (insert_interval) { - et = taosGetTimestampMs(); - - if (insert_interval > ((et - st)) ) { - int sleep_time = insert_interval - (et -st); - performancePrint("%s() LN%d sleep: %d ms for insert interval\n", - __func__, __LINE__, sleep_time); - taosMsleep(sleep_time); // ms - } - } - */ } // num_of_DPT - if (g_args.verbose_print) { - if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo && + if ((g_args.verbose_print) && + (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) && (0 == strncasecmp( superTblInfo->dataSource, "sample", strlen("sample")))) { verbosePrint("%s() LN%d samplePos=%"PRId64"\n", __func__, __LINE__, pThreadInfo->samplePos); - } } } // tableSeq @@ -5557,7 +5599,6 @@ static void* syncWrite(void *sarg) { // progressive mode return syncWriteProgressive(pThreadInfo); } - } static void callBack(void *param, TAOS_RES *res, int code) { @@ -5595,10 +5636,12 @@ static void callBack(void *param, TAOS_RES *res, int code) { int rand_num = taosRandom() % 100; if (0 != pThreadInfo->superTblInfo->disorderRatio && rand_num < pThreadInfo->superTblInfo->disorderRatio) { - int64_t d = pThreadInfo->lastTs - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); - generateRowData(data, d, pThreadInfo->superTblInfo); + int64_t d = pThreadInfo->lastTs + - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); + generateStbRowData(pThreadInfo->superTblInfo, data, d); } else { - generateRowData(data, pThreadInfo->lastTs += 1000, pThreadInfo->superTblInfo); + generateStbRowData(pThreadInfo->superTblInfo, + data, pThreadInfo->lastTs += 1000); } pstr += sprintf(pstr, "%s", data); pThreadInfo->counter++; From 4697cdfdafbc3e639a253f65fbae6bae4ea5c1a8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 20 May 2021 14:01:32 +0800 Subject: [PATCH 126/157] Hotfix/sangshuduo/td 4238 taosdemo async subscribe (#6170) * [TD-4238]: taosdemo async subscribe. * [TD-4238]: taosdemo async subscribe subsribe sql command do not use aggregation functions. * [TD-4238]: taosdemo async subscribe. interval. * [TD-4238]: taosdemo async subscribe. fix super table sub result file. * fix specified sub sync mode. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f8d3f83d47..ae7ffd16b9 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6682,13 +6682,18 @@ static void *superSubscribe(void *sarg) { continue; } - taosMsleep(g_queryInfo.superQueryInfo.subscribeInterval); // ms res = taos_consume(tsub[i]); if (res) { if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(pThreadInfo->fp, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + appendResultToFile(res, pThreadInfo->fp); + } + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); appendResultToFile(res, pThreadInfo->fp); } consumed[i] ++; @@ -6783,9 +6788,15 @@ static void *specifiedSubscribe(void *sarg) { continue; } - taosMsleep(g_queryInfo.specifiedQueryInfo.subscribeInterval); // ms res = taos_consume(tsub); if (res) { + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + appendResultToFile(res, pThreadInfo->fp); + } + consumed ++; if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) && (consumed >= From c69604fdb4c7e68e70bb61c33fa610d20c313c55 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 20 May 2021 14:51:59 +0800 Subject: [PATCH 127/157] Hotfix/sangshuduo/td 4238 taosdemo async for develop (#6171) * [TD-4238]: taosdemo async sub. for develop branch. * [TD-4238]: taosdemo async subscribe subsribe sql command do not use aggregation functions. * cherry pick f5e4cdd1f7cffc4ee04fc7d043e4841bd1dd9961 * [TD-4238]: taosdemo async subscribe. fix super table sub result file. * fix specified sub sync mode. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 7a444c839d..ebc1074cba 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6755,13 +6755,18 @@ static void *superSubscribe(void *sarg) { continue; } - taosMsleep(g_queryInfo.superQueryInfo.subscribeInterval); // ms res = taos_consume(tsub[i]); if (res) { if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(pThreadInfo->fp, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + appendResultToFile(res, pThreadInfo->fp); + } + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); appendResultToFile(res, pThreadInfo->fp); } consumed[i] ++; @@ -6856,9 +6861,15 @@ static void *specifiedSubscribe(void *sarg) { continue; } - taosMsleep(g_queryInfo.specifiedQueryInfo.subscribeInterval); // ms res = taos_consume(tsub); if (res) { + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + appendResultToFile(res, pThreadInfo->fp); + } + consumed ++; if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) && (consumed >= From a12b0e71755e2d3fce52c3c646e427aa86e02f69 Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 15:12:34 +0800 Subject: [PATCH 128/157] [TD-1920]add mnode compact wal functions --- src/inc/mnode.h | 1 + src/mnode/inc/mnodeAcct.h | 2 ++ src/mnode/inc/mnodeCluster.h | 2 ++ src/mnode/inc/mnodeDb.h | 2 ++ src/mnode/inc/mnodeDnode.h | 1 + src/mnode/inc/mnodeMnode.h | 1 + src/mnode/inc/mnodeSdb.h | 1 + src/mnode/inc/mnodeTable.h | 1 + src/mnode/inc/mnodeUser.h | 2 ++ src/mnode/inc/mnodeVgroup.h | 1 + src/mnode/src/mnodeAcct.c | 26 +++++++++++++++ src/mnode/src/mnodeCluster.c | 24 ++++++++++++++ src/mnode/src/mnodeDb.c | 27 ++++++++++++++++ src/mnode/src/mnodeDnode.c | 27 ++++++++++++++++ src/mnode/src/mnodeMain.c | 16 ++++++++-- src/mnode/src/mnodeMnode.c | 27 ++++++++++++++++ src/mnode/src/mnodeSdb.c | 27 ++++++++++++++++ src/mnode/src/mnodeTable.c | 62 ++++++++++++++++++++++++++++++++++++ src/mnode/src/mnodeUser.c | 27 ++++++++++++++++ src/mnode/src/mnodeVgroup.c | 27 ++++++++++++++++ 20 files changed, 301 insertions(+), 3 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index c66dc60a1b..203ac57469 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -74,6 +74,7 @@ void mnodeProcessPeerRsp(SRpcMsg *pMsg); int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); int32_t mnodeCompactWal(); +int32_t mnodeCompactComponents(); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeAcct.h b/src/mnode/inc/mnodeAcct.h index 595dcca413..522070e909 100644 --- a/src/mnode/inc/mnodeAcct.h +++ b/src/mnode/inc/mnodeAcct.h @@ -35,6 +35,8 @@ void mnodeDropDbFromAcct(SAcctObj *pAcct, SDbObj *pDb); void mnodeAddUserToAcct(SAcctObj *pAcct, SUserObj *pUser); void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser); +int32_t mnodeCompactAccts(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeCluster.h b/src/mnode/inc/mnodeCluster.h index a5af544dc2..db258ae6d6 100644 --- a/src/mnode/inc/mnodeCluster.h +++ b/src/mnode/inc/mnodeCluster.h @@ -25,6 +25,8 @@ void mnodeCleanupCluster(); void mnodeUpdateClusterId(); const char* mnodeGetClusterId(); +int32_t mnodeCompactCluster(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index da0865833d..0fa1a15e2d 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -41,6 +41,8 @@ void mnodeDecDbRef(SDbObj *pDb); bool mnodeCheckIsMonitorDB(char *db, char *monitordb); void mnodeDropAllDbs(SAcctObj *pAcct); +int32_t mnodeCompactDbs(); + // util func void mnodeAddSuperTableIntoDb(SDbObj *pDb); void mnodeRemoveSuperTableFromDb(SDbObj *pDb); diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index fa1995254e..2db794a173 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -77,6 +77,7 @@ void * mnodeGetDnodeByEp(char *ep); void mnodeUpdateDnode(SDnodeObj *pDnode); int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg); +int32_t mnodeCompactDnodes(); extern int32_t tsAccessSquence; #ifdef __cplusplus diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index ffdec02eb6..66e9eb1e0e 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -50,6 +50,7 @@ char* mnodeGetMnodeMasterEp(); void mnodeGetMnodeInfos(void *mnodes); void mnodeUpdateMnodeEpSet(SMInfos *pMnodes); +int32_t mnodeCompactMnodes(); #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 13dd8bd06b..2ae0c47902 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -92,6 +92,7 @@ void sdbUpdateMnodeRoles(); int32_t sdbGetReplicaNum(); int32_t sdbInsertRow(SSdbRow *pRow); +int32_t sdbInsertCompactRow(SSdbRow *pRow); int32_t sdbDeleteRow(SSdbRow *pRow); int32_t sdbUpdateRow(SSdbRow *pRow); int32_t sdbInsertRowToQueue(SSdbRow *pRow); diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h index bf04f26a90..c583a60c7a 100644 --- a/src/mnode/inc/mnodeTable.h +++ b/src/mnode/inc/mnodeTable.h @@ -36,6 +36,7 @@ void mnodeCancelGetNextSuperTable(void *pIter); void mnodeDropAllChildTables(SDbObj *pDropDb); void mnodeDropAllSuperTables(SDbObj *pDropDb); void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup); +int32_t mnodeCompactTables(); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeUser.h b/src/mnode/inc/mnodeUser.h index 156bc7aaeb..b8f0805120 100644 --- a/src/mnode/inc/mnodeUser.h +++ b/src/mnode/inc/mnodeUser.h @@ -33,6 +33,8 @@ char * mnodeGetUserFromMsg(void *pMnodeMsg); int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg); void mnodeDropAllUsers(SAcctObj *pAcct); +int32_t mnodeCompactUsers(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 7b798c23f8..73b0e6ae1b 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -32,6 +32,7 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb); void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb); void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode); //void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb); +int32_t mnodeCompactVgroups(); void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup); void mnodeCancelGetNextVgroup(void *pIter); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index afe474df6b..64cfa28917 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -238,6 +238,32 @@ static int32_t mnodeCreateRootAcct() { return sdbInsertRow(&row); } +int32_t mnodeCompactAccts() { + void *pIter = NULL; + SAcctObj *pAcct = NULL; + + mInfo("start to compact accts table..."); + + while (1) { + pIter = mnodeGetNextAcct(pIter, &pAcct); + if (pAcct == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsAcctSdb, + .pObj = pAcct, + }; + + mInfo("compact accts %s", pAcct->user); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact accts table..."); + + return 0; +} + #ifndef _ACCT int32_t acctInit() { return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 169d2ebd9d..8b9af6d493 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -237,3 +237,27 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, pShow->numOfReads += numOfRows; return numOfRows; } + +int32_t mnodeCompactCluster() { + SClusterObj *pCluster = NULL; + void *pIter; + + mInfo("start to compact cluster table..."); + + pIter = mnodeGetNextCluster(NULL, &pCluster); + while (pCluster) { + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsClusterSdb, + .pObj = pCluster, + }; + + sdbInsertCompactRow(&row); + + pIter = mnodeGetNextCluster(pIter, &pCluster); + } + + mInfo("end to compact cluster table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8af20aa862..b5663bc00c 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -1271,3 +1271,30 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { mInfo("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs); } + +int32_t mnodeCompactDbs() { + void *pIter = NULL; + SDbObj *pDb = NULL; + + mInfo("start to compact dbs table..."); + + while (1) { + pIter = mnodeGetNextDb(pIter, &pDb); + if (pDb == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj), + }; + + mInfo("compact dbs %s", pDb->name); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact dbs table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 3525bcac18..51f16e4bc6 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -1270,3 +1270,30 @@ char* dnodeRoles[] = { "vnode", "any" }; + +int32_t mnodeCompactDnodes() { + SDnodeObj *pDnode = NULL; + void * pIter = NULL; + + mInfo("start to compact dnodes table..."); + + while (1) { + pIter = mnodeGetNextDnode(pIter, &pDnode); + if (pDnode == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDnodeSdb, + .pObj = pDnode, + .rowSize = sizeof(SDnodeObj), + }; + + mInfo("compact dnode %d", pDnode->dnodeId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact dnodes table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 76eebe0d01..01b83bec19 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -57,11 +57,16 @@ static SStep tsMnodeSteps[] = { {"show", mnodeInitShow, mnodeCleanUpShow} }; -/* static SStep tsMnodeCompactSteps[] = { - {"cluster", mnodeInitCluster, NULL}, + {"cluster", mnodeCompactCluster, NULL}, + {"dnodes", mnodeCompactDnodes, NULL}, + {"mnodes", mnodeCompactMnodes, NULL}, + {"accts", mnodeCompactAccts, NULL}, + {"users", mnodeCompactUsers, NULL}, + {"dbs", mnodeCompactDbs, NULL}, + {"vgroups", mnodeCompactVgroups, NULL}, + {"tables", mnodeCompactTables, NULL}, }; -*/ static void mnodeInitTimer(); static void mnodeCleanupTimer(); @@ -77,6 +82,11 @@ static int32_t mnodeInitComponents() { return dnodeStepInit(tsMnodeSteps, stepSize); } +int32_t mnodeCompactComponents() { + int32_t stepSize = sizeof(tsMnodeCompactSteps) / sizeof(SStep); + return dnodeStepInit(tsMnodeCompactSteps, stepSize); +} + int32_t mnodeStartSystem() { if (tsMgmtIsRunning) { mInfo("mnode module already started..."); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index ca6d6400ae..ddc9ea59c4 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -566,3 +566,30 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo return numOfRows; } + +int32_t mnodeCompactMnodes() { + void *pIter = NULL; + SMnodeObj *pMnode = NULL; + + mInfo("start to compact mnodes table..."); + + while (1) { + pIter = mnodeGetNextMnode(pIter, &pMnode); + if (pMnode == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsMnodeSdb, + .pObj = pMnode, + .rowSize = sizeof(SMnodeObj), + }; + + mInfo("compact mnode %d", pMnode->mnodeId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact mnodes table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 64c7713164..79081945d9 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -732,6 +732,12 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * } } +int32_t sdbInsertCompactRow(SSdbRow *pRow) { + SSdbTable *pTable = pRow->pTable; + if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; + return sdbWriteRowToQueue(pRow, SDB_ACTION_INSERT); +} + int32_t sdbInsertRow(SSdbRow *pRow) { SSdbTable *pTable = pRow->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; @@ -1148,5 +1154,26 @@ int32_t sdbGetReplicaNum() { int32_t mnodeCompactWal() { sdbInfo("vgId:1, start compact mnode wal..."); + // close wal + walFsync(tsSdbMgmt.wal, true); + walClose(tsSdbMgmt.wal); + + // change wal to wal_bak dir + char temp[TSDB_FILENAME_LEN] = {0}; + SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; + sprintf(temp, "%s/wal_tmp", tsMnodeDir); + if (mkdir(temp, 0755) != 0 && errno != EEXIST) { + return -1; + } + tsSdbMgmt.wal = walOpen(temp, &walCfg); + walRenew(tsSdbMgmt.wal); + + // compact memory tables info to wal tmp dir + mnodeCompactComponents(); + + // close wal + walFsync(tsSdbMgmt.wal, true); + walClose(tsSdbMgmt.wal); + return 0; } \ No newline at end of file diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2a8e941fcb..2e5d80b787 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -3242,3 +3242,65 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro return numOfRows; } + +static int32_t mnodeCompactSuperTables() { + void *pIter = NULL; + SSTableObj *pTable = NULL; + + mInfo("start to compact super table..."); + + while (1) { + pIter = mnodeGetNextSuperTable(pIter, &pTable); + if (pTable == NULL) break; + + int32_t schemaSize = (pTable->numOfColumns + pTable->numOfTags) * sizeof(SSchema); + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pTable, + .rowSize = sizeof(SSTableObj) + schemaSize, + }; + + mInfo("compact super %ld", pTable->uid); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact super table..."); + + return 0; +} + +static int32_t mnodeCompactChildTables() { + void *pIter = NULL; + SCTableObj *pTable = NULL; + + mInfo("start to compact child table..."); + + while (1) { + pIter = mnodeGetNextChildTable(pIter, &pTable); + if (pTable == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pObj = pTable, + .pTable = tsChildTableSdb, + }; + + mInfo("compact child %ld:%d", pTable->uid, pTable->tid); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact child table..."); + + return 0; +} + +int32_t mnodeCompactTables() { + mnodeCompactSuperTables(); + + mnodeCompactChildTables(); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index e77c1b3e59..c5c54791cf 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -617,3 +617,30 @@ static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg) { return mnodeRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey); } + +int32_t mnodeCompactUsers() { + void *pIter = NULL; + SUserObj *pUser = NULL; + + mInfo("start to compact users table..."); + + while (1) { + pIter = mnodeGetNextUser(pIter, &pUser); + if (pUser == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsUserSdb, + .pObj = pUser, + .rowSize = sizeof(SUserObj), + }; + + mInfo("compact users %s", pUser->user); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact users table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 7222c8d1a0..67532ad85a 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -1302,3 +1302,30 @@ void mnodeSetVgidVer(int8_t *cver, uint64_t iver) { cver[1] = (int8_t)((int32_t)(iver % 100000) / 100); cver[2] = (int8_t)(iver % 100); } + +int32_t mnodeCompactVgroups() { + void *pIter = NULL; + SVgObj *pVgroup = NULL; + + mInfo("start to compact vgroups table..."); + + while (1) { + pIter = mnodeGetNextVgroup(pIter, &pVgroup); + if (pVgroup == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj), + }; + + mInfo("compact vgroups %d", pVgroup->vgId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact vgroups table..."); + + return 0; +} \ No newline at end of file From 99f7359390539e659b7de331344015582aa1fbbe Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 16:04:53 +0800 Subject: [PATCH 129/157] [TD-1920]add mnode compact wal functions --- src/mnode/src/mnodeCluster.c | 2 +- src/mnode/src/mnodeMain.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 8b9af6d493..553e8446ab 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -249,7 +249,7 @@ int32_t mnodeCompactCluster() { SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsClusterSdb, - .pObj = pCluster, + .pObj = pCluster, }; sdbInsertCompactRow(&row); diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 01b83bec19..d9abe16120 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -59,6 +59,8 @@ static SStep tsMnodeSteps[] = { static SStep tsMnodeCompactSteps[] = { {"cluster", mnodeCompactCluster, NULL}, + +/* {"dnodes", mnodeCompactDnodes, NULL}, {"mnodes", mnodeCompactMnodes, NULL}, {"accts", mnodeCompactAccts, NULL}, @@ -66,6 +68,7 @@ static SStep tsMnodeCompactSteps[] = { {"dbs", mnodeCompactDbs, NULL}, {"vgroups", mnodeCompactVgroups, NULL}, {"tables", mnodeCompactTables, NULL}, +*/ }; static void mnodeInitTimer(); From f0b77fa3c3af7564054fe1c4be9210be0d5e8805 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Thu, 20 May 2021 17:16:09 +0800 Subject: [PATCH 130/157] [TD-4258]: speedup nodejs fetchall() with taos_fetch_block (#6165) --- src/connector/nodejs/nodetaos/cinterface.js | 254 ++++++++--------- src/connector/nodejs/nodetaos/cursor.js | 121 ++++----- src/connector/nodejs/package-lock.json | 285 -------------------- src/connector/nodejs/package.json | 2 +- 4 files changed, 188 insertions(+), 474 deletions(-) delete mode 100644 src/connector/nodejs/package-lock.json diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 43a08a800a..f3961e3787 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -9,7 +9,7 @@ const ffi = require('ffi-napi'); const ArrayType = require('ref-array-napi'); const Struct = require('ref-struct-napi'); const FieldTypes = require('./constants'); -const errors = require ('./error'); +const errors = require('./error'); const TaosObjects = require('./taosobjects'); const { NULL_POINTER } = require('ref-napi'); @@ -22,7 +22,7 @@ function convertMicrosecondsToDatetime(time) { return new TaosObjects.TaosTimestamp(time * 0.001, true); } -function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { timestampConverter = convertMillisecondsToDatetime; if (micro == true) { timestampConverter = convertMicrosecondsToDatetime; @@ -44,14 +44,14 @@ function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro=false } return res; } -function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = new Array(data.length); for (let i = 0; i < data.length; i++) { if (data[i] == 0) { res[i] = false; } - else if (data[i] == 1){ + else if (data[i] == 1) { res[i] = true; } else if (data[i] == FieldTypes.C_BOOL_NULL) { @@ -60,29 +60,29 @@ function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; while (currOffset < data.length) { - let d = data.readIntLE(currOffset,1); + let d = data.readIntLE(currOffset, 1); res.push(d == FieldTypes.C_TINYINT_NULL ? null : d); currOffset += nbytes; } return res; } -function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; while (currOffset < data.length) { - let d = data.readIntLE(currOffset,2); + let d = data.readIntLE(currOffset, 2); res.push(d == FieldTypes.C_SMALLINT_NULL ? null : d); currOffset += nbytes; } return res; } -function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -93,7 +93,7 @@ function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -104,7 +104,7 @@ function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -115,7 +115,7 @@ function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -126,7 +126,7 @@ function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -142,7 +142,7 @@ function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let dataEntry = data.slice(0, nbytes); //one entry in a row under a column; @@ -153,23 +153,23 @@ function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { // Object with all the relevant converters from pblock data to javascript readable data let convertFunctions = { - [FieldTypes.C_BOOL] : convertBool, - [FieldTypes.C_TINYINT] : convertTinyint, - [FieldTypes.C_SMALLINT] : convertSmallint, - [FieldTypes.C_INT] : convertInt, - [FieldTypes.C_BIGINT] : convertBigint, - [FieldTypes.C_FLOAT] : convertFloat, - [FieldTypes.C_DOUBLE] : convertDouble, - [FieldTypes.C_BINARY] : convertBinary, - [FieldTypes.C_TIMESTAMP] : convertTimestamp, - [FieldTypes.C_NCHAR] : convertNchar + [FieldTypes.C_BOOL]: convertBool, + [FieldTypes.C_TINYINT]: convertTinyint, + [FieldTypes.C_SMALLINT]: convertSmallint, + [FieldTypes.C_INT]: convertInt, + [FieldTypes.C_BIGINT]: convertBigint, + [FieldTypes.C_FLOAT]: convertFloat, + [FieldTypes.C_DOUBLE]: convertDouble, + [FieldTypes.C_BINARY]: convertBinary, + [FieldTypes.C_TIMESTAMP]: convertTimestamp, + [FieldTypes.C_NCHAR]: convertNchar } // Define TaosField structure var char_arr = ArrayType(ref.types.char); var TaosField = Struct({ - 'name': char_arr, - }); + 'name': char_arr, +}); TaosField.fields.name.type.size = 65; TaosField.defineProperty('type', ref.types.char); TaosField.defineProperty('bytes', ref.types.short); @@ -183,7 +183,7 @@ TaosField.defineProperty('bytes', ref.types.short); * @classdesc The CTaosInterface is the interface through which Node.JS communicates data back and forth with TDengine. It is not advised to * access this class directly and use it unless you understand what these functions do. */ -function CTaosInterface (config = null, pass = false) { +function CTaosInterface(config = null, pass = false) { ref.types.char_ptr = ref.refType(ref.types.char); ref.types.void_ptr = ref.refType(ref.types.void); ref.types.void_ptr2 = ref.refType(ref.types.void_ptr); @@ -196,64 +196,65 @@ function CTaosInterface (config = null, pass = false) { taoslibname = 'libtaos'; } this.libtaos = ffi.Library(taoslibname, { - 'taos_options': [ ref.types.int, [ ref.types.int , ref.types.void_ptr ] ], - 'taos_init': [ ref.types.void, [ ] ], + 'taos_options': [ref.types.int, [ref.types.int, ref.types.void_ptr]], + 'taos_init': [ref.types.void, []], //TAOS *taos_connect(char *ip, char *user, char *pass, char *db, int port) - 'taos_connect': [ ref.types.void_ptr, [ ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int ] ], + 'taos_connect': [ref.types.void_ptr, [ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int]], //void taos_close(TAOS *taos) - 'taos_close': [ ref.types.void, [ ref.types.void_ptr ] ], - //int *taos_fetch_lengths(TAOS_RES *taos); - 'taos_fetch_lengths': [ ref.types.void_ptr, [ ref.types.void_ptr ] ], + 'taos_close': [ref.types.void, [ref.types.void_ptr]], + //int *taos_fetch_lengths(TAOS_RES *res); + 'taos_fetch_lengths': [ref.types.void_ptr, [ref.types.void_ptr]], //int taos_query(TAOS *taos, char *sqlstr) - 'taos_query': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.char_ptr ] ], - //int taos_affected_rows(TAOS *taos) - 'taos_affected_rows': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_query': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.char_ptr]], + //int taos_affected_rows(TAOS_RES *res) + 'taos_affected_rows': [ref.types.int, [ref.types.void_ptr]], //int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) - 'taos_fetch_block': [ ref.types.int, [ ref.types.void_ptr, ref.types.void_ptr] ], + 'taos_fetch_block': [ref.types.int, [ref.types.void_ptr, ref.types.void_ptr]], //int taos_num_fields(TAOS_RES *res); - 'taos_num_fields': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_num_fields': [ref.types.int, [ref.types.void_ptr]], //TAOS_ROW taos_fetch_row(TAOS_RES *res) //TAOS_ROW is void **, but we set the return type as a reference instead to get the row - 'taos_fetch_row': [ ref.refType(ref.types.void_ptr2), [ ref.types.void_ptr ] ], + 'taos_fetch_row': [ref.refType(ref.types.void_ptr2), [ref.types.void_ptr]], + 'taos_print_row': [ref.types.int, [ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int]], //int taos_result_precision(TAOS_RES *res) - 'taos_result_precision': [ ref.types.int, [ ref.types.void_ptr ] ], + 'taos_result_precision': [ref.types.int, [ref.types.void_ptr]], //void taos_free_result(TAOS_RES *res) - 'taos_free_result': [ ref.types.void, [ ref.types.void_ptr] ], + 'taos_free_result': [ref.types.void, [ref.types.void_ptr]], //int taos_field_count(TAOS *taos) - 'taos_field_count': [ ref.types.int, [ ref.types.void_ptr ] ], + 'taos_field_count': [ref.types.int, [ref.types.void_ptr]], //TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) - 'taos_fetch_fields': [ ref.refType(TaosField), [ ref.types.void_ptr ] ], + 'taos_fetch_fields': [ref.refType(TaosField), [ref.types.void_ptr]], //int taos_errno(TAOS *taos) - 'taos_errno': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_errno': [ref.types.int, [ref.types.void_ptr]], //char *taos_errstr(TAOS *taos) - 'taos_errstr': [ ref.types.char_ptr, [ ref.types.void_ptr] ], + 'taos_errstr': [ref.types.char_ptr, [ref.types.void_ptr]], //void taos_stop_query(TAOS_RES *res); - 'taos_stop_query': [ ref.types.void, [ ref.types.void_ptr] ], + 'taos_stop_query': [ref.types.void, [ref.types.void_ptr]], //char *taos_get_server_info(TAOS *taos); - 'taos_get_server_info': [ ref.types.char_ptr, [ ref.types.void_ptr ] ], + 'taos_get_server_info': [ref.types.char_ptr, [ref.types.void_ptr]], //char *taos_get_client_info(); - 'taos_get_client_info': [ ref.types.char_ptr, [ ] ], + 'taos_get_client_info': [ref.types.char_ptr, []], // ASYNC // void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *, TAOS_RES *, int), void *param) - 'taos_query_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr ] ], + 'taos_query_a': [ref.types.void, [ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr]], // void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); - 'taos_fetch_rows_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr ]], + 'taos_fetch_rows_a': [ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr]], // Subscription //TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) - 'taos_subscribe': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int] ], + 'taos_subscribe': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int]], // TAOS_RES *taos_consume(TAOS_SUB *tsub) - 'taos_consume': [ ref.types.void_ptr, [ref.types.void_ptr] ], + 'taos_consume': [ref.types.void_ptr, [ref.types.void_ptr]], //void taos_unsubscribe(TAOS_SUB *tsub); - 'taos_unsubscribe': [ ref.types.void, [ ref.types.void_ptr ] ], + 'taos_unsubscribe': [ref.types.void, [ref.types.void_ptr]], // Continuous Query //TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), // int64_t stime, void *param, void (*callback)(void *)); - 'taos_open_stream': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.int64, ref.types.void_ptr, ref.types.void_ptr ] ], + 'taos_open_stream': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.int64, ref.types.void_ptr, ref.types.void_ptr]], //void taos_close_stream(TAOS_STREAM *tstr); - 'taos_close_stream': [ ref.types.void, [ ref.types.void_ptr ] ] + 'taos_close_stream': [ref.types.void, [ref.types.void_ptr]] }); if (pass == false) { @@ -264,7 +265,7 @@ function CTaosInterface (config = null, pass = false) { try { this._config = ref.allocCString(config); } - catch(err){ + catch (err) { throw "Attribute Error: config is expected as a str"; } } @@ -276,38 +277,38 @@ function CTaosInterface (config = null, pass = false) { return this; } CTaosInterface.prototype.config = function config() { - return this._config; - } -CTaosInterface.prototype.connect = function connect(host=null, user="root", password="taosdata", db=null, port=0) { - let _host,_user,_password,_db,_port; - try { + return this._config; +} +CTaosInterface.prototype.connect = function connect(host = null, user = "root", password = "taosdata", db = null, port = 0) { + let _host, _user, _password, _db, _port; + try { _host = host != null ? ref.allocCString(host) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: host is expected as a str"; } try { _user = ref.allocCString(user) } - catch(err) { + catch (err) { throw "Attribute Error: user is expected as a str"; } try { _password = ref.allocCString(password); } - catch(err) { + catch (err) { throw "Attribute Error: password is expected as a str"; } try { _db = db != null ? ref.allocCString(db) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: db is expected as a str"; } try { _port = ref.alloc(ref.types.int, port); } - catch(err) { + catch (err) { throw TypeError("port is expected as an int") } let connection = this.libtaos.taos_connect(_host, _user, _password, _db, _port); @@ -324,10 +325,10 @@ CTaosInterface.prototype.close = function close(connection) { console.log("Connection is closed"); } CTaosInterface.prototype.query = function query(connection, sql) { - return this.libtaos.taos_query(connection, ref.allocCString(sql)); + return this.libtaos.taos_query(connection, ref.allocCString(sql)); } -CTaosInterface.prototype.affectedRows = function affectedRows(connection) { - return this.libtaos.taos_affected_rows(connection); +CTaosInterface.prototype.affectedRows = function affectedRows(result) { + return this.libtaos.taos_affected_rows(result); } CTaosInterface.prototype.useResult = function useResult(result) { @@ -337,8 +338,8 @@ CTaosInterface.prototype.useResult = function useResult(result) { pfields = ref.reinterpret(pfields, this.fieldsCount(result) * 68, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,65,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 65, i)), type: pfields[i + 65], bytes: pfields[i + 66] }) @@ -347,11 +348,10 @@ CTaosInterface.prototype.useResult = function useResult(result) { return fields; } CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { - //let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data - let pblock = this.libtaos.taos_fetch_row(result); - let num_of_rows = 1; - if (ref.isNull(pblock) == true) { - return {block:null, num_of_rows:0}; + let pblock = ref.NULL_POINTER; + let num_of_rows = this.libtaos.taos_fetch_block(result, pblock); + if (ref.isNull(pblock.deref()) == true) { + return { block: null, num_of_rows: 0 }; } var fieldL = this.libtaos.taos_fetch_lengths(result); @@ -359,10 +359,10 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO); var fieldlens = []; - + if (ref.isNull(fieldL) == false) { - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 4, i*4); + for (let i = 0; i < fields.length; i++) { + let plen = ref.reinterpret(fieldL, 4, i * 4); let len = plen.readInt32LE(0); fieldlens.push(len); } @@ -370,21 +370,23 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { let blocks = new Array(fields.length); blocks.fill(null); - //num_of_rows = Math.abs(num_of_rows); + num_of_rows = Math.abs(num_of_rows); let offset = 0; + let ptr = pblock.deref(); + for (let i = 0; i < fields.length; i++) { - pdata = ref.reinterpret(pblock,8,i*8); - if(ref.isNull(pdata.readPointer())){ - blocks[i] = new Array(); - }else{ - pdata = ref.ref(pdata.readPointer()); - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); - } + pdata = ref.reinterpret(ptr, 8, i * 8); + if (ref.isNull(pdata.readPointer())) { + blocks[i] = new Array(); + } else { + pdata = ref.ref(pdata.readPointer()); + if (!convertFunctions[fields[i]['type']]) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + blocks[i] = convertFunctions[fields[i]['type']](pdata, num_of_rows, fieldlens[i], offset, isMicro); + } } - return {blocks: blocks, num_of_rows:Math.abs(num_of_rows)} + return { blocks: blocks, num_of_rows } } CTaosInterface.prototype.fetchRow = function fetchRow(result, fields) { let row = this.libtaos.taos_fetch_row(result); @@ -414,7 +416,7 @@ CTaosInterface.prototype.errStr = function errStr(result) { // Async CTaosInterface.prototype.query_a = function query_a(connection, sql, callback, param = ref.ref(ref.NULL)) { // void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, int), void *param) - callback = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.int ], callback); + callback = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.int], callback); this.libtaos.taos_query_a(connection, ref.allocCString(sql), callback, param); return param; } @@ -439,46 +441,46 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback, var fieldL = cti.libtaos.taos_fetch_lengths(result); var fieldlens = []; if (ref.isNull(fieldL) == false) { - - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 8, i*8); - let len = ref.get(plen,0,ref.types.int32); + + for (let i = 0; i < fields.length; i++) { + let plen = ref.reinterpret(fieldL, 8, i * 8); + let len = ref.get(plen, 0, ref.types.int32); fieldlens.push(len); } } - if (numOfRows2 > 0){ + if (numOfRows2 > 0) { for (let i = 0; i < fields.length; i++) { - if(ref.isNull(pdata.readPointer())){ - blocks[i] = new Array(); - }else{ - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - let prow = ref.reinterpret(row,8,i*8); - prow = prow.readPointer(); - prow = ref.ref(prow); - blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); - //offset += fields[i]['bytes'] * numOfRows2; - } + if (ref.isNull(pdata.readPointer())) { + blocks[i] = new Array(); + } else { + if (!convertFunctions[fields[i]['type']]) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + let prow = ref.reinterpret(row, 8, i * 8); + prow = prow.readPointer(); + prow = ref.ref(prow); + blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); + //offset += fields[i]['bytes'] * numOfRows2; + } } } callback(param2, result2, numOfRows2, blocks); } - asyncCallbackWrapper = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.int ], asyncCallbackWrapper); + asyncCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.int], asyncCallbackWrapper); this.libtaos.taos_fetch_rows_a(result, asyncCallbackWrapper, param); return param; } // Fetch field meta data by result handle -CTaosInterface.prototype.fetchFields_a = function fetchFields_a (result) { +CTaosInterface.prototype.fetchFields_a = function fetchFields_a(result) { let pfields = this.fetchFields(result); let pfieldscount = this.numFields(result); let fields = []; if (ref.isNull(pfields) == false) { - pfields = ref.reinterpret(pfields, 68 * pfieldscount , 0); + pfields = ref.reinterpret(pfields, 68 * pfieldscount, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 64 = name //65 = type, 66 - 67 = bytes - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,65,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 65, i)), type: pfields[i + 65], bytes: pfields[i + 66] }) @@ -488,7 +490,7 @@ CTaosInterface.prototype.fetchFields_a = function fetchFields_a (result) { } // Stop a query by result handle CTaosInterface.prototype.stopQuery = function stopQuery(result) { - if (result != null){ + if (result != null) { this.libtaos.taos_stop_query(result); } else { @@ -509,13 +511,13 @@ CTaosInterface.prototype.subscribe = function subscribe(connection, restart, top try { sql = sql != null ? ref.allocCString(sql) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: sql is expected as a str"; } try { topic = topic != null ? ref.allocCString(topic) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw TypeError("topic is expected as a str"); } @@ -539,8 +541,8 @@ CTaosInterface.prototype.consume = function consume(subscription) { pfields = ref.reinterpret(pfields, this.numFields(result) * 68, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,64,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 64, i)), bytes: pfields[i + 64], type: pfields[i + 66] }) @@ -548,7 +550,7 @@ CTaosInterface.prototype.consume = function consume(subscription) { } let data = []; - while(true) { + while (true) { let { blocks, num_of_rows } = this.fetchBlock(result, fields); if (num_of_rows == 0) { break; @@ -559,7 +561,7 @@ CTaosInterface.prototype.consume = function consume(subscription) { for (let j = 0; j < fields.length; j++) { rowBlock[j] = blocks[j][i]; } - data[data.length-1] = (rowBlock); + data[data.length - 1] = (rowBlock); } } return { data: data, fields: fields, result: result }; @@ -570,11 +572,11 @@ CTaosInterface.prototype.unsubscribe = function unsubscribe(subscription) { } // Continuous Query -CTaosInterface.prototype.openStream = function openStream(connection, sql, callback, stime,stoppingCallback, param = ref.ref(ref.NULL)) { +CTaosInterface.prototype.openStream = function openStream(connection, sql, callback, stime, stoppingCallback, param = ref.ref(ref.NULL)) { try { sql = ref.allocCString(sql); } - catch(err) { + catch (err) { throw "Attribute Error: sql string is expected as a str"; } var cti = this; @@ -587,7 +589,7 @@ CTaosInterface.prototype.openStream = function openStream(connection, sql, callb let offset = 0; if (numOfRows2 > 0) { for (let i = 0; i < fields.length; i++) { - if (!convertFunctions[fields[i]['type']] ) { + if (!convertFunctions[fields[i]['type']]) { throw new errors.DatabaseError("Invalid data type returned from database"); } blocks[i] = convertFunctions[fields[i]['type']](row, numOfRows2, fields[i]['bytes'], offset, isMicro); @@ -596,8 +598,8 @@ CTaosInterface.prototype.openStream = function openStream(connection, sql, callb } callback(param2, result2, blocks, fields); } - asyncCallbackWrapper = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.refType(ref.types.void_ptr2) ], asyncCallbackWrapper); - asyncStoppingCallbackWrapper = ffi.Callback( ref.types.void, [ ref.types.void_ptr ], stoppingCallback); + asyncCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.refType(ref.types.void_ptr2)], asyncCallbackWrapper); + asyncStoppingCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr], stoppingCallback); let streamHandle = this.libtaos.taos_open_stream(connection, sql, asyncCallbackWrapper, stime, param, asyncStoppingCallbackWrapper); if (ref.isNull(streamHandle)) { throw new errors.TDError('Failed to open a stream with TDengine'); diff --git a/src/connector/nodejs/nodetaos/cursor.js b/src/connector/nodejs/nodetaos/cursor.js index e18e6c2500..f879d89d48 100644 --- a/src/connector/nodejs/nodetaos/cursor.js +++ b/src/connector/nodejs/nodetaos/cursor.js @@ -1,7 +1,7 @@ const ref = require('ref-napi'); require('./globalfunc.js') const CTaosInterface = require('./cinterface') -const errors = require ('./error') +const errors = require('./error') const TaosQuery = require('./taosquery') const { PerformanceObserver, performance } = require('perf_hooks'); module.exports = TDengineCursor; @@ -22,7 +22,7 @@ module.exports = TDengineCursor; * @property {fields} - Array of the field objects in order from left to right of the latest data retrieved * @since 1.0.0 */ -function TDengineCursor(connection=null) { +function TDengineCursor(connection = null) { //All parameters are store for sync queries only. this._rowcount = -1; this._connection = null; @@ -91,7 +91,7 @@ TDengineCursor.prototype.execute = function execute(operation, options, callback return null; } - if (typeof options == 'function') { + if (typeof options == 'function') { callback = options; } if (typeof options != 'object') options = {} @@ -144,10 +144,10 @@ TDengineCursor.prototype.execute = function execute(operation, options, callback } TDengineCursor.prototype._createAffectedResponse = function (num, time) { - return "Query OK, " + num + " row(s) affected (" + (time * 0.001).toFixed(8) + "s)"; + return "Query OK, " + num + " row(s) affected (" + (time * 0.001).toFixed(8) + "s)"; } TDengineCursor.prototype._createSetResponse = function (num, time) { - return "Query OK, " + num + " row(s) in set (" + (time * 0.001).toFixed(8) + "s)"; + return "Query OK, " + num + " row(s) in set (" + (time * 0.001).toFixed(8) + "s)"; } TDengineCursor.prototype.executemany = function executemany() { @@ -176,27 +176,22 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { throw new errors.OperationalError("Invalid use of fetchall, either result or fields from query are null. First execute a query first"); } - let data = []; + let num_of_rows = this._chandle.affectedRows(this._result); + let data = new Array(num_of_rows); + this._rowcount = 0; - //let nodetime = 0; + let time = 0; const obs = new PerformanceObserver((items) => { time += items.getEntries()[0].duration; performance.clearMarks(); }); - /* - const obs2 = new PerformanceObserver((items) => { - nodetime += items.getEntries()[0].duration; - performance.clearMarks(); - }); - obs2.observe({ entryTypes: ['measure'] }); - performance.mark('nodea'); - */ obs.observe({ entryTypes: ['measure'] }); performance.mark('A'); - while(true) { - + while (true) { let blockAndRows = this._chandle.fetchBlock(this._result, this._fields); + // console.log(blockAndRows); + // break; let block = blockAndRows.blocks; let num_of_rows = blockAndRows.num_of_rows; if (num_of_rows == 0) { @@ -205,22 +200,24 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { this._rowcount += num_of_rows; let numoffields = this._fields.length; for (let i = 0; i < num_of_rows; i++) { - data.push([]); - + // data.push([]); + let rowBlock = new Array(numoffields); for (let j = 0; j < numoffields; j++) { rowBlock[j] = block[j][i]; } - data[data.length-1] = (rowBlock); + data[this._rowcount - num_of_rows + i] = (rowBlock); + // data.push(rowBlock); } } + performance.mark('B'); performance.measure('query', 'A', 'B'); let response = this._createSetResponse(this._rowcount, time) console.log(response); - // this._connection._clearResultSet(); + // this._connection._clearResultSet(); let fields = this.fields; this._reset_result(); this.data = data; @@ -239,12 +236,12 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { * @return {number | Buffer} Number of affected rows or a Buffer that points to the results of the query * @since 1.0.0 */ -TDengineCursor.prototype.execute_a = function execute_a (operation, options, callback, param) { +TDengineCursor.prototype.execute_a = function execute_a(operation, options, callback, param) { if (operation == undefined) { throw new errors.ProgrammingError('No operation passed as argument'); return null; } - if (typeof options == 'function') { + if (typeof options == 'function') { //we expect the parameter after callback to be param param = callback; callback = options; @@ -265,14 +262,14 @@ TDengineCursor.prototype.execute_a = function execute_a (operation, options, cal } if (resCode >= 0) { -// let fieldCount = cr._chandle.numFields(res2); -// if (fieldCount == 0) { -// //cr._chandle.freeResult(res2); -// return res2; -// } -// else { -// return res2; -// } + // let fieldCount = cr._chandle.numFields(res2); + // if (fieldCount == 0) { + // //cr._chandle.freeResult(res2); + // return res2; + // } + // else { + // return res2; + // } return res2; } @@ -317,7 +314,7 @@ TDengineCursor.prototype.execute_a = function execute_a (operation, options, cal * }) */ TDengineCursor.prototype.fetchall_a = function fetchall_a(result, options, callback, param = {}) { - if (typeof options == 'function') { + if (typeof options == 'function') { //we expect the parameter after callback to be param param = callback; callback = options; @@ -360,17 +357,17 @@ TDengineCursor.prototype.fetchall_a = function fetchall_a(result, options, callb for (let k = 0; k < fields.length; k++) { rowBlock[k] = block[k][j]; } - data[data.length-1] = rowBlock; + data[data.length - 1] = rowBlock; } } cr._chandle.freeResult(result2); // free result, avoid seg faults and mem leaks! - callback(param2, result2, numOfRows2, {data:data,fields:fields}); + callback(param2, result2, numOfRows2, { data: data, fields: fields }); } } ref.writeObject(buf, 0, param); param = this._chandle.fetch_rows_a(result, asyncCallbackWrapper, buf); //returned param - return {param:param,result:result}; + return { param: param, result: result }; } /** * Stop a query given the result handle. @@ -428,7 +425,7 @@ TDengineCursor.prototype.subscribe = function subscribe(config) { */ TDengineCursor.prototype.consumeData = async function consumeData(subscription, callback) { while (true) { - let { data, fields, result} = this._chandle.consume(subscription); + let { data, fields, result } = this._chandle.consume(subscription); callback(data, fields, result); } } @@ -450,30 +447,30 @@ TDengineCursor.prototype.unsubscribe = function unsubscribe(subscription) { * @return {Buffer} A buffer pointing to the stream handle * @since 1.3.0 */ - TDengineCursor.prototype.openStream = function openStream(sql, callback, stime = 0, stoppingCallback, param = {}) { - let buf = ref.alloc('Object'); - ref.writeObject(buf, 0, param); +TDengineCursor.prototype.openStream = function openStream(sql, callback, stime = 0, stoppingCallback, param = {}) { + let buf = ref.alloc('Object'); + ref.writeObject(buf, 0, param); - let asyncCallbackWrapper = function (param2, result2, blocks, fields) { - let data = []; - let num_of_rows = blocks[0].length; - for (let j = 0; j < num_of_rows; j++) { - data.push([]); - let rowBlock = new Array(fields.length); - for (let k = 0; k < fields.length; k++) { - rowBlock[k] = blocks[k][j]; - } - data[data.length-1] = rowBlock; - } - callback(param2, result2, blocks, fields); - } - return this._chandle.openStream(this._connection._conn, sql, asyncCallbackWrapper, stime, stoppingCallback, buf); - } - /** - * Close a stream - * @param {Buffer} - A buffer pointing to the handle of the stream to be closed - * @since 1.3.0 - */ - TDengineCursor.prototype.closeStream = function closeStream(stream) { - this._chandle.closeStream(stream); - } + let asyncCallbackWrapper = function (param2, result2, blocks, fields) { + let data = []; + let num_of_rows = blocks[0].length; + for (let j = 0; j < num_of_rows; j++) { + data.push([]); + let rowBlock = new Array(fields.length); + for (let k = 0; k < fields.length; k++) { + rowBlock[k] = blocks[k][j]; + } + data[data.length - 1] = rowBlock; + } + callback(param2, result2, blocks, fields); + } + return this._chandle.openStream(this._connection._conn, sql, asyncCallbackWrapper, stime, stoppingCallback, buf); +} +/** + * Close a stream + * @param {Buffer} - A buffer pointing to the handle of the stream to be closed + * @since 1.3.0 + */ +TDengineCursor.prototype.closeStream = function closeStream(stream) { + this._chandle.closeStream(stream); +} diff --git a/src/connector/nodejs/package-lock.json b/src/connector/nodejs/package-lock.json deleted file mode 100644 index 9ca174ccd1..0000000000 --- a/src/connector/nodejs/package-lock.json +++ /dev/null @@ -1,285 +0,0 @@ -{ - "name": "td2.0-connector", - "version": "2.0.6", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "array-index": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-index/-/array-index-1.0.0.tgz", - "integrity": "sha1-7FanSe4QPk4Ix5C5w1PfFgVbl/k=", - "requires": { - "debug": "^2.2.0", - "es6-symbol": "^3.0.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } - } - }, - "d": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", - "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", - "requires": { - "es5-ext": "^0.10.50", - "type": "^1.0.1" - } - }, - "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", - "requires": { - "ms": "2.1.2" - } - }, - "es5-ext": { - "version": "0.10.53", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", - "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", - "requires": { - "es6-iterator": "~2.0.3", - "es6-symbol": "~3.1.3", - "next-tick": "~1.0.0" - } - }, - "es6-iterator": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", - "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=", - "requires": { - "d": "1", - "es5-ext": "^0.10.35", - "es6-symbol": "^3.1.1" - } - }, - "es6-symbol": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", - "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", - "requires": { - "d": "^1.0.1", - "ext": "^1.1.2" - } - }, - "ext": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", - "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", - "requires": { - "type": "^2.0.0" - }, - "dependencies": { - "type": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/type/-/type-2.1.0.tgz", - "integrity": "sha512-G9absDWvhAWCV2gmF1zKud3OyC61nZDwWvBL2DApaVFogI07CprggiQAOOjvp2NRjYWFzPyu7vwtDrQFq8jeSA==" - } - } - }, - "ffi-napi": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ffi-napi/-/ffi-napi-3.1.0.tgz", - "integrity": "sha512-EsHO+sP2p/nUC/3l/l8m9niee1BLm4asUFDzkkBGR4kYVgp2KqdAYUomZhkKtzim4Fq7mcYHjpUaIHsMqs+E1g==", - "requires": { - "debug": "^4.1.1", - "get-uv-event-loop-napi-h": "^1.0.5", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1", - "ref-napi": "^2.0.1", - "ref-struct-di": "^1.1.0" - }, - "dependencies": { - "ref-napi": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-2.1.2.tgz", - "integrity": "sha512-aFl+vrIuLWUXMUTQGAwGAuSNLX3Ub5W3iVP8b7KyFFZUdn4+i4U1TXXTop0kCTUfGNu8glBGVz4lowkwMcPVVA==", - "requires": { - "debug": "^4.1.1", - "get-symbol-from-current-process-h": "^1.0.2", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - } - } - } - }, - "get-symbol-from-current-process-h": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz", - "integrity": "sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw==" - }, - "get-uv-event-loop-napi-h": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz", - "integrity": "sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg==", - "requires": { - "get-symbol-from-current-process-h": "^1.0.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "next-tick": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", - "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" - }, - "node-addon-api": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", - "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" - }, - "node-gyp-build": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.2.3.tgz", - "integrity": "sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg==" - }, - "ref-array-napi": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ref-array-napi/-/ref-array-napi-1.2.1.tgz", - "integrity": "sha512-jQp2WWSucmxkqVfoNfm7yDlDeGu3liAbzqfwjNybL80ooLOCnCZpAK2woDInY+lxNOK/VlIVSqeDEYb4gVPuNQ==", - "requires": { - "array-index": "1", - "debug": "2", - "ref-napi": "^1.4.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "ref-napi": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", - "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", - "requires": { - "debug": "^3.1.0", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - } - } - }, - "ref-napi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.1.tgz", - "integrity": "sha512-W3rcb0E+tlO9u9ySFnX5vifInwwPGToOfFgTZUHJBNiOBsW0NNvgHz2zJN7ctABo/2yIlgdPQUvuqqfORIF4LA==", - "requires": { - "debug": "^4.1.1", - "get-symbol-from-current-process-h": "^1.0.2", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - } - }, - "ref-struct-di": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz", - "integrity": "sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g==", - "requires": { - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - } - } - }, - "ref-struct-napi": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ref-struct-napi/-/ref-struct-napi-1.1.1.tgz", - "integrity": "sha512-YgS5/d7+kT5zgtySYI5ieH0hREdv+DabgDvoczxsui0f9VLm0rrDcWEj4DHKehsH+tJnVMsLwuyctWgvdEcVRw==", - "requires": { - "debug": "2", - "ref-napi": "^1.4.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "ref-napi": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", - "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", - "requires": { - "debug": "^3.1.0", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - } - } - }, - "type": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", - "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" - } - } -} diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index b39ce2c17d..d21b62108b 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "td2.0-connector", - "version": "2.0.6", + "version": "2.0.7", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", "directories": { From 9958b3d8d27130525b1c3b0a4f95bf08d23b620a Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 18:19:27 +0800 Subject: [PATCH 131/157] [TD-1920]add mnode compact wal functions --- src/common/inc/tglobal.h | 2 ++ src/common/src/tglobal.c | 2 ++ src/dnode/src/dnodeMain.c | 11 +++++++++++ src/mnode/src/mnodeMain.c | 4 +--- src/mnode/src/mnodeSdb.c | 33 +++++++++++++++++++++++++-------- 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index abb4d39ee1..e07c3611d7 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -149,6 +149,8 @@ extern char configDir[]; extern char tsVnodeDir[]; extern char tsDnodeDir[]; extern char tsMnodeDir[]; +extern char tsMnodeBakDir[]; +extern char tsMnodeTmpDir[]; extern char tsDataDir[]; extern char tsLogDir[]; extern char tsScriptDir[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index ab036b021c..d6bbc288ad 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -183,6 +183,8 @@ char configDir[TSDB_FILENAME_LEN] = {0}; char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeTmpDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeBakDir[TSDB_FILENAME_LEN] = {0}; char tsDataDir[TSDB_FILENAME_LEN] = {0}; char tsScriptDir[TSDB_FILENAME_LEN] = {0}; char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 48680e20d9..ee4dfcf852 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -217,6 +217,17 @@ static int32_t dnodeInitStorage() { sprintf(tsDnodeDir, "%s/dnode", tsDataDir); // sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir); + if (tsCompactMnodeWal == 1) { + sprintf(tsMnodeTmpDir, "%s/mnode_tmp", tsDataDir); + tfsRmdir(tsMnodeTmpDir); + if (dnodeCreateDir(tsMnodeTmpDir) < 0) { + dError("failed to create dir: %s, reason: %s", tsMnodeTmpDir, strerror(errno)); + return -1; + } + + sprintf(tsMnodeBakDir, "%s/mnode_bak", tsDataDir); + //tfsRmdir(tsMnodeBakDir); + } //TODO(dengyihao): no need to init here if (dnodeCreateDir(tsMnodeDir) < 0) { dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno)); diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index d9abe16120..d3511a4e62 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -59,8 +59,6 @@ static SStep tsMnodeSteps[] = { static SStep tsMnodeCompactSteps[] = { {"cluster", mnodeCompactCluster, NULL}, - -/* {"dnodes", mnodeCompactDnodes, NULL}, {"mnodes", mnodeCompactMnodes, NULL}, {"accts", mnodeCompactAccts, NULL}, @@ -68,7 +66,7 @@ static SStep tsMnodeCompactSteps[] = { {"dbs", mnodeCompactDbs, NULL}, {"vgroups", mnodeCompactVgroups, NULL}, {"tables", mnodeCompactTables, NULL}, -*/ + }; static void mnodeInitTimer(); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 79081945d9..9e3b87671e 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -20,6 +20,7 @@ #include "tutil.h" #include "tref.h" #include "tbn.h" +#include "tfs.h" #include "tqueue.h" #include "twal.h" #include "tsync.h" @@ -1154,26 +1155,42 @@ int32_t sdbGetReplicaNum() { int32_t mnodeCompactWal() { sdbInfo("vgId:1, start compact mnode wal..."); - // close wal + // close old wal walFsync(tsSdbMgmt.wal, true); walClose(tsSdbMgmt.wal); - // change wal to wal_bak dir - char temp[TSDB_FILENAME_LEN] = {0}; + // reset version,then compacted wal log can start from version 1 + tsSdbMgmt.version = 0; + + // change wal to wal_tmp dir SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; - sprintf(temp, "%s/wal_tmp", tsMnodeDir); - if (mkdir(temp, 0755) != 0 && errno != EEXIST) { - return -1; - } + char temp[TSDB_FILENAME_LEN] = {0}; + sprintf(temp, "%s/wal", tsMnodeTmpDir); tsSdbMgmt.wal = walOpen(temp, &walCfg); walRenew(tsSdbMgmt.wal); // compact memory tables info to wal tmp dir - mnodeCompactComponents(); + if (mnodeCompactComponents() != 0) { + tfsRmdir(tsMnodeTmpDir); + return -1; + } // close wal walFsync(tsSdbMgmt.wal, true); walClose(tsSdbMgmt.wal); + // rename old wal to wal_bak + if (taosRename(tsMnodeDir, tsMnodeBakDir) != 0) { + return -1; + } + + // rename wal_tmp to wal + if (taosRename(tsMnodeTmpDir, tsMnodeDir) != 0) { + return -1; + } + + // del wal_tmp dir + sdbInfo("vgId:1, compact mnode wal success"); + return 0; } \ No newline at end of file From 824eb0f3133168b47f2850fc3968914be87b030e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 20 May 2021 18:41:35 +0800 Subject: [PATCH 132/157] [TD-4272]: taosdemo specify columns. (#6174) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 70 ++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index ebc1074cba..4ca4031b1f 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -198,6 +198,8 @@ typedef struct { } SColDes; /* Used by main to communicate with parse_opt. */ +static char *g_dupstr = NULL; + typedef struct SArguments_S { char * metaFile; uint32_t test_mode; @@ -537,6 +539,7 @@ static int taosRandom() #endif // ifdef Windows +static void prompt(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); @@ -692,8 +695,9 @@ static void printHelp() { "The data_type of columns, default: INT,INT,INT,INT."); printf("%s%s%s%s\n", indent, "-w", indent, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); - printf("%s%s%s%s\n", indent, "-l", indent, - "The number of columns per record. Default is 4."); + printf("%s%s%s%s%d\n", indent, "-l", indent, + "The number of columns per record. Default is 4. Max values is ", + MAX_NUM_DATATYPE); printf("%s%s%s%s\n", indent, "-T", indent, "The number of threads. Default is 10."); printf("%s%s%s%s\n", indent, "-i", indent, @@ -737,7 +741,6 @@ static bool isStringNumber(char *input) } static void parse_args(int argc, char *argv[], SArguments *arguments) { - char **sptr; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-f") == 0) { @@ -881,20 +884,31 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->database = argv[++i]; } else if (strcmp(argv[i], "-l") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-l need a number following!\n"); - exit(EXIT_FAILURE); + if (argc == i+1) { + if (!isStringNumber(argv[i+1])) { + printHelp(); + errorPrint("%s", "\n\t-l need a number following!\n"); + exit(EXIT_FAILURE); + } } arguments->num_of_CPR = atoi(argv[++i]); + + if (arguments->num_of_CPR > MAX_NUM_DATATYPE) { + printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE); + prompt(); + arguments->num_of_CPR = MAX_NUM_DATATYPE; + } + + for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) { + arguments->datatype[col] = NULL; + } + } else if (strcmp(argv[i], "-b") == 0) { if (argc == i+1) { printHelp(); errorPrint("%s", "\n\t-b need valid string following!\n"); exit(EXIT_FAILURE); } - sptr = arguments->datatype; ++i; if (strstr(argv[i], ",") == NULL) { // only one col @@ -911,12 +925,12 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - sptr[0] = argv[i]; + arguments->datatype[0] = argv[i]; } else { // more than one col int index = 0; - char *dupstr = strdup(argv[i]); - char *running = dupstr; + g_dupstr = strdup(argv[i]); + char *running = g_dupstr; char *token = strsep(&running, ","); while(token != NULL) { if (strcasecmp(token, "INT") @@ -929,16 +943,15 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { printHelp(); - free(dupstr); + free(g_dupstr); errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - sptr[index++] = token; + arguments->datatype[index++] = token; token = strsep(&running, ","); if (index >= MAX_NUM_DATATYPE) break; } - free(dupstr); - sptr[index] = NULL; + arguments->datatype[index] = NULL; } } else if (strcmp(argv[i], "-w") == 0) { if ((argc == i+1) || @@ -3464,10 +3477,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.interlace_rows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } + prompt(); g_args.interlace_rows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -5812,19 +5822,13 @@ static void startMultiThreadInsertData(int threads, char* db_name, && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit ) > superTblInfo->childTblCount)) { printf("WARNING: specified offset + limit > child table count!\n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + prompt(); } if ((superTblInfo->childTblExists != TBL_NO_EXISTS) && (0 == superTblInfo->childTblLimit)) { printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + prompt(); } superTblInfo->childTblName = (char*)calloc(1, @@ -6172,6 +6176,13 @@ static void *readMetric(void *sarg) { return NULL; } +static void prompt() +{ + if (!g_args.answer_yes) { + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); + (void)getchar(); + } +} static int insertTestProcess() { @@ -7397,6 +7408,9 @@ int main(int argc, char *argv[]) { } else { testCmdLine(); } + + if (g_dupstr) + free(g_dupstr); } return 0; From 608e67826de2c23cddab295f5fc9c7498c4bcc7a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 20 May 2021 18:41:55 +0800 Subject: [PATCH 133/157] [TD-4272]: taosdemo specify columns. (#6173) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 70 ++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index ae7ffd16b9..34302ac2b0 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -189,6 +189,8 @@ typedef struct { } SColDes; /* Used by main to communicate with parse_opt. */ +static char *g_dupstr = NULL; + typedef struct SArguments_S { char * metaFile; uint32_t test_mode; @@ -526,7 +528,6 @@ static int taosRandom() #endif // ifdef Windows static void prompt(); -static void prompt2(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); @@ -679,8 +680,9 @@ static void printHelp() { "The data_type of columns, default: INT,INT,INT,INT."); printf("%s%s%s%s\n", indent, "-w", indent, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); - printf("%s%s%s%s\n", indent, "-l", indent, - "The number of columns per record. Default is 4."); + printf("%s%s%s%s%d\n", indent, "-l", indent, + "The number of columns per record. Default is 4. Max values is ", + MAX_NUM_DATATYPE); printf("%s%s%s%s\n", indent, "-T", indent, "The number of threads. Default is 10."); printf("%s%s%s%s\n", indent, "-i", indent, @@ -724,7 +726,6 @@ static bool isStringNumber(char *input) } static void parse_args(int argc, char *argv[], SArguments *arguments) { - char **sptr; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-f") == 0) { @@ -851,20 +852,31 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->database = argv[++i]; } else if (strcmp(argv[i], "-l") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-l need a number following!\n"); - exit(EXIT_FAILURE); + if (argc == i+1) { + if (!isStringNumber(argv[i+1])) { + printHelp(); + errorPrint("%s", "\n\t-l need a number following!\n"); + exit(EXIT_FAILURE); + } } arguments->num_of_CPR = atoi(argv[++i]); + + if (arguments->num_of_CPR > MAX_NUM_DATATYPE) { + printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE); + prompt(); + arguments->num_of_CPR = MAX_NUM_DATATYPE; + } + + for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) { + arguments->datatype[col] = NULL; + } + } else if (strcmp(argv[i], "-b") == 0) { if (argc == i+1) { printHelp(); errorPrint("%s", "\n\t-b need valid string following!\n"); exit(EXIT_FAILURE); } - sptr = arguments->datatype; ++i; if (strstr(argv[i], ",") == NULL) { // only one col @@ -881,12 +893,12 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - sptr[0] = argv[i]; + arguments->datatype[0] = argv[i]; } else { // more than one col int index = 0; - char *dupstr = strdup(argv[i]); - char *running = dupstr; + g_dupstr = strdup(argv[i]); + char *running = g_dupstr; char *token = strsep(&running, ","); while(token != NULL) { if (strcasecmp(token, "INT") @@ -899,16 +911,15 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { printHelp(); - free(dupstr); + free(g_dupstr); errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - sptr[index++] = token; + arguments->datatype[index++] = token; token = strsep(&running, ","); if (index >= MAX_NUM_DATATYPE) break; } - free(dupstr); - sptr[index] = NULL; + arguments->datatype[index] = NULL; } } else if (strcmp(argv[i], "-w") == 0) { if ((argc == i+1) || @@ -3449,7 +3460,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.interlace_rows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - prompt2(); + prompt(); g_args.interlace_rows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -5759,19 +5770,13 @@ static void startMultiThreadInsertData(int threads, char* db_name, && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit ) > superTblInfo->childTblCount)) { printf("WARNING: specified offset + limit > child table count!\n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + prompt(); } if ((superTblInfo->childTblExists != TBL_NO_EXISTS) && (0 == superTblInfo->childTblLimit)) { printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + prompt(); } superTblInfo->childTblName = (char*)calloc(1, @@ -6093,19 +6098,11 @@ static void *readMetric(void *sarg) { static void prompt() { if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); (void)getchar(); } } -static void prompt2() -{ - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } -} - static int insertTestProcess() { setupForAnsiEscape(); @@ -7325,6 +7322,9 @@ int main(int argc, char *argv[]) { } else { testCmdLine(); } + + if (g_dupstr) + free(g_dupstr); } return 0; From 4b016b75bb026f86bd5db56910409d6e2c17fa95 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 20 May 2021 19:08:42 +0800 Subject: [PATCH 134/157] [TD-4263]: update performance test script --- tests/perftest-scripts/perftest-query.sh | 2 +- tests/pytest/tools/taosdemoPerformance.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index 9a16084683..c6d4687ed7 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -74,7 +74,7 @@ function runQueryPerfTest { python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT - python3 perfbenchmark/joinPerformance.py | tee -a $PERFORMANCE_TEST_REPORT + #python3 perfbenchmark/joinPerformance.py | tee -a $PERFORMANCE_TEST_REPORT } diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index b50180e2b3..a32cba167e 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -104,7 +104,7 @@ class taosdemoPerformace: return output def insertData(self): - os.system("taosdemo -f %s > taosdemoperf.txt" % self.generateJson()) + os.system("taosdemo -f %s > taosdemoperf.txt 2>&1" % self.generateJson()) self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'") self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'") self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'") From 243a3d16b6c23a0c4cce60a0fc07fe53d6d54dd8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 20 May 2021 23:25:12 +0800 Subject: [PATCH 135/157] Feature/sangshuduo/td 4068 taosdemo stmt (#6178) * merge with develop branch. change query/tests/CMakeLists.txt to allow unused function and variable. * refactor data generating. * refactor. * refactor. * refactor. * refactor. * refactor Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 252 ++++++++++++++++++++++-------------- 1 file changed, 156 insertions(+), 96 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 4ca4031b1f..1d3b0d1ade 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -61,6 +61,8 @@ extern char configDir[]; #define QUERY_JSON_NAME "query.json" #define SUBSCRIBE_JSON_NAME "subscribe.json" +#define STR_INSERT_INTO "INSERT INTO " + enum TEST_MODE { INSERT_TEST, // 0 QUERY_TEST, // 1 @@ -5093,7 +5095,8 @@ static int generateStbSQLHead( return len; } -static int64_t generateInterlaceDataBuffer( +static int64_t generateStbInterlaceData( + SSuperTable *superTblInfo, char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes, uint64_t tableSeq, threadInfo *pThreadInfo, char *buffer, @@ -5103,12 +5106,11 @@ static int64_t generateInterlaceDataBuffer( { assert(buffer); char *pstr = buffer; - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; int headLen = generateStbSQLHead( superTblInfo, tableName, tableSeq, pThreadInfo->db_name, - pstr, *pRemainderBufLen); + pstr, *pRemainderBufLen); if (headLen <= 0) { return 0; @@ -5126,24 +5128,15 @@ static int64_t generateInterlaceDataBuffer( pThreadInfo->threadID, __func__, __LINE__, i, batchPerTblTimes, batchPerTbl); - int64_t k; - if (superTblInfo) { - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { startTime = taosGetTimestamp(pThreadInfo->time_precision); - } + } - k = generateStbDataTail( + int64_t k = generateStbDataTail( superTblInfo, batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, startTime, &(pThreadInfo->samplePos), &dataLen); - } else { - startTime = 1500000000000; - k = generateDataTailWithoutStb( - batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, - startTime, - /* &(pThreadInfo->samplePos), */&dataLen); - } if (k == batchPerTbl) { pstr += dataLen; @@ -5159,33 +5152,67 @@ static int64_t generateInterlaceDataBuffer( return k; } -static int64_t generateProgressiveDataBuffer( +static int64_t generateInterlaceDataWithoutStb( + char *tableName, uint64_t batchPerTbl, + uint64_t tableSeq, + char *dbName, char *buffer, + int64_t insertRows, + uint64_t *pRemainderBufLen) +{ + assert(buffer); + char *pstr = buffer; + + int headLen = generateSQLHeadWithoutStb( + tableName, dbName, + pstr, *pRemainderBufLen); + + if (headLen <= 0) { + return 0; + } + + pstr += headLen; + *pRemainderBufLen -= headLen; + + int64_t dataLen = 0; + + int64_t startTime = 1500000000000; + int64_t k = generateDataTailWithoutStb( + batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, + startTime, + &dataLen); + + if (k == batchPerTbl) { + pstr += dataLen; + *pRemainderBufLen -= dataLen; + } else { + debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n", + __func__, __LINE__, k, batchPerTbl); + pstr -= headLen; + pstr[0] = '\0'; + k = 0; + } + + return k; +} + +static int64_t generateStbProgressiveData( + SSuperTable *superTblInfo, char *tableName, int64_t tableSeq, - threadInfo *pThreadInfo, char *buffer, + char *dbName, char *buffer, int64_t insertRows, uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *pRemainderBufLen) { - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - assert(buffer != NULL); char *pstr = buffer; memset(buffer, 0, *pRemainderBufLen); - int64_t headLen; - - if (superTblInfo) { - headLen = generateStbSQLHead( + int64_t headLen = generateStbSQLHead( superTblInfo, - tableName, tableSeq, pThreadInfo->db_name, + tableName, tableSeq, dbName, buffer, *pRemainderBufLen); - } else { - headLen = generateSQLHeadWithoutStb( - tableName, pThreadInfo->db_name, - buffer, *pRemainderBufLen); - } if (headLen <= 0) { return 0; @@ -5194,22 +5221,43 @@ static int64_t generateProgressiveDataBuffer( *pRemainderBufLen -= headLen; int64_t dataLen; - int64_t k; - if (superTblInfo) { - k = generateStbDataTail(superTblInfo, + return generateStbDataTail(superTblInfo, g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom, startTime, pSamplePos, &dataLen); - } else { - k = generateDataTailWithoutStb( +} + +static int64_t generateProgressiveDataWithoutStb( + char *tableName, + int64_t tableSeq, + threadInfo *pThreadInfo, char *buffer, + int64_t insertRows, + uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, + int64_t *pRemainderBufLen) +{ + assert(buffer != NULL); + char *pstr = buffer; + + memset(buffer, 0, *pRemainderBufLen); + + int64_t headLen = generateSQLHeadWithoutStb( + tableName, pThreadInfo->db_name, + buffer, *pRemainderBufLen); + + if (headLen <= 0) { + return 0; + } + pstr += headLen; + *pRemainderBufLen -= headLen; + + int64_t dataLen; + + return generateDataTailWithoutStb( g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom, startTime, /*pSamplePos, */&dataLen); - } - - return k; } static void printStatPerThread(threadInfo *pThreadInfo) @@ -5228,6 +5276,9 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int64_t insertRows; uint64_t interlaceRows; + uint64_t maxSqlLen; + int64_t nTimeStampStep; + uint64_t insert_interval; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; @@ -5240,58 +5291,27 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } else { interlaceRows = superTblInfo->interlaceRows; } + maxSqlLen = superTblInfo->maxSqlLen; + nTimeStampStep = superTblInfo->timeStampStep; + insert_interval = superTblInfo->insertInterval; } else { insertRows = g_args.num_of_DPT; interlaceRows = g_args.interlace_rows; + maxSqlLen = g_args.max_sql_len; + nTimeStampStep = DEFAULT_TIMESTAMP_STEP; + insert_interval = g_args.insert_interval; } - if (interlaceRows > insertRows) - interlaceRows = insertRows; - - if (interlaceRows > g_args.num_of_RPR) - interlaceRows = g_args.num_of_RPR; - - int progOrInterlace; - - if (interlaceRows > 0) { - progOrInterlace= INTERLACE_INSERT_MODE; - } else { - progOrInterlace = PROGRESSIVE_INSERT_MODE; - } - - uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; - pThreadInfo->buffer = calloc(maxSqlLen, 1); - if (NULL == pThreadInfo->buffer) { - errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", - __func__, __LINE__, maxSqlLen, strerror(errno)); - return NULL; - } - - char tableName[TSDB_TABLE_NAME_LEN]; - - pThreadInfo->totalInsertRows = 0; - pThreadInfo->totalAffectedRows = 0; - - int64_t nTimeStampStep = - superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; - - uint64_t insert_interval = - superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; - uint64_t st = 0; - uint64_t et = UINT64_MAX; - - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); - uint64_t endTs; - - uint64_t tableSeq = pThreadInfo->start_table_from; - debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, pThreadInfo->ntables, insertRows); - int64_t startTime = pThreadInfo->start_time; + if (interlaceRows > insertRows) + interlaceRows = insertRows; + + if (interlaceRows > g_args.num_of_RPR) + interlaceRows = g_args.num_of_RPR; uint64_t batchPerTbl = interlaceRows; uint64_t batchPerTblTimes; @@ -5303,13 +5323,30 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { batchPerTblTimes = 1; } + pThreadInfo->buffer = calloc(maxSqlLen, 1); + if (NULL == pThreadInfo->buffer) { + errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", + __func__, __LINE__, maxSqlLen, strerror(errno)); + return NULL; + } + + pThreadInfo->totalInsertRows = 0; + pThreadInfo->totalAffectedRows = 0; + + uint64_t st = 0; + uint64_t et = UINT64_MAX; + + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + uint64_t endTs; + + uint64_t tableSeq = pThreadInfo->start_table_from; + int64_t startTime = pThreadInfo->start_time; + uint64_t generatedRecPerTbl = 0; bool flagSleep = true; uint64_t sleepTimeTotal = 0; - char *strInsertInto = "insert into "; - int nInsertBufLen = strlen(strInsertInto); - while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { if ((flagSleep) && (insert_interval)) { st = taosGetTimestampMs(); @@ -5321,13 +5358,16 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { char *pstr = pThreadInfo->buffer; - int len = snprintf(pstr, nInsertBufLen + 1, "%s", strInsertInto); + int len = snprintf(pstr, + strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); pstr += len; remainderBufLen -= len; uint64_t recOfBatch = 0; for (uint64_t i = 0; i < batchPerTblTimes; i ++) { + char tableName[TSDB_TABLE_NAME_LEN]; + getTableName(tableName, pThreadInfo, tableSeq); if (0 == strlen(tableName)) { errorPrint("[%d] %s() LN%d, getTableName return null\n", @@ -5337,13 +5377,25 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } uint64_t oldRemainderLen = remainderBufLen; - int64_t generated = generateInterlaceDataBuffer( - tableName, batchPerTbl, i, batchPerTblTimes, - tableSeq, - pThreadInfo, pstr, - insertRows, - startTime, - &remainderBufLen); + + int64_t generated; + if (superTblInfo) { + generated = generateStbInterlaceData( + superTblInfo, + tableName, batchPerTbl, i, batchPerTblTimes, + tableSeq, + pThreadInfo, pstr, + insertRows, + startTime, + &remainderBufLen); + } else { + generated = generateInterlaceDataWithoutStb( + tableName, batchPerTbl, + tableSeq, + pThreadInfo->db_name, pstr, + insertRows, + &remainderBufLen); + } debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n", pThreadInfo->threadID, __func__, __LINE__, generated); @@ -5364,8 +5416,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__, batchPerTbl, recOfBatch); - if (progOrInterlace == INTERLACE_INSERT_MODE) { - if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { + if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { // turn to first table tableSeq = pThreadInfo->start_table_from; generatedRecPerTbl += batchPerTbl; @@ -5383,7 +5434,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR) break; - } } verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n", @@ -5511,18 +5561,28 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int64_t remainderBufLen = maxSqlLen; char *pstr = pThreadInfo->buffer; - int nInsertBufLen = strlen("insert into "); - int len = snprintf(pstr, nInsertBufLen + 1, "%s", "insert into "); + int len = snprintf(pstr, + strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); pstr += len; remainderBufLen -= len; - int64_t generated = generateProgressiveDataBuffer( + int64_t generated; + if (superTblInfo) { + generated = generateStbProgressiveData( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, pstr, insertRows, + i, start_time, + &(pThreadInfo->samplePos), + &remainderBufLen); + } else { + generated = generateProgressiveDataWithoutStb( tableName, tableSeq, pThreadInfo, pstr, insertRows, i, start_time, &(pThreadInfo->samplePos), &remainderBufLen); + } if (generated > 0) i += generated; else From d324edadfe03a70a77cf73523802e8caf5e1086e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 May 2021 06:24:17 +0800 Subject: [PATCH 136/157] Hotfix/sangshuduo/td 4136 taosdemo records morethan 32767 (#6177) * [TD-4136]: taosdemo max records per req < 32767 * [TD-4136]: taosdemo check insert rows not more than 32767. check insert rows for progressive. * fix with answer_yes. * add extra prompt. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 34302ac2b0..34f5c87c44 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3494,6 +3494,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { __func__, __LINE__); goto PARSE_OVER; } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) { + printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n", + numRecPerReq->valueint, MAX_RECORDS_PER_REQ); + printf(" number of records per request value will be set to %d\n\n", + MAX_RECORDS_PER_REQ); + prompt(); numRecPerReq->valueint = MAX_RECORDS_PER_REQ; } g_args.num_of_RPR = numRecPerReq->valueint; @@ -3979,10 +3984,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } + prompt(); g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; } } else if (!interlaceRows) { From c820edc32ad6578148c2e3de21a7fa3f3caba1e9 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Fri, 21 May 2021 08:33:44 +0800 Subject: [PATCH 137/157] fix bug --- src/query/inc/qHistogram.h | 2 +- src/query/src/qHistogram.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/inc/qHistogram.h b/src/query/inc/qHistogram.h index 7742d151a0..3b5c2b4cfb 100644 --- a/src/query/inc/qHistogram.h +++ b/src/query/inc/qHistogram.h @@ -40,7 +40,7 @@ typedef struct SHeapEntry { } SHeapEntry; typedef struct SHistogramInfo { - int32_t numOfElems; + int64_t numOfElems; int32_t numOfEntries; int32_t maxEntries; double min; diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index ae25a75234..5fa35d0ee5 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -446,7 +446,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto) { } void tHistogramPrint(SHistogramInfo* pHisto) { - printf("total entries: %d, elements: %d\n", pHisto->numOfEntries, pHisto->numOfElems); + printf("total entries: %d, elements: %"PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems); #if defined(USE_ARRAYLIST) for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num); From d2de6c5fa3db80271944965fe1fcc6f19baed7fc Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 21 May 2021 08:56:27 +0800 Subject: [PATCH 138/157] [TD-3746]:save grant objects to monitor database --- src/inc/monitor.h | 3 ++- src/inc/taoserror.h | 3 +++ src/plugins/monitor/src/monMain.c | 10 ++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/inc/monitor.h b/src/inc/monitor.h index 1aefb0b848..d2e5e06487 100644 --- a/src/inc/monitor.h +++ b/src/inc/monitor.h @@ -54,7 +54,8 @@ void monCleanupSystem(); void monSaveAcctLog(SAcctMonitorObj *pMonObj); void monSaveLog(int32_t level, const char *const format, ...); void monExecuteSQL(char *sql); - +typedef void (*MonExecuteSQLCbFP)(void *param, TAOS_RES *, int code); +void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param); #ifdef __cplusplus } #endif diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index ce6f7c4f22..d3ac0ce280 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -427,6 +427,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) //"tfs invalid level") #define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) //"tfs no valid disk") +// monitor +#define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) //"monitor invalid monitor db connection") + #ifdef __cplusplus } #endif diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index 94af8e3ecd..d9f7d81ebd 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -417,3 +417,13 @@ void monExecuteSQL(char *sql) { monDebug("execute sql:%s", sql); taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "sql"); } + +void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param) { + if (tsMonitor.conn == NULL) { + callback(param, NULL, TSDB_CODE_MON_CONNECTION_INVALID); + return; + } + + monDebug("execute sql:%s", sql); + taos_query_a(tsMonitor.conn, sql, callback, param); +} From e38362c037c864f1752975fdf792202983a3c306 Mon Sep 17 00:00:00 2001 From: lichuang Date: Fri, 21 May 2021 09:43:05 +0800 Subject: [PATCH 139/157] [TD-1920]fix compile error --- src/mnode/src/mnodeTable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2e5d80b787..4d3125a2d1 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -3261,7 +3261,7 @@ static int32_t mnodeCompactSuperTables() { .rowSize = sizeof(SSTableObj) + schemaSize, }; - mInfo("compact super %ld", pTable->uid); + mInfo("compact super %" PRIu64, pTable->uid); sdbInsertCompactRow(&row); } @@ -3287,7 +3287,7 @@ static int32_t mnodeCompactChildTables() { .pTable = tsChildTableSdb, }; - mInfo("compact child %ld:%d", pTable->uid, pTable->tid); + mInfo("compact child %" PRIu64 ":%d", pTable->uid, pTable->tid); sdbInsertCompactRow(&row); } From 891886ce56565d88bec3e0bb461a50f1c0888917 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 May 2021 13:11:52 +0800 Subject: [PATCH 140/157] Hotfix/sangshuduo/td 4136 taosdemo records morethan 32767 for develop (#6175) * [TD-4136]: taosdemo records per req more than 32767. for develop branch. * change taosRandom() to rand_bool() for bool data. * [TD-4136]: taosdemo check insert rows not more than 32767. check insert rows for progressive. * fix with answer_yes. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 35 ++++++++-------------- tests/pytest/tools/taosdemoTestWithJson.py | 22 +++++++------- 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 1d3b0d1ade..b0e1dd28a6 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1088,10 +1088,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { printf("# Print debug info: %d\n", arguments->debug_print); printf("# Print verbose info: %d\n", arguments->verbose_print); printf("###################################################################\n"); - if (!arguments->answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + + prompt(); } } @@ -3513,6 +3511,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { __func__, __LINE__); goto PARSE_OVER; } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) { + printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n", + numRecPerReq->valueint, MAX_RECORDS_PER_REQ); + printf(" number of records per request value will be set to %d\n\n", + MAX_RECORDS_PER_REQ); + prompt(); numRecPerReq->valueint = MAX_RECORDS_PER_REQ; } g_args.num_of_RPR = numRecPerReq->valueint; @@ -3536,9 +3539,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.answer_yes = false; } } else if (!answerPrompt) { - g_args.answer_yes = false; + g_args.answer_yes = true; // default is no, mean answer_yes. } else { - printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); + errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n"); goto PARSE_OVER; } @@ -4007,10 +4010,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } + prompt(); g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; } } else if (!stbInterlaceRows) { @@ -6263,10 +6263,7 @@ static int insertTestProcess() { if (g_fpOfInsertResult) printfInsertMetaToFile(g_fpOfInsertResult); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); init_rand_data(); @@ -6536,10 +6533,7 @@ static int queryTestProcess() { &g_queryInfo.superQueryInfo.childTblCount); } - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); if (g_args.debug_print || g_args.verbose_print) { printfQuerySystemInfo(taos); @@ -6977,10 +6971,7 @@ static int subscribeTestProcess() { printfQueryMeta(); resetAfterAnsiEscape(); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + prompt(); TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, diff --git a/tests/pytest/tools/taosdemoTestWithJson.py b/tests/pytest/tools/taosdemoTestWithJson.py index f57af9ce5c..b2ecd54976 100644 --- a/tests/pytest/tools/taosdemoTestWithJson.py +++ b/tests/pytest/tools/taosdemoTestWithJson.py @@ -23,32 +23,32 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] else: - projPath = selfPath[:selfPath.find("tests")] + projPath = selfPath[: selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if "taosd" in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + if "packaging" not in rootRealPath: + buildPath = root[: len(root) - len("/build/bin")] break return buildPath - + def run(self): tdSql.prepare() buildPath = self.getBuildPath() - if (buildPath == ""): + if buildPath == "": tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - os.system("yes | %staosdemo -f tools/insert.json" % binPath) + binPath = buildPath + "/build/bin/" + os.system("%staosdemo -f tools/insert.json -y" % binPath) tdSql.execute("use db01") tdSql.query("select count(*) from stb01") From 972a75c3faaa9f05f7b978e614f055e31c1d6449 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 May 2021 15:15:28 +0800 Subject: [PATCH 141/157] Hotfix/sangshuduo/td 3913 mips compile support for develop (#6184) * [TD-3913]: mips compile support. * [TD-3913]: mips64 support. * fix osSignal.c * fix tcrc32c.c * fix cmake. * fix osInc.h * fix gcc 4.8 compile error. Co-authored-by: Shuduo Sang --- cmake/platform.inc | 6 +++ src/os/inc/osMips64.h | 87 ++++++++++++++++++++++++++++++++++++ src/os/src/detail/osSignal.c | 10 ++--- src/util/src/tcrc32c.c | 2 +- 4 files changed, 99 insertions(+), 6 deletions(-) create mode 100644 src/os/inc/osMips64.h diff --git a/cmake/platform.inc b/cmake/platform.inc index dcd0183e27..5f7391c996 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -102,6 +102,12 @@ IF ("${CPUTYPE}" STREQUAL "") SET(TD_LINUX TRUE) SET(TD_LINUX_64 FALSE) SET(TD_ARM_64 TRUE) + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") + SET(CPUTYPE "mips64") + MESSAGE(STATUS "Set CPUTYPE to mips64") + SET(TD_LINUX TRUE) + SET(TD_LINUX_64 FALSE) + SET(TD_MIPS_64 TRUE) ENDIF () ELSE () diff --git a/src/os/inc/osMips64.h b/src/os/inc/osMips64.h new file mode 100644 index 0000000000..ed7b08a311 --- /dev/null +++ b/src/os/inc/osMips64.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_MIPS64_H +#define TDENGINE_OS_MIPS64_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/src/detail/osSignal.c b/src/os/src/detail/osSignal.c index 4467a607b2..33cc39e112 100644 --- a/src/os/src/detail/osSignal.c +++ b/src/os/src/detail/osSignal.c @@ -25,14 +25,14 @@ typedef void (*FLinuxSignalHandler)(int32_t signum, siginfo_t *sigInfo, void *context); void taosSetSignal(int32_t signum, FSignalHandler sigfp) { - struct sigaction act = {{0}}; + struct sigaction act; memset(&act, 0, sizeof(act)); #if 1 act.sa_flags = SA_SIGINFO; act.sa_sigaction = (FLinuxSignalHandler)sigfp; -#else - act.sa_handler = sigfp; -#endif - sigaction(signum, &act, NULL); +#else + act.sa_handler = sigfp; +#endif + sigaction(signum, &act, NULL); } void taosIgnSignal(int32_t signum) { diff --git a/src/util/src/tcrc32c.c b/src/util/src/tcrc32c.c index 4009973a9f..d2b63eb9ee 100644 --- a/src/util/src/tcrc32c.c +++ b/src/util/src/tcrc32c.c @@ -18,7 +18,7 @@ 3. This notice may not be removed or altered from any source distribution. */ -#ifndef _TD_ARM_ +#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) #include #endif From 83187703a8d4ee3f8d9f0a44eaebc358b2e894bd Mon Sep 17 00:00:00 2001 From: root Date: Fri, 21 May 2021 11:51:33 +0000 Subject: [PATCH 142/157] [TD-4182] specify column for single batch bind --- tests/script/api/stmtBatchTest.c | 975 ++++++++++++++++++++++++++++++- 1 file changed, 950 insertions(+), 25 deletions(-) diff --git a/tests/script/api/stmtBatchTest.c b/tests/script/api/stmtBatchTest.c index 3ce2fefb0d..8bd296db61 100644 --- a/tests/script/api/stmtBatchTest.c +++ b/tests/script/api/stmtBatchTest.c @@ -37,7 +37,7 @@ typedef struct { //void taosMsleep(int mseconds); -int g_rows = 0; +int g_runTimes = 5; unsigned long long getCurrentTime(){ @@ -1336,7 +1336,7 @@ static void checkResult(TAOS *taos, char *tname, int printr, int expected) { result = taos_query(taos, sql); int code = taos_errno(result); if (code != 0) { - printf("failed to query table, reason:%s\n", taos_errstr(result)); + printf("failed to query table: %s, reason:%s\n", tname, taos_errstr(result)); taos_free_result(result); return; } @@ -1358,9 +1358,9 @@ static void checkResult(TAOS *taos, char *tname, int printr, int expected) { } if (rows == expected) { - printf("%d rows are fetched as expectation\n", rows); + printf("%d rows are fetched as expectation from %s\n", rows, tname); } else { - printf("!!!expect %d rows, but %d rows are fetched\n", expected, rows); + printf("!!!expect %d rows, but %d rows are fetched from %s\n", expected, rows, tname); return; } @@ -1451,15 +1451,52 @@ static void prepareV_long(TAOS *taos, int schemaCase, int tableNum, int lenO -//void runcase(TAOS *taos, int idx) { -static void* runCase(void *para) { - ThreadInfo* tInfo = (ThreadInfo *)para; - TAOS *taos = tInfo->taos; - int idx = tInfo->idx; - - TAOS_STMT *stmt = NULL; +static void prepareVcolumn(TAOS *taos, int schemaCase, int tableNum, int lenOfBinaryDef, char* dbName) { + TAOS_RES *result; + int code; + char sqlstr[1024] = {0}; + sprintf(sqlstr, "drop database if exists %s;", dbName); + result = taos_query(taos, sqlstr); + taos_free_result(result); - (void)idx; + sprintf(sqlstr, "create database %s;", dbName); + result = taos_query(taos, sqlstr); + code = taos_errno(result); + if (code != 0) { + printf("failed to create database, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + sprintf(sqlstr, "use %s;", dbName); + result = taos_query(taos, sqlstr); + taos_free_result(result); + + // create table + for (int i = 0 ; i < tableNum; i++) { + char buf[1024]; + if (schemaCase) { + sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp)", i, lenOfBinaryDef, lenOfBinaryDef) ; + } else { + sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; + } + + result = taos_query(taos, buf); + code = taos_errno(result); + if (code != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + } + +} + +//void runcase(TAOS *taos, int idx) { +static void runCase(TAOS *taos) { + TAOS_STMT *stmt = NULL; int tableNum; int lenOfBinaryDef; @@ -1975,7 +2012,7 @@ static void* runCase(void *para) { } #endif - return NULL; + return ; } @@ -2166,14 +2203,8 @@ static int stmt_bind_case_001_long(TAOS_STMT *stmt, int tableNum, int rowsOfPerC return 0; } -static void* runCase_long(void *para) { - ThreadInfo* tInfo = (ThreadInfo *)para; - TAOS *taos = tInfo->taos; - int idx = tInfo->idx; - +static void runCase_long(TAOS *taos) { TAOS_STMT *stmt = NULL; - - (void)idx; int tableNum; int lenOfBinaryDef; @@ -2228,7 +2259,7 @@ static void* runCase_long(void *para) { prepareV_long(taos, 1, tableNum, lenOfBinaryDef); totalRowsPerTbl = 0; - for (int i = 0; i < 30000; i++) { + for (int i = 0; i < g_runTimes; i++) { stmt = taos_stmt_init(taos); stmt_bind_case_001_long(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum, &startTs); @@ -2245,17 +2276,895 @@ static void* runCase_long(void *para) { } #endif - return NULL; + return; } +/*=======================*/ +/* +test scene: insert into tb1 (ts,f1) values (?,?) +*/ +static int stmt_specifyCol_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d-0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d-0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_INT; + params[i+2].buffer_length = sizeof(int32_t); + params[i+2].buffer = v->v4; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+3].buffer_length = sizeof(float); + params[i+3].buffer = v->f4; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+4].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+4].buffer = v->br; + params[i+4].length = lb; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + +// create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp) + char *sql = "insert into m0 (ts,b,v4,f4,br) values(?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + //char buf[32]; + //sprintf(buf, "m%d", zz); + //code = taos_stmt_set_tbname(stmt, buf); + //if (code != 0){ + // printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + // return -1; + //} + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +/*=======================*/ +/* +test scene: insert into ? (ts,f1) values (?,?) +*/ +static int stmt_specifyCol_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_INT; + params[i+2].buffer_length = sizeof(int32_t); + params[i+2].buffer = v->v4; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+3].buffer_length = sizeof(float); + params[i+3].buffer = v->f4; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+4].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+4].buffer = v->br; + params[i+4].length = lb; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + +// create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp) + char *sql = "insert into ? (ts,b,v4,f4,br) values(?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +/*=======================*/ +/* +test scene: insert into tb1 (ts,f1) values (?,?) +*/ +static int stmt_specifyCol_bind_case_001_maxRows(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d-0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d-0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_INT; + params[i+1].buffer_length = sizeof(int32_t); + params[i+1].buffer = v->v4; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + +// create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp) + char *sql = "insert into m0 (ts,b,v4,f4,br) values(?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + //char buf[32]; + //sprintf(buf, "m%d", zz); + //code = taos_stmt_set_tbname(stmt, buf); + //if (code != 0){ + // printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + // return -1; + //} + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +static void SpecifyColumnBatchCase(TAOS *taos) { + TAOS_STMT *stmt = NULL; + + int tableNum; + int lenOfBinaryDef; + int rowsOfPerColum; + int bingNum; + int lenOfBinaryAct; + int columnNum; + + int totalRowsPerTbl; + +//=======================================================================// +//=============================== single table ==========================// +//========== case 1: ======================// +#if 1 +{ + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 1; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db1"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 1 check result end\n\n"); +} +#endif + + //========== case 2: ======================// +#if 1 +{ + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 5; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 15; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db2"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 2 check result end\n\n"); +} +#endif + + //========== case 2-1: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 32767; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 15; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db2_1"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 2-1 check result end\n\n"); + } +#endif + //========== case 2-2: ======================// +#if 1 + { + printf("====case 2-2 error test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 32768; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 15; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db2_2"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("====case 2-2 check result end\n\n"); + } +#endif + + + //========== case 3: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 1; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 20; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db3"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 3 check result end\n\n"); + } +#endif + + //========== case 4: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 5; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db4"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 4 check result end\n\n"); + } +#endif + +//=======================================================================// +//=============================== multi tables ==========================// + //========== case 5: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 1; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 16; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db5"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 5 check result end\n\n"); + } +#endif + + //========== case 6: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 5; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 20; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db6"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 6 check result end\n\n"); + } +#endif + + //========== case 7: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 1; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db7"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 7 check result end\n\n"); + } +#endif + + //========== case 8: ======================// +#if 1 +{ + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 5; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 40; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db8"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 8 check result end\n\n"); +} +#endif + + //=======================================================================// + //=============================== multi-rows to single table ==========================// + //========== case 9: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 23740; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db9"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 9 check result end\n\n"); + } +#endif + + //========== case 10: ======================// +#if 0 + { + printf("====case 10 error test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 23741; // WAL size exceeds limit + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("====case 10 check result end\n\n"); + } +#endif + + + //=======================================================================// + //=============================== multi tables, multi bind one same table ==========================// + //========== case 13: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 1; + bingNum = 5; + lenOfBinaryDef = 40; + lenOfBinaryAct = 28; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db13"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 13 check result end\n\n"); + } +#endif + + //========== case 14: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 5; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db14"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 14 check result end\n\n"); + } +#endif + + + //========== case 15: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1000; + rowsOfPerColum = 10; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db15"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m111", 0, totalRowsPerTbl); + checkResult(taos, "m222", 0, totalRowsPerTbl); + checkResult(taos, "m333", 0, totalRowsPerTbl); + checkResult(taos, "m500", 0, totalRowsPerTbl); + checkResult(taos, "m999", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 15 check result end\n\n"); + } +#endif + + //========== case 17: ======================// +#if 1 + { + //printf("case 17 test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 10; + rowsOfPerColum = 100; + bingNum = 1; + lenOfBinaryDef = 100; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db17"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m5", 0, totalRowsPerTbl); + checkResult(taos, "m8", 0, totalRowsPerTbl); + checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 17 check result end\n\n"); + } +#endif + + return ; + +} int main(int argc, char *argv[]) { TAOS *taos; char host[32] = "127.0.0.1"; char* serverIp = NULL; - int threadNum = 2; + int threadNum = 1; // connect to server if (argc == 1) { @@ -2268,9 +3177,12 @@ int main(int argc, char *argv[]) } else if (argc == 4) { serverIp = argv[1]; threadNum = atoi(argv[2]); - g_rows = atoi(argv[3]); + g_runTimes = atoi(argv[3]); } + printf("server:%s, runTimes:%d\n\n", serverIp, g_runTimes); + +#if 0 printf("server:%s, threadNum:%d, rows:%d\n\n", serverIp, threadNum, g_rows); pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), (size_t)threadNum); @@ -2287,7 +3199,8 @@ int main(int argc, char *argv[]) tInfo->taos = taos; tInfo->idx = i; if (0 == i) { - pthread_create(&(pThreadList[0]), NULL, runCase, (void *)tInfo); + //pthread_create(&(pThreadList[0]), NULL, runCase, (void *)tInfo); + pthread_create(&(pThreadList[0]), NULL, SpecifyColumnBatchCase, (void *)tInfo); } else if (1 == i){ pthread_create(&(pThreadList[0]), NULL, runCase_long, (void *)tInfo); } @@ -2300,6 +3213,18 @@ int main(int argc, char *argv[]) free(pThreadList); free(threadInfo); +#endif + + taos = taos_connect(serverIp, "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to TDengine, reason:%s\n", taos_errstr(taos)); + return -1; + } + + runCase(taos); + runCase_long(taos); + SpecifyColumnBatchCase(taos); + return 0; } From c74651a6cb906f71368357fe7acd61a260bae8d0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 May 2021 21:35:41 +0800 Subject: [PATCH 143/157] Hotfix/sangshuduo/td 4136 taosdemo records morethan 32767 for develop (#6189) * [TD-4136]: taosdemo records per req more than 32767. for develop branch. * change taosRandom() to rand_bool() for bool data. * [TD-4136]: taosdemo check insert rows not more than 32767. check insert rows for progressive. * fix with answer_yes. * fix interlace rows checking position. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index e8075c69ba..6c1f56e44a 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3492,16 +3492,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } g_args.interlace_rows = interlaceRows->valueint; - - // rows per table need be less than insert batch - if (g_args.interlace_rows > g_args.num_of_RPR) { - printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n", - g_args.interlace_rows, g_args.num_of_RPR); - printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", - g_args.num_of_RPR); - prompt(); - g_args.interlace_rows = g_args.num_of_RPR; - } } else if (!interlaceRows) { g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req } else { @@ -3567,6 +3557,16 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } + // rows per table need be less than insert batch + if (g_args.interlace_rows > g_args.num_of_RPR) { + printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n", + g_args.interlace_rows, g_args.num_of_RPR); + printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", + g_args.num_of_RPR); + prompt(); + g_args.interlace_rows = g_args.num_of_RPR; + } + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); if (!dbs || dbs->type != cJSON_Array) { printf("ERROR: failed to read json, databases not found\n"); From fc394200e4a7bb01f36810da14cb93bf12ba2d01 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 May 2021 23:20:42 +0800 Subject: [PATCH 144/157] Hotfix/sangshuduo/td 4240 taosdemo subscribe toomuch for develop (#6193) * [TD-4240]: taosdemo subscribe more than max query sql count. * [TD-4240]: taosdemo subscribe more than 100. fix tsub sequence bug. * [TD-4240]: taosdemo subscribe more than 100. fix auto create table. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 102 +++++++++++++++++++++--------------- 1 file changed, 59 insertions(+), 43 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6c1f56e44a..feab1aab90 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4890,10 +4890,12 @@ static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k) return affectedRows; } -static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t tableSeq) +static void getTableName(char *pTblName, + threadInfo* pThreadInfo, uint64_t tableSeq) { SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - if (superTblInfo) { + if ((superTblInfo) + && (AUTO_CREATE_SUBTBL != superTblInfo->autoCreateTable)) { if (superTblInfo->childTblLimit > 0) { snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s", superTblInfo->childTblName + @@ -6768,6 +6770,7 @@ static void *superSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; char subSqlstr[MAX_QUERY_SQL_LENGTH]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; + uint64_t tsubSeq; if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n", @@ -6776,6 +6779,15 @@ static void *superSubscribe(void *sarg) { exit(-1); } + if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { + errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n", + g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables, + g_queryInfo.superQueryInfo.sqlCount, + pThreadInfo->ntables, + MAX_QUERY_SQL_COUNT); + exit(-1); + } + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, @@ -6804,6 +6816,8 @@ static void *superSubscribe(void *sarg) { char topic[32] = {0}; for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { + + tsubSeq = i - pThreadInfo->start_table_from; verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n", __func__, __LINE__, pThreadInfo->threadID, @@ -6823,12 +6837,12 @@ static void *superSubscribe(void *sarg) { debugPrint("%s() LN%d, [%d] subSqlstr: %s\n", __func__, __LINE__, pThreadInfo->threadID, subSqlstr); - tsub[i] = subscribeImpl( + tsub[tsubSeq] = subscribeImpl( STABLE_CLASS, pThreadInfo, subSqlstr, topic, g_queryInfo.superQueryInfo.subscribeRestart, g_queryInfo.superQueryInfo.subscribeInterval); - if (NULL == tsub[i]) { + if (NULL == tsub[tsubSeq]) { taos_close(pThreadInfo->taos); return NULL; } @@ -6844,54 +6858,56 @@ static void *superSubscribe(void *sarg) { while(1) { for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { - continue; - } + tsubSeq = i - pThreadInfo->start_table_from; + if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { + continue; + } - res = taos_consume(tsub[i]); - if (res) { - if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - appendResultToFile(res, pThreadInfo->fp); - } - if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - appendResultToFile(res, pThreadInfo->fp); - } - consumed[i] ++; + res = taos_consume(tsub[tsubSeq]); + if (res) { + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + appendResultToFile(res, pThreadInfo->fp); + } + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + appendResultToFile(res, pThreadInfo->fp); + } + consumed[tsubSeq] ++; - if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) - && (consumed[i] >= - g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { - printf("keepProgress:%d, resub super table query: %"PRIu64"\n", - g_queryInfo.superQueryInfo.subscribeKeepProgress, - pThreadInfo->querySeq); - taos_unsubscribe(tsub, + if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) + && (consumed[tsubSeq] >= + g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + printf("keepProgress:%d, resub super table query: %"PRIu64"\n", + g_queryInfo.superQueryInfo.subscribeKeepProgress, + pThreadInfo->querySeq); + taos_unsubscribe(tsub[tsubSeq], g_queryInfo.superQueryInfo.subscribeKeepProgress); - consumed[i]= 0; - tsub[i] = subscribeImpl( - STABLE_CLASS, - pThreadInfo, subSqlstr, topic, - g_queryInfo.superQueryInfo.subscribeRestart, - g_queryInfo.superQueryInfo.subscribeInterval - ); - if (NULL == tsub[i]) { - taos_close(pThreadInfo->taos); - return NULL; - } - } - } + consumed[tsubSeq]= 0; + tsub[tsubSeq] = subscribeImpl( + STABLE_CLASS, + pThreadInfo, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart, + g_queryInfo.superQueryInfo.subscribeInterval + ); + if (NULL == tsub[tsubSeq]) { + taos_close(pThreadInfo->taos); + return NULL; + } + } + } } } taos_free_result(res); for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - taos_unsubscribe(tsub[i], 0); + tsubSeq = i - pThreadInfo->start_table_from; + taos_unsubscribe(tsub[tsubSeq], 0); } taos_close(pThreadInfo->taos); From 171ca11a0ad9be9e2973512d4dd63d8592e4a921 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 22 May 2021 11:54:50 +0800 Subject: [PATCH 145/157] Update README-CN.md (#6191) --- README-CN.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README-CN.md b/README-CN.md index d4c10e71d6..afb242d621 100644 --- a/README-CN.md +++ b/README-CN.md @@ -116,7 +116,7 @@ mkdir debug && cd debug cmake .. && cmake --build . ``` -在X86-64、X86、arm64 和 arm32 平台上，TDengine 生成脚本可以自动检测机器架构。也可以手动配置 CPUTYPE 参数来指定 CPU 类型，如 aarch64 或 aarch32 等。 +在X86-64、X86、arm64、arm32 和 mips64 平台上，TDengine 生成脚本可以自动检测机器架构。也可以手动配置 CPUTYPE 参数来指定 CPU 类型，如 aarch64 或 aarch32 等。 aarch64： @@ -130,6 +130,12 @@ aarch32： cmake .. -DCPUTYPE=aarch32 && cmake --build . ``` +mips64： + +```bash +cmake .. -DCPUTYPE=mips64 && cmake --build . +``` + ### Windows 系统 如果你使用的是 Visual Studio 2013 版本： From 2d43e7f06abf5d373edc1f654cd1f12f89b52a01 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 22 May 2021 11:55:12 +0800 Subject: [PATCH 146/157] add mips64 build instruction. (#6190) Co-authored-by: Shuduo Sang --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 78f902babe..2c33119e34 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ mkdir debug && cd debug cmake .. && cmake --build . ``` -TDengine build script can detect the host machine's architecture on X86-64, X86, arm64 and arm32 platform. +TDengine build script can detect the host machine's architecture on X86-64, X86, arm64, arm32 and mips64 platform. You can also specify CPUTYPE option like aarch64 or aarch32 too if the detection result is not correct: aarch64: @@ -123,6 +123,11 @@ aarch32: cmake .. -DCPUTYPE=aarch32 && cmake --build . ``` +mips64: +```bash +cmake .. -DCPUTYPE=mips64 && cmake --build . +``` + ### On Windows platform If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". From 795503ccbe4f9ebc5e970e00414283822d209328 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 22 May 2021 16:17:03 +0800 Subject: [PATCH 147/157] [TD-4296]: taosdemo sub keepProgress. (#6201) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index feab1aab90..aa7a022f91 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -403,7 +403,7 @@ typedef struct SuperQueryInfo_S { uint64_t sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; - int resubAfterConsume[MAX_QUERY_SQL_COUNT]; + int resubAfterConsume; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; char* childTblName; @@ -4495,6 +4495,18 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } + cJSON* superResubAfterConsume = + cJSON_GetObjectItem(superQuery, "resubAfterConsume"); + if (superResubAfterConsume + && superResubAfterConsume->type == cJSON_Number) { + g_queryInfo.superQueryInfo.resubAfterConsume = + superResubAfterConsume->valueint; + } else if (!superResubAfterConsume) { + //printf("failed to read json, subscribe interval no found\n"); + ////goto PARSE_OVER; + g_queryInfo.superQueryInfo.resubAfterConsume = 1; + } + // sqls cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls"); if (!subsqls) { @@ -4526,18 +4538,6 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); - cJSON* superResubAfterConsume = - cJSON_GetObjectItem(sql, "resubAfterConsume"); - if (superResubAfterConsume - && superResubAfterConsume->type == cJSON_Number) { - g_queryInfo.superQueryInfo.resubAfterConsume[j] = - superResubAfterConsume->valueint; - } else if (!superResubAfterConsume) { - //printf("failed to read json, subscribe interval no found\n"); - //goto PARSE_OVER; - g_queryInfo.superQueryInfo.resubAfterConsume[j] = 1; - } - cJSON *result = cJSON_GetObjectItem(sql, "result"); if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ @@ -6855,6 +6855,8 @@ static void *superSubscribe(void *sarg) { } TAOS_RES* res = NULL; + uint64_t st = 0, et = 0; + while(1) { for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { @@ -6863,7 +6865,12 @@ static void *superSubscribe(void *sarg) { continue; } + st = taosGetTimestampMs(); + performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et)); res = taos_consume(tsub[tsubSeq]); + et = taosGetTimestampMs(); + performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st)); + if (res) { if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(pThreadInfo->fp, "%s-%d", @@ -6881,7 +6888,7 @@ static void *superSubscribe(void *sarg) { if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) && (consumed[tsubSeq] >= - g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + g_queryInfo.superQueryInfo.resubAfterConsume)) { printf("keepProgress:%d, resub super table query: %"PRIu64"\n", g_queryInfo.superQueryInfo.subscribeKeepProgress, pThreadInfo->querySeq); From 091b8e6e565a8dc53b6b5ff4001cf62611438bce Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Sat, 22 May 2021 19:54:28 +0800 Subject: [PATCH 148/157] [TD-4299]Circle CI startup (#6197) * [TD-4299]Circle CI startup * test * test * revert * udpate case --- .circleci/config.yml | 13 +++++++++++++ tests/script/unique/cluster/cache.sim | 2 +- tests/script/unique/dnode/monitor.sim | 2 +- tests/script/unique/dnode/monitor_bug.sim | 6 +++--- 4 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..6f98693add --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,13 @@ +# Use the latest 2.1 version of CircleCI pipeline process engine. See: https://circleci.com/docs/2.0/configuration-reference +version: 2.1 +# Use a package of configuration called an orb. +orbs: + # Declare a dependency on the welcome-orb + welcome: circleci/welcome-orb@0.4.1 +# Orchestrate or schedule a set of jobs +workflows: + # Name the workflow "welcome" + welcome: + # Run the welcome/run job in its own container + jobs: + - welcome/run diff --git a/tests/script/unique/cluster/cache.sim b/tests/script/unique/cluster/cache.sim index 33aaea425c..740eddfb0d 100644 --- a/tests/script/unique/cluster/cache.sim +++ b/tests/script/unique/cluster/cache.sim @@ -41,7 +41,7 @@ sql create dnode $hostname2 sleep 10000 sql show log.tables; -if $rows != 5 then +if $rows > 6 then return -1 endi diff --git a/tests/script/unique/dnode/monitor.sim b/tests/script/unique/dnode/monitor.sim index b9b5e41889..0b41a4137c 100644 --- a/tests/script/unique/dnode/monitor.sim +++ b/tests/script/unique/dnode/monitor.sim @@ -56,7 +56,7 @@ print $data30 print $data40 print $data50 -if $rows != 5 then +if $rows > 6 then return -1 endi diff --git a/tests/script/unique/dnode/monitor_bug.sim b/tests/script/unique/dnode/monitor_bug.sim index efdf5e94b9..60c6524d9c 100644 --- a/tests/script/unique/dnode/monitor_bug.sim +++ b/tests/script/unique/dnode/monitor_bug.sim @@ -19,7 +19,7 @@ sleep 3000 sql show dnodes print dnode1 openVnodes $data2_1 -if $data2_1 != 1 then +if $data2_1 > 2 then return -1 endi @@ -41,7 +41,7 @@ print dnode2 openVnodes $data2_2 if $data2_1 != 0 then goto show2 endi -if $data2_2 != 1 then +if $data2_2 > 2 then goto show2 endi @@ -55,7 +55,7 @@ print $data30 print $data40 print $data50 -if $rows != 4 then +if $rows > 5 then return -1 endi From beac10a45094ea2c20e224beec4db920b4477619 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 22 May 2021 21:08:13 +0800 Subject: [PATCH 149/157] Hotfix/sangshuduo/td 4240 taosdemo subscribe toomuch for develop (#6203) * [TD-4240]: taosdemo subscribe more than max query sql count. * [TD-4240]: taosdemo subscribe more than 100. fix tsub sequence bug. * [TD-4240]: taosdemo subscribe more than 100. fix auto create table. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index aa7a022f91..eace05af8d 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6779,15 +6779,6 @@ static void *superSubscribe(void *sarg) { exit(-1); } - if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { - errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n", - g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables, - g_queryInfo.superQueryInfo.sqlCount, - pThreadInfo->ntables, - MAX_QUERY_SQL_COUNT); - exit(-1); - } - if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, From 4729a13ed94c073aa49b34ff467efd7a7a805b37 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Sun, 23 May 2021 14:40:29 +0800 Subject: [PATCH 150/157] change version number (#6205) Co-authored-by: plum-lihui --- .gitmodules | 6 +++--- cmake/version.inc | 2 +- snap/snapcraft.yaml | 4 ++-- src/connector/go | 2 +- src/connector/grafanaplugin | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitmodules b/.gitmodules index 74afbbf997..0e65b02221 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,12 @@ [submodule "src/connector/go"] path = src/connector/go - url = https://github.com/taosdata/driver-go + url = git@github.com:taosdata/driver-go.git [submodule "src/connector/grafanaplugin"] path = src/connector/grafanaplugin - url = https://github.com/taosdata/grafanaplugin + url = git@github.com:taosdata/grafanaplugin.git [submodule "src/connector/hivemq-tdengine-extension"] path = src/connector/hivemq-tdengine-extension - url = https://github.com/huskar-t/hivemq-tdengine-extension.git + url = git@github.com:taosdata/hivemq-tdengine-extension.git [submodule "tests/examples/rust"] path = tests/examples/rust url = https://github.com/songtianyi/tdengine-rust-bindings.git diff --git a/cmake/version.inc b/cmake/version.inc index 0ee23f319a..ed8e7a156c 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.1.0.0") + SET(TD_VER_NUMBER "2.1.1.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 43006928a6..d85c8e5871 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,7 +1,7 @@ name: tdengine base: core18 -version: '2.1.0.0' +version: '2.1.1.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -73,7 +73,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.1.0.0 + - usr/lib/libtaos.so.2.1.1.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/connector/go b/src/connector/go index 7a26c432f8..8ce6d86558 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5 +Subproject commit 8ce6d86558afc8c0b50c10f990fd2b4270cf06fc diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index 32e2c97a4c..3530c6df09 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df +Subproject commit 3530c6df097134a410bacec6b3cd013ef38a61aa From 105d6b040a60e99ad0c0300b63e1494b6f73255f Mon Sep 17 00:00:00 2001 From: yaofei517 <41509375+yaofei517@users.noreply.github.com> Date: Sun, 23 May 2021 17:11:06 +0800 Subject: [PATCH 151/157] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dwindows=E4=B8=8Bvscod?= =?UTF-8?q?e=E8=B0=83=E8=AF=95go=E7=A8=8B=E5=BA=8F=E5=90=8D=E6=97=A0?= =?UTF-8?q?=E5=90=8E=E7=BC=80=E7=9A=84=E6=83=85=E5=86=B5=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E7=A8=8B=E5=BA=8Fcrash=20(#6176)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 修复windows下vscode调试go程序名无后缀的情况导致程序crash * 修复windows下vscode调试go程序时临时程序_debug_bin无后缀的情况，导致调用接口发生crash * 修复由于头文件排序导致编译错误 --- src/os/src/windows/wSemphone.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/os/src/windows/wSemphone.c b/src/os/src/windows/wSemphone.c index a3f0367ee1..878ceba791 100644 --- a/src/os/src/windows/wSemphone.c +++ b/src/os/src/windows/wSemphone.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE + #include "os.h" #include "taosdef.h" #include "tglobal.h" @@ -24,7 +25,7 @@ bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } -void taosResetPthread(pthread_t *thread) { thread->p = 0; } +void taosResetPthread(pthread_t* thread) { thread->p = 0; } int64_t taosGetPthreadId(pthread_t thread) { #ifdef PTW32_VERSION @@ -34,27 +35,24 @@ int64_t taosGetPthreadId(pthread_t thread) { #endif } -int64_t taosGetSelfPthreadId() { - return GetCurrentThreadId(); -} +int64_t taosGetSelfPthreadId() { return GetCurrentThreadId(); } -bool taosComparePthread(pthread_t first, pthread_t second) { - return first.p == second.p; -} +bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == second.p; } -int32_t taosGetPId() { - return GetCurrentProcessId(); -} +int32_t taosGetPId() { return GetCurrentProcessId(); } -int32_t taosGetCurrentAPPName(char *name, int32_t* len) { +int32_t taosGetCurrentAPPName(char* name, int32_t* len) { char filepath[1024] = {0}; GetModuleFileName(NULL, filepath, MAX_PATH); - *strrchr(filepath,'.') = '\0'; + char* sub = strrchr(filepath, '.'); + if (sub != NULL) { + *sub = '\0'; + } strcpy(name, filepath); if (len != NULL) { - *len = (int32_t) strlen(filepath); + *len = (int32_t)strlen(filepath); } return 0; From d26ff7cede73bdc099c23b292638d4ea9e3d7479 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 23 May 2021 19:35:13 +0800 Subject: [PATCH 152/157] [TD-4313]: odbc example compile failed on Windows. (#6206) * [TD-4313]: odbc example compile failed on Windows. * [TD-4313]: odbc example compile failed on Windows. few functions also need change. --- src/connector/odbc/examples/c/main.c | 6 +----- src/os/src/detail/osMemory.c | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/connector/odbc/examples/c/main.c b/src/connector/odbc/examples/c/main.c index 3ad32dbe53..de01d2b85e 100644 --- a/src/connector/odbc/examples/c/main.c +++ b/src/connector/odbc/examples/c/main.c @@ -312,11 +312,7 @@ static int test_sqls_in_stmt(SQLHENV env, SQLHDBC conn, SQLHSTMT stmt, const cha size_t len = 0; ssize_t n = 0; -#ifdef _MSC_VER - n = taosGetlineImp(&line, &len, f); -#else - n = getline(&line, &len, f); -#endif + n = tgetline(&line, &len, f); if (n==-1) break; const char *p = NULL; diff --git a/src/os/src/detail/osMemory.c b/src/os/src/detail/osMemory.c index 291a54b669..d8194feab4 100644 --- a/src/os/src/detail/osMemory.c +++ b/src/os/src/detail/osMemory.c @@ -62,7 +62,7 @@ static void* taosRandomRealloc(void* ptr, size_t size, const char* file, uint32_ static char* taosRandomStrdup(const char* str, const char* file, uint32_t line) { size_t len = strlen(str); - return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrdupImp(str); + return taosRandomAllocFail(len + 1, file, line) ? NULL : tstrdup(str); } static char* taosRandomStrndup(const char* str, size_t size, const char* file, uint32_t line) { @@ -70,11 +70,11 @@ static char* taosRandomStrndup(const char* str, size_t size, const char* file, u if (len > size) { len = size; } - return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrndupImp(str, len); + return taosRandomAllocFail(len + 1, file, line) ? NULL : tstrndup(str, len); } static ssize_t taosRandomGetline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { - return taosRandomAllocFail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream); + return taosRandomAllocFail(*n, file, line) ? -1 : tgetline(lineptr, n, stream); } //////////////////////////////////////////////////////////////////////////////// @@ -242,7 +242,7 @@ static char* taosStrndupDetectLeak(const char* str, size_t size, const char* fil static ssize_t taosGetlineDetectLeak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { char* buf = NULL; size_t bufSize = 0; - ssize_t size = taosGetlineImp(&buf, &bufSize, stream); + ssize_t size = tgetline(&buf, &bufSize, stream); if (size != -1) { if (*n < size + 1) { void* p = taosReallocDetectLeak(*lineptr, size + 1, file, line); @@ -372,7 +372,7 @@ void taosFreeMem(void* ptr, const char* file, uint32_t line) { char* taosStrdupMem(const char* str, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: - return taosStrdupImp(str); + return tstrdup(str); case TAOS_ALLOC_MODE_RANDOM_FAIL: return taosRandomStrdup(str, file, line); @@ -380,13 +380,13 @@ char* taosStrdupMem(const char* str, const char* file, uint32_t line) { case TAOS_ALLOC_MODE_DETECT_LEAK: return taosStrdupDetectLeak(str, file, line); } - return taosStrdupImp(str); + return tstrdup(str); } char* taosStrndupMem(const char* str, size_t size, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: - return taosStrndupImp(str, size); + return tstrndup(str, size); case TAOS_ALLOC_MODE_RANDOM_FAIL: return taosRandomStrndup(str, size, file, line); @@ -394,13 +394,13 @@ char* taosStrndupMem(const char* str, size_t size, const char* file, uint32_t li case TAOS_ALLOC_MODE_DETECT_LEAK: return taosStrndupDetectLeak(str, size, file, line); } - return taosStrndupImp(str, size); + return tstrndup(str, size); } ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: - return taosGetlineImp(lineptr, n, stream); + return tgetline(lineptr, n, stream); case TAOS_ALLOC_MODE_RANDOM_FAIL: return taosRandomGetline(lineptr, n, stream, file, line); @@ -408,7 +408,7 @@ ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char* file case TAOS_ALLOC_MODE_DETECT_LEAK: return taosGetlineDetectLeak(lineptr, n, stream, file, line); } - return taosGetlineImp(lineptr, n, stream); + return tgetline(lineptr, n, stream); } static void taosCloseAllocLog() { @@ -517,4 +517,4 @@ void* taosTZfree(void* ptr) { free((void*)((char*)ptr - sizeof(size_t))); } return NULL; -} \ No newline at end of file +} From 8e6a5acbc39b2b3fa88a8db8835f8bfbd3447080 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 24 May 2021 12:07:45 +0800 Subject: [PATCH 153/157] Hotfix/sangshuduo/td 3197 fix taosdemo coverity scan (#6208) * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix coverity scan issues. check super tbl info pointer. * [TD-3197] : fix coverity scan issues. move sub tbl query thread join into loop * [TD-3197] : fix coverity scan issues. remove unused variable * [TD-3197] : fix coverity scan issues. use more secure random library * [TD-3197] : fix coverity scan issues. use strncpy for more safe * [TD-3197] : fix taosdemo coverity scan issue. replace arc4random with rand(). * [TD-3197] : fix coverity scan issues. check stb info pointer for start time * [TD-3197] : fix coverity scan issues. fix strcpy vulnerability * [TD-3197] : fix taosdemo coverity scan issue. modify taosdemoTest2. try to check database continously. * [TD-3197] : taosdemo coverity scan issues. * [TD-3197] : fix memory leak when parsing arguments. * [TD-3197] : fix cmake strip arguments. * [TD-3197] : taosdemo coverity scan. fix cmake string manipulation. * [TD-3197]: taosdemo coverity scan issue. configDir buffer overwrite. * [TD-3197]: coverity scan issue. taosdump argument validation. * [TD-3197]: taosdemo and taosdump coverity scan issues. * [TD-3197]: taosdemo coverity scan. append result buf to file. for develop branch. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index eace05af8d..32c1be372c 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1147,19 +1147,18 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { static void appendResultBufToFile(char *resultBuf, char *resultFile) { - FILE *fp = NULL; - if (resultFile[0] != 0) { - fp = fopen(resultFile, "at"); - if (fp == NULL) { - errorPrint( - "%s() LN%d, failed to open result file: %s, result will not save to file\n", - __func__, __LINE__, resultFile); - return; + FILE *fp = NULL; + if (resultFile[0] != 0) { + fp = fopen(resultFile, "at"); + if (fp == NULL) { + errorPrint( + "%s() LN%d, failed to open result file: %s, result will not save to file\n", + __func__, __LINE__, resultFile); + return; + } + fprintf(fp, "%s", resultBuf); + tmfclose(fp); } - } - - fprintf(fp, "%s", resultBuf); - tmfclose(fp); } static void appendResultToFile(TAOS_RES *res, char* resultFile) { From 9facb54b3ff22b54ed2b83bdec53bf6a4d815da0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 24 May 2021 15:21:20 +0800 Subject: [PATCH 154/157] [TD-4322]: taosdemo query then fetch result. (#6211) * [TD-4322]: taosdemo query then fetch result. * build again. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 56 ++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 32c1be372c..adb6e836de 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1147,21 +1147,20 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { static void appendResultBufToFile(char *resultBuf, char *resultFile) { - FILE *fp = NULL; - if (resultFile[0] != 0) { - fp = fopen(resultFile, "at"); - if (fp == NULL) { - errorPrint( - "%s() LN%d, failed to open result file: %s, result will not save to file\n", - __func__, __LINE__, resultFile); - return; - } - fprintf(fp, "%s", resultBuf); - tmfclose(fp); - } + FILE *fp = NULL; + fp = fopen(resultFile, "at"); + if (fp == NULL) { + errorPrint( + "%s() LN%d, failed to open result file: %s, result will not save to file\n", + __func__, __LINE__, resultFile); + return; + } + + fprintf(fp, "%s", resultBuf); + tmfclose(fp); } -static void appendResultToFile(TAOS_RES *res, char* resultFile) { +static void fetchResult(TAOS_RES *res, char* resultFile) { TAOS_ROW row = NULL; int num_rows = 0; int num_fields = taos_field_count(res); @@ -1179,10 +1178,11 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { // fetch the records row by row while((row = taos_fetch_row(res))) { - if (totalLen >= 100*1024*1024 - 32000) { - appendResultBufToFile(databuf, resultFile); - totalLen = 0; - memset(databuf, 0, 100*1024*1024); + if ((resultFile) + && (totalLen >= 100*1024*1024 - 32000)) { + appendResultBufToFile(databuf, resultFile); + totalLen = 0; + memset(databuf, 0, 100*1024*1024); } num_rows++; int len = taos_print_row(temp, row, fields, num_fields); @@ -1194,7 +1194,9 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile); - appendResultBufToFile(databuf, resultFile); + if (resultFile) { + appendResultBufToFile(databuf, resultFile); + } free(databuf); } @@ -1210,9 +1212,7 @@ static void selectAndGetResult( return; } - if ((strlen(pThreadInfo->fp))) { - appendResultToFile(res, pThreadInfo->fp); - } + fetchResult(res, pThreadInfo->fp); taos_free_result(res); } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { @@ -2053,13 +2053,13 @@ static void printfQuerySystemInfo(TAOS * taos) { // show variables res = taos_query(taos, "show variables;"); - //appendResultToFile(res, filename); + //fetchResult(res, filename); xDumpResultToFile(filename, res); // show dnodes res = taos_query(taos, "show dnodes;"); xDumpResultToFile(filename, res); - //appendResultToFile(res, filename); + //fetchResult(res, filename); // show databases res = taos_query(taos, "show databases;"); @@ -6712,7 +6712,7 @@ static void stable_sub_callback( } if (param) - appendResultToFile(res, ((threadInfo *)param)->fp); + fetchResult(res, ((threadInfo *)param)->fp); // tao_unscribe() will free result. } @@ -6725,7 +6725,7 @@ static void specified_sub_callback( } if (param) - appendResultToFile(res, ((threadInfo *)param)->fp); + fetchResult(res, ((threadInfo *)param)->fp); // tao_unscribe() will free result. } @@ -6866,13 +6866,13 @@ static void *superSubscribe(void *sarg) { sprintf(pThreadInfo->fp, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - appendResultToFile(res, pThreadInfo->fp); + fetchResult(res, pThreadInfo->fp); } if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(pThreadInfo->fp, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - appendResultToFile(res, pThreadInfo->fp); + fetchResult(res, pThreadInfo->fp); } consumed[tsubSeq] ++; @@ -6973,7 +6973,7 @@ static void *specifiedSubscribe(void *sarg) { sprintf(pThreadInfo->fp, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - appendResultToFile(res, pThreadInfo->fp); + fetchResult(res, pThreadInfo->fp); } consumed ++; From 432f961e6122c96d62de0c5e94cd83fc0650853e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 24 May 2021 16:22:25 +0800 Subject: [PATCH 155/157] Hotfix/sangshuduo/td 4296 taosdemo sub keepprogress for develop (#6215) * [TD-4296]: taosdemo sub keepProgress. * make print sqlcount quiet. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index adb6e836de..5afc8c44a1 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -7082,7 +7082,7 @@ static int subscribeTestProcess() { //==== create threads for super table query if (g_queryInfo.superQueryInfo.sqlCount <= 0) { - printf("%s() LN%d, super table query sqlCount %"PRIu64".\n", + debugPrint("%s() LN%d, super table query sqlCount %"PRIu64".\n", __func__, __LINE__, g_queryInfo.superQueryInfo.sqlCount); } else { From d1f15a82eb825f12f4d422ab853951c12e73f289 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 24 May 2021 20:16:26 +0800 Subject: [PATCH 156/157] Hotfix/sangshuduo/td 4322 taosdemo fetch result for develop (#6220) * [TD-4322]: taosdemo query then fetch result. * build again. * [TD-4322]: taosdemo fetch result. check result file name length. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 5afc8c44a1..0f3adc29be 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1178,7 +1178,7 @@ static void fetchResult(TAOS_RES *res, char* resultFile) { // fetch the records row by row while((row = taos_fetch_row(res))) { - if ((resultFile) + if ((resultFile) && (strlen(resultFile) > 0) && (totalLen >= 100*1024*1024 - 32000)) { appendResultBufToFile(databuf, resultFile); totalLen = 0; @@ -1194,7 +1194,7 @@ static void fetchResult(TAOS_RES *res, char* resultFile) { verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile); - if (resultFile) { + if ((resultFile) && (strlen(resultFile) > 0)) { appendResultBufToFile(databuf, resultFile); } free(databuf); @@ -2231,7 +2231,7 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port response_buf[RESP_BUF_LEN - 1] = '\0'; printf("Response:\n%s\n", response_buf); - if (resultFile) { + if ((resultFile) && (strlen(resultFile) > 0)) { appendResultBufToFile(response_buf, resultFile); } From 21c0341b32142cebbd06d8b3909049013839cf00 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 24 May 2021 22:10:55 +0800 Subject: [PATCH 157/157] Hotfix/sangshuduo/td 4296 taosdemo sub keepprogress for develop (#6222) * [TD-4296]: taosdemo sub keepProgress. * make print sqlcount quiet. * [TD-4296]: taosdemo subscribe keepprogress. specified query topic thread safe. * cherry pick from master branch. * cherry pick from master branch. * fix compile warning for mac. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 407 ++++++++++++++++++------------------ 1 file changed, 206 insertions(+), 201 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 0f3adc29be..6f651c18cf 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -375,7 +375,7 @@ typedef struct SDbs_S { typedef struct SpecifiedQueryInfo_S { uint64_t queryInterval; // 0: unlimit > 0 loop/s - uint64_t concurrent; + uint32_t concurrent; uint64_t sqlCount; uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms @@ -386,6 +386,9 @@ typedef struct SpecifiedQueryInfo_S { char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; int resubAfterConsume[MAX_QUERY_SQL_COUNT]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; + char topic[MAX_QUERY_SQL_COUNT][32]; + int consumed[MAX_QUERY_SQL_COUNT]; + TAOS_RES* res[MAX_QUERY_SQL_COUNT]; uint64_t totalQueried; } SpecifiedQueryInfo; @@ -431,7 +434,8 @@ typedef struct SThreadInfo_S { int threadID; char db_name[MAX_DB_NAME_SIZE+1]; uint32_t time_precision; - char fp[4096]; + char filePath[4096]; + FILE *fp; char tb_prefix[MAX_TB_NAME_SIZE]; uint64_t start_table_from; uint64_t end_table_to; @@ -546,7 +550,7 @@ static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); static int postProceSql(char *host, struct sockaddr_in *pServAddr, - uint16_t port, char* sqlstr, char *resultFile); + uint16_t port, char* sqlstr, threadInfo *pThreadInfo); /* ************ Global variables ************ */ @@ -1128,7 +1132,8 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { if (code != 0) { if (!quiet) { debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command); - errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res)); + errorPrint("Failed to execute %s, command length: %d, reason: %s\n", + command, (int)strlen(command), taos_errstr(res)); } taos_free_result(res); //taos_close(taos); @@ -1145,22 +1150,22 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { return 0; } -static void appendResultBufToFile(char *resultBuf, char *resultFile) +static void appendResultBufToFile(char *resultBuf, threadInfo *pThreadInfo) { - FILE *fp = NULL; - fp = fopen(resultFile, "at"); - if (fp == NULL) { + pThreadInfo->fp = fopen(pThreadInfo->filePath, "at"); + if (pThreadInfo->fp == NULL) { errorPrint( "%s() LN%d, failed to open result file: %s, result will not save to file\n", - __func__, __LINE__, resultFile); + __func__, __LINE__, pThreadInfo->filePath); return; } - fprintf(fp, "%s", resultBuf); - tmfclose(fp); + fprintf(pThreadInfo->fp, "%s", resultBuf); + tmfclose(pThreadInfo->fp); + pThreadInfo->fp = NULL; } -static void fetchResult(TAOS_RES *res, char* resultFile) { +static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { TAOS_ROW row = NULL; int num_rows = 0; int num_fields = taos_field_count(res); @@ -1178,9 +1183,9 @@ static void fetchResult(TAOS_RES *res, char* resultFile) { // fetch the records row by row while((row = taos_fetch_row(res))) { - if ((resultFile) && (strlen(resultFile) > 0) + if ((strlen(pThreadInfo->filePath) > 0) && (totalLen >= 100*1024*1024 - 32000)) { - appendResultBufToFile(databuf, resultFile); + appendResultBufToFile(databuf, pThreadInfo); totalLen = 0; memset(databuf, 0, 100*1024*1024); } @@ -1193,9 +1198,9 @@ static void fetchResult(TAOS_RES *res, char* resultFile) { } verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", - __func__, __LINE__, databuf, resultFile); - if ((resultFile) && (strlen(resultFile) > 0)) { - appendResultBufToFile(databuf, resultFile); + __func__, __LINE__, databuf, pThreadInfo->filePath); + if (strlen(pThreadInfo->filePath) > 0) { + appendResultBufToFile(databuf, pThreadInfo); } free(databuf); } @@ -1212,14 +1217,14 @@ static void selectAndGetResult( return; } - fetchResult(res, pThreadInfo->fp); + fetchResult(res, pThreadInfo); taos_free_result(res); } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { int retCode = postProceSql( g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port, command, - pThreadInfo->fp); + pThreadInfo); if (0 != retCode) { printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID); } @@ -1756,7 +1761,7 @@ static void printfQueryMeta() { printf("query interval: \033[33m%"PRIu64" ms\033[0m\n", g_queryInfo.specifiedQueryInfo.queryInterval); printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times); - printf("concurrent: \033[33m%"PRIu64"\033[0m\n", + printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.concurrent); printf("mod: \033[33m%s\033[0m\n", (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync"); @@ -2095,7 +2100,7 @@ static void printfQuerySystemInfo(TAOS * taos) { } static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, - char* sqlstr, char *resultFile) + char* sqlstr, threadInfo *pThreadInfo) { char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s"; @@ -2231,8 +2236,8 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port response_buf[RESP_BUF_LEN - 1] = '\0'; printf("Response:\n%s\n", response_buf); - if ((resultFile) && (strlen(resultFile) > 0)) { - appendResultBufToFile(response_buf, resultFile); + if (strlen(pThreadInfo->filePath) > 0) { + appendResultBufToFile(response_buf, pThreadInfo); } free(request_buf); @@ -2994,19 +2999,19 @@ static int startMultiThreadCreateChildTable( int64_t b = 0; b = ntables % threads; - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); - t_info->superTblInfo = superTblInfo; + for (int64_t i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE); + pThreadInfo->superTblInfo = superTblInfo; verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name); - t_info->taos = taos_connect( + pThreadInfo->taos = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); - if (t_info->taos == NULL) { + if (pThreadInfo->taos == NULL) { errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n", __func__, __LINE__, taos_errstr(NULL)); free(pids); @@ -3014,14 +3019,14 @@ static int startMultiThreadCreateChildTable( return -1; } - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; - t_info->use_metric = true; - t_info->cols = cols; - t_info->minDelay = UINT64_MAX; - pthread_create(pids + i, NULL, createTable, t_info); + pThreadInfo->start_table_from = startFrom; + pThreadInfo->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; + startFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->use_metric = true; + pThreadInfo->cols = cols; + pThreadInfo->minDelay = UINT64_MAX; + pthread_create(pids + i, NULL, createTable, pThreadInfo); } for (int i = 0; i < threads; i++) { @@ -3029,8 +3034,8 @@ static int startMultiThreadCreateChildTable( } for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - taos_close(t_info->taos); + threadInfo *pThreadInfo = infos + i; + taos_close(pThreadInfo->taos); } free(pids); @@ -4246,7 +4251,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (concurrent && concurrent->type == cJSON_Number) { if (concurrent->valueint <= 0) { errorPrint( - "%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n", + "%s() LN%d, query sqlCount %"PRIu64" or concurrent %d is not correct.\n", __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount, g_queryInfo.specifiedQueryInfo.concurrent); @@ -4313,24 +4318,28 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } // sqls - cJSON* superSqls = cJSON_GetObjectItem(specifiedQuery, "sqls"); - if (!superSqls) { + cJSON* specifiedSqls = cJSON_GetObjectItem(specifiedQuery, "sqls"); + if (!specifiedSqls) { g_queryInfo.specifiedQueryInfo.sqlCount = 0; - } else if (superSqls->type != cJSON_Array) { + } else if (specifiedSqls->type != cJSON_Array) { errorPrint("%s() LN%d, failed to read json, super sqls not found\n", __func__, __LINE__); goto PARSE_OVER; } else { - int superSqlSize = cJSON_GetArraySize(superSqls); - if (superSqlSize > MAX_QUERY_SQL_COUNT) { - errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n", - __func__, __LINE__, MAX_QUERY_SQL_COUNT); + int superSqlSize = cJSON_GetArraySize(specifiedSqls); + if (superSqlSize * g_queryInfo.specifiedQueryInfo.concurrent + > MAX_QUERY_SQL_COUNT) { + errorPrint("%s() LN%d, failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n", + __func__, __LINE__, + superSqlSize, + g_queryInfo.specifiedQueryInfo.concurrent, + MAX_QUERY_SQL_COUNT); goto PARSE_OVER; } g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize; for (int j = 0; j < superSqlSize; ++j) { - cJSON* sql = cJSON_GetArrayItem(superSqls, j); + cJSON* sql = cJSON_GetArrayItem(specifiedSqls, j); if (sql == NULL) continue; cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); @@ -4506,16 +4515,16 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.superQueryInfo.resubAfterConsume = 1; } - // sqls - cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls"); - if (!subsqls) { + // supert table sqls + cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls"); + if (!superSqls) { g_queryInfo.superQueryInfo.sqlCount = 0; - } else if (subsqls->type != cJSON_Array) { + } else if (superSqls->type != cJSON_Array) { errorPrint("%s() LN%d: failed to read json, super sqls not found\n", __func__, __LINE__); goto PARSE_OVER; } else { - int superSqlSize = cJSON_GetArraySize(subsqls); + int superSqlSize = cJSON_GetArraySize(superSqls); if (superSqlSize > MAX_QUERY_SQL_COUNT) { errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n", __func__, __LINE__, MAX_QUERY_SQL_COUNT); @@ -4524,7 +4533,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.superQueryInfo.sqlCount = superSqlSize; for (int j = 0; j < superSqlSize; ++j) { - cJSON* sql = cJSON_GetArrayItem(subsqls, j); + cJSON* sql = cJSON_GetArrayItem(superSqls, j); if (sql == NULL) continue; cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); @@ -5970,22 +5979,22 @@ static void startMultiThreadInsertData(int threads, char* db_name, memset(infos, 0, threads * sizeof(threadInfo)); for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); - t_info->time_precision = timePrec; - t_info->superTblInfo = superTblInfo; + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE); + pThreadInfo->time_precision = timePrec; + pThreadInfo->superTblInfo = superTblInfo; - t_info->start_time = start_time; - t_info->minDelay = UINT64_MAX; + pThreadInfo->start_time = start_time; + pThreadInfo->minDelay = UINT64_MAX; if ((NULL == superTblInfo) || (superTblInfo->insertMode != REST_IFACE)) { - //t_info->taos = taos; - t_info->taos = taos_connect( + //pThreadInfo->taos = taos; + pThreadInfo->taos = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); - if (NULL == t_info->taos) { + if (NULL == pThreadInfo->taos) { errorPrint( "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n", __func__, __LINE__, @@ -5995,8 +6004,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, } if ((superTblInfo) && (superTblInfo->insertMode == STMT_IFACE)) { - t_info->stmt = taos_stmt_init(t_info->taos); - if (NULL == t_info->stmt) { + pThreadInfo->stmt = taos_stmt_init(pThreadInfo->taos); + if (NULL == pThreadInfo->stmt) { errorPrint( "%s() LN%d, failed init stmt, reason: %s\n", __func__, __LINE__, @@ -6007,27 +6016,27 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } } else { - t_info->taos = NULL; + pThreadInfo->taos = NULL; } /* if ((NULL == superTblInfo) || (0 == superTblInfo->multiThreadWriteOneTbl)) { */ - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; + pThreadInfo->start_table_from = startFrom; + pThreadInfo->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; + startFrom = pThreadInfo->end_table_to + 1; /* } else { - t_info->start_table_from = 0; - t_info->ntables = superTblInfo->childTblCount; - t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint(); + pThreadInfo->start_table_from = 0; + pThreadInfo->ntables = superTblInfo->childTblCount; + pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint(); } */ - tsem_init(&(t_info->lock_sem), 0, 0); + tsem_init(&(pThreadInfo->lock_sem), 0, 0); if (ASYNC_MODE == g_Dbs.asyncMode) { - pthread_create(pids + i, NULL, asyncWrite, t_info); + pthread_create(pids + i, NULL, asyncWrite, pThreadInfo); } else { - pthread_create(pids + i, NULL, syncWrite, t_info); + pthread_create(pids + i, NULL, syncWrite, pThreadInfo); } } @@ -6042,31 +6051,32 @@ static void startMultiThreadInsertData(int threads, char* db_name, double avgDelay = 0; for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; + threadInfo *pThreadInfo = infos + i; - tsem_destroy(&(t_info->lock_sem)); + tsem_destroy(&(pThreadInfo->lock_sem)); - if (t_info->stmt) { - taos_stmt_close(t_info->stmt); + if (pThreadInfo->stmt) { + taos_stmt_close(pThreadInfo->stmt); } - taos_close(t_info->taos); + tsem_destroy(&(pThreadInfo->lock_sem)); + taos_close(pThreadInfo->taos); debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n", __func__, __LINE__, - t_info->threadID, t_info->totalInsertRows, - t_info->totalAffectedRows); + pThreadInfo->threadID, pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows); if (superTblInfo) { - superTblInfo->totalAffectedRows += t_info->totalAffectedRows; - superTblInfo->totalInsertRows += t_info->totalInsertRows; + superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows; + superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows; } else { - g_args.totalAffectedRows += t_info->totalAffectedRows; - g_args.totalInsertRows += t_info->totalInsertRows; + g_args.totalAffectedRows += pThreadInfo->totalAffectedRows; + g_args.totalInsertRows += pThreadInfo->totalInsertRows; } - totalDelay += t_info->totalDelay; - cntDelay += t_info->cntDelay; - if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay; - if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; + totalDelay += pThreadInfo->totalDelay; + cntDelay += pThreadInfo->cntDelay; + if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay; + if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay; } cntDelay -= 1; @@ -6122,26 +6132,26 @@ static void startMultiThreadInsertData(int threads, char* db_name, static void *readTable(void *sarg) { #if 1 - threadInfo *rinfo = (threadInfo *)sarg; - TAOS *taos = rinfo->taos; + threadInfo *pThreadInfo = (threadInfo *)sarg; + TAOS *taos = pThreadInfo->taos; char command[BUFFER_SIZE] = "\0"; - uint64_t sTime = rinfo->start_time; - char *tb_prefix = rinfo->tb_prefix; - FILE *fp = fopen(rinfo->fp, "a"); + uint64_t sTime = pThreadInfo->start_time; + char *tb_prefix = pThreadInfo->tb_prefix; + FILE *fp = fopen(pThreadInfo->filePath, "a"); if (NULL == fp) { - errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + errorPrint( "fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno)); return NULL; } int64_t num_of_DPT; -/* if (rinfo->superTblInfo) { - num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; +/* if (pThreadInfo->superTblInfo) { + num_of_DPT = pThreadInfo->superTblInfo->insertRows; // nrecords_per_table; } else { */ num_of_DPT = g_args.num_of_DPT; // } - int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; @@ -6194,17 +6204,17 @@ static void *readTable(void *sarg) { static void *readMetric(void *sarg) { #if 1 - threadInfo *rinfo = (threadInfo *)sarg; - TAOS *taos = rinfo->taos; + threadInfo *pThreadInfo = (threadInfo *)sarg; + TAOS *taos = pThreadInfo->taos; char command[BUFFER_SIZE] = "\0"; - FILE *fp = fopen(rinfo->fp, "a"); + FILE *fp = fopen(pThreadInfo->filePath, "a"); if (NULL == fp) { - printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + printf("fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno)); return NULL; } - int64_t num_of_DPT = rinfo->superTblInfo->insertRows; - int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t num_of_DPT = pThreadInfo->superTblInfo->insertRows; + int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; @@ -6403,8 +6413,8 @@ static void *specifiedTableQuery(void *sarg) { uint64_t lastPrintTime = taosGetTimestampMs(); uint64_t startTs = taosGetTimestampMs(); - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq] != NULL) { + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); } @@ -6504,8 +6514,8 @@ static void *superTableQuery(void *sarg) { for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { memset(sqlstr,0,sizeof(sqlstr)); replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i); - if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + if (g_queryInfo.superQueryInfo.result[j] != NULL) { + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } @@ -6595,9 +6605,9 @@ static int queryTestProcess() { for (uint64_t i = 0; i < nSqlCount; i++) { for (int j = 0; j < nConcurrent; j++) { uint64_t seq = i * nConcurrent + j; - threadInfo *t_info = infos + seq; - t_info->threadID = seq; - t_info->querySeq = i; + threadInfo *pThreadInfo = infos + seq; + pThreadInfo->threadID = seq; + pThreadInfo->querySeq = i; if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { @@ -6614,10 +6624,10 @@ static int queryTestProcess() { } } - t_info->taos = NULL;// TODO: workaround to use separate taos connection; + pThreadInfo->taos = NULL;// TODO: workaround to use separate taos connection; pthread_create(pids + seq, NULL, specifiedTableQuery, - t_info); + pThreadInfo); } } } else { @@ -6657,15 +6667,15 @@ static int queryTestProcess() { uint64_t startFrom = 0; for (int i = 0; i < threads; i++) { - threadInfo *t_info = infosOfSub + i; - t_info->threadID = i; + threadInfo *pThreadInfo = infosOfSub + i; + pThreadInfo->threadID = i; - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pidsOfSub + i, NULL, superTableQuery, t_info); + pThreadInfo->start_table_from = startFrom; + pThreadInfo->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; + startFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo); } g_queryInfo.superQueryInfo.threadCnt = threads; @@ -6712,7 +6722,7 @@ static void stable_sub_callback( } if (param) - fetchResult(res, ((threadInfo *)param)->fp); + fetchResult(res, (threadInfo *)param); // tao_unscribe() will free result. } @@ -6725,7 +6735,7 @@ static void specified_sub_callback( } if (param) - fetchResult(res, ((threadInfo *)param)->fp); + fetchResult(res, (threadInfo *)param); // tao_unscribe() will free result. } @@ -6779,18 +6789,15 @@ static void *superSubscribe(void *sarg) { } if (pThreadInfo->taos == NULL) { - TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, + pThreadInfo->taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); - if (taos == NULL) { + if (pThreadInfo->taos == NULL) { errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", pThreadInfo->threadID, taos_errstr(NULL)); return NULL; - } else { - pThreadInfo->taos = taos; } } @@ -6820,7 +6827,7 @@ static void *superSubscribe(void *sarg) { g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq], subSqlstr, i); if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); } @@ -6863,16 +6870,16 @@ static void *superSubscribe(void *sarg) { if (res) { if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - fetchResult(res, pThreadInfo->fp); + fetchResult(res, pThreadInfo); } if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - fetchResult(res, pThreadInfo->fp); + fetchResult(res, pThreadInfo); } consumed[tsubSeq] ++; @@ -6913,21 +6920,18 @@ static void *superSubscribe(void *sarg) { static void *specifiedSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; - TAOS_SUB* tsub = NULL; +// TAOS_SUB* tsub = NULL; if (pThreadInfo->taos == NULL) { - TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, + pThreadInfo->taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); - if (taos == NULL) { + if (pThreadInfo->taos == NULL) { errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", pThreadInfo->threadID, taos_errstr(NULL)); return NULL; - } else { - pThreadInfo->taos = taos; } } @@ -6939,69 +6943,70 @@ static void *specifiedSubscribe(void *sarg) { return NULL; } - char topic[32] = {0}; - sprintf(topic, "taosdemo-subscribe-%"PRIu64"", pThreadInfo->querySeq); - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], + "taosdemo-subscribe-%"PRIu64"-%d", + pThreadInfo->querySeq, + pThreadInfo->threadID); + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq] != NULL) { + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); } - tsub = subscribeImpl( - SPECIFIED_CLASS, pThreadInfo, + g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl( + SPECIFIED_CLASS, pThreadInfo, g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], - topic, + g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], g_queryInfo.specifiedQueryInfo.subscribeRestart, g_queryInfo.specifiedQueryInfo.subscribeInterval); - if (NULL == tsub) { + if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) { taos_close(pThreadInfo->taos); return NULL; } // start loop to consume result - TAOS_RES* res = NULL; - - int consumed; + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0; while(1) { if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { continue; } - res = taos_consume(tsub); - if (res) { + g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = taos_consume( + g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]); + if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) { if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - fetchResult(res, pThreadInfo->fp); + fetchResult(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], pThreadInfo); } - consumed ++; + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++; if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) - && (consumed >= + && (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] >= g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { printf("keepProgress:%d, resub specified query: %"PRIu64"\n", g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, pThreadInfo->querySeq); - consumed = 0; - taos_unsubscribe(tsub, + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0; + taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID], g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); - tsub = subscribeImpl( + g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl( SPECIFIED_CLASS, pThreadInfo, g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], - topic, + g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], g_queryInfo.specifiedQueryInfo.subscribeRestart, g_queryInfo.specifiedQueryInfo.subscribeInterval); - if (NULL == tsub) { + if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) { taos_close(pThreadInfo->taos); return NULL; } } } } - taos_free_result(res); - taos_unsubscribe(tsub, 0); + taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]); + taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->querySeq], 0); taos_close(pThreadInfo->taos); return NULL; @@ -7071,11 +7076,11 @@ static int subscribeTestProcess() { for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; - threadInfo *t_info = infos + seq; - t_info->threadID = seq; - t_info->querySeq = i; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pids + seq, NULL, specifiedSubscribe, t_info); + threadInfo *pThreadInfo = infos + seq; + pThreadInfo->threadID = seq; + pThreadInfo->querySeq = i; + pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pids + seq, NULL, specifiedSubscribe, pThreadInfo); } } } @@ -7121,17 +7126,17 @@ static int subscribeTestProcess() { uint64_t startFrom = 0; for (int j = 0; j < threads; j++) { uint64_t seq = i * threads + j; - threadInfo *t_info = infosOfStable + seq; - t_info->threadID = seq; - t_info->querySeq = i; + threadInfo *pThreadInfo = infosOfStable + seq; + pThreadInfo->threadID = seq; + pThreadInfo->querySeq = i; - t_info->start_table_from = startFrom; - t_info->ntables = jend_table_to = jend_table_to + 1; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; + pThreadInfo->start_table_from = startFrom; + pThreadInfo->ntables = jend_table_to = jend_table_to + 1; + pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; pthread_create(pidsOfStable + seq, - NULL, superSubscribe, t_info); + NULL, superSubscribe, pThreadInfo); } } @@ -7410,47 +7415,47 @@ static void queryResult() { // query data pthread_t read_id; - threadInfo *rInfo = malloc(sizeof(threadInfo)); - assert(rInfo); - rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 - rInfo->start_table_from = 0; + threadInfo *pThreadInfo = malloc(sizeof(threadInfo)); + assert(pThreadInfo); + pThreadInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 + pThreadInfo->start_table_from = 0; - //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; + //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc; if (g_args.use_metric) { - rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount; - rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; - rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; - tstrncpy(rInfo->tb_prefix, + pThreadInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount; + pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; + pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; + tstrncpy(pThreadInfo->tb_prefix, g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE); } else { - rInfo->ntables = g_args.num_of_tables; - rInfo->end_table_to = g_args.num_of_tables -1; - tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); + pThreadInfo->ntables = g_args.num_of_tables; + pThreadInfo->end_table_to = g_args.num_of_tables -1; + tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); } - rInfo->taos = taos_connect( + pThreadInfo->taos = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, g_Dbs.db[0].dbName, g_Dbs.port); - if (rInfo->taos == NULL) { + if (pThreadInfo->taos == NULL) { errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); - free(rInfo); + free(pThreadInfo); exit(-1); } - tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN); + tstrncpy(pThreadInfo->filePath, g_Dbs.resultFile, MAX_FILE_NAME_LEN); if (!g_Dbs.use_metric) { - pthread_create(&read_id, NULL, readTable, rInfo); + pthread_create(&read_id, NULL, readTable, pThreadInfo); } else { - pthread_create(&read_id, NULL, readMetric, rInfo); + pthread_create(&read_id, NULL, readMetric, pThreadInfo); } pthread_join(read_id, NULL); - taos_close(rInfo->taos); - free(rInfo); + taos_close(pThreadInfo->taos); + free(pThreadInfo); } static void testCmdLine() {