diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 3aa1ffc07e..e4d440d76e 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -87,7 +87,7 @@ IF ("${CPUTYPE}" STREQUAL "") SET(TD_ARM_32 TRUE) ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_32") - ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)") MESSAGE(STATUS "The current platform is aarch64") SET(PLATFORM_ARCH_STR "arm64") SET(TD_ARM_64 TRUE) diff --git a/cmake/cmake.version b/cmake/cmake.version index 0873e59e92..7c895b12ff 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.0.2") + SET(TD_VER_NUMBER "3.0.1.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index f182beed33..eb0faf6d5d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG abed566 + GIT_TAG 71e7ccf SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index 15d3764876..04b1262caf 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 0609b50 + GIT_TAG e771403 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index e2be419517..35e7ad1933 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -62,7 +62,7 @@ taos> ## 体验查询 -使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。。 +使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。 查询超级表下记录总条数： diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index ef1cd4797a..3239eae49b 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -116,7 +116,7 @@ aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。 参数的具体含义是： - inputDataBlock: 输入的数据块 - - resultColumn: 输出列。输出列 + - resultColumn: 输出列 ### 聚合接口函数 diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 2337f2a272..f6790e3c69 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -23,10 +23,7 @@ create_subtable_clause: { } create_definition: - col_name column_definition - -column_definition: - type_name [comment 'string_value'] + col_name column_type table_options: table_option ... diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index e3e1463131..a6ec560d3c 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -6,11 +6,7 @@ description: TDengine 发布历史、Release Notes 及下载链接 import Release from "/components/ReleaseV3"; -## 3.0.0.1 +## 3.0.1.0 - - - + diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 61129d74e5..9e8757cc4e 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -6,6 +6,6 @@ description: taosTools 的发布历史、Release Notes 和下载链接 import Release from "/components/ReleaseV3"; -## 2.1.2 +## 2.1.3 - \ No newline at end of file + diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 006ba7f21b..3f917ff0d1 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -272,6 +272,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_SYNC_LEADER_TRANSFER, "sync-leader-transfer", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_SET_MNODE_STANDBY, "set-mnode-standby", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_SET_VNODE_STANDBY, "set-vnode-standby", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT, "sync-heartbeat", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT_REPLY, "sync-heartbeat-reply", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL) #if defined(TD_MSG_NUMBER_) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 9c2fdf587b..6500d3d183 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -320,6 +320,9 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode); int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen); int32_t nodesStringToList(const char* pStr, SNodeList** pList); +int32_t nodesNodeToMsg(const SNode* pNode, char** pMsg, int32_t* pLen); +int32_t nodesMsgToNode(const char* pStr, int32_t len, SNode** pNode); + int32_t nodesNodeToSQL(SNode* pNode, char* buf, int32_t bufSize, int32_t* len); char* nodesGetNameFromColumnNode(SNode* pNode); int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 2c27509008..afd8de6b1c 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -554,6 +554,8 @@ typedef struct { int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateDel(SStreamState* pState, const SWinKey* key); +int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal); void streamFreeVal(void* val); SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key); diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 6c95c3c6d7..de2271554d 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -444,6 +444,70 @@ void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg); void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg); void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg); +// --------------------------------------------- +typedef struct SyncHeartbeat { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + + // private data + SyncTerm term; + SyncIndex commitIndex; + SyncTerm privateTerm; +} SyncHeartbeat; + +SyncHeartbeat* syncHeartbeatBuild(int32_t vgId); +void syncHeartbeatDestroy(SyncHeartbeat* pMsg); +void syncHeartbeatSerialize(const SyncHeartbeat* pMsg, char* buf, uint32_t bufLen); +void syncHeartbeatDeserialize(const char* buf, uint32_t len, SyncHeartbeat* pMsg); +char* syncHeartbeatSerialize2(const SyncHeartbeat* pMsg, uint32_t* len); +SyncHeartbeat* syncHeartbeatDeserialize2(const char* buf, uint32_t len); +void syncHeartbeat2RpcMsg(const SyncHeartbeat* pMsg, SRpcMsg* pRpcMsg); +void syncHeartbeatFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeat* pMsg); +SyncHeartbeat* syncHeartbeatFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncHeartbeat2Json(const SyncHeartbeat* pMsg); +char* syncHeartbeat2Str(const SyncHeartbeat* pMsg); + +// for debug ---------------------- +void syncHeartbeatPrint(const SyncHeartbeat* pMsg); +void syncHeartbeatPrint2(char* s, const SyncHeartbeat* pMsg); +void syncHeartbeatLog(const SyncHeartbeat* pMsg); +void syncHeartbeatLog2(char* s, const SyncHeartbeat* pMsg); + +// --------------------------------------------- +typedef struct SyncHeartbeatReply { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + + // private data + SyncTerm term; + SyncTerm privateTerm; + int64_t startTime; +} SyncHeartbeatReply; + +SyncHeartbeatReply* syncHeartbeatReplyBuild(int32_t vgId); +void syncHeartbeatReplyDestroy(SyncHeartbeatReply* pMsg); +void syncHeartbeatReplySerialize(const SyncHeartbeatReply* pMsg, char* buf, uint32_t bufLen); +void syncHeartbeatReplyDeserialize(const char* buf, uint32_t len, SyncHeartbeatReply* pMsg); +char* syncHeartbeatReplySerialize2(const SyncHeartbeatReply* pMsg, uint32_t* len); +SyncHeartbeatReply* syncHeartbeatReplyDeserialize2(const char* buf, uint32_t len); +void syncHeartbeatReply2RpcMsg(const SyncHeartbeatReply* pMsg, SRpcMsg* pRpcMsg); +void syncHeartbeatReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeatReply* pMsg); +SyncHeartbeatReply* syncHeartbeatReplyFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncHeartbeatReply2Json(const SyncHeartbeatReply* pMsg); +char* syncHeartbeatReply2Str(const SyncHeartbeatReply* pMsg); + +// for debug ---------------------- +void syncHeartbeatReplyPrint(const SyncHeartbeatReply* pMsg); +void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg); +void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg); +void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg); + // --------------------------------------------- typedef struct SyncApplyMsg { uint32_t bytes; diff --git a/include/util/tdef.h b/include/util/tdef.h index 57fde32bed..21f879cd93 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -359,8 +359,8 @@ typedef enum ELogicConditionType { #define TSDB_DB_SCHEMALESS_ON 1 #define TSDB_DB_SCHEMALESS_OFF 0 #define TSDB_DEFAULT_DB_SCHEMALESS TSDB_DB_SCHEMALESS_OFF -#define TSDB_MIN_SST_TRIGGER 1 -#define TSDB_MAX_SST_TRIGGER 128 +#define TSDB_MIN_STT_TRIGGER 1 +#define TSDB_MAX_STT_TRIGGER 16 #define TSDB_DEFAULT_SST_TRIGGER 8 #define TSDB_MIN_HASH_PREFIX 0 #define TSDB_MAX_HASH_PREFIX 128 diff --git a/include/util/tutil.h b/include/util/tutil.h index a0c7b3d7ad..32a88b37ec 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -69,7 +69,18 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar memcpy(target, buf, TSDB_PASSWORD_LEN); } -#define taosGetTbHashVal(tbname, tblen, method, prefix, suffix) MurmurHash3_32((tbname), (tblen)) +static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix, + int32_t suffix) { + if (prefix == 0 && suffix == 0) { + return MurmurHash3_32(tbname, tblen); + } else { + if (tblen <= (prefix + suffix)) { + return MurmurHash3_32(tbname, tblen); + } else { + return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix); + } + } +} #ifdef __cplusplus } diff --git a/packaging/MPtestJenkinsfile b/packaging/MPtestJenkinsfile index 77f642180a..5b79345916 100644 --- a/packaging/MPtestJenkinsfile +++ b/packaging/MPtestJenkinsfile @@ -5,13 +5,6 @@ def sync_source(branch_name) { echo ''' + branch_name + ''' ''' sh ''' - cd ${TDINTERNAL_ROOT_DIR} - git reset --hard - git fetch || git fetch - git checkout ''' + branch_name + ''' -f - git branch - git pull || git pull - git log | head -n 20 cd ${TDENGINE_ROOT_DIR} git reset --hard git fetch || git fetch @@ -64,17 +57,12 @@ pipeline { defaultValue:'2.1.2', description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1' ) - string ( - name:'nasPassword', - defaultValue:'password', - description: 'the pasword of the NAS server which has installPackage-192.168.1.131' - ) } environment{ WORK_DIR = '/var/lib/jenkins/workspace' TDINTERNAL_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal' TDENGINE_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal/community' - BRANCH_NAME = '3.0' + BRANCH_NAME = 'test/chr/TD-14699' TD_SERVER_TAR = "TDengine-server-${version}-Linux-x64.tar.gz" BASE_TD_SERVER_TAR = "TDengine-server-${baseVersion}-Linux-x64.tar.gz" @@ -107,7 +95,7 @@ pipeline { } stages { - stage ('RUN') { + stage ('Test Server') { parallel { stage('ubuntu16') { agent{label " ubuntu16 "} @@ -116,17 +104,17 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' } @@ -139,24 +127,21 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} - python3 checkPackageRuning.py - ''' - sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client ${nasPassword} + bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py + dpkg -r tdengine ''' + } } } @@ -167,17 +152,17 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' } @@ -190,28 +175,23 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py - ''' - sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client ${nasPassword} - python3 checkPackageRuning.py - ''' + sudo rpm -e tdengine + ''' } } - } - + } stage('arm64') { agent{label 'linux_arm64'} steps { @@ -219,18 +199,53 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' - sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client ${nasPassword} - python3 checkPackageRuning.py - ''' } } } } } - } + stage ('Test Client') { + parallel { + stage('ubuntu18') { + agent{label " ubuntu18 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client + python3 checkPackageRuning.py 192.168.0.21 + ''' + } + } + } + stage('centos8') { + agent{label " centos8_3 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client + python3 checkPackageRuning.py 192.168.0.24 + ''' + } + } + } + } + } + stage('arm64-client') { + agent{label " linux_arm64 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client + python3 checkPackageRuning.py 192.168.0.21 + ''' + } + } + } + } } \ No newline at end of file diff --git a/packaging/checkPackageRuning.py b/packaging/checkPackageRuning.py index c0d1e8b86c..2edeeb6dbb 100755 --- a/packaging/checkPackageRuning.py +++ b/packaging/checkPackageRuning.py @@ -19,12 +19,19 @@ import subprocess # from this import d import time + +if( len(sys.argv)>1 ): + serverHost=sys.argv[1] +else: + serverHost="localhost" + + # install taospy out = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ") print("taospy version %s "%out) if (out == "" ): - os.system("pip install git+https://github.com/taosdata/taos-connector-python.git") + os.system("pip3 install git+https://github.com/taosdata/taos-connector-python.git") print("install taos python connector") else: os.system("pip3 install --upgrade taospy ") @@ -32,19 +39,19 @@ else: # start taosd prepare -os.system("rm -rf /var/lib/taos/*") -os.system("systemctl restart taosd ") +# os.system("rm -rf /var/lib/taos/*") +# os.system("systemctl restart taosd ") # wait a moment ,at least 5 seconds time.sleep(5) # prepare data by taosBenchmark -os.system("taosBenchmark -y -n 100 -t 100") +os.system("taosBenchmark -y -n 100 -t 100 -h %s "%serverHost ) import taos -conn = taos.connect(host="localhost", +conn = taos.connect(host="%s"%serverHost, user="root", password="taosdata", database="test", @@ -80,15 +87,15 @@ os.system("rm -rf /tmp/dumpdata/*") # dump data out print("taosdump dump out data") -os.system("taosdump -o /tmp/dumpdata -D test -y ") +os.system("taosdump -o /tmp/dumpdata -D test -y -h %s "%serverHost) # drop database of test print("drop database test") -os.system(" taos -s ' drop database test ;' ") +os.system(" taos -s ' drop database test ;' -h %s "%serverHost) # dump data in print("taosdump dump data in") -os.system("taosdump -i /tmp/dumpdata -y ") +os.system("taosdump -i /tmp/dumpdata -y -h %s "%serverHost) result = conn.query("SELECT count(*) from test.meters") diff --git a/packaging/debRpmAutoInstall.sh b/packaging/debRpmAutoInstall.sh index 1f51378c91..3579f813e5 100755 --- a/packaging/debRpmAutoInstall.sh +++ b/packaging/debRpmAutoInstall.sh @@ -11,3 +11,5 @@ expect "*one:" send "\r" expect "*skip:" send "\r" + +expect eof \ No newline at end of file diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 56da9e59be..794b3968fe 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -7,7 +7,6 @@ originPackageName=$3 originversion=$4 testFile=$5 subFile="taos.tar.gz" -password=$6 # Color setting RED='\033[41;30m' @@ -68,11 +67,37 @@ fi } +function wgetFile { + +file=$1 + +if [ ! -f ${file} ];then + echoColor BD "wget https://www.taosdata.com/assets-download/3.0/${file}" + wget https://www.taosdata.com/assets-download/3.0/${file} +else + echoColor YD "${file} already exists " +fi +} + +function newPath { + +buildPath=$1 + +if [ ! -d ${buildPath} ] ;then + echoColor BD "mkdir -p ${buildPath}" + mkdir -p ${buildPath} +else + echoColor YD "${buildPath} already exists" +fi + +} + + echoColor G "===== install basesoft =====" cmdInstall tree cmdInstall wget -cmdInstall sshpass +cmdInstall expect echoColor G "===== Uninstall all components of TDeingne =====" @@ -97,11 +122,14 @@ echoColor G "===== new workroom path =====" installPath="/usr/local/src/packageTest" oriInstallPath="/usr/local/src/packageTest/3.1" -if [ ! -d ${installPath} ] ;then - echoColor BD "mkdir -p ${installPath}" - mkdir -p ${installPath} -else - echoColor YD "${installPath} already exists" +newPath ${installPath} + +newPath ${oriInstallPath} + + +if [ -d ${oriInstallPath}/${originTdpPath} ] ;then + echoColor BD "rm -rf ${oriInstallPath}/${originTdpPath}/*" + rm -rf ${oriInstallPath}/${originTdpPath}/* fi if [ -d ${installPath}/${tdPath} ] ;then @@ -109,33 +137,13 @@ if [ -d ${installPath}/${tdPath} ] ;then rm -rf ${installPath}/${tdPath}/* fi -if [ ! -d ${oriInstallPath} ] ;then - echoColor BD "mkdir -p ${oriInstallPath}" - mkdir -p ${oriInstallPath} -else - echoColor YD "${oriInstallPath} already exists" -fi - -if [ -d ${oriInstallPath}/${originTdpPath} ] ;then - echoColor BD "rm -rf ${oriInstallPath}/${originTdpPath}/*" - rm -rf ${oriInstallPath}/${originTdpPath}/* -fi - - echoColor G "===== download installPackage =====" -# cd ${installPath} -# wget https://www.taosdata.com/assets-download/3.0/${packgeName} -# cd ${oriInstallPath} -# wget https://www.taosdata.com/assets-download/3.0/${originPackageName} +cd ${installPath} && wgetFile ${packgeName} +cd ${oriInstallPath} && wgetFile ${originPackageName} cd ${installPath} cp -r ${scriptDir}/debRpmAutoInstall.sh . -if [ ! -f {packgeName} ];then - echoColor BD "sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/${packgeName} ." - sshpass -p ${password} scp -oStrictHostKeyChecking=no -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/${packgeName} . -fi - packageSuffix=$(echo ${packgeName} | awk -F '.' '{print $NF}') @@ -181,8 +189,7 @@ elif [[ ${packgeName} =~ "tar" ]];then cd ${oriInstallPath} if [ ! -f {originPackageName} ];then echoColor YD "download base installPackage" - echoColor BD "sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${originversion}/community/${originPackageName} ." - sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${originversion}/community/${originPackageName} . + wgetFile ${originPackageName} fi echoColor YD "unzip the base installation package" echoColor BD "tar -xf ${originPackageName}" && tar -xf ${originPackageName} @@ -222,24 +229,45 @@ fi cd ${installPath} -if ([[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "tar" ]]) || [[ ${packgeName} =~ "client" ]] ;then +if [[ ${packgeName} =~ "Lite" ]] || ([[ ${packgeName} =~ "x64" ]] && [[ ${packgeName} =~ "client" ]]) || ([[ ${packgeName} =~ "deb" ]] && [[ ${packgeName} =~ "server" ]]) || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then echoColor G "===== install taos-tools when package is lite or client =====" cd ${installPath} - sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz . - # wget https://www.taosdata.com/assets-download/3.0/taosTools-2.1.2-Linux-x64.tar.gz - tar xf taosTools-2.1.2-Linux-x64.tar.gz - cd taosTools-2.1.2 && bash install-taostools.sh -elif [[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "deb" ]] ;then - echoColor G "===== install taos-tools when package is lite or client =====" + wgetFile taosTools-2.1.3-Linux-x64.tar.gz . + tar xf taosTools-2.1.3-Linux-x64.tar.gz + cd taosTools-2.1.3 && bash install-taostools.sh +elif ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then + echoColor G "===== install taos-tools arm when package is arm64-client =====" cd ${installPath} - sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz . - tar xf taosTools-2.1.2-Linux-x64.tar.gz - cd taosTools-2.1.2 && bash install-taostools.sh -elif [[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "rpm" ]] ;then - echoColor G "===== install taos-tools when package is lite or client =====" - cd ${installPath} - sshpass -p ${password} scp -oStrictHostKeyChecking=no -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz . - tar xf taosTools-2.1.2-Linux-x64.tar.gz - cd taosTools-2.1.2 && bash install-taostools.sh + wgetFile taosTools-2.1.3-Linux-arm64.tar.gz . + tar xf taosTools-2.1.3-Linux-arm64.tar.gz + cd taosTools-2.1.3 && bash install-taostools.sh fi +echoColor G "===== start TDengine =====" + +if [[ ${packgeName} =~ "server" ]] ;then + echoColor BD " rm -rf /var/lib/taos/* && systemctl restart taosd " + rm -rf /var/lib/taos/* + systemctl restart taosd +fi + +# if ([[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "tar" ]]) || [[ ${packgeName} =~ "client" ]] ;then +# echoColor G "===== install taos-tools when package is lite or client =====" +# cd ${installPath} +# wgetFile taosTools-2.1.2-Linux-x64.tar.gz . +# tar xf taosTools-2.1.2-Linux-x64.tar.gz +# cd taosTools-2.1.2 && bash install-taostools.sh +# elif [[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "deb" ]] ;then +# echoColor G "===== install taos-tools when package is lite or client =====" +# cd ${installPath} +# wgetFile taosTools-2.1.2-Linux-x64.tar.gz . +# tar xf taosTools-2.1.2-Linux-x64.tar.gz +# cd taosTools-2.1.2 && bash install-taostools.sh +# elif [[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "rpm" ]] ;then +# echoColor G "===== install taos-tools when package is lite or client =====" +# cd ${installPath} +# wgetFile taosTools-2.1.2-Linux-x64.tar.gz . +# tar xf taosTools-2.1.2-Linux-x64.tar.gz +# cd taosTools-2.1.2 && bash install-taostools.sh +# fi + diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 61fc530f57..f77320c5ac 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -103,6 +103,8 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "wal_roll_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "wal_segment_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true}, {.name = "sst_trigger", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, + {.name = "table_prefix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, + {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, }; static const SSysDbTableSchema userFuncSchema[] = { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 830bf9523a..981576efec 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -167,7 +167,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->walCfg.segSize = pCreate->walSegmentSize; pCfg->walCfg.level = pCreate->walLevel; - pCfg->sstTrigger = pCreate->sstTrigger; + pCfg->sttTrigger = pCreate->sstTrigger; pCfg->hashBegin = pCreate->hashBegin; pCfg->hashEnd = pCreate->hashEnd; pCfg->hashMethod = pCreate->hashMethod; @@ -223,10 +223,10 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } dInfo("vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d", - createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize, - createReq.sstTrigger); + createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize, + createReq.sstTrigger); dInfo("vgId:%d, hashMethod:%d begin:%u end:%u prefix:%d surfix:%d", createReq.vgId, createReq.hashMethod, - createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix); + createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix); vmGenerateVnodeCfg(&createReq, &vnodeCfg); if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c598b188b7..9e58b74017 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDb.h" +#include "mndCluster.h" #include "mndDnode.h" #include "mndOffset.h" #include "mndPrivilege.h" @@ -337,7 +338,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->walRetentionSize < TSDB_DB_MIN_WAL_RETENTION_SIZE) return -1; if (pCfg->walRollPeriod < TSDB_DB_MIN_WAL_ROLL_PERIOD) return -1; if (pCfg->walSegmentSize < TSDB_DB_MIN_WAL_SEGMENT_SIZE) return -1; - if (pCfg->sstTrigger < TSDB_MIN_SST_TRIGGER || pCfg->sstTrigger > TSDB_MAX_SST_TRIGGER) return -1; + if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1; if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1; if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1; @@ -512,6 +513,12 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, return -1; } + if (dbObj.cfg.hashPrefix > 0) { + int32_t dbLen = strlen(dbObj.name) + 1; + mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix + dbLen); + dbObj.cfg.hashPrefix += dbLen; + } + SVgObj *pVgroups = NULL; if (mndAllocVgroup(pMnode, &dbObj, &pVgroups) != 0) { mError("db:%s, failed to create since %s", pCreate->db, terrstr()); @@ -1709,23 +1716,33 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.sstTrigger, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + int16_t hashPrefix = pDb->cfg.hashPrefix; + if (hashPrefix > 0) { + hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1; + } + colDataAppend(pColInfo, rows, (const char *)&hashPrefix, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.hashSuffix, false); } taosMemoryFree(buf); } -static void setInformationSchemaDbCfg(SDbObj *pDbObj) { +static void setInformationSchemaDbCfg(SMnode *pMnode, SDbObj *pDbObj) { tstrncpy(pDbObj->name, TSDB_INFORMATION_SCHEMA_DB, tListLen(pDbObj->name)); - pDbObj->createdTime = 0; + pDbObj->createdTime = mndGetClusterCreateTime(pMnode); pDbObj->cfg.numOfVgroups = 0; pDbObj->cfg.strict = 1; pDbObj->cfg.replications = 1; pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI; } -static void setPerfSchemaDbCfg(SDbObj *pDbObj) { +static void setPerfSchemaDbCfg(SMnode *pMnode, SDbObj *pDbObj) { tstrncpy(pDbObj->name, TSDB_PERFORMANCE_SCHEMA_DB, tListLen(pDbObj->name)); - pDbObj->createdTime = 0; + pDbObj->createdTime = mndGetClusterCreateTime(pMnode); pDbObj->cfg.numOfVgroups = 0; pDbObj->cfg.strict = 1; pDbObj->cfg.replications = 1; @@ -1756,7 +1773,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc // Append the information_schema database into the result. if (!pShow->sysDbRsp) { SDbObj infoschemaDb = {0}; - setInformationSchemaDbCfg(&infoschemaDb); + setInformationSchemaDbCfg(pMnode, &infoschemaDb); size_t numOfTables = 0; getVisibleInfosTablesNum(sysinfo, &numOfTables); mndDumpDbInfoData(pMnode, pBlock, &infoschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); @@ -1764,7 +1781,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc numOfRows += 1; SDbObj perfschemaDb = {0}; - setPerfSchemaDbCfg(&perfschemaDb); + setPerfSchemaDbCfg(pMnode, &perfschemaDb); numOfTables = 0; getPerfDbMeta(NULL, &numOfTables); mndDumpDbInfoData(pMnode, pBlock, &perfschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 8e545eb527..6ba10641f5 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -125,6 +125,9 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur); // typedef struct STsdb STsdb; typedef struct STsdbReader STsdbReader; +#define TSDB_DEFAULT_STT_FILE 8 +#define TSDB_DEFAULT_PAGE_SIZE 4096 + #define TIMEWINDOW_RANGE_CONTAINED 1 #define TIMEWINDOW_RANGE_EXTERNAL 2 @@ -288,9 +291,10 @@ struct SVnodeCfg { SVnodeStats vndStats; uint32_t hashBegin; uint32_t hashEnd; - int16_t sstTrigger; + int16_t sttTrigger; int16_t hashPrefix; int16_t hashSuffix; + int32_t tsdbPageSize; }; typedef struct { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 8bca22fea1..a836fa2bc5 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -67,12 +67,9 @@ typedef struct SBlockCol SBlockCol; typedef struct SVersionRange SVersionRange; typedef struct SLDataIter SLDataIter; -#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) -#define TSDB_MAX_SUBBLOCKS 8 -#define TSDB_MAX_STT_FILE 16 -#define TSDB_DEFAULT_STT_FILE 8 -#define TSDB_FHDR_SIZE 512 -#define TSDB_DEFAULT_PAGE_SIZE 4096 +#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) +#define TSDB_MAX_SUBBLOCKS 8 +#define TSDB_FHDR_SIZE 512 #define HAS_NONE ((int8_t)0x1) #define HAS_NULL ((int8_t)0x2) @@ -84,13 +81,26 @@ typedef struct SLDataIter SLDataIter; #define TSDBKEY_MIN ((TSDBKEY){.ts = TSKEY_MIN, .version = VERSION_MIN}) #define TSDBKEY_MAX ((TSDBKEY){.ts = TSKEY_MAX, .version = VERSION_MAX}) +#define TABLE_SAME_SCHEMA(SUID1, UID1, SUID2, UID2) ((SUID1) ? (SUID1) == (SUID2) : (UID1) == (UID2)) + #define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM)) #define LOGIC_TO_FILE_OFFSET(LOFFSET, PAGE) \ ((LOFFSET) / PAGE_CONTENT_SIZE(PAGE) * (PAGE) + (LOFFSET) % PAGE_CONTENT_SIZE(PAGE)) #define FILE_TO_LOGIC_OFFSET(OFFSET, PAGE) ((OFFSET) / (PAGE)*PAGE_CONTENT_SIZE(PAGE) + (OFFSET) % (PAGE)) #define PAGE_OFFSET(PGNO, PAGE) (((PGNO)-1) * (PAGE)) #define OFFSET_PGNO(OFFSET, PAGE) ((OFFSET) / (PAGE) + 1) -#define LOGIC_TO_FILE_SIZE(LSIZE, PAGE) OFFSET_PGNO(LOGIC_TO_FILE_OFFSET(LSIZE, PAGE), PAGE) * (PAGE) + +static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { + int64_t fOffSet = LOGIC_TO_FILE_OFFSET(lSize, szPage); + int64_t pgno = OFFSET_PGNO(fOffSet, szPage); + int32_t szPageCont = PAGE_CONTENT_SIZE(szPage); + + if (fOffSet % szPageCont == 0) { + pgno--; + } + + return pgno * szPage; +} // tsdbUtil.c ============================================================================================== // TSDBROW @@ -262,7 +272,7 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync); int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); -int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx); +int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx); int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast); @@ -272,7 +282,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); -int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); +int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk); int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk); int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData); @@ -572,7 +582,7 @@ struct SDFileSet { SDataFile *pDataF; SSmaFile *pSmaF; uint8_t nSttF; - SSttFile *aSttF[TSDB_MAX_STT_FILE]; + SSttFile *aSttF[TSDB_MAX_STT_TRIGGER]; }; struct SRowIter { @@ -622,7 +632,7 @@ struct SDataFWriter { SHeadFile fHead; SDataFile fData; SSmaFile fSma; - SSttFile fStt[TSDB_MAX_STT_FILE]; + SSttFile fStt[TSDB_MAX_STT_TRIGGER]; uint8_t *aBuf[4]; }; @@ -633,7 +643,7 @@ struct SDataFReader { STsdbFD *pHeadFD; STsdbFD *pDataFD; STsdbFD *pSmaFD; - STsdbFD *aSttFD[TSDB_MAX_STT_FILE]; + STsdbFD *aSttFD[TSDB_MAX_STT_TRIGGER]; uint8_t *aBuf[3]; }; @@ -644,31 +654,41 @@ typedef struct { } SRowInfo; typedef struct SSttBlockLoadInfo { - SBlockData blockData[2]; - SArray *aSttBlk; - int32_t blockIndex[2]; // to denote the loaded block in the corresponding position. - int32_t currentLoadBlockIndex; + SBlockData blockData[2]; + SArray *aSttBlk; + int32_t blockIndex[2]; // to denote the loaded block in the corresponding position. + int32_t currentLoadBlockIndex; + int32_t loadBlocks; + double elapsedTime; } SSttBlockLoadInfo; typedef struct SMergeTree { - int8_t backward; - SRBTree rbt; - SArray *pIterList; - SLDataIter *pIter; - bool destroyLoadInfo; - SSttBlockLoadInfo* pLoadInfo; + int8_t backward; + SRBTree rbt; + SArray *pIterList; + SLDataIter *pIter; + bool destroyLoadInfo; + SSttBlockLoadInfo *pLoadInfo; + const char *idStr; } SMergeTree; +typedef struct { + int64_t suid; + int64_t uid; + STSchema *pTSchema; +} SSkmInfo; + int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, - STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pLoadInfo); + STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pLoadInfo, const char *idStr); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); -SSttBlockLoadInfo* tCreateLastBlockLoadInfo(); -void resetLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo); -void* destroyLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo); +SSttBlockLoadInfo *tCreateLastBlockLoadInfo(); +void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); +void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el); +void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 62aba649a3..0e85e7bfb6 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -104,7 +104,7 @@ int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp); -int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t *tbUid); +int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid); int metaTtlDropTable(SMeta* pMeta, int64_t ttl, SArray* tbUids); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 8af764c4bc..8da783a5bd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -457,7 +457,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid, &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, - &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, NULL); + &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, NULL, NULL); bool hasVal = tMergeTreeNext(&state->mergeTree); if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; @@ -589,7 +589,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { } tMapDataReset(&state->blockMap); - code = tsdbReadBlock(state->pDataFReader, state->pBlockIdx, &state->blockMap); + code = tsdbReadDataBlk(state->pDataFReader, state->pBlockIdx, &state->blockMap); if (code) goto _err; state->nBlock = state->blockMap.nItem; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index fb06203605..a619b9f2e4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -14,13 +14,8 @@ */ #include "tsdb.h" -typedef struct { - int64_t suid; - int64_t uid; - STSchema *pTSchema; -} SSkmInfo; -typedef enum { MEMORY_DATA_ITER = 0, LAST_DATA_ITER } EDataIterT; +typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT; typedef struct { SRBTreeNode n; @@ -50,7 +45,7 @@ typedef struct { int32_t minRow; int32_t maxRow; int8_t cmprAlg; - int8_t maxLast; + int8_t sttTrigger; SArray *aTbDataP; // memory STsdbFS fs; // disk // -------------- @@ -71,7 +66,7 @@ typedef struct { SDataIter *pIter; SRBTree rbt; SDataIter dataIter; - SDataIter aDataIter[TSDB_MAX_STT_FILE]; + SDataIter aDataIter[TSDB_MAX_STT_TRIGGER]; int8_t toLastOnly; }; struct { @@ -99,7 +94,7 @@ static int32_t tsdbCommitCache(SCommitter *pCommitter); static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno); static int32_t tsdbNextCommitRow(SCommitter *pCommitter); -static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { +int32_t tRowInfoCmprFn(const void *p1, const void *p2) { SRowInfo *pInfo1 = (SRowInfo *)p1; SRowInfo *pInfo2 = (SRowInfo *)p2; @@ -325,22 +320,22 @@ _err: return code; } -static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t suid, int64_t uid) { +int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo) { int32_t code = 0; if (suid) { - if (pCommitter->skmTable.suid == suid) { - pCommitter->skmTable.uid = uid; + if (pSkmInfo->suid == suid) { + pSkmInfo->uid = uid; goto _exit; } } else { - if (pCommitter->skmTable.uid == uid) goto _exit; + if (pSkmInfo->uid == uid) goto _exit; } - pCommitter->skmTable.suid = suid; - pCommitter->skmTable.uid = uid; - tTSchemaDestroy(pCommitter->skmTable.pTSchema); - code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, -1, &pCommitter->skmTable.pTSchema); + pSkmInfo->suid = suid; + pSkmInfo->uid = uid; + tTSchemaDestroy(pSkmInfo->pTSchema); + code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); if (code) goto _exit; _exit: @@ -382,7 +377,7 @@ static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); - code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); + code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); if (code) goto _exit; ASSERT(pCommitter->dReader.mBlock.nItem > 0); @@ -428,11 +423,11 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { pCommitter->toLastOnly = 0; SDataFReader *pReader = pCommitter->dReader.pReader; if (pReader) { - if (pReader->pSet->nSttF >= pCommitter->maxLast) { + if (pReader->pSet->nSttF >= pCommitter->sttTrigger) { int8_t iIter = 0; for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) { pIter = &pCommitter->aDataIter[iIter]; - pIter->type = LAST_DATA_ITER; + pIter->type = STT_DATA_ITER; pIter->iStt = iStt; code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk); @@ -498,7 +493,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { pCommitter->dReader.iBlockIdx = 0; if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) { pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0); - code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); + code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); if (code) goto _err; } else { pCommitter->dReader.pBlockIdx = NULL; @@ -515,11 +510,11 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { SSttFile fStt = {.commitID = pCommitter->commitID}; SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma}; if (pRSet) { - ASSERT(pRSet->nSttF <= pCommitter->maxLast); + ASSERT(pRSet->nSttF <= pCommitter->sttTrigger); fData = *pRSet->pDataF; fSma = *pRSet->pSmaF; wSet.diskId = pRSet->diskId; - if (pRSet->nSttF < pCommitter->maxLast) { + if (pRSet->nSttF < pCommitter->sttTrigger) { for (int32_t iStt = 0; iStt < pRSet->nSttF; iStt++) { wSet.aSttF[iStt] = pRSet->aSttF[iStt]; } @@ -556,46 +551,45 @@ _err: return code; } -static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) { - int32_t code = 0; - SBlockData *pBlockData = &pCommitter->dWriter.bData; - SDataBlk block; +int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg) { + int32_t code = 0; - ASSERT(pBlockData->nRow > 0); + if (pBlockData->nRow == 0) return code; - tDataBlkReset(&block); + SDataBlk dataBlk; + tDataBlkReset(&dataBlk); // info - block.nRow += pBlockData->nRow; + dataBlk.nRow += pBlockData->nRow; for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; if (iRow == 0) { - if (tsdbKeyCmprFn(&block.minKey, &key) > 0) { - block.minKey = key; + if (tsdbKeyCmprFn(&dataBlk.minKey, &key) > 0) { + dataBlk.minKey = key; } } else { if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) { - block.hasDup = 1; + dataBlk.hasDup = 1; } } - if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&block.maxKey, &key) < 0) { - block.maxKey = key; + if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&dataBlk.maxKey, &key) < 0) { + dataBlk.maxKey = key; } - block.minVer = TMIN(block.minVer, key.version); - block.maxVer = TMAX(block.maxVer, key.version); + dataBlk.minVer = TMIN(dataBlk.minVer, key.version); + dataBlk.maxVer = TMAX(dataBlk.maxVer, key.version); } // write - block.nSubBlock++; - code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock - 1], - ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0); + dataBlk.nSubBlock++; + code = tsdbWriteBlockData(pWriter, pBlockData, &dataBlk.aSubBlock[dataBlk.nSubBlock - 1], + ((dataBlk.nSubBlock == 1) && !dataBlk.hasDup) ? &dataBlk.smaInfo : NULL, cmprAlg, 0); if (code) goto _err; // put SDataBlk - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutDataBlk); + code = tMapDataPutItem(mDataBlk, &dataBlk, tPutDataBlk); if (code) goto _err; // clear @@ -604,39 +598,38 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) { return code; _err: - tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { - int32_t code = 0; - SSttBlk blockL; - SBlockData *pBlockData = &pCommitter->dWriter.bDatal; +int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg) { + int32_t code = 0; + SSttBlk sstBlk; - ASSERT(pBlockData->nRow > 0); + if (pBlockData->nRow == 0) return code; // info - blockL.suid = pBlockData->suid; - blockL.nRow = pBlockData->nRow; - blockL.minKey = TSKEY_MAX; - blockL.maxKey = TSKEY_MIN; - blockL.minVer = VERSION_MAX; - blockL.maxVer = VERSION_MIN; + sstBlk.suid = pBlockData->suid; + sstBlk.nRow = pBlockData->nRow; + sstBlk.minKey = TSKEY_MAX; + sstBlk.maxKey = TSKEY_MIN; + sstBlk.minVer = VERSION_MAX; + sstBlk.maxVer = VERSION_MIN; for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - blockL.minKey = TMIN(blockL.minKey, pBlockData->aTSKEY[iRow]); - blockL.maxKey = TMAX(blockL.maxKey, pBlockData->aTSKEY[iRow]); - blockL.minVer = TMIN(blockL.minVer, pBlockData->aVersion[iRow]); - blockL.maxVer = TMAX(blockL.maxVer, pBlockData->aVersion[iRow]); + sstBlk.minKey = TMIN(sstBlk.minKey, pBlockData->aTSKEY[iRow]); + sstBlk.maxKey = TMAX(sstBlk.maxKey, pBlockData->aTSKEY[iRow]); + sstBlk.minVer = TMIN(sstBlk.minVer, pBlockData->aVersion[iRow]); + sstBlk.maxVer = TMAX(sstBlk.maxVer, pBlockData->aVersion[iRow]); } - blockL.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0]; - blockL.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1]; + sstBlk.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0]; + sstBlk.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1]; // write - code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1); + code = tsdbWriteBlockData(pWriter, pBlockData, &sstBlk.bInfo, NULL, cmprAlg, 1); if (code) goto _err; // push SSttBlk - if (taosArrayPush(pCommitter->dWriter.aSttBlk, &blockL) == NULL) { + if (taosArrayPush(aSttBlk, &sstBlk) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -647,7 +640,7 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { return code; _err: - tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } @@ -692,7 +685,7 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) { SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; - code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); + code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); if (code) goto _err; if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { @@ -757,7 +750,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; - pCommitter->maxLast = TSDB_DEFAULT_STT_FILE; // TODO: make it as a config + pCommitter->sttTrigger = pTsdb->pVnode->config.sttTrigger; pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); if (pCommitter->aTbDataP == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -787,7 +780,7 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { if (code) goto _exit; // merger - for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) { + for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { SDataIter *pIter = &pCommitter->aDataIter[iStt]; pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pIter->aSttBlk == NULL) { @@ -829,7 +822,7 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { tBlockDataDestroy(&pCommitter->dReader.bData, 1); // merger - for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) { + for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { SDataIter *pIter = &pCommitter->aDataIter[iStt]; taosArrayDestroy(pIter->aSttBlk); tBlockDataDestroy(&pIter->bData, 1); @@ -1046,7 +1039,7 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { break; } } - } else if (pCommitter->pIter->type == LAST_DATA_ITER) { // last file + } else if (pCommitter->pIter->type == STT_DATA_ITER) { // last file pIter->iRow++; if (pIter->iRow < pIter->bData.nRow) { pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; @@ -1124,15 +1117,14 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) } if (pBlockData->nRow >= pCommitter->maxRow) { - code = tsdbCommitDataBlock(pCommitter); + code = + tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); if (code) goto _err; } } - if (pBlockData->nRow) { - code = tsdbCommitDataBlock(pCommitter); - if (code) goto _err; - } + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); + if (code) goto _err; return code; @@ -1193,7 +1185,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) } if (pBDataW->nRow >= pCommitter->maxRow) { - code = tsdbCommitDataBlock(pCommitter); + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); if (code) goto _err; } } @@ -1210,15 +1202,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) } if (pBDataW->nRow >= pCommitter->maxRow) { - code = tsdbCommitDataBlock(pCommitter); + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); if (code) goto _err; } } - if (pBDataW->nRow) { - code = tsdbCommitDataBlock(pCommitter); - if (code) goto _err; - } + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); + if (code) goto _err; return code; @@ -1306,10 +1296,8 @@ static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { SBlockData *pBDatal = &pCommitter->dWriter.bDatal; if (pBDatal->suid || pBDatal->uid) { if ((pBDatal->suid != id.suid) || (id.suid == 0)) { - if (pBDatal->nRow) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _exit; - } + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + if (code) goto _exit; tBlockDataReset(pBDatal); } } @@ -1341,7 +1329,7 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { if (code) goto _err; if (pBDatal->nRow >= pCommitter->maxRow) { - code = tsdbCommitLastBlock(pCommitter); + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); if (code) goto _err; } } @@ -1393,10 +1381,11 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pBData->nRow >= pCommitter->maxRow) { if (pCommitter->toLastOnly) { - code = tsdbCommitLastBlock(pCommitter); + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); if (code) goto _err; } else { - code = tsdbCommitDataBlock(pCommitter); + code = + tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); if (code) goto _err; } } @@ -1404,7 +1393,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (!pCommitter->toLastOnly && pBData->nRow) { if (pBData->nRow > pCommitter->minRow) { - code = tsdbCommitDataBlock(pCommitter); + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); if (code) goto _err; } else { code = tsdbAppendLastBlock(pCommitter); @@ -1437,7 +1426,7 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { tMapDataReset(&pCommitter->dWriter.mBlock); // impl - code = tsdbCommitterUpdateTableSchema(pCommitter, id.suid, id.uid); + code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable); if (code) goto _err; code = tBlockDataInit(&pCommitter->dReader.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema); if (code) goto _err; @@ -1455,7 +1444,7 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { // end if (pCommitter->dWriter.mBlock.nItem > 0) { SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid}; - code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); + code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); if (code) goto _err; if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { @@ -1470,10 +1459,9 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { code = tsdbMoveCommitData(pCommitter, id); if (code) goto _err; - if (pCommitter->dWriter.bDatal.nRow > 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, + pCommitter->cmprAlg); + if (code) goto _err; return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 8ab733aa56..6b4134f416 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -295,7 +295,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size != LOGIC_TO_FILE_SIZE(pSet->pHeadF->size, TSDB_DEFAULT_PAGE_SIZE)) { + if (size != tsdbLogicToFileSize(pSet->pHeadF->size, pTsdb->pVnode->config.tsdbPageSize)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -306,10 +306,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size < LOGIC_TO_FILE_SIZE(pSet->pDataF->size, TSDB_DEFAULT_PAGE_SIZE)) { + if (size < tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; - } else if (size > LOGIC_TO_FILE_SIZE(pSet->pDataF->size, TSDB_DEFAULT_PAGE_SIZE)) { + } else if (size > tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) { code = tsdbDFileRollback(pTsdb, pSet, TSDB_DATA_FILE); if (code) goto _err; } @@ -320,10 +320,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size < LOGIC_TO_FILE_SIZE(pSet->pSmaF->size, TSDB_DEFAULT_PAGE_SIZE)) { + if (size < tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; - } else if (size > LOGIC_TO_FILE_SIZE(pSet->pSmaF->size, TSDB_DEFAULT_PAGE_SIZE)) { + } else if (size > tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) { code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE); if (code) goto _err; } @@ -335,7 +335,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size != LOGIC_TO_FILE_SIZE(pSet->aSttF[iStt]->size, TSDB_DEFAULT_PAGE_SIZE)) { + if (size != tsdbLogicToFileSize(pSet->aSttF[iStt]->size, pTsdb->pVnode->config.tsdbPageSize)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 619fc17f5b..3c944584de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -148,7 +148,7 @@ int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) { } // ftruncate - if (taosFtruncateFile(pFD, LOGIC_TO_FILE_SIZE(size, TSDB_DEFAULT_PAGE_SIZE)) < 0) { + if (taosFtruncateFile(pFD, tsdbLogicToFileSize(size, pTsdb->pVnode->config.tsdbPageSize)) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 950c9348af..45fe29f0fa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -67,6 +67,16 @@ void resetLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo) { pLoadInfo[i].blockIndex[1] = -1; taosArrayClear(pLoadInfo[i].aSttBlk); + + pLoadInfo[i].elapsedTime = 0; + pLoadInfo[i].loadBlocks = 0; + } +} + +void getLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo, int64_t* blocks, double* el) { + for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) { + *el += pLoadInfo[i].elapsedTime; + *blocks += pLoadInfo[i].loadBlocks; } } @@ -86,7 +96,7 @@ void* destroyLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo) { return NULL; } -static SBlockData* loadBlockIfMissing(SLDataIter *pIter) { +static SBlockData* loadLastBlock(SLDataIter *pIter, const char* idStr) { int32_t code = 0; SSttBlockLoadInfo* pInfo = pIter->pBlockLoadInfo; @@ -100,8 +110,13 @@ static SBlockData* loadBlockIfMissing(SLDataIter *pIter) { pInfo->currentLoadBlockIndex ^= 1; if (pIter->pSttBlk != NULL) { // current block not loaded yet + int64_t st = taosGetTimestampUs(); code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, &pInfo->blockData[pInfo->currentLoadBlockIndex]); - tsdbDebug("read last block, index:%d, last file index:%d", pIter->iSttBlk, pIter->iStt); + double el = (taosGetTimestampUs() - st)/ 1000.0; + pInfo->elapsedTime += el; + pInfo->loadBlocks += 1; + + tsdbDebug("read last block, index:%d, last file index:%d, elapsed time:%.2f ms, %s", pIter->iSttBlk, pIter->iStt, el, idStr); if (code != TSDB_CODE_SUCCESS) { goto _exit; } @@ -120,6 +135,92 @@ static SBlockData* loadBlockIfMissing(SLDataIter *pIter) { return NULL; } +// find the earliest block that contains the required records +static FORCE_INLINE int32_t findEarliestIndex(int32_t index, uint64_t uid, const SSttBlk* pBlockList, int32_t num, int32_t backward) { + int32_t i = index; + int32_t step = backward? 1:-1; + while (i >= 0 && i < num && uid >= pBlockList[i].minUid && uid <= pBlockList[i].maxUid) { + i += step; + } + return i - step; +} + +static int32_t binarySearchForStartBlock(SSttBlk*pBlockList, int32_t num, uint64_t uid, int32_t backward) { + int32_t midPos = -1; + if (num <= 0) { + return -1; + } + + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + // find the first position which is bigger than the key + if ((uid > pBlockList[lastPos].maxUid) || (uid < pBlockList[firstPos].minUid)) { + return -1; + } + + while (1) { + if (uid >= pBlockList[firstPos].minUid && uid <= pBlockList[firstPos].maxUid) { + return findEarliestIndex(firstPos, uid, pBlockList, num, backward); + } + + if (uid > pBlockList[lastPos].maxUid || uid < pBlockList[firstPos].minUid) { + return -1; + } + + int32_t numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1u) + firstPos; + + if (uid < pBlockList[midPos].minUid) { + lastPos = midPos - 1; + } else if (uid > pBlockList[midPos].maxUid) { + firstPos = midPos + 1; + } else { + return findEarliestIndex(midPos, uid, pBlockList, num, backward); + } + } +} + +static FORCE_INLINE int32_t findEarliestRow(int32_t index, uint64_t uid, const uint64_t* uidList, int32_t num, int32_t backward) { + int32_t i = index; + int32_t step = backward? 1:-1; + while (i >= 0 && i < num && uid == uidList[i]) { + i += step; + } + return i - step; +} + +static int32_t binarySearchForStartRowIndex(uint64_t* uidList, int32_t num, uint64_t uid, int32_t backward) { + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + // find the first position which is bigger than the key + if ((uid > uidList[lastPos]) || (uid < uidList[firstPos])) { + return -1; + } + + while (1) { + if (uid == uidList[firstPos]) { + return findEarliestRow(firstPos, uid, uidList, num, backward); + } + + if (uid > uidList[lastPos] || uid < uidList[firstPos]) { + return -1; + } + + int32_t numOfRows = lastPos - firstPos + 1; + int32_t midPos = (numOfRows >> 1u) + firstPos; + + if (uid < uidList[midPos]) { + lastPos = midPos - 1; + } else if (uid > uidList[midPos]) { + firstPos = midPos + 1; + } else { + return findEarliestRow(midPos, uid, uidList, num, backward); + } + } +} + int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo* pBlockLoadInfo) { int32_t code = 0; @@ -141,41 +242,26 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk); if (code) { goto _exit; + } else { + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + SArray* pTmp = taosArrayInit(size, sizeof(SSttBlk)); + for(int32_t i = 0; i < size; ++i) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + if (p->suid == suid) { + taosArrayPush(pTmp, p); + } + } + + taosArrayDestroy(pBlockLoadInfo->aSttBlk); + pBlockLoadInfo->aSttBlk = pTmp; } } size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); // find the start block - int32_t index = -1; - if (!backward) { // asc - for (int32_t i = 0; i < size; ++i) { - SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); - if (p->suid != suid) { - continue; - } - - if (p->minUid <= uid && p->maxUid >= uid) { - index = i; - break; - } - } - } else { // desc - for (int32_t i = size - 1; i >= 0; --i) { - SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); - if (p->suid != suid) { - continue; - } - - if (p->minUid <= uid && p->maxUid >= uid) { - index = i; - break; - } - } - } - - (*pIter)->iSttBlk = index; - if (index != -1) { + (*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward); + if ((*pIter)->iSttBlk != -1) { (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk); (*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1; } @@ -193,7 +279,7 @@ void tLDataIterNextBlock(SLDataIter *pIter) { pIter->iSttBlk += step; int32_t index = -1; - size_t size = taosArrayGetSize(pIter->pBlockLoadInfo->aSttBlk); + size_t size = pIter->pBlockLoadInfo->aSttBlk->size; for (int32_t i = pIter->iSttBlk; i < size && i >= 0; i += step) { SSttBlk *p = taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, i); if ((!pIter->backward) && p->minUid > pIter->uid) { @@ -232,33 +318,41 @@ void tLDataIterNextBlock(SLDataIter *pIter) { } } - if (index == -1) { - pIter->pSttBlk = NULL; - } else { + pIter->pSttBlk = NULL; + if (index != -1) { pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk); } } -static void findNextValidRow(SLDataIter *pIter) { +static void findNextValidRow(SLDataIter *pIter, const char* idStr) { int32_t step = pIter->backward ? -1 : 1; bool hasVal = false; int32_t i = pIter->iRow; - SBlockData *pBlockData = loadBlockIfMissing(pIter); + SBlockData *pBlockData = loadLastBlock(pIter, idStr); + + // mostly we only need to find the start position for a given table + if ((((i == 0) && (!pIter->backward)) || (i == pBlockData->nRow - 1 && pIter->backward)) && pBlockData->aUid != NULL) { + i = binarySearchForStartRowIndex((uint64_t*)pBlockData->aUid, pBlockData->nRow, pIter->uid, pIter->backward); + if (i == -1) { + pIter->iRow = -1; + return; + } + } for (; i < pBlockData->nRow && i >= 0; i += step) { if (pBlockData->aUid != NULL) { if (!pIter->backward) { - if (pBlockData->aUid[i] < pIter->uid) { + /*if (pBlockData->aUid[i] < pIter->uid) { continue; - } else if (pBlockData->aUid[i] > pIter->uid) { + } else */if (pBlockData->aUid[i] > pIter->uid) { break; } } else { - if (pBlockData->aUid[i] > pIter->uid) { + /*if (pBlockData->aUid[i] > pIter->uid) { continue; - } else if (pBlockData->aUid[i] < pIter->uid) { + } else */if (pBlockData->aUid[i] < pIter->uid) { break; } } @@ -296,7 +390,7 @@ static void findNextValidRow(SLDataIter *pIter) { pIter->iRow = (hasVal) ? i : -1; } -bool tLDataIterNextRow(SLDataIter *pIter) { +bool tLDataIterNextRow(SLDataIter *pIter, const char* idStr) { int32_t code = 0; int32_t step = pIter->backward ? -1 : 1; @@ -306,11 +400,11 @@ bool tLDataIterNextRow(SLDataIter *pIter) { } int32_t iBlockL = pIter->iSttBlk; - SBlockData *pBlockData = loadBlockIfMissing(pIter); + SBlockData *pBlockData = loadLastBlock(pIter, idStr); pIter->iRow += step; while (1) { - findNextValidRow(pIter); + findNextValidRow(pIter, idStr); if (pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) { tLDataIterNextBlock(pIter); @@ -322,7 +416,7 @@ bool tLDataIterNextRow(SLDataIter *pIter) { } if (iBlockL != pIter->iSttBlk) { - pBlockData = loadBlockIfMissing(pIter); + pBlockData = loadLastBlock(pIter, idStr); pIter->iRow += step; } } @@ -365,7 +459,7 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { } int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, - STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pBlockLoadInfo) { + STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pBlockLoadInfo, const char* idStr) { pMTree->backward = backward; pMTree->pIter = NULL; pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); @@ -373,6 +467,8 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead return TSDB_CODE_OUT_OF_MEMORY; } + pMTree->idStr = idStr; + tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); int32_t code = TSDB_CODE_SUCCESS; @@ -395,7 +491,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead goto _end; } - bool hasVal = tLDataIterNextRow(pIter); + bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr); if (hasVal) { taosArrayPush(pMTree->pIterList, &pIter); tMergeTreeAddIter(pMTree, pIter); @@ -418,7 +514,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) { if (pMTree->pIter) { SLDataIter *pIter = pMTree->pIter; - bool hasVal = tLDataIterNextRow(pIter); + bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr); if (!hasVal) { pMTree->pIter = NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index facfdb9a62..41c06bc500 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -16,7 +16,7 @@ #include "osDef.h" #include "tsdb.h" -#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) typedef enum { EXTERNAL_ROWS_PREV = 0x1, @@ -70,6 +70,8 @@ typedef struct SIOCostSummary { double smaLoadTime; int64_t lastBlockLoad; double lastBlockLoadTime; + int64_t composedBlocks; + double buildComposedBlockTime; } SIOCostSummary; typedef struct SBlockLoadSuppInfo { @@ -81,11 +83,11 @@ typedef struct SBlockLoadSuppInfo { } SBlockLoadSuppInfo; typedef struct SLastBlockReader { - STimeWindow window; - SVersionRange verRange; - int32_t order; - uint64_t uid; - SMergeTree mergeTree; + STimeWindow window; + SVersionRange verRange; + int32_t order; + uint64_t uid; + SMergeTree mergeTree; SSttBlockLoadInfo* pInfo; } SLastBlockReader; @@ -190,6 +192,9 @@ static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader); static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static int32_t doBuildDataBlock(STsdbReader* pReader); +static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); + +static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; @@ -229,10 +234,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid}; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { int64_t skey = pTsdbReader->window.skey; - info.lastKey = (skey > INT64_MIN)? (skey - 1):skey; + info.lastKey = (skey > INT64_MIN) ? (skey - 1) : skey; } else { int64_t ekey = pTsdbReader->window.ekey; - info.lastKey = (ekey < INT64_MAX)? (ekey + 1):ekey; + info.lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey; } taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); @@ -365,6 +370,9 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { return false; } + SIOCostSummary* pSum = &pReader->cost; + getLastBlockLoadInfo(pIter->pLastBlockReader->pInfo, &pSum->lastBlockLoad, &pReader->cost.lastBlockLoadTime); + pIter->pLastBlockReader->uid = 0; tMergeTreeClose(&pIter->pLastBlockReader->mergeTree); resetLastBlockLoadInfo(pIter->pLastBlockReader->pInfo); @@ -596,7 +604,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(int64_t)); tMapDataReset(&pScanInfo->mapData); - tsdbReadBlock(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); + tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); sizeInDisk += pScanInfo->mapData.nData; for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { @@ -677,12 +685,153 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { static SDataBlk* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } +int32_t binarySearchForTs(char* pValue, int num, TSKEY key, int order) { + int32_t midPos = -1; + int32_t numOfRows; + + ASSERT(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + + TSKEY* keyList = (TSKEY*)pValue; + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + if (order == TSDB_ORDER_DESC) { + // find the first position which is smaller than the key + while (1) { + if (key >= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key < keyList[lastPos]) { + lastPos += 1; + if (lastPos >= num) { + return -1; + } else { + return lastPos; + } + } + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1) + firstPos; + + if (key < keyList[midPos]) { + firstPos = midPos + 1; + } else if (key > keyList[midPos]) { + lastPos = midPos - 1; + } else { + break; + } + } + + } else { + // find the first position which is bigger than the key + while (1) { + if (key <= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key > keyList[lastPos]) { + lastPos = lastPos + 1; + if (lastPos >= num) + return -1; + else + return lastPos; + } + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1u) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + } + + return midPos; +} + +static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int order) { + // start end position + int s, e; + s = pos; + + // check + assert(pos >=0 && pos < num); + assert(num > 0); + + if (order == TSDB_ORDER_ASC) { + // find the first position which is smaller than the key + e = num - 1; + if (key < keyList[pos]) + return -1; + while (1) { + // check can return + if (key >= keyList[e]) + return e; + if (key <= keyList[s]) + return s; + if (e - s <= 1) + return s; + + // change start or end position + int mid = s + (e - s + 1)/2; + if (keyList[mid] > key) + e = mid; + else if(keyList[mid] < key) + s = mid; + else + return mid; + } + } else { // DESC + // find the first position which is bigger than the key + e = 0; + if (key > keyList[pos]) + return -1; + while (1) { + // check can return + if (key <= keyList[e]) + return e; + if (key >= keyList[s]) + return s; + if (s - e <= 1) + return s; + + // change start or end position + int mid = s - (s - e + 1)/2; + if (keyList[mid] < key) + e = mid; + else if(keyList[mid] > key) + s = mid; + else + return mid; + } + } +} + +int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SDataBlk* pBlock, int32_t pos) { + // NOTE: reverse the order to find the end position in data block + int32_t endPos = -1; + bool asc = ASCENDING_TRAVERSE(pReader->order); + + if (asc && pReader->window.ekey >= pBlock->maxKey.ts) { + endPos = pBlock->nRow - 1; + } else if (!asc && pReader->window.skey <= pBlock->minKey.ts) { + endPos = 0; + } else { + endPos = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, pReader->window.ekey, pReader->order); + } + + return endPos; +} + static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; SBlockData* pBlockData = &pStatus->fileBlockData; - SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter); SSDataBlock* pResBlock = pReader->pResBlock; int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock); @@ -695,23 +844,42 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn bool asc = ASCENDING_TRAVERSE(pReader->order); int32_t step = asc ? 1 : -1; - int32_t rowIndex = 0; - int32_t remain = asc ? (pBlockData->nRow - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex + 1); - - int32_t endIndex = 0; - if (remain <= pReader->capacity) { - endIndex = pBlockData->nRow; + if (asc && pReader->window.skey <= pBlock->minKey.ts) { + pDumpInfo->rowIndex = 0; + } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts) { + pDumpInfo->rowIndex = pBlock->nRow - 1; } else { - endIndex = pDumpInfo->rowIndex + step * pReader->capacity; + int32_t pos = asc? pBlock->nRow-1:0; + int32_t order = (pReader->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, pReader->window.skey, order); + } + + // time window check + int32_t endIndex = getEndPosInDataBlock(pReader, pBlockData, pBlock, pDumpInfo->rowIndex); + if (endIndex == -1) { + setBlockAllDumped(pDumpInfo, pReader->window.ekey, pReader->order); + return TSDB_CODE_SUCCESS; + } + + endIndex += step; + int32_t remain = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); + if (remain > pReader->capacity) { // output buffer check remain = pReader->capacity; } - int32_t i = 0; + int32_t rowIndex = 0; + + int32_t i = 0; SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - for (int32_t j = pDumpInfo->rowIndex; j < endIndex && j >= 0; j += step) { - colDataAppend(pColData, rowIndex++, (const char*)&pBlockData->aTSKEY[j], false); + if (asc) { + memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], remain * sizeof(int64_t)); + } else { + for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step) { + colDataAppendInt64(pColData, rowIndex++, &pBlockData->aTSKEY[j]); + } } + i += 1; } @@ -725,13 +893,32 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn if (pData->cid < pColData->info.colId) { colIndex += 1; } else if (pData->cid == pColData->info.colId) { - for (int32_t j = pDumpInfo->rowIndex; j < endIndex && j >= 0; j += step) { - tColDataGetValue(pData, j, &cv); - doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); + if (pData->flag == HAS_NONE || pData->flag == HAS_NULL) { + colDataAppendNNULL(pColData, 0, remain); + } else { + if (IS_NUMERIC_TYPE(pColData->info.type) && asc) { + uint8_t* p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; + memcpy(pColData->pData, p, remain * tDataTypes[pData->type].bytes); + + // null value exists, check one-by-one + if (pData->flag != HAS_VALUE) { + for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step, rowIndex++) { + uint8_t v = GET_BIT2(pData->pBitMap, j); + if (v == 0 || v == 1) { + colDataSetNull_f(pColData->nullbitmap, rowIndex); + } + } + } + } else { + for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step) { + tColDataGetValue(pData, j, &cv); + doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); + } + } } + colIndex += 1; i += 1; - ASSERT(rowIndex == remain); } else { // the specified column does not exist in file block, fill with null data colDataAppendNNULL(pColData, 0, remain); i += 1; @@ -747,7 +934,13 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn pResBlock->info.rows = remain; pDumpInfo->rowIndex += step * remain; - setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); + if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pBlock->nRow) { + int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + setBlockAllDumped(pDumpInfo, ts, pReader->order); + } else { + int64_t k = asc? pBlock->maxKey.ts:pBlock->minKey.ts; + setBlockAllDumped(pDumpInfo, k, pReader->order); + } double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; pReader->cost.blockLoadTime += elapsedTime; @@ -755,7 +948,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t unDumpedRows = asc ? pBlock->nRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1; tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", - pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, unDumpedRows, + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, unDumpedRows, pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); return TSDB_CODE_SUCCESS; @@ -1143,26 +1336,31 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SDa } } -// 1. the version of all rows should be less than the endVersion -// 2. current block should not overlap with next neighbor block -// 3. current timestamp should not be overlap with each other -// 4. output buffer should be large enough to hold all rows in current block -// 5. delete info should not overlap with current block data -static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SDataBlk* pBlock, - STableBlockScanInfo* pScanInfo, TSDBKEY key, SLastBlockReader* pLastBlockReader) { +typedef struct { + bool overlapWithNeighborBlock; + bool hasDupTs; + bool overlapWithDelInfo; + bool overlapWithLastBlock; + bool overlapWithKeyInBuf; + bool partiallyRequired; + bool moreThanCapcity; +} SDataBlockToLoadInfo; + +static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, SDataBlk* pBlock, + STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader, + STsdbReader* pReader) { int32_t neighborIndex = 0; - SDataBlk* pNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &neighborIndex, pReader->order); + SDataBlk* pNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pScanInfo, &neighborIndex, pReader->order); // overlap with neighbor - bool overlapWithNeighbor = false; if (pNeighbor) { - overlapWithNeighbor = overlapWithNeighborBlock(pBlock, pNeighbor, pReader->order); + pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock(pBlock, pNeighbor, pReader->order); taosMemoryFree(pNeighbor); } // has duplicated ts of different version in this block - bool hasDup = (pBlock->nSubBlock == 1) ? pBlock->hasDup : true; - bool overlapWithDel = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order); + pInfo->hasDupTs = (pBlock->nSubBlock == 1) ? pBlock->hasDup : true; + pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order); // todo here we need to each key in the last files to identify if it is really overlapped with last block // todo @@ -1174,25 +1372,48 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc } #endif - bool moreThanOutputCapacity = pBlock->nRow > pReader->capacity; - bool partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock); - bool overlapWithKey = keyOverlapFileBlock(key, pBlock, &pReader->verRange); + pInfo->moreThanCapcity = pBlock->nRow > pReader->capacity; + pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock); + pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlock, &pReader->verRange); +} - bool loadDataBlock = (overlapWithNeighbor || hasDup || partiallyRequired || overlapWithKey || - moreThanOutputCapacity || overlapWithDel || overlapWithlastBlock); +// 1. the version of all rows should be less than the endVersion +// 2. current block should not overlap with next neighbor block +// 3. current timestamp should not be overlap with each other +// 4. output buffer should be large enough to hold all rows in current block +// 5. delete info should not overlap with current block data +// 6. current block should not contain the duplicated ts +static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, SDataBlk* pBlock, + STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) { + SDataBlockToLoadInfo info = {0}; + getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader); + + bool loadDataBlock = + (info.overlapWithNeighborBlock || info.hasDupTs || info.partiallyRequired || info.overlapWithKeyInBuf || + info.moreThanCapcity || info.overlapWithDelInfo || info.overlapWithLastBlock); // log the reason why load the datablock for profile if (loadDataBlock) { tsdbDebug("%p uid:%" PRIu64 " need to load the datablock, overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, " "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s", - pReader, pFBlock->uid, overlapWithNeighbor, hasDup, partiallyRequired, overlapWithKey, - moreThanOutputCapacity, overlapWithDel, overlapWithlastBlock, pReader->idStr); + pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired, + info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithLastBlock, + pReader->idStr); } return loadDataBlock; } +static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, SDataBlk* pBlock, + STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) { + SDataBlockToLoadInfo info = {0}; + getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader); + bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || + info.overlapWithDelInfo || info.overlapWithLastBlock); + return isCleanFileBlock; +} + static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, int64_t endKey) { if (!(pBlockScanInfo->iiter.hasVal || pBlockScanInfo->iter.hasVal)) { return TSDB_CODE_SUCCESS; @@ -1238,6 +1459,38 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB return false; } +static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { + while (1) { + bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); + if (!hasVal) { + return false; + } + + TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBKEY k = TSDBROW_KEY(&row); + if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) { + return true; + } + } +} + +static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader, + STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader) { + bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo); + if (hasVal) { + int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); + if (next1 != ts) { + doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); + return true; + } + } else { + doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); + return true; + } + + return false; +} + static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { // always set the newest schema version in pReader->pSchema if (pReader->pSchema == NULL) { @@ -1385,29 +1638,54 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, bool mergeBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); STSRow* pTSRow = NULL; SRowMerger merge = {0}; + TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); + // only last block exists + if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { + if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) { + return TSDB_CODE_SUCCESS; + } else { + tRowMergerInit(&merge, &fRow, pReader->pSchema); - // merge with block data if ts == key - if (mergeBlockData && (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex])) { - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tRowMerge(&merge, &fRow1); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); + + int32_t code = tRowMergerGetRow(&merge, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + } + } else { // not merge block data + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); + ASSERT(mergeBlockData); + + // merge with block data if ts == key + if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + } + + int32_t code = tRowMergerGetRow(&merge, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); } - int32_t code = tRowMergerGetRow(&merge, &pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); - - taosMemoryFree(pTSRow); - tRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } @@ -1856,23 +2134,6 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum return true; } -static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } - -static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { - while (1) { - bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); - if (!hasVal) { - return false; - } - - TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - TSDBKEY k = TSDBROW_KEY(&row); - if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) { - return true; - } - } -} - static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { // the last block reader has been initialized for this table. if (pLBlockReader->uid == pScanInfo->uid) { @@ -1886,7 +2147,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan initMemDataIterator(pScanInfo, pReader); pLBlockReader->uid = pScanInfo->uid; - int32_t step = ASCENDING_TRAVERSE(pLBlockReader->order)? 1:-1; + int32_t step = ASCENDING_TRAVERSE(pLBlockReader->order) ? 1 : -1; STimeWindow w = pLBlockReader->window; if (ASCENDING_TRAVERSE(pLBlockReader->order)) { w.skey = pScanInfo->lastKey + step; @@ -1896,7 +2157,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan int32_t code = tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, - pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo); + pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo, pReader->idStr); if (code != TSDB_CODE_SUCCESS) { return false; } @@ -1906,8 +2167,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - TSDBKEY key = TSDBROW_KEY(&row); - return key.ts; + return TSDBROW_TS(&row); } static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } @@ -1915,12 +2175,11 @@ static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLas int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { return TSDB_CODE_SUCCESS; } else { + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + STSRow* pTSRow = NULL; SRowMerger merge = {0}; @@ -1937,21 +2196,25 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc tRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } - - return TSDB_CODE_SUCCESS; } static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int64_t key = (pBlockData->nRow > 0) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; - TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); - TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); - + int64_t key = (pBlockData->nRow > 0) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) { return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); } else { + TSDBROW *pRow = NULL, *piRow = NULL; + if (pBlockScanInfo->iter.hasVal) { + pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); + } + + if (pBlockScanInfo->iiter.hasVal) { + piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); + } + // imem + file + last block if (pBlockScanInfo->iiter.hasVal) { return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader); @@ -1971,20 +2234,29 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->pResBlock; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + + int64_t st = taosGetTimestampUs(); STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo != NULL) { pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); - } else { + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); + + // it is a clean block, load it directly + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) { + copyBlockDataToSDataBlock(pReader, pBlockScanInfo); + goto _end; + } + } else { // file blocks not exist pBlockScanInfo = pReader->status.pTableIter; } - SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; - int64_t st = taosGetTimestampUs(); while (1) { // todo check the validate of row in file block @@ -2027,17 +2299,21 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } } + _end: pResBlock->info.uid = pBlockScanInfo->uid; blockDataUpdateTsWindow(pResBlock, 0); setComposedBlockFlag(pReader, true); - int64_t et = taosGetTimestampUs(); + double el = (taosGetTimestampUs() - st)/1000.0; + + pReader->cost.composedBlocks += 1; + pReader->cost.buildComposedBlockTime += el; if (pResBlock->info.rows > 0) { tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%d, elapsed time:%.2f ms %s", pReader, pBlockScanInfo->uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, - pResBlock->info.rows, (et - st) / 1000.0, pReader->idStr); + pResBlock->info.rows, el, pReader->idStr); } return TSDB_CODE_SUCCESS; @@ -2127,7 +2403,7 @@ _err: return code; } -static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { +TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); if (pRow != NULL) { @@ -2323,12 +2599,12 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } initLastBlockReader(pLastBlockReader, pScanInfo, pReader); - TSDBKEY key = getCurrentKeyInBuf(pScanInfo, pReader); + TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); if (pBlockInfo == NULL) { // build data block from last data file ASSERT(pBlockIter->numOfBlocks == 0); code = buildComposedDataBlock(pReader); - } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, key, pLastBlockReader)) { + } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { tBlockDataReset(&pStatus->fileBlockData); code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pScanInfo->uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { @@ -2342,7 +2618,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // build composed data block code = buildComposedDataBlock(pReader); - } else if (bufferDataInFileBlockGap(pReader->order, key, pBlock)) { + } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { // data in memory that are earlier than current file block // todo rows in buffer should be less than the file block in asc, greater than file block in desc int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; @@ -3073,18 +3349,23 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S } SColVal cv = {0}; - int32_t numOfInputCols = taosArrayGetSize(pBlockData->aIdx); - int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock); + int32_t numOfInputCols = pBlockData->aIdx->size; + int32_t numOfOutputCols = pResBlock->pDataBlock->size; while (i < numOfOutputCols && j < numOfInputCols) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); + SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, i); SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); + if (pData->cid < pCol->info.colId) { + j += 1; + continue; + } + if (pData->cid == pCol->info.colId) { tColDataGetValue(pData, rowIndex, &cv); doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); j += 1; - } else { // the specified column does not exist in file block, fill with null data + } else if (pData->cid > pCol->info.colId) { // the specified column does not exist in file block, fill with null data colDataAppendNULL(pCol, outputRowIndex); } @@ -3206,11 +3487,18 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl } } + // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here. if (pCond->suid != 0) { - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, pCond->endVersion); + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, /*pCond->endVersion*/ -1); + if (pReader->pSchema == NULL) { + tsdbError("failed to get table schema, suid:%"PRIu64", ver:%"PRId64" , %s", pReader->suid, -1, pReader->idStr); + } } else if (taosArrayGetSize(pTableList) > 0) { STableKeyInfo* pKey = taosArrayGet(pTableList, 0); - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, pCond->endVersion); + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, /*pCond->endVersion*/ -1); + if (pReader->pSchema == NULL) { + tsdbError("failed to get table schema, uid:%"PRIu64", ver:%"PRId64" , %s", pKey->uid, -1, pReader->idStr); + } } int32_t numOfTables = taosArrayGetSize(pTableList); @@ -3307,24 +3595,27 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap); taosMemoryFree(pReader->status.uidCheckInfo.tableUidList); + SIOCostSummary* pCost = &pReader->cost; SFilesetIter* pFilesetIter = &pReader->status.fileIter; if (pFilesetIter->pLastBlockReader != NULL) { - tMergeTreeClose(&pFilesetIter->pLastBlockReader->mergeTree); - pFilesetIter->pLastBlockReader->pInfo = destroyLastBlockLoadInfo(pFilesetIter->pLastBlockReader->pInfo); - taosMemoryFree(pFilesetIter->pLastBlockReader); + SLastBlockReader* pLReader = pFilesetIter->pLastBlockReader; + tMergeTreeClose(&pLReader->mergeTree); + + getLastBlockLoadInfo(pLReader->pInfo, &pCost->lastBlockLoad, &pCost->lastBlockLoadTime); + + pLReader->pInfo = destroyLastBlockLoadInfo(pLReader->pInfo); + taosMemoryFree(pLReader); } - SIOCostSummary* pCost = &pReader->cost; - - tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 - " SMA-time:%.2f ms, fileBlocks:%" PRId64 - ", fileBlocks-time:%.2f ms, " - "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 - ", lastBlocks-time:%.2f ms, STableBlockScanInfo size:%.2f Kb %s", - pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, - pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, - pCost->lastBlockLoadTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); + tsdbDebug( + "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 + " SMA-time:%.2f ms, fileBlocks:%" PRId64 ", fileBlocks-load-time:%.2f ms, " + "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 + ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb %s", + pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks, + pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks, + pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->pSchema); @@ -3559,7 +3850,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); resetDataBlockIterator(&pReader->status.blockIter, pReader->order); - int64_t ts = ASCENDING_TRAVERSE(pReader->order)?pReader->window.skey-1:pReader->window.ekey+1; + int64_t ts = ASCENDING_TRAVERSE(pReader->order) ? pReader->window.skey - 1 : pReader->window.ekey + 1; resetDataBlockScanInfo(pReader->status.pTableMap, ts); int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 128bfe37da..5fe0b408b1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -209,7 +209,7 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS int32_t code = 0; int32_t flag; int64_t n; - int32_t szPage = TSDB_DEFAULT_PAGE_SIZE; + int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; SDataFWriter *pWriter = NULL; char fname[TSDB_FILENAME_LEN]; char hdr[TSDB_FHDR_SIZE] = {0}; @@ -418,21 +418,21 @@ _err: return code; } -int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBlockIdx) { +int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx) { int32_t code = 0; SHeadFile *pHeadFile = &pWriter->fHead; int64_t size; int64_t n; - ASSERT(mBlock->nItem > 0); + ASSERT(mDataBlk->nItem > 0); // alloc - size = tPutMapData(NULL, mBlock); + size = tPutMapData(NULL, mDataBlk); code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; // build - n = tPutMapData(pWriter->aBuf[0], mBlock); + n = tPutMapData(pWriter->aBuf[0], mDataBlk); // write code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size); @@ -446,7 +446,7 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBloc tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 " size:%" PRId64 " nItem:%d", TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, - pBlockIdx->offset, pBlockIdx->size, mBlock->nItem); + pBlockIdx->offset, pBlockIdx->size, mDataBlk->nItem); return code; _err: @@ -611,28 +611,26 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int32_t code = 0; int64_t n; int64_t size; - TdFilePtr pOutFD = NULL; // TODO - TdFilePtr PInFD = NULL; // TODO + TdFilePtr pOutFD = NULL; + TdFilePtr PInFD = NULL; + int32_t szPage = pTsdb->pVnode->config.szPage; char fNameFrom[TSDB_FILENAME_LEN]; char fNameTo[TSDB_FILENAME_LEN]; // head tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); if (PInFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pHeadF->size); + n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -643,44 +641,17 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { // data tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); if (PInFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pDataF->size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // stt - tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[0], fNameFrom); - tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[0], fNameTo); - - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aSttF[0]->size); + n = taosFSendFile(pOutFD, PInFD, 0, LOGIC_TO_FILE_OFFSET(pSetFrom->pDataF->size, szPage)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -691,20 +662,17 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { // sma tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); if (PInFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pSmaF->size); + n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -712,6 +680,29 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { taosCloseFile(&pOutFD); taosCloseFile(&PInFD); + // stt + for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) { + tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom); + tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo); + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); + } + return code; _err: @@ -723,7 +714,7 @@ _err: int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; SDataFReader *pReader; - int32_t szPage = TSDB_DEFAULT_PAGE_SIZE; + int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; char fname[TSDB_FILENAME_LEN]; // alloc @@ -780,7 +771,7 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { tsdbCloseFile(&(*ppReader)->pSmaFD); // stt - for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) { + for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { if ((*ppReader)->aSttFD[iStt]) { tsdbCloseFile(&(*ppReader)->aSttFD[iStt]); } @@ -872,7 +863,7 @@ _err: return code; } -int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock) { +int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk) { int32_t code = 0; int64_t offset = pBlockIdx->offset; int64_t size = pBlockIdx->size; @@ -886,7 +877,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl if (code) goto _err; // decode - int64_t n = tGetMapData(pReader->aBuf[0], mBlock); + int64_t n = tGetMapData(pReader->aBuf[0], mDataBlk); if (n < 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -1053,6 +1044,29 @@ _err: return code; } +int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) { + int32_t code = 0; + SBlockInfo *pBlockInfo = &pDataBlk->aSubBlock[0]; + + // alloc + code = tRealloc(&pReader->aBuf[0], pBlockInfo->szBlock); + if (code) goto _err; + + // read + code = tsdbReadFile(pReader->pDataFD, pBlockInfo->offset, pReader->aBuf[0], pBlockInfo->szBlock); + if (code) goto _err; + + // decmpr + code = tDecmprBlockData(pReader->aBuf[0], pBlockInfo->szBlock, pBlockData, &pReader->aBuf[1]); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d tsdb read data block ex failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) { int32_t code = 0; @@ -1147,8 +1161,8 @@ int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb pDelFWriter->fDel = *pFile; tsdbDelFileName(pTsdb, pFile, fname); - code = - tsdbOpenFile(fname, TSDB_DEFAULT_PAGE_SIZE, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, &pDelFWriter->pWriteH); + code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, + &pDelFWriter->pWriteH); if (code) goto _err; // update header @@ -1315,7 +1329,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb pDelFReader->fDel = *pFile; tsdbDelFileName(pTsdb, pFile, fname); - code = tsdbOpenFile(fname, TSDB_DEFAULT_PAGE_SIZE, TD_FILE_READ, &pDelFReader->pReadH); + code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ, &pDelFReader->pReadH); if (code) goto _err; *ppReader = pDelFReader; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index d10613e719..d99bf2aa5c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -82,8 +82,6 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { code = tsdbFSUpsertFSet(&fs, &fSet); if (code) goto _err; } - - /* code */ } // do change fs diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 9fc5639c5e..99e88a442c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -16,6 +16,29 @@ #include "tsdb.h" // STsdbSnapReader ======================================== +typedef enum { SNAP_DATA_FILE_ITER = 0, SNAP_STT_FILE_ITER } EFIterT; +typedef struct { + SRBTreeNode n; + SRowInfo rInfo; + EFIterT type; + union { + struct { + SArray* aBlockIdx; + int32_t iBlockIdx; + SBlockIdx* pBlockIdx; + SMapData mBlock; + int32_t iBlock; + }; // .data file + struct { + int32_t iStt; + SArray* aSttBlk; + int32_t iSttBlk; + }; // .stt file + }; + SBlockData bData; + int32_t iRow; +} SFDataIter; + struct STsdbSnapReader { STsdb* pTsdb; int64_t sver; @@ -26,146 +49,301 @@ struct STsdbSnapReader { int8_t dataDone; int32_t fid; SDataFReader* pDataFReader; - SArray* aBlockIdx; // SArray - SArray* aSstBlk; // SArray - SBlockIdx* pBlockIdx; - SSttBlk* pSstBlk; - - int32_t iBlockIdx; - int32_t iBlockL; - SMapData mBlock; // SMapData - int32_t iBlock; - SBlockData oBlockData; - SBlockData nBlockData; + SFDataIter* pIter; + SRBTree rbt; + SFDataIter aFDataIter[TSDB_MAX_STT_TRIGGER + 1]; + SBlockData bData; + SSkmInfo skmTable; // for del file int8_t delDone; SDelFReader* pDelFReader; SArray* aDelIdx; // SArray int32_t iDelIdx; SArray* aDelData; // SArray + uint8_t* aBuf[5]; }; +extern int32_t tRowInfoCmprFn(const void* p1, const void* p2); +extern int32_t tsdbReadDataBlockEx(SDataFReader* pReader, SDataBlk* pDataBlk, SBlockData* pBlockData); +extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo); + +static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { + int32_t code = 0; + + SDFileSet dFileSet = {.fid = pReader->fid}; + SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT); + if (pSet == NULL) return code; + + pReader->fid = pSet->fid; + code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet); + if (code) goto _err; + + pReader->pIter = NULL; + tRBTreeCreate(&pReader->rbt, tRowInfoCmprFn); + + // .data file + SFDataIter* pIter = &pReader->aFDataIter[0]; + pIter->type = SNAP_DATA_FILE_ITER; + + code = tsdbReadBlockIdx(pReader->pDataFReader, pIter->aBlockIdx); + if (code) goto _err; + + for (pIter->iBlockIdx = 0; pIter->iBlockIdx < taosArrayGetSize(pIter->aBlockIdx); pIter->iBlockIdx++) { + pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx); + + code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock); + if (code) goto _err; + + for (pIter->iBlock = 0; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) { + SDataBlk dataBlk; + tMapDataGetItemByIdx(&pIter->mBlock, pIter->iBlock, &dataBlk, tGetDataBlk); + + if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue; + + code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData); + if (code) goto _err; + + ASSERT(pIter->pBlockIdx->suid == pIter->bData.suid); + ASSERT(pIter->pBlockIdx->uid == pIter->bData.uid); + + for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) { + int64_t rowVer = pIter->bData.aVersion[pIter->iRow]; + + if (rowVer >= pReader->sver && rowVer <= pReader->ever) { + pIter->rInfo.suid = pIter->pBlockIdx->suid; + pIter->rInfo.uid = pIter->pBlockIdx->uid; + pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); + goto _add_iter_and_break; + } + } + } + + continue; + + _add_iter_and_break: + tRBTreePut(&pReader->rbt, (SRBTreeNode*)pIter); + break; + } + + // .stt file + pIter = &pReader->aFDataIter[1]; + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + pIter->type = SNAP_STT_FILE_ITER; + pIter->iStt = iStt; + + code = tsdbReadSttBlk(pReader->pDataFReader, iStt, pIter->aSttBlk); + if (code) goto _err; + + for (pIter->iSttBlk = 0; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) { + SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); + + if (pSttBlk->minVer > pReader->ever) continue; + if (pSttBlk->maxVer < pReader->sver) continue; + + code = tsdbReadSttBlock(pReader->pDataFReader, iStt, pSttBlk, &pIter->bData); + if (code) goto _err; + + for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) { + int64_t rowVer = pIter->bData.aVersion[pIter->iRow]; + + if (rowVer >= pReader->sver && rowVer <= pReader->ever) { + pIter->rInfo.suid = pIter->bData.suid; + pIter->rInfo.uid = pIter->bData.uid; + pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); + goto _add_iter; + } + } + } + + continue; + + _add_iter: + tRBTreePut(&pReader->rbt, (SRBTreeNode*)pIter); + pIter++; + } + + tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pReader->pTsdb->pVnode), + pReader->pTsdb->path, pReader->fid); + return code; + +_err: + tsdbError("vgId:%d vnode snapshot tsdb snap read open file failed since %s", TD_VID(pReader->pTsdb->pVnode), + tstrerror(code)); + return code; +} + +static SRowInfo* tsdbSnapGetRow(STsdbSnapReader* pReader) { return pReader->pIter ? &pReader->pIter->rInfo : NULL; } + +static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) { + int32_t code = 0; + + if (pReader->pIter) { + SFDataIter* pIter = pReader->pIter; + + while (true) { + _find_row: + for (pIter->iRow++; pIter->iRow < pIter->bData.nRow; pIter->iRow++) { + int64_t rowVer = pIter->bData.aVersion[pIter->iRow]; + + if (rowVer >= pReader->sver && rowVer <= pReader->ever) { + pIter->rInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; + pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); + goto _out; + } + } + + if (pIter->type == SNAP_DATA_FILE_ITER) { + while (true) { + for (pIter->iBlock++; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) { + SDataBlk dataBlk; + tMapDataGetItemByIdx(&pIter->mBlock, pIter->iBlock, &dataBlk, tGetDataBlk); + + if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue; + + code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData); + if (code) goto _err; + + pIter->iRow = -1; + goto _find_row; + } + + pIter->iBlockIdx++; + if (pIter->iBlockIdx >= taosArrayGetSize(pIter->aBlockIdx)) break; + + pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx); + code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock); + if (code) goto _err; + pIter->iBlock = -1; + } + + pReader->pIter = NULL; + } else if (pIter->type == SNAP_STT_FILE_ITER) { + for (pIter->iSttBlk++; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) { + SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); + + if (pSttBlk->minVer > pReader->ever || pSttBlk->maxVer < pReader->sver) continue; + + code = tsdbReadSttBlock(pReader->pDataFReader, pIter->iStt, pSttBlk, &pIter->bData); + if (code) goto _err; + + pIter->iRow = -1; + goto _find_row; + } + + pReader->pIter = NULL; + } else { + ASSERT(0); + } + } + + _out: + pIter = (SFDataIter*)tRBTreeMin(&pReader->rbt); + if (pReader->pIter && pIter) { + int32_t c = tRowInfoCmprFn(&pReader->pIter->rInfo, &pIter->rInfo); + if (c > 0) { + tRBTreePut(&pReader->rbt, (SRBTreeNode*)pReader->pIter); + pReader->pIter = NULL; + } else { + ASSERT(c); + } + } + } + + if (pReader->pIter == NULL) { + pReader->pIter = (SFDataIter*)tRBTreeMin(&pReader->rbt); + if (pReader->pIter) { + tRBTreeDrop(&pReader->rbt, (SRBTreeNode*)pReader->pIter); + } + } + + return code; + +_err: + return code; +} + +static int32_t tsdbSnapCmprData(STsdbSnapReader* pReader, uint8_t** ppData) { + int32_t code = 0; + + ASSERT(pReader->bData.nRow); + + int32_t aBufN[5] = {0}; + code = tCmprBlockData(&pReader->bData, TWO_STAGE_COMP, NULL, NULL, pReader->aBuf, aBufN); + if (code) goto _exit; + + int32_t size = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; + *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); + if (*ppData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + SSnapDataHdr* pHdr = (SSnapDataHdr*)*ppData; + pHdr->type = SNAP_DATA_TSDB; + pHdr->size = size; + + memcpy(pHdr->data, pReader->aBuf[3], aBufN[3]); + memcpy(pHdr->data + aBufN[3], pReader->aBuf[2], aBufN[2]); + if (aBufN[1]) { + memcpy(pHdr->data + aBufN[3] + aBufN[2], pReader->aBuf[1], aBufN[1]); + } + if (aBufN[0]) { + memcpy(pHdr->data + aBufN[3] + aBufN[2] + aBufN[1], pReader->aBuf[0], aBufN[0]); + } + +_exit: + return code; +} + static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; STsdb* pTsdb = pReader->pTsdb; while (true) { if (pReader->pDataFReader == NULL) { - // next - SDFileSet dFileSet = {.fid = pReader->fid}; - SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT); - if (pSet == NULL) goto _exit; - pReader->fid = pSet->fid; - - // load - code = tsdbDataFReaderOpen(&pReader->pDataFReader, pTsdb, pSet); + code = tsdbSnapReadOpenFile(pReader); if (code) goto _err; - - code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); - if (code) goto _err; - - code = tsdbReadSttBlk(pReader->pDataFReader, 0, pReader->aSstBlk); - if (code) goto _err; - - // init - pReader->iBlockIdx = 0; - if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) { - pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); - - code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock); - if (code) goto _err; - - pReader->iBlock = 0; - } else { - pReader->pBlockIdx = NULL; - } - - pReader->iBlockL = 0; - while (true) { - if (pReader->iBlockL >= taosArrayGetSize(pReader->aSstBlk)) { - pReader->pSstBlk = NULL; - break; - } - - pReader->pSstBlk = (SSttBlk*)taosArrayGet(pReader->aSstBlk, pReader->iBlockL); - if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) { - // TODO - break; - } - - pReader->iBlockL++; - } - - tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pTsdb->pVnode), pTsdb->path, - pReader->fid); } - while (true) { - if (pReader->pBlockIdx && pReader->pSstBlk) { - TABLEID id = {.suid = pReader->pSstBlk->suid, .uid = pReader->pSstBlk->minUid}; + if (pReader->pDataFReader == NULL) break; - ASSERT(0); + SRowInfo* pRowInfo = tsdbSnapGetRow(pReader); + if (pRowInfo == NULL) { + tsdbDataFReaderClose(&pReader->pDataFReader); + continue; + } - // if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) { - // // TODO - // } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) { - // // TODO - // } else { - // // TODO - // } - } else if (pReader->pBlockIdx) { - while (pReader->iBlock < pReader->mBlock.nItem) { - SDataBlk block; - tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetDataBlk); + TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; + SBlockData* pBlockData = &pReader->bData; - if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) { - // load data (todo) - } + code = tsdbUpdateTableSchema(pTsdb->pVnode->pMeta, id.suid, id.uid, &pReader->skmTable); + if (code) goto _err; - // next - pReader->iBlock++; - if (*ppData) break; - } + code = tBlockDataInit(pBlockData, id.suid, id.uid, pReader->skmTable.pTSchema); + if (code) goto _err; - if (pReader->iBlock >= pReader->mBlock.nItem) { - pReader->iBlockIdx++; - if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) { - pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); + while (pRowInfo->suid == id.suid && pRowInfo->uid == id.uid) { + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pRowInfo->uid); + if (code) goto _err; - code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock); - if (code) goto _err; + code = tsdbSnapNextRow(pReader); + if (code) goto _err; - pReader->iBlock = 0; - } else { - pReader->pBlockIdx = NULL; - } - } - - if (*ppData) goto _exit; - } else if (pReader->pSstBlk) { - while (pReader->pSstBlk) { - if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) { - // load data (todo) - } - - // next - pReader->iBlockL++; - if (pReader->iBlockL < taosArrayGetSize(pReader->aSstBlk)) { - pReader->pSstBlk = (SSttBlk*)taosArrayGetSize(pReader->aSstBlk); - } else { - pReader->pSstBlk = NULL; - } - - if (*ppData) goto _exit; - } - } else { + pRowInfo = tsdbSnapGetRow(pReader); + if (pRowInfo == NULL) { tsdbDataFReaderClose(&pReader->pDataFReader); break; } + + if (pBlockData->nRow >= 4096) break; } + + code = tsdbSnapCmprData(pReader, ppData); + if (code) goto _err; + + break; } -_exit: return code; _err: @@ -216,7 +394,6 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { size += tPutDelData(NULL, pDelData); } } - if (size == 0) continue; // org data @@ -292,23 +469,33 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type goto _err; } + // data pReader->fid = INT32_MIN; - pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pReader->aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + for (int32_t iIter = 0; iIter < sizeof(pReader->aFDataIter) / sizeof(pReader->aFDataIter[0]); iIter++) { + SFDataIter* pIter = &pReader->aFDataIter[iIter]; + + if (iIter == 0) { + pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pIter->aBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } else { + pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); + if (pIter->aSttBlk == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + + code = tBlockDataCreate(&pIter->bData); + if (code) goto _err; } - pReader->aSstBlk = taosArrayInit(0, sizeof(SSttBlk)); - if (pReader->aSstBlk == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - pReader->mBlock = tMapDataInit(); - code = tBlockDataCreate(&pReader->oBlockData); - if (code) goto _err; - code = tBlockDataCreate(&pReader->nBlockData); + + code = tBlockDataCreate(&pReader->bData); if (code) goto _err; + // del pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); if (pReader->aDelIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -335,18 +522,26 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { int32_t code = 0; STsdbSnapReader* pReader = *ppReader; - if (pReader->pDataFReader) { - tsdbDataFReaderClose(&pReader->pDataFReader); - } - taosArrayDestroy(pReader->aSstBlk); - taosArrayDestroy(pReader->aBlockIdx); - tMapDataClear(&pReader->mBlock); - tBlockDataDestroy(&pReader->oBlockData, 1); - tBlockDataDestroy(&pReader->nBlockData, 1); + // data + if (pReader->pDataFReader) tsdbDataFReaderClose(&pReader->pDataFReader); + for (int32_t iIter = 0; iIter < sizeof(pReader->aFDataIter) / sizeof(pReader->aFDataIter[0]); iIter++) { + SFDataIter* pIter = &pReader->aFDataIter[iIter]; - if (pReader->pDelFReader) { - tsdbDelFReaderClose(&pReader->pDelFReader); + if (iIter == 0) { + taosArrayDestroy(pIter->aBlockIdx); + tMapDataClear(&pIter->mBlock); + } else { + taosArrayDestroy(pIter->aSttBlk); + } + + tBlockDataDestroy(&pIter->bData, 1); } + + tBlockDataDestroy(&pReader->bData, 1); + tTSchemaDestroy(pReader->skmTable.pTSchema); + + // del + if (pReader->pDelFReader) tsdbDelFReaderClose(&pReader->pDelFReader); taosArrayDestroy(pReader->aDelIdx); taosArrayDestroy(pReader->aDelData); @@ -354,6 +549,10 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { tsdbInfo("vgId:%d, vnode snapshot tsdb reader closed for %s", TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path); + for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(pReader->aBuf[0]); iBuf++) { + tFree(pReader->aBuf[iBuf]); + } + taosMemoryFree(pReader); *ppReader = NULL; return code; @@ -410,40 +609,37 @@ struct STsdbSnapWriter { STsdbFS fs; // config - int32_t minutes; - int8_t precision; - int32_t minRow; - int32_t maxRow; - int8_t cmprAlg; - int64_t commitID; - + int32_t minutes; + int8_t precision; + int32_t minRow; + int32_t maxRow; + int8_t cmprAlg; + int64_t commitID; uint8_t* aBuf[5]; + // for data file SBlockData bData; - - int32_t fid; - SDataFReader* pDataFReader; - SArray* aBlockIdx; // SArray - int32_t iBlockIdx; - SBlockIdx* pBlockIdx; - SMapData mBlock; // SMapData - int32_t iBlock; - SBlockData* pBlockData; - int32_t iRow; - SBlockData bDataR; - SArray* aSstBlk; // SArray - int32_t iBlockL; - SBlockData lDataR; - - SDataFWriter* pDataFWriter; - SBlockIdx* pBlockIdxW; // NULL when no committing table - SDataBlk blockW; - SBlockData bDataW; - SBlockIdx blockIdxW; - - SMapData mBlockW; // SMapData - SArray* aBlockIdxW; // SArray - SArray* aBlockLW; // SArray + int32_t fid; + TABLEID id; + SSkmInfo skmTable; + struct { + SDataFReader* pReader; + SArray* aBlockIdx; + int32_t iBlockIdx; + SBlockIdx* pBlockIdx; + SMapData mDataBlk; + int32_t iDataBlk; + SBlockData bData; + int32_t iRow; + } dReader; + struct { + SDataFWriter* pWriter; + SArray* aBlockIdx; + SMapData mDataBlk; + SArray* aSttBlk; + SBlockData bData; + SBlockData sData; + } dWriter; // for del file SDelFReader* pDelFReader; @@ -454,520 +650,447 @@ struct STsdbSnapWriter { SArray* aDelIdxW; }; +// SNAP_DATA_TSDB +extern int32_t tsdbWriteDataBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SMapData* mDataBlk, int8_t cmprAlg); +extern int32_t tsdbWriteSttBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SArray* aSttBlk, int8_t cmprAlg); + +static int32_t tsdbSnapNextTableData(STsdbSnapWriter* pWriter) { + int32_t code = 0; + + ASSERT(pWriter->dReader.iRow >= pWriter->dReader.bData.nRow); + + if (pWriter->dReader.iBlockIdx < taosArrayGetSize(pWriter->dReader.aBlockIdx)) { + pWriter->dReader.pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->dReader.aBlockIdx, pWriter->dReader.iBlockIdx); + + code = tsdbReadDataBlk(pWriter->dReader.pReader, pWriter->dReader.pBlockIdx, &pWriter->dReader.mDataBlk); + if (code) goto _exit; + + pWriter->dReader.iBlockIdx++; + } else { + pWriter->dReader.pBlockIdx = NULL; + tMapDataReset(&pWriter->dReader.mDataBlk); + } + pWriter->dReader.iDataBlk = 0; // point to the next one + tBlockDataReset(&pWriter->dReader.bData); + pWriter->dReader.iRow = 0; + +_exit: + return code; +} + +static int32_t tsdbSnapWriteCopyData(STsdbSnapWriter* pWriter, TABLEID* pId) { + int32_t code = 0; + + while (true) { + if (pWriter->dReader.pBlockIdx == NULL) break; + if (tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, pId) >= 0) break; + + SBlockIdx blkIdx = *pWriter->dReader.pBlockIdx; + code = tsdbWriteDataBlk(pWriter->dWriter.pWriter, &pWriter->dReader.mDataBlk, &blkIdx); + if (code) goto _exit; + + if (taosArrayPush(pWriter->dWriter.aBlockIdx, &blkIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + code = tsdbSnapNextTableData(pWriter); + if (code) goto _exit; + } + +_exit: + return code; +} + +static int32_t tsdbSnapWriteTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pId) { + int32_t code = 0; + + code = tsdbSnapWriteCopyData(pWriter, pId); + if (code) goto _err; + + pWriter->id.suid = pId->suid; + pWriter->id.uid = pId->uid; + + code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pId->suid, pId->uid, &pWriter->skmTable); + if (code) goto _err; + + tMapDataReset(&pWriter->dWriter.mDataBlk); + code = tBlockDataInit(&pWriter->dWriter.bData, pId->suid, pId->uid, pWriter->skmTable.pTSchema); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code)); + return code; +} + static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { int32_t code = 0; - ASSERT(pWriter->pDataFWriter); + if (pWriter->id.suid == 0 && pWriter->id.uid == 0) return code; - if (pWriter->pBlockIdxW == NULL) goto _exit; + int32_t c = 1; + if (pWriter->dReader.pBlockIdx) { + c = tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, &pWriter->id); + ASSERT(c >= 0); + } - // consume remain rows - if (pWriter->pBlockData) { - ASSERT(pWriter->iRow < pWriter->pBlockData->nRow); - while (pWriter->iRow < pWriter->pBlockData->nRow) { - code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL, - 0); // todo + if (c == 0) { + SBlockData* pBData = &pWriter->dWriter.bData; + + for (; pWriter->dReader.iRow < pWriter->dReader.bData.nRow; pWriter->dReader.iRow++) { + TSDBROW row = tsdbRowFromBlockData(&pWriter->dReader.bData, pWriter->dReader.iRow); + + code = tBlockDataAppendRow(pBData, &row, NULL, pWriter->id.uid); if (code) goto _err; - if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) { - // pWriter->blockW.last = 0; - // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - // &pWriter->blockW, pWriter->cmprAlg); + if (pBData->nRow >= pWriter->maxRow) { + code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, pBData, &pWriter->dWriter.mDataBlk, pWriter->cmprAlg); if (code) goto _err; - - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk); - if (code) goto _err; - - tDataBlkReset(&pWriter->blockW); - tBlockDataClear(&pWriter->bDataW); - } - - pWriter->iRow++; - } - } - - // write remain data if has - if (pWriter->bDataW.nRow > 0) { - // pWriter->blockW.last = 0; - if (pWriter->bDataW.nRow < pWriter->minRow) { - if (pWriter->iBlock > pWriter->mBlock.nItem) { - // pWriter->blockW.last = 1; } } - // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - // &pWriter->blockW, pWriter->cmprAlg); - // if (code) goto _err; - - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk); - if (code) goto _err; - } - - while (true) { - if (pWriter->iBlock >= pWriter->mBlock.nItem) break; - - SDataBlk block; - tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetDataBlk); - - // if (block.last) { - // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); - // if (code) goto _err; - - // tBlockReset(&block); - // block.last = 1; - // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block, - // pWriter->cmprAlg); - // if (code) goto _err; - // } - - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk); + code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, pBData, &pWriter->dWriter.mDataBlk, pWriter->cmprAlg); if (code) goto _err; - pWriter->iBlock++; - } + for (; pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem; pWriter->dReader.iDataBlk++) { + SDataBlk dataBlk; + tMapDataGetItemByIdx(&pWriter->dReader.mDataBlk, pWriter->dReader.iDataBlk, &dataBlk, tGetDataBlk); - // SDataBlk - // code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW); - // if (code) goto _err; - - // SBlockIdx - if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - -_exit: - tsdbInfo("vgId:%d, tsdb snapshot write table data end for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); - return code; - -_err: - tsdbError("vgId:%d, tsdb snapshot write table data end for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), - pWriter->pTsdb->path, tstrerror(code)); - return code; -} - -static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* pBlockIdx) { - int32_t code = 0; - - code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock); - if (code) goto _err; - - // SBlockData - SDataBlk block; - tMapDataReset(&pWriter->mBlockW); - for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) { - tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetDataBlk); - - // if (block.last) { - // code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); - // if (code) goto _err; - - // tBlockReset(&block); - // block.last = 1; - // code = - // tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block, - // pWriter->cmprAlg); - // if (code) goto _err; - // } - - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk); - if (code) goto _err; - } - - // SDataBlk - SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; - code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx); - if (code) goto _err; - - // SBlockIdx - if (taosArrayPush(pWriter->aBlockIdxW, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - -_exit: - return code; - -_err: - tsdbError("vgId:%d, tsdb snapshot move write table data for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), - pWriter->pTsdb->path, tstrerror(code)); - return code; -} - -static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { - int32_t code = 0; - SBlockData* pBlockData = &pWriter->bData; - int32_t iRow = 0; - TSDBROW row; - TSDBROW* pRow = &row; - - // // correct schema - // code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData); - // if (code) goto _err; - - // loop to merge - *pRow = tsdbRowFromBlockData(pBlockData, iRow); - while (true) { - if (pRow == NULL) break; - - if (pWriter->pBlockData) { - ASSERT(pWriter->iRow < pWriter->pBlockData->nRow); - - int32_t c = tsdbRowCmprFn(pRow, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow)); - - ASSERT(c); - - if (c < 0) { - // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); - // if (code) goto _err; - - iRow++; - if (iRow < pWriter->pBlockData->nRow) { - *pRow = tsdbRowFromBlockData(pBlockData, iRow); - } else { - pRow = NULL; - } - } else if (c > 0) { - // code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), - // NULL); if (code) goto _err; - - pWriter->iRow++; - if (pWriter->iRow >= pWriter->pBlockData->nRow) { - pWriter->pBlockData = NULL; - } - } - } else { - TSDBKEY key = TSDBROW_KEY(pRow); - - while (true) { - if (pWriter->iBlock >= pWriter->mBlock.nItem) break; - - SDataBlk block; - int32_t c; - - tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetDataBlk); - - // if (block.last) { - // pWriter->pBlockData = &pWriter->bDataR; - - // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, - // NULL); if (code) goto _err; pWriter->iRow = 0; - - // pWriter->iBlock++; - // break; - // } - - c = tsdbKeyCmprFn(&block.maxKey, &key); - - ASSERT(c); - - if (c < 0) { - if (pWriter->bDataW.nRow) { - // pWriter->blockW.last = 0; - // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - // &pWriter->blockW, pWriter->cmprAlg); - // if (code) goto _err; - - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk); - if (code) goto _err; - - tDataBlkReset(&pWriter->blockW); - tBlockDataClear(&pWriter->bDataW); - } - - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk); - if (code) goto _err; - - pWriter->iBlock++; - } else { - c = tsdbKeyCmprFn(&tBlockDataLastKey(pBlockData), &block.minKey); - - ASSERT(c); - - if (c > 0) { - pWriter->pBlockData = &pWriter->bDataR; - // code = - // tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, - // NULL); - // if (code) goto _err; - pWriter->iRow = 0; - - pWriter->iBlock++; - } - break; - } - } - - if (pWriter->pBlockData) continue; - - // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); - // if (code) goto _err; - - iRow++; - if (iRow < pBlockData->nRow) { - *pRow = tsdbRowFromBlockData(pBlockData, iRow); - } else { - pRow = NULL; - } - } - - _check_write: - if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue; - - _write_block: - // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - // &pWriter->blockW, pWriter->cmprAlg); - // if (code) goto _err; - - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk); - if (code) goto _err; - - tDataBlkReset(&pWriter->blockW); - tBlockDataClear(&pWriter->bDataW); - } - - return code; - -_err: - tsdbError("vgId:%d, vnode snapshot tsdb write table data impl for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), - pWriter->pTsdb->path, tstrerror(code)); - return code; -} - -static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { - int32_t code = 0; - SBlockData* pBlockData = &pWriter->bData; - TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData); - TSDBKEY keyLast = tBlockDataLastKey(pBlockData); - - // end last table write if should - if (pWriter->pBlockIdxW) { - int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id); - if (c < 0) { - // end - code = tsdbSnapWriteTableDataEnd(pWriter); + code = tMapDataPutItem(&pWriter->dWriter.mDataBlk, &dataBlk, tPutDataBlk); if (code) goto _err; + } - // reset - pWriter->pBlockIdxW = NULL; - } else if (c > 0) { - ASSERT(0); + code = tsdbSnapNextTableData(pWriter); + if (code) goto _err; + } + + if (pWriter->dWriter.mDataBlk.nItem) { + SBlockIdx blockIdx = {.suid = pWriter->id.suid, .uid = pWriter->id.uid}; + code = tsdbWriteDataBlk(pWriter->dWriter.pWriter, &pWriter->dWriter.mDataBlk, &blockIdx); + + if (taosArrayPush(pWriter->dWriter.aBlockIdx, &blockIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } } - // start new table data write if need - if (pWriter->pBlockIdxW == NULL) { - // write table data ahead - while (true) { - if (pWriter->iBlockIdx >= taosArrayGetSize(pWriter->aBlockIdx)) break; + pWriter->id.suid = 0; + pWriter->id.uid = 0; - SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); - int32_t c = tTABLEIDCmprFn(pBlockIdx, &id); - - if (c >= 0) break; - - code = tsdbSnapMoveWriteTableData(pWriter, pBlockIdx); - if (code) goto _err; - - pWriter->iBlockIdx++; - } - - // reader - pWriter->pBlockIdx = NULL; - if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { - ASSERT(pWriter->pDataFReader); - - SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); - int32_t c = tTABLEIDCmprFn(pBlockIdx, &id); - - ASSERT(c >= 0); - - if (c == 0) { - pWriter->pBlockIdx = pBlockIdx; - pWriter->iBlockIdx++; - } - } - - if (pWriter->pBlockIdx) { - code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock); - if (code) goto _err; - } else { - tMapDataReset(&pWriter->mBlock); - } - pWriter->iBlock = 0; - pWriter->pBlockData = NULL; - pWriter->iRow = 0; - - // writer - pWriter->pBlockIdxW = &pWriter->blockIdxW; - pWriter->pBlockIdxW->suid = id.suid; - pWriter->pBlockIdxW->uid = id.uid; - - tDataBlkReset(&pWriter->blockW); - tBlockDataReset(&pWriter->bDataW); - tMapDataReset(&pWriter->mBlockW); - } - - ASSERT(pWriter->pBlockIdxW && pWriter->pBlockIdxW->suid == id.suid && pWriter->pBlockIdxW->uid == id.uid); - ASSERT(pWriter->pBlockIdx == NULL || (pWriter->pBlockIdx->suid == id.suid && pWriter->pBlockIdx->uid == id.uid)); - - code = tsdbSnapWriteTableDataImpl(pWriter); - if (code) goto _err; - -_exit: - tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode), - pWriter->pTsdb->path); return code; _err: - tsdbError("vgId:%d, vnode snapshot tsdb write data impl for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), - pWriter->pTsdb->path, tstrerror(code)); return code; } -static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { +static int32_t tsdbSnapWriteOpenFile(STsdbSnapWriter* pWriter, int32_t fid) { int32_t code = 0; STsdb* pTsdb = pWriter->pTsdb; - if (pWriter->pDataFWriter == NULL) goto _exit; + ASSERT(pWriter->dWriter.pWriter == NULL); + + pWriter->fid = fid; + pWriter->id = (TABLEID){0}; + SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ); + + // Reader + if (pSet) { + code = tsdbDataFReaderOpen(&pWriter->dReader.pReader, pWriter->pTsdb, pSet); + if (code) goto _err; + + code = tsdbReadBlockIdx(pWriter->dReader.pReader, pWriter->dReader.aBlockIdx); + if (code) goto _err; + } else { + ASSERT(pWriter->dReader.pReader == NULL); + taosArrayClear(pWriter->dReader.aBlockIdx); + } + pWriter->dReader.iBlockIdx = 0; // point to the next one + code = tsdbSnapNextTableData(pWriter); + if (code) goto _err; + + // Writer + SHeadFile fHead = {.commitID = pWriter->commitID}; + SDataFile fData = {.commitID = pWriter->commitID}; + SSmaFile fSma = {.commitID = pWriter->commitID}; + SSttFile fStt = {.commitID = pWriter->commitID}; + SDFileSet wSet = {.fid = pWriter->fid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma}; + if (pSet) { + wSet.diskId = pSet->diskId; + fData = *pSet->pDataF; + fSma = *pSet->pSmaF; + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + wSet.aSttF[iStt] = pSet->aSttF[iStt]; + } + wSet.nSttF = pSet->nSttF + 1; // TODO: fix pSet->nSttF == pTsdb->maxFile + } else { + SDiskID did = {0}; + tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did); + tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); + wSet.diskId = did; + wSet.nSttF = 1; + } + wSet.aSttF[wSet.nSttF - 1] = &fStt; + + code = tsdbDataFWriterOpen(&pWriter->dWriter.pWriter, pWriter->pTsdb, &wSet); + if (code) goto _err; + taosArrayClear(pWriter->dWriter.aBlockIdx); + tMapDataReset(&pWriter->dWriter.mDataBlk); + taosArrayClear(pWriter->dWriter.aSttBlk); + tBlockDataReset(&pWriter->dWriter.bData); + tBlockDataReset(&pWriter->dWriter.sData); + + return code; + +_err: + return code; +} + +static int32_t tsdbSnapWriteCloseFile(STsdbSnapWriter* pWriter) { + int32_t code = 0; + + ASSERT(pWriter->dWriter.pWriter); - // finish current table code = tsdbSnapWriteTableDataEnd(pWriter); if (code) goto _err; - // move remain table - while (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { - code = tsdbSnapMoveWriteTableData(pWriter, (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx)); - if (code) goto _err; - - pWriter->iBlockIdx++; - } - - // write remain stuff - if (taosArrayGetSize(pWriter->aBlockLW) > 0) { - code = tsdbWriteSttBlk(pWriter->pDataFWriter, pWriter->aBlockIdxW); - if (code) goto _err; - } - - if (taosArrayGetSize(pWriter->aBlockIdx) > 0) { - code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW); - if (code) goto _err; - } - - code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet); + // copy remain table data + TABLEID id = {.suid = INT64_MAX, .uid = INT64_MAX}; + code = tsdbSnapWriteCopyData(pWriter, &id); if (code) goto _err; - code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1); + code = + tsdbWriteSttBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.sData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg); if (code) goto _err; - if (pWriter->pDataFReader) { - code = tsdbDataFReaderClose(&pWriter->pDataFReader); + // Indices + code = tsdbWriteBlockIdx(pWriter->dWriter.pWriter, pWriter->dWriter.aBlockIdx); + if (code) goto _err; + + code = tsdbWriteSttBlk(pWriter->dWriter.pWriter, pWriter->dWriter.aSttBlk); + if (code) goto _err; + + code = tsdbUpdateDFileSetHeader(pWriter->dWriter.pWriter); + if (code) goto _err; + + code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->dWriter.pWriter->wSet); + if (code) goto _err; + + code = tsdbDataFWriterClose(&pWriter->dWriter.pWriter, 1); + if (code) goto _err; + + if (pWriter->dReader.pReader) { + code = tsdbDataFReaderClose(&pWriter->dReader.pReader); if (code) goto _err; } _exit: - tsdbInfo("vgId:%d, vnode snapshot tsdb writer data end for %s", TD_VID(pTsdb->pVnode), pTsdb->path); return code; _err: - tsdbError("vgId:%d, vnode snapshot tsdb writer data end for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); + return code; +} + +static int32_t tsdbSnapWriteToDataFile(STsdbSnapWriter* pWriter, int32_t iRow, int8_t* done) { + int32_t code = 0; + + SBlockData* pBData = &pWriter->bData; + TABLEID id = {.suid = pBData->suid, .uid = pBData->uid ? pBData->uid : pBData->aUid[iRow]}; + TSDBROW row = tsdbRowFromBlockData(pBData, iRow); + TSDBKEY key = TSDBROW_KEY(&row); + + *done = 0; + while (pWriter->dReader.iRow < pWriter->dReader.bData.nRow || + pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem) { + // Merge row by row + for (; pWriter->dReader.iRow < pWriter->dReader.bData.nRow; pWriter->dReader.iRow++) { + TSDBROW trow = tsdbRowFromBlockData(&pWriter->dReader.bData, pWriter->dReader.iRow); + TSDBKEY tKey = TSDBROW_KEY(&trow); + + ASSERT(pWriter->dReader.bData.suid == id.suid && pWriter->dReader.bData.uid == id.uid); + + int32_t c = tsdbKeyCmprFn(&key, &tKey); + if (c < 0) { + code = tBlockDataAppendRow(&pWriter->dWriter.bData, &row, NULL, id.uid); + if (code) goto _err; + } else if (c > 0) { + code = tBlockDataAppendRow(&pWriter->dWriter.bData, &trow, NULL, id.uid); + if (code) goto _err; + } else { + ASSERT(0); + } + + if (pWriter->dWriter.bData.nRow >= pWriter->maxRow) { + code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk, + pWriter->cmprAlg); + if (code) goto _err; + } + + if (c < 0) { + *done = 1; + goto _exit; + } + } + + // Merge row by block + SDataBlk tDataBlk = {.minKey = key, .maxKey = key}; + for (; pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem; pWriter->dReader.iDataBlk++) { + SDataBlk dataBlk; + tMapDataGetItemByIdx(&pWriter->dReader.mDataBlk, pWriter->dReader.iDataBlk, &dataBlk, tGetDataBlk); + + int32_t c = tDataBlkCmprFn(&dataBlk, &tDataBlk); + if (c < 0) { + code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk, + pWriter->cmprAlg); + if (code) goto _err; + + code = tMapDataPutItem(&pWriter->dWriter.mDataBlk, &dataBlk, tPutDataBlk); + if (code) goto _err; + } else if (c > 0) { + code = tBlockDataAppendRow(&pWriter->dWriter.bData, &row, NULL, id.uid); + if (code) goto _err; + + if (pWriter->dWriter.bData.nRow >= pWriter->maxRow) { + code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk, + pWriter->cmprAlg); + if (code) goto _err; + } + + *done = 1; + goto _exit; + } else { + code = tsdbReadDataBlockEx(pWriter->dReader.pReader, &dataBlk, &pWriter->dReader.bData); + if (code) goto _err; + pWriter->dReader.iRow = 0; + + pWriter->dReader.iDataBlk++; + break; + } + } + } + +_exit: + return code; + +_err: + tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code)); + return code; +} + +static int32_t tsdbSnapWriteToSttFile(STsdbSnapWriter* pWriter, int32_t iRow) { + int32_t code = 0; + + TABLEID id = {.suid = pWriter->bData.suid, + .uid = pWriter->bData.uid ? pWriter->bData.uid : pWriter->bData.aUid[iRow]}; + TSDBROW row = tsdbRowFromBlockData(&pWriter->bData, iRow); + SBlockData* pBData = &pWriter->dWriter.sData; + + if (pBData->suid || pBData->uid) { + if (!TABLE_SAME_SCHEMA(pBData->suid, pBData->uid, id.suid, id.uid)) { + code = tsdbWriteSttBlock(pWriter->dWriter.pWriter, pBData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg); + if (code) goto _err; + + pBData->suid = 0; + pBData->uid = 0; + } + } + + if (pBData->suid == 0 && pBData->uid == 0) { + code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pWriter->id.suid, pWriter->id.uid, &pWriter->skmTable); + if (code) goto _err; + + code = tBlockDataInit(pBData, pWriter->id.suid, pWriter->id.suid ? 0 : pWriter->id.uid, pWriter->skmTable.pTSchema); + if (code) goto _err; + } + + code = tBlockDataAppendRow(pBData, &row, NULL, id.uid); + if (code) goto _err; + + if (pBData->nRow >= pWriter->maxRow) { + code = tsdbWriteSttBlock(pWriter->dWriter.pWriter, pBData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg); + if (code) goto _err; + } + +_exit: + return code; + +_err: + return code; +} + +static int32_t tsdbSnapWriteRowData(STsdbSnapWriter* pWriter, int32_t iRow) { + int32_t code = 0; + + SBlockData* pBlockData = &pWriter->bData; + TABLEID id = {.suid = pBlockData->suid, .uid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[iRow]}; + + // End last table data write if need + if (tTABLEIDCmprFn(&pWriter->id, &id) != 0) { + code = tsdbSnapWriteTableDataEnd(pWriter); + if (code) goto _err; + } + + // Start new table data write if need + if (pWriter->id.suid == 0 && pWriter->id.uid == 0) { + code = tsdbSnapWriteTableDataStart(pWriter, &id); + if (code) goto _err; + } + + // Merge with .data file data + int8_t done = 0; + if (pWriter->dReader.pBlockIdx && tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, &id) == 0) { + code = tsdbSnapWriteToDataFile(pWriter, iRow, &done); + if (code) goto _err; + } + + // Append to the .stt data block (todo: check if need to set/reload sst block) + if (!done) { + code = tsdbSnapWriteToSttFile(pWriter, iRow); + if (code) goto _err; + } + +_exit: + return code; + +_err: + tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code)); return code; } static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - STsdb* pTsdb = pWriter->pTsdb; - SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; - TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); - int64_t n; - - // decode + int32_t code = 0; + STsdb* pTsdb = pWriter->pTsdb; SBlockData* pBlockData = &pWriter->bData; - code = tDecmprBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pHdr->size - sizeof(TABLEID), pBlockData, - pWriter->aBuf); + + // Decode data + SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; + code = tDecmprBlockData(pHdr->data, pHdr->size, pBlockData, pWriter->aBuf); if (code) goto _err; - // open file - TSDBKEY keyFirst = {.version = pBlockData->aVersion[0], .ts = pBlockData->aTSKEY[0]}; - TSDBKEY keyLast = {.version = pBlockData->aVersion[pBlockData->nRow - 1], - .ts = pBlockData->aTSKEY[pBlockData->nRow - 1]}; + ASSERT(pBlockData->nRow > 0); - int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision); - ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision)); - if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) { - // end last file data write if need - code = tsdbSnapWriteDataEnd(pWriter); - if (code) goto _err; + // Loop to handle each row + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + TSKEY ts = pBlockData->aTSKEY[iRow]; + int32_t fid = tsdbKeyFid(ts, pWriter->minutes, pWriter->precision); - pWriter->fid = fid; + if (pWriter->dWriter.pWriter == NULL || pWriter->fid != fid) { + if (pWriter->dWriter.pWriter) { + ASSERT(fid > pWriter->fid); - // read - SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ); - if (pSet) { - code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet); + code = tsdbSnapWriteCloseFile(pWriter); + if (code) goto _err; + } + + code = tsdbSnapWriteOpenFile(pWriter, fid); if (code) goto _err; - - code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx); - if (code) goto _err; - - code = tsdbReadSttBlk(pWriter->pDataFReader, 0, pWriter->aSstBlk); - if (code) goto _err; - } else { - ASSERT(pWriter->pDataFReader == NULL); - taosArrayClear(pWriter->aBlockIdx); - taosArrayClear(pWriter->aSstBlk); - } - pWriter->iBlockIdx = 0; - pWriter->pBlockIdx = NULL; - tMapDataReset(&pWriter->mBlock); - pWriter->iBlock = 0; - pWriter->pBlockData = NULL; - pWriter->iRow = 0; - pWriter->iBlockL = 0; - tBlockDataReset(&pWriter->bDataR); - tBlockDataReset(&pWriter->lDataR); - - // write - SHeadFile fHead; - SDataFile fData; - SSttFile fLast; - SSmaFile fSma; - SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aSttF[0] = &fLast, .pSmaF = &fSma}; - - if (pSet) { - wSet.diskId = pSet->diskId; - wSet.fid = fid; - wSet.nSttF = 1; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; - fData = *pSet->pDataF; - fLast = (SSttFile){.commitID = pWriter->commitID, .size = 0}; - fSma = *pSet->pSmaF; - } else { - wSet.diskId = (SDiskID){.level = 0, .id = 0}; - wSet.fid = fid; - wSet.nSttF = 1; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; - fData = (SDataFile){.commitID = pWriter->commitID, .size = 0}; - fLast = (SSttFile){.commitID = pWriter->commitID, .size = 0, .offset = 0}; - fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0}; } - code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet); + code = tsdbSnapWriteRowData(pWriter, iRow); if (code) goto _err; - - taosArrayClear(pWriter->aBlockIdxW); - taosArrayClear(pWriter->aBlockLW); - tMapDataReset(&pWriter->mBlockW); - pWriter->pBlockIdxW = NULL; - tBlockDataReset(&pWriter->bDataW); } - code = tsdbSnapWriteTableData(pWriter, id); - if (code) goto _err; - - tsdbInfo("vgId:%d, vnode snapshot tsdb write data for %s, fid:%d suid:%" PRId64 " uid:%" PRId64 " nRow:%d", - TD_VID(pTsdb->pVnode), pTsdb->path, fid, id.suid, id.suid, pBlockData->nRow); return code; _err: @@ -976,10 +1099,41 @@ _err: return code; } +// SNAP_DATA_DEL +static int32_t tsdbSnapMoveWriteDelData(STsdbSnapWriter* pWriter, TABLEID* pId) { + int32_t code = 0; + + while (true) { + if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break; + + SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); + + if (tTABLEIDCmprFn(pDelIdx, pId) >= 0) break; + + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); + if (code) goto _exit; + + SDelIdx delIdx = *pDelIdx; + code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); + if (code) goto _exit; + + if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pWriter->iDelIdx++; + } + +_exit: + return code; +} + static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; STsdb* pTsdb = pWriter->pTsdb; + // Open del file if not opened yet if (pWriter->pDelFWriter == NULL) { SDelFile* pDelFile = pWriter->fs.pDelFile; @@ -990,38 +1144,28 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR); if (code) goto _err; + } else { + taosArrayClear(pWriter->aDelIdxR); } + pWriter->iDelIdx = 0; // writer - SDelFile delFile = {.commitID = pWriter->commitID, .offset = 0, .size = 0}; + SDelFile delFile = {.commitID = pWriter->commitID}; code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &delFile, pTsdb); if (code) goto _err; + taosArrayClear(pWriter->aDelIdxW); } - // process the del data - TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); + SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; + TABLEID id = *(TABLEID*)pHdr->data; - while (true) { - if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break; - if (tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) >= 0) break; + ASSERT(pHdr->size + sizeof(SSnapDataHdr) == nData); - SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); - - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); - if (code) goto _err; - - SDelIdx delIdx = *pDelIdx; - code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); - if (code) goto _err; - - if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - pWriter->iDelIdx++; - } + // Move write data < id + code = tsdbSnapMoveWriteDelData(pWriter, &id); + if (code) goto _err; + // Merge incoming data with current if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR) && tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) == 0) { SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); @@ -1055,7 +1199,6 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 goto _err; } -_exit: return code; _err: @@ -1068,23 +1211,14 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { int32_t code = 0; STsdb* pTsdb = pWriter->pTsdb; - if (pWriter->pDelFWriter == NULL) goto _exit; + if (pWriter->pDelFWriter == NULL) return code; - for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) { - SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); + TABLEID id = {.suid = INT64_MAX, .uid = INT64_MAX}; + code = tsdbSnapMoveWriteDelData(pWriter, &id); + if (code) goto _err; - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); - if (code) goto _err; - - SDelIdx delIdx = *pDelIdx; - code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); - if (code) goto _err; - - if (taosArrayPush(pWriter->aDelIdxR, &delIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } + code = tsdbWriteDelIdx(pWriter->pDelFWriter, pWriter->aDelIdxW); + if (code) goto _err; code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter); if (code) goto _err; @@ -1100,7 +1234,6 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { if (code) goto _err; } -_exit: tsdbInfo("vgId:%d, vnode snapshot tsdb write del for %s end", TD_VID(pTsdb->pVnode), pTsdb->path); return code; @@ -1110,6 +1243,7 @@ _err: return code; } +// APIs int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) { int32_t code = 0; STsdbSnapWriter* pWriter = NULL; @@ -1135,39 +1269,38 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr pWriter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; pWriter->commitID = pTsdb->pVnode->state.commitID; - // for data file + // SNAP_DATA_TSDB code = tBlockDataCreate(&pWriter->bData); - - if (code) goto _err; - pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pWriter->aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - code = tBlockDataCreate(&pWriter->bDataR); if (code) goto _err; - pWriter->aSstBlk = taosArrayInit(0, sizeof(SSttBlk)); - if (pWriter->aSstBlk == NULL) { + pWriter->fid = INT32_MIN; + pWriter->id = (TABLEID){0}; + // Reader + pWriter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pWriter->dReader.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - - pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx)); - if (pWriter->aBlockIdxW == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - code = tBlockDataCreate(&pWriter->bDataW); + code = tBlockDataCreate(&pWriter->dReader.bData); if (code) goto _err; - pWriter->aBlockLW = taosArrayInit(0, sizeof(SSttBlk)); - if (pWriter->aBlockLW == NULL) { + // Writer + pWriter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pWriter->dWriter.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } + pWriter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); + if (pWriter->dWriter.aSttBlk == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + code = tBlockDataCreate(&pWriter->dWriter.bData); + if (code) goto _err; + code = tBlockDataCreate(&pWriter->dWriter.sData); + if (code) goto _err; - // for del file + // SNAP_DATA_DEL pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx)); if (pWriter->aDelIdxR == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1188,6 +1321,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr tsdbInfo("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path); return code; + _err: tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); @@ -1198,14 +1332,17 @@ _err: int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { int32_t code = 0; STsdbSnapWriter* pWriter = *ppWriter; + STsdb* pTsdb = pWriter->pTsdb; if (rollback) { ASSERT(0); // code = tsdbFSRollback(pWriter->pTsdb->pFS); // if (code) goto _err; } else { - code = tsdbSnapWriteDataEnd(pWriter); - if (code) goto _err; + if (pWriter->dWriter.pWriter) { + code = tsdbSnapWriteCloseFile(pWriter); + if (code) goto _err; + } code = tsdbSnapWriteDelEnd(pWriter); if (code) goto _err; @@ -1213,14 +1350,44 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { code = tsdbFSCommit1(pWriter->pTsdb, &pWriter->fs); if (code) goto _err; + // lock + taosThreadRwlockWrlock(&pTsdb->rwLock); + code = tsdbFSCommit2(pWriter->pTsdb, &pWriter->fs); - if (code) goto _err; + if (code) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _err; + } + + // unlock + taosThreadRwlockUnlock(&pTsdb->rwLock); } + // SNAP_DATA_DEL + taosArrayDestroy(pWriter->aDelIdxW); + taosArrayDestroy(pWriter->aDelData); + taosArrayDestroy(pWriter->aDelIdxR); + + // SNAP_DATA_TSDB + + // Writer + tBlockDataDestroy(&pWriter->dWriter.sData, 1); + tBlockDataDestroy(&pWriter->dWriter.bData, 1); + taosArrayDestroy(pWriter->dWriter.aSttBlk); + tMapDataClear(&pWriter->dWriter.mDataBlk); + taosArrayDestroy(pWriter->dWriter.aBlockIdx); + + // Reader + tBlockDataDestroy(&pWriter->dReader.bData, 1); + tMapDataClear(&pWriter->dReader.mDataBlk); + taosArrayDestroy(pWriter->dReader.aBlockIdx); + + tBlockDataDestroy(&pWriter->bData, 1); + tTSchemaDestroy(pWriter->skmTable.pTSchema); + for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { tFree(pWriter->aBuf[iBuf]); } - tsdbInfo("vgId:%d, vnode snapshot tsdb writer close for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); taosMemoryFree(pWriter); *ppWriter = NULL; @@ -1245,8 +1412,8 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) goto _exit; } else { - if (pWriter->pDataFWriter) { - code = tsdbSnapWriteDataEnd(pWriter); + if (pWriter->dWriter.pWriter) { + code = tsdbSnapWriteCloseFile(pWriter); if (code) goto _err; } } @@ -1259,7 +1426,6 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) _exit: tsdbDebug("vgId:%d, tsdb snapshot write for %s succeed", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); - return code; _err: diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index f379d9f71a..9ed72eeecd 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "vnd.h" #include "tutil.h" +#include "vnd.h" const SVnodeCfg vnodeCfgDefault = {.vgId = -1, .dbname = "", @@ -48,7 +48,9 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1, }, .hashBegin = 0, .hashEnd = 0, - .hashMethod = 0}; + .hashMethod = 0, + .sttTrigger = TSDB_DEFAULT_STT_FILE, + .tsdbPageSize = TSDB_DEFAULT_PAGE_SIZE}; int vnodeCheckCfg(const SVnodeCfg *pCfg) { // TODO @@ -107,7 +109,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sstTrigger) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; @@ -132,6 +134,9 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { tjsonAddItemToArray(pNodeInfoArr, pNodeInfo); } + // add tsdb page size config + if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1; + return 0; } @@ -209,7 +214,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code); if (code < 0) return -1; - tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sstTrigger, code); + tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code); if (code < 0) return -1; tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code); if (code < 0) return -1; @@ -249,6 +254,8 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetStringValue(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn); } + tjsonGetNumberValue(pJson, "tsdbPageSize", pCfg->tsdbPageSize, code); + return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index a4fd984fb7..4ccfea4051 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -60,6 +60,8 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path); + info.config = vnodeCfgDefault; + // load vnode info ret = vnodeLoadInfo(dir, &info); if (ret < 0) { diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 03ab5e8285..08c3a34699 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -354,7 +354,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { code = metaSnapWrite(pWriter->pMetaSnapWriter, pData, nData); if (code) goto _err; } break; - case SNAP_DATA_TSDB: { + case SNAP_DATA_TSDB: + case SNAP_DATA_DEL: { // tsdb if (pWriter->pTsdbSnapWriter == NULL) { code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 0722c2b306..a26f1c74f8 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -126,6 +126,7 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList); SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type); +void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a9826e0018..fc8f942015 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -411,7 +411,7 @@ typedef enum EStreamScanMode { STREAM_SCAN_FROM_READERHANDLE = 1, STREAM_SCAN_FROM_RES, STREAM_SCAN_FROM_UPDATERES, - STREAM_SCAN_FROM_DELETERES, + STREAM_SCAN_FROM_DELETE_DATA, STREAM_SCAN_FROM_DATAREADER_RETRIEVE, STREAM_SCAN_FROM_DATAREADER_RANGE, } EStreamScanMode; @@ -794,6 +794,7 @@ typedef struct SStreamPartitionOperatorInfo { void* parIte; SSDataBlock* pInputDataBlock; int32_t tsColIndex; + SSDataBlock* pDelRes; } SStreamPartitionOperatorInfo; typedef struct STimeSliceOperatorInfo { @@ -1108,6 +1109,13 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol bool groupbyTbname(SNodeList* pGroupList); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); +int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, + SGroupResInfo* pGroupResInfo); +int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, + SExecTaskInfo* pTaskInfo); +int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult); +int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 3965b7e5b2..0c1728386b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -475,7 +475,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray if (code != TSDB_CODE_SUCCESS) { terrno = code; qError("failed to create result, reason:%s", tstrerror(code)); - terrno = code; goto end; } @@ -1012,6 +1011,100 @@ static SColumn* createColumn(int32_t blockId, int32_t slotId, int32_t colId, SDa return pCol; } +void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode) { + pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode)); + pExp->pExpr->_function.num = 1; + pExp->pExpr->_function.functionId = -1; + + int32_t type = nodeType(pTargetNode->pExpr); + // it is a project query, or group by column + if (type == QUERY_NODE_COLUMN) { + pExp->pExpr->nodeType = QUERY_NODE_COLUMN; + SColumnNode* pColNode = (SColumnNode*)pTargetNode->pExpr; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pColNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pColNode->colName); + pExp->base.pParam[0].pCol = + createColumn(pColNode->dataBlockId, pColNode->slotId, pColNode->colId, pType, pColNode->colType); + pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; + } else if (type == QUERY_NODE_VALUE) { + pExp->pExpr->nodeType = QUERY_NODE_VALUE; + SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pValNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pValNode->node.aliasName); + pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE; + nodesValueNodeToVariant(pValNode, &pExp->base.pParam[0].param); + } else if (type == QUERY_NODE_FUNCTION) { + pExp->pExpr->nodeType = QUERY_NODE_FUNCTION; + SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; + + SDataType* pType = &pFuncNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pFuncNode->node.aliasName); + + pExp->pExpr->_function.functionId = pFuncNode->funcId; + pExp->pExpr->_function.pFunctNode = pFuncNode; + + strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, + tListLen(pExp->pExpr->_function.functionName)); +#if 1 + // todo refactor: add the parameter for tbname function + if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) { + pFuncNode->pParameterList = nodesMakeList(); + ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); + SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { // todo handle error + } else { + res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; + nodesListAppend(pFuncNode->pParameterList, (SNode*)res); + } + } +#endif + + int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList); + + pExp->base.pParam = taosMemoryCalloc(numOfParam, sizeof(SFunctParam)); + pExp->base.numOfParams = numOfParam; + + for (int32_t j = 0; j < numOfParam; ++j) { + SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); + if (p1->type == QUERY_NODE_COLUMN) { + SColumnNode* pcn = (SColumnNode*)p1; + + pExp->base.pParam[j].type = FUNC_PARAM_TYPE_COLUMN; + pExp->base.pParam[j].pCol = + createColumn(pcn->dataBlockId, pcn->slotId, pcn->colId, &pcn->node.resType, pcn->colType); + } else if (p1->type == QUERY_NODE_VALUE) { + SValueNode* pvn = (SValueNode*)p1; + pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE; + nodesValueNodeToVariant(pvn, &pExp->base.pParam[j].param); + } + } + } else if (type == QUERY_NODE_OPERATOR) { + pExp->pExpr->nodeType = QUERY_NODE_OPERATOR; + SOperatorNode* pNode = (SOperatorNode*)pTargetNode->pExpr; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pNode->node.aliasName); + pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr; + } else { + ASSERT(0); + } +} + SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs) { int32_t numOfFuncs = LIST_LENGTH(pNodeList); int32_t numOfGroupKeys = 0; @@ -1035,98 +1128,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* } SExprInfo* pExp = &pExprs[i]; - - pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode)); - pExp->pExpr->_function.num = 1; - pExp->pExpr->_function.functionId = -1; - - int32_t type = nodeType(pTargetNode->pExpr); - // it is a project query, or group by column - if (type == QUERY_NODE_COLUMN) { - pExp->pExpr->nodeType = QUERY_NODE_COLUMN; - SColumnNode* pColNode = (SColumnNode*)pTargetNode->pExpr; - - pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); - pExp->base.numOfParams = 1; - - SDataType* pType = &pColNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, - pType->precision, pColNode->colName); - pExp->base.pParam[0].pCol = - createColumn(pColNode->dataBlockId, pColNode->slotId, pColNode->colId, pType, pColNode->colType); - pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; - } else if (type == QUERY_NODE_VALUE) { - pExp->pExpr->nodeType = QUERY_NODE_VALUE; - SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr; - - pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); - pExp->base.numOfParams = 1; - - SDataType* pType = &pValNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, - pType->precision, pValNode->node.aliasName); - pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE; - nodesValueNodeToVariant(pValNode, &pExp->base.pParam[0].param); - } else if (type == QUERY_NODE_FUNCTION) { - pExp->pExpr->nodeType = QUERY_NODE_FUNCTION; - SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; - - SDataType* pType = &pFuncNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, - pType->precision, pFuncNode->node.aliasName); - - pExp->pExpr->_function.functionId = pFuncNode->funcId; - pExp->pExpr->_function.pFunctNode = pFuncNode; - - strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, - tListLen(pExp->pExpr->_function.functionName)); -#if 1 - // todo refactor: add the parameter for tbname function - if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) { - pFuncNode->pParameterList = nodesMakeList(); - ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); - SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); - if (NULL == res) { // todo handle error - } else { - res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; - nodesListAppend(pFuncNode->pParameterList, (SNode*)res); - } - } -#endif - - int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList); - - pExp->base.pParam = taosMemoryCalloc(numOfParam, sizeof(SFunctParam)); - pExp->base.numOfParams = numOfParam; - - for (int32_t j = 0; j < numOfParam; ++j) { - SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); - if (p1->type == QUERY_NODE_COLUMN) { - SColumnNode* pcn = (SColumnNode*)p1; - - pExp->base.pParam[j].type = FUNC_PARAM_TYPE_COLUMN; - pExp->base.pParam[j].pCol = - createColumn(pcn->dataBlockId, pcn->slotId, pcn->colId, &pcn->node.resType, pcn->colType); - } else if (p1->type == QUERY_NODE_VALUE) { - SValueNode* pvn = (SValueNode*)p1; - pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE; - nodesValueNodeToVariant(pvn, &pExp->base.pParam[j].param); - } - } - } else if (type == QUERY_NODE_OPERATOR) { - pExp->pExpr->nodeType = QUERY_NODE_OPERATOR; - SOperatorNode* pNode = (SOperatorNode*)pTargetNode->pExpr; - - pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); - pExp->base.numOfParams = 1; - - SDataType* pType = &pNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, - pType->precision, pNode->node.aliasName); - pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr; - } else { - ASSERT(0); - } + createExprFromTargetNode(pExp, pTargetNode); } return pExprs; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 205bcd58df..815d6a71ef 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3439,6 +3439,44 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t } } +static bool isWstartColumnExist(SFillOperatorInfo* pInfo) { + if (pInfo->numOfNotFillExpr == 0) { + return false; + } + for (int32_t i = 0; i < pInfo->numOfNotFillExpr; ++i) { + SExprInfo* exprInfo = pInfo->pNotFillExprInfo + i; + if (exprInfo->pExpr->nodeType == QUERY_NODE_COLUMN && + exprInfo->base.numOfParams == 1 && + exprInfo->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_START) { + return true; + } + } + return false; +} + +static int32_t createWStartTsAsNotFillExpr(SFillOperatorInfo* pInfo, SFillPhysiNode* pPhyFillNode) { + bool wstartExist = isWstartColumnExist(pInfo); + if (wstartExist == false) { + if (pPhyFillNode->pWStartTs->type != QUERY_NODE_TARGET) { + qError("pWStartTs of fill physical node is not a target node"); + return TSDB_CODE_QRY_SYS_ERROR; + } + + SExprInfo* notFillExprs = taosMemoryRealloc(pInfo->pNotFillExprInfo, (pInfo->numOfNotFillExpr + 1) * sizeof(SExprInfo)); + if (notFillExprs == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + createExprFromTargetNode(notFillExprs + pInfo->numOfNotFillExpr, (STargetNode*)pPhyFillNode->pWStartTs); + + ++pInfo->numOfNotFillExpr; + pInfo->pNotFillExprInfo = notFillExprs; + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_SUCCESS; +} + SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo) { SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); @@ -3450,7 +3488,10 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* SSDataBlock* pResBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc); SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr); pInfo->pNotFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pInfo->numOfNotFillExpr); - + int32_t code = createWStartTsAsNotFillExpr(pInfo, pPhyFillNode); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } SInterval* pInterval = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval @@ -3471,7 +3512,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* SArray* pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); - int32_t code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr, + code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr, (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity, pTaskInfo->id.str, pInterval, type, order); if (code != TSDB_CODE_SUCCESS) { @@ -3938,7 +3979,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode, pTaskInfo, isStream); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode; @@ -4410,3 +4451,108 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF return code; } + +int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, + SExecTaskInfo* pTaskInfo) { + SWinKey key = { + .ts = win->skey, + .groupId = tableGroupId, + }; + char* value = NULL; + int32_t size = pAggSup->resultRowSize; + if (streamStateAddIfNotExist(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + *pResult = (SResultRow*)value; + ASSERT(*pResult); + // set time window for current result + (*pResult)->win = (*win); + setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset); + return TSDB_CODE_SUCCESS; +} + +int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult) { + streamStateReleaseBuf(pTaskInfo->streamInfo.pState, pKey, pResult); + return TSDB_CODE_SUCCESS; +} + +int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { + streamStatePut(pTaskInfo->streamInfo.pState, pKey, pResult, resSize); + return TSDB_CODE_SUCCESS; +} + +int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, + SGroupResInfo* pGroupResInfo) { + SExprInfo* pExprInfo = pSup->pExprInfo; + int32_t numOfExprs = pSup->numOfExprs; + int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + SqlFunctionCtx* pCtx = pSup->pCtx; + + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + + for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { + SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); + int32_t size = 0; + void* pVal = NULL; + SWinKey key = { + .ts = *(TSKEY*)pPos->key, + .groupId = pPos->groupId, + }; + int32_t code = streamStateGet(pTaskInfo->streamInfo.pState, &key, &pVal, &size); + ASSERT(code == 0); + SResultRow* pRow = (SResultRow*)pVal; + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); + // no results, continue to check the next one + if (pRow->numOfRows == 0) { + pGroupResInfo->index += 1; + releaseOutputBuf(pTaskInfo, &key, pRow); + continue; + } + + if (pBlock->info.groupId == 0) { + pBlock->info.groupId = pPos->groupId; + } else { + // current value belongs to different group, it can't be packed into one datablock + if (pBlock->info.groupId != pPos->groupId) { + releaseOutputBuf(pTaskInfo, &key, pRow); + break; + } + } + + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + ASSERT(pBlock->info.rows > 0); + releaseOutputBuf(pTaskInfo, &key, pRow); + break; + } + + pGroupResInfo->index += 1; + + for (int32_t j = 0; j < numOfExprs; ++j) { + int32_t slotId = pExprInfo[j].base.resSchema.slotId; + + pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); + if (pCtx[j].fpSet.finalize) { + int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (TAOS_FAILED(code)) { + qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); + T_LONG_JMP(pTaskInfo->env, code); + } + } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { + // do nothing, todo refactor + } else { + // expand the result into multiple rows. E.g., _wstart, top(k, 20) + // the _wstart needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); + for (int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + } + } + } + releaseOutputBuf(pTaskInfo, &key, pRow); + pBlock->info.rows += pRow->numOfRows; + } + blockDataUpdateTsWindow(pBlock, 0); + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 599edb0722..5eb6557dbd 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -13,26 +13,26 @@ * along with this program. If not, see . */ -#include "os.h" #include "function.h" +#include "os.h" #include "tname.h" #include "tdatablock.h" #include "tmsg.h" +#include "executorInt.h" #include "executorimpl.h" #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "executorInt.h" static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len); static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity); -static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes, - uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup); +static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, + int16_t bytes, uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup); static void freeGroupKey(void* param) { - SGroupKeys* pKey = (SGroupKeys*) param; + SGroupKeys* pKey = (SGroupKeys*)param; taosMemoryFree(pKey->pData); } @@ -62,13 +62,13 @@ static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = taosArrayGet(pGroupColList, i); - (*keyLen) += pCol->bytes; // actual data + null_flag + (*keyLen) += pCol->bytes; // actual data + null_flag SGroupKeys key = {0}; - key.bytes = pCol->bytes; - key.type = pCol->type; + key.bytes = pCol->bytes; + key.type = pCol->type; key.isNull = false; - key.pData = taosMemoryCalloc(1, pCol->bytes); + key.pData = taosMemoryCalloc(1, pCol->bytes); if (key.pData == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -87,7 +87,8 @@ static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** return TSDB_CODE_SUCCESS; } -static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { +static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex, + int32_t numOfGroupCols) { SColumnDataAgg* pColAgg = NULL; for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = taosArrayGet(pGroupCols, i); @@ -112,7 +113,7 @@ static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlo if (pkey->type == TSDB_DATA_TYPE_JSON) { int32_t dataLen = getJsonValueLen(val); - if (memcmp(pkey->pData, val, dataLen) == 0){ + if (memcmp(pkey->pData, val, dataLen) == 0) { continue; } else { return false; @@ -154,7 +155,7 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData pkey->isNull = false; char* val = colDataGetData(pColInfoData, rowIndex); if (pkey->type == TSDB_DATA_TYPE_JSON) { - if(tTagIsJson(val)){ + if (tTagIsJson(val)) { terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; return; } @@ -198,13 +199,13 @@ static int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { } } - return (int32_t) (pStart - (char*)pKey); + return (int32_t)(pStart - (char*)pKey); } // assign the group keys or user input constant values if required static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t totalRows, int32_t rowIndex) { for (int32_t i = 0; i < numOfOutput; ++i) { - if (pCtx[i].functionId == -1) { // select count(*),key from t group by key. + if (pCtx[i].functionId == -1) { // select count(*),key from t group by key. SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[i]); SColumnInfoData* pColInfoData = pCtx[i].input.pData[0]; @@ -221,7 +222,7 @@ static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t } else { memcpy(dest, data, pColInfoData->info.bytes); } - } else { // it is a NULL value + } else { // it is a NULL value pEntryInfo->isNullRes = 1; } @@ -275,7 +276,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); - int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); + int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, + len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -291,9 +293,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { if (num > 0) { len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); - int32_t ret = - setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, - pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); + int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, + len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -308,7 +309,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { SGroupbyOperatorInfo* pInfo = pOperator->info; SSDataBlock* pRes = pInfo->binfo.pRes; - while(1) { + while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pRes, NULL); @@ -323,7 +324,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { } pOperator->resultInfo.totalRows += pRes->info.rows; - return (pRes->info.rows == 0)? NULL:pRes; + return (pRes->info.rows == 0) ? NULL : pRes; } static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { @@ -334,7 +335,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SGroupbyOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pRes = pInfo->binfo.pRes; + SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { return buildGroupResultDataBlock(pOperator); @@ -343,7 +344,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -362,7 +363,8 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { // there is an scalar expression that needs to be calculated right before apply the group aggregation. if (pInfo->scalarSup.pExprInfo != NULL) { - pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL); + pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, + pInfo->scalarSup.numOfExprs, NULL); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -403,8 +405,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx goto _error; } - pInfo->pGroupCols = pGroupColList; - pInfo->pCondition = pCondition; + pInfo->pGroupCols = pGroupColList; + pInfo->pCondition = pCondition; int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); if (code != TSDB_CODE_SUCCESS) { @@ -425,14 +427,15 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx initBasicInfo(&pInfo->binfo, pResultBlock); initResultRowInfo(&pInfo->binfo.resultRowInfo); - pOperator->name = "GroupbyAggOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; + pOperator->name = "GroupbyAggOperator"; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; // pOperator->operatorType = OP_Groupby; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, + destroyGroupOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -440,7 +443,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; destroyGroupOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); @@ -448,7 +451,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx } static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { -// SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + // SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SPartitionOperatorInfo* pInfo = pOperator->info; @@ -457,7 +460,7 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); SDataGroupInfo* pGroupInfo = NULL; - void *pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len); + void* pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len); pGroupInfo->numOfRows += 1; @@ -467,32 +470,32 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } // number of rows - int32_t* rows = (int32_t*) pPage; + int32_t* rows = (int32_t*)pPage; size_t numOfCols = pOperator->exprSupp.numOfExprs; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; - int32_t slotId = pExpr->base.pParam[0].pCol->slotId; + int32_t slotId = pExpr->base.pParam[0].pCol->slotId; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); int32_t bytes = pColInfoData->info.bytes; int32_t startOffset = pInfo->columnOffset[i]; - int32_t* columnLen = NULL; + int32_t* columnLen = NULL; int32_t contentLen = 0; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { int32_t* offset = (int32_t*)((char*)pPage + startOffset); - columnLen = (int32_t*) ((char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity); - char* data = (char*)((char*) columnLen + sizeof(int32_t)); + columnLen = (int32_t*)((char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity); + char* data = (char*)((char*)columnLen + sizeof(int32_t)); if (colDataIsNull_s(pColInfoData, j)) { offset[(*rows)] = -1; contentLen = 0; - } else if(pColInfoData->info.type == TSDB_DATA_TYPE_JSON){ + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { offset[*rows] = (*columnLen); - char* src = colDataGetData(pColInfoData, j); + char* src = colDataGetData(pColInfoData, j); int32_t dataLen = getJsonValueLen(src); memcpy(data + (*columnLen), src, dataLen); @@ -511,8 +514,8 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } } else { char* bitmap = (char*)pPage + startOffset; - columnLen = (int32_t*) ((char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity)); - char* data = (char*) columnLen + sizeof(int32_t); + columnLen = (int32_t*)((char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity)); + char* data = (char*)columnLen + sizeof(int32_t); bool isNull = colDataIsNull_f(pColInfoData->nullbitmap, j); if (isNull) { @@ -539,7 +542,7 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf SDataGroupInfo* p = taosHashGet(pInfo->pGroupSet, pInfo->keyBuf, len); void* pPage = NULL; - if (p == NULL) { // it is a new group + if (p == NULL) { // it is a new group SDataGroupInfo gi = {0}; gi.pPageList = taosArrayInit(100, sizeof(int32_t)); taosHashPut(pInfo->pGroupSet, pInfo->keyBuf, len, &gi, sizeof(SDataGroupInfo)); @@ -550,12 +553,12 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf pPage = getNewBufPage(pInfo->pBuf, &pageId); taosArrayPush(p->pPageList, &pageId); - *(int32_t *) pPage = 0; + *(int32_t*)pPage = 0; } else { int32_t* curId = taosArrayGetLast(p->pPageList); pPage = getBufPage(pInfo->pBuf, *curId); - int32_t *rows = (int32_t*) pPage; + int32_t* rows = (int32_t*)pPage; if (*rows >= pInfo->rowCapacity) { // release buffer releaseBufPage(pInfo->pBuf, pPage); @@ -585,17 +588,18 @@ uint64_t calcGroupId(char* pData, int32_t len) { } int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t)); - offset[0] = sizeof(int32_t) + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format + offset[0] = sizeof(int32_t) + + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format - for(int32_t i = 0; i < numOfCols - 1; ++i) { + for (int32_t i = 0; i < numOfCols - 1; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); int32_t bytes = pColInfoData->info.bytes; int32_t payloadLen = bytes * rowCapacity; - + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { // offset segment + content length + payload offset[i + 1] = rowCapacity * sizeof(int32_t) + sizeof(int32_t) + payloadLen + offset[i]; @@ -609,9 +613,9 @@ int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { } static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) { - void *ite = NULL; - while( (ite = taosHashIterate(pInfo->pGroupSet, ite)) != NULL ) { - taosArrayDestroy( ((SDataGroupInfo *)ite)->pPageList); + void* ite = NULL; + while ((ite = taosHashIterate(pInfo->pGroupSet, ite)) != NULL) { + taosArrayDestroy(((SDataGroupInfo*)ite)->pPageList); } taosArrayClear(pInfo->sortedGroupArray); clearDiskbasedBuf(pInfo->pBuf); @@ -626,13 +630,14 @@ static int compareDataGroupInfo(const void* group1, const void* group2) { return 0; } - return (pGroupInfo1->groupId < pGroupInfo2->groupId)? -1:1; + return (pGroupInfo1->groupId < pGroupInfo2->groupId) ? -1 : 1; } static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SPartitionOperatorInfo* pInfo = pOperator->info; - SDataGroupInfo* pGroupInfo = (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; + SDataGroupInfo* pGroupInfo = + (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { // try next group data ++pInfo->groupIndex; @@ -647,7 +652,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { } int32_t* pageId = taosArrayGet(pGroupInfo->pPageList, pInfo->pageIndex); - void* page = getBufPage(pInfo->pBuf, *pageId); + void* page = getBufPage(pInfo->pBuf, *pageId); blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity); blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity); @@ -670,14 +675,14 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SPartitionOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pRes = pInfo->binfo.pRes; + SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { blockDataCleanup(pRes); return buildPartitionResult(pOperator); } - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -688,7 +693,8 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { // there is an scalar expression that needs to be calculated right before apply the group aggregation. if (pInfo->scalarSup.pExprInfo != NULL) { - pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL); + pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, + pInfo->scalarSup.numOfExprs, NULL); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -727,7 +733,7 @@ static void destroyPartitionOperatorInfo(void* param) { cleanupBasicInfo(&pInfo->binfo); taosArrayDestroy(pInfo->pGroupCols); - for(int i = 0; i < taosArrayGetSize(pInfo->pGroupColVals); i++){ + for (int i = 0; i < taosArrayGetSize(pInfo->pGroupColVals); i++) { SGroupKeys key = *(SGroupKeys*)taosArrayGet(pInfo->pGroupColVals, i); taosMemoryFree(key.pData); } @@ -743,24 +749,25 @@ static void destroyPartitionOperatorInfo(void* param) { taosMemoryFreeClear(param); } -SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, + SExecTaskInfo* pTaskInfo) { SPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SPartitionOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } SSDataBlock* pResBlock = createResDataBlock(pPartNode->node.pOutputDataBlockDesc); - int32_t numOfCols = 0; + int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols); pInfo->pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys); if (pPartNode->pExprs != NULL) { - int32_t num = 0; + int32_t num = 0; SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num); - int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num); + int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -772,7 +779,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - uint32_t defaultPgsz = 0; + uint32_t defaultPgsz = 0; uint32_t defaultBufsz = 0; getBufferPgSize(pResBlock->info.rowSize, &defaultPgsz, &defaultBufsz); @@ -794,15 +801,15 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - pOperator->name = "PartitionOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; + pOperator->name = "PartitionOperator"; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; - pInfo->binfo.pRes = pResBlock; - pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pInfo->binfo.pRes = pResBlock; + pOperator->exprSupp.numOfExprs = numOfCols; + pOperator->exprSupp.pExprInfo = pExprInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, NULL, destroyPartitionOperatorInfo, NULL, NULL, NULL); @@ -810,16 +817,16 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pOperator); return NULL; } -int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes, - uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; +int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, + int16_t bytes, uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; @@ -833,37 +840,36 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId) { if (pExprSup->pExprInfo != NULL) { - int32_t code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + int32_t code = + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); if (code != TSDB_CODE_SUCCESS) { qError("calaculate group id error, code:%d", code); } } recordNewGroupKeys(pParSup->pGroupCols, pParSup->pGroupColVals, pBlock, rowId); - int32_t len = buildGroupKeys(pParSup->keyBuf, pParSup->pGroupColVals); + int32_t len = buildGroupKeys(pParSup->keyBuf, pParSup->pGroupColVals); uint64_t groupId = calcGroupId(pParSup->keyBuf, len); return groupId; } -static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { - return pInfo->parIte != NULL; -} +static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo->parIte != NULL; } static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { SStreamPartitionOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pDest = pInfo->binfo.pRes; + SSDataBlock* pDest = pInfo->binfo.pRes; ASSERT(hasRemainPartion(pInfo)); SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->parIte; blockDataCleanup(pDest); - int32_t rows = taosArrayGetSize(pParInfo->rowIds); + int32_t rows = taosArrayGetSize(pParInfo->rowIds); SSDataBlock* pSrc = pInfo->pInputDataBlock; for (int32_t i = 0; i < rows; i++) { int32_t rowIndex = *(int32_t*)taosArrayGet(pParInfo->rowIds, i); for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; j++) { - int32_t slotId = pOperator->exprSupp.pExprInfo[j].base.pParam[0].pCol->slotId; + int32_t slotId = pOperator->exprSupp.pExprInfo[j].base.pParam[0].pCol->slotId; SColumnInfoData* pSrcCol = taosArrayGet(pSrc->pDataBlock, slotId); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, j); - bool isNull = colDataIsNull(pSrcCol, pSrc->info.rows, rowIndex, NULL); - char* pSrcData = colDataGetData(pSrcCol, rowIndex); + bool isNull = colDataIsNull(pSrcCol, pSrc->info.rows, rowIndex, NULL); + char* pSrcData = colDataGetData(pSrcCol, rowIndex); colDataAppend(pDestCol, pDest->info.rows, pSrcData, isNull); } pDest->info.rows++; @@ -881,9 +887,9 @@ static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDat pInfo->pInputDataBlock = pBlock; for (int32_t i = 0; i < pBlock->info.rows; ++i) { recordNewGroupKeys(pInfo->partitionSup.pGroupCols, pInfo->partitionSup.pGroupColVals, pBlock, i); - int32_t keyLen = buildGroupKeys(pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupColVals); - SPartitionDataInfo* pParData = - (SPartitionDataInfo*) taosHashGet(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen); + int32_t keyLen = buildGroupKeys(pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupColVals); + SPartitionDataInfo* pParData = + (SPartitionDataInfo*)taosHashGet(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen); if (pParData) { taosArrayPush(pParData->rowIds, &i); } else { @@ -891,8 +897,7 @@ static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDat newParData.groupId = calcGroupId(pInfo->partitionSup.keyBuf, keyLen); newParData.rowIds = taosArrayInit(64, sizeof(int32_t)); taosArrayPush(newParData.rowIds, &i); - taosHashPut(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen, &newParData, - sizeof(SPartitionDataInfo)); + taosHashPut(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen, &newParData, sizeof(SPartitionDataInfo)); } } } @@ -902,13 +907,13 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { return NULL; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamPartitionOperatorInfo* pInfo = pOperator->info; if (hasRemainPartion(pInfo)) { return buildStreamPartitionResult(pOperator); } - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; { pInfo->pInputDataBlock = NULL; @@ -924,14 +929,19 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { case STREAM_INVALID: pInfo->binfo.pRes->info.type = pBlock->info.type; break; + case STREAM_DELETE_DATA: { + copyDataBlock(pInfo->pDelRes, pBlock); + pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; + return pInfo->pDelRes; + } break; default: return pBlock; } // there is an scalar expression that needs to be calculated right before apply the group aggregation. if (pInfo->scalarSup.pExprInfo != NULL) { - pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, - pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL); + pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, + pInfo->scalarSup.numOfExprs, NULL); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, pTaskInfo->code); } @@ -940,7 +950,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { doStreamHashPartitionImpl(pInfo, pBlock); } pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - + pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL); return buildStreamPartitionResult(pOperator); } @@ -950,7 +960,7 @@ static void destroyStreamPartitionOperatorInfo(void* param) { cleanupBasicInfo(&pInfo->binfo); taosArrayDestroy(pInfo->partitionSup.pGroupCols); - for(int i = 0; i < taosArrayGetSize(pInfo->partitionSup.pGroupColVals); i++){ + for (int i = 0; i < taosArrayGetSize(pInfo->partitionSup.pGroupColVals); i++) { SGroupKeys key = *(SGroupKeys*)taosArrayGet(pInfo->partitionSup.pGroupColVals, i); taosMemoryFree(key.pData); } @@ -958,6 +968,7 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosMemoryFree(pInfo->partitionSup.keyBuf); cleanupExprSupp(&pInfo->scalarSup); + blockDataDestroy(pInfo->pDelRes); taosMemoryFreeClear(param); } @@ -970,7 +981,8 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup pScanInfo->pPartScalarSup = pExpr; } -SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, + SExecTaskInfo* pTaskInfo) { SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -980,7 +992,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr pInfo->partitionSup.pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys); if (pPartNode->pExprs != NULL) { - int32_t num = 0; + int32_t num = 0; SExprInfo* pCalExprInfo = createExprInfo(pPartNode->pExprs, NULL, &num); code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num); if (code != TSDB_CODE_SUCCESS) { @@ -989,7 +1001,8 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } int32_t keyLen = 0; - code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupCols); + code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, + pInfo->partitionSup.pGroupCols); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1000,35 +1013,35 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr goto _error; } blockDataEnsureCapacity(pResBlock, 4096); - pInfo->binfo.pRes = pResBlock; - pInfo->parIte = NULL; - pInfo->pInputDataBlock = NULL; + pInfo->binfo.pRes = pResBlock; + pInfo->parIte = NULL; + pInfo->pInputDataBlock = NULL; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pInfo->pPartitions = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); - pInfo->tsColIndex = 0; + pInfo->pPartitions = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + pInfo->tsColIndex = 0; + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); - int32_t numOfCols = 0; + int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols); - pOperator->name = "StreamPartitionOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; + pOperator->name = "StreamPartitionOperator"; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION; - pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL, destroyStreamPartitionOperatorInfo, - NULL, NULL, NULL); + pOperator->exprSupp.numOfExprs = numOfCols; + pOperator->exprSupp.pExprInfo = pExprInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL, + destroyStreamPartitionOperatorInfo, NULL, NULL, NULL); initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFreeClear(pInfo); + destroyStreamPartitionOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); return NULL; } - diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c099c1c24c..ad9cd1ffe7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1057,24 +1057,24 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ return true; } -static STimeWindow getSlidingWindow(TSKEY* tsCol, SInterval* pInterval, SDataBlockInfo* pDataBlockInfo, - int32_t* pRowIndex, bool hasGroup) { +static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, SInterval* pInterval, + SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); STimeWindow endWin = win; STimeWindow preWin = win; while (1) { if (hasGroup) { (*pRowIndex) += 1; } else { - (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, tsCol, *pRowIndex, endWin.ekey, binarySearchForKey, NULL, - TSDB_ORDER_ASC); + (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, startTsCol, *pRowIndex, endWin.ekey, binarySearchForKey, + NULL, TSDB_ORDER_ASC); } do { preWin = endWin; getNextTimeWindow(pInterval, &endWin, TSDB_ORDER_ASC); - } while (tsCol[(*pRowIndex) - 1] >= endWin.skey); + } while (endTsCol[(*pRowIndex) - 1] >= endWin.skey); endWin = preWin; if (win.ekey == endWin.ekey || (*pRowIndex) == pDataBlockInfo->rows) { win.ekey = endWin.ekey; @@ -1102,6 +1102,11 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 return NULL; } + doFilter(pInfo->pCondition, pResult, NULL); + if (pResult->info.rows == 0) { + continue; + } + if (pInfo->partitionSup.needCalc) { SSDataBlock* tmpBlock = createOneDataBlock(pResult, true); blockDataCleanup(pResult); @@ -1188,13 +1193,15 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS return code; } - SColumnInfoData* pSrcTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); uint64_t* srcUidData = (uint64_t*)pSrcUidCol->pData; SColumnInfoData* pSrcGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* srcGp = (uint64_t*)pSrcGpCol->pData; - ASSERT(pSrcTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); - TSKEY* tsCol = (TSKEY*)pSrcTsCol->pData; + ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); + TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; + TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; SColumnInfoData* pStartTsCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndTsCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pDeUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX); @@ -1204,12 +1211,13 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS int64_t version = pSrcBlock->info.version - 1; for (int32_t i = 0; i < rows;) { uint64_t srcUid = srcUidData[i]; - uint64_t groupId = getGroupIdByData(pInfo, srcUid, tsCol[i], version); + uint64_t groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); uint64_t srcGpId = srcGp[i]; - TSKEY calStartTs = tsCol[i]; + TSKEY calStartTs = srcStartTsCol[i]; colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); - STimeWindow win = getSlidingWindow(tsCol, &pInfo->interval, &pSrcBlock->info, &i, pInfo->partitionSup.needCalc); - TSKEY calEndTs = tsCol[i - 1]; + STimeWindow win = getSlidingWindow(srcStartTsCol, srcEndTsCol, &pInfo->interval, &pSrcBlock->info, &i, + pInfo->partitionSup.needCalc); + TSKEY calEndTs = srcStartTsCol[i - 1]; colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false); colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false); colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false); @@ -1229,11 +1237,49 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS return TSDB_CODE_SUCCESS; } +static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { + if (pSrcBlock->info.rows == 0) { + return TSDB_CODE_SUCCESS; + } + blockDataCleanup(pDestBlock); + int32_t code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + ASSERT(taosArrayGetSize(pSrcBlock->pDataBlock) >= 3); + SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* startData = (TSKEY*)pStartTsCol->pData; + SColumnInfoData* pEndTsCol = taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); + TSKEY* endData = (TSKEY*)pEndTsCol->pData; + SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); + uint64_t* uidCol = (uint64_t*)pUidCol->pData; + + SColumnInfoData* pDestStartCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pDestEndCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pDestUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX); + SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + int32_t dummy = 0; + int64_t version = pSrcBlock->info.version - 1; + for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { + uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); + colDataAppend(pDestStartCol, i, (const char*)(startData + i), false); + colDataAppend(pDestEndCol, i, (const char*)(endData + i), false); + colDataAppendNULL(pDestUidCol, i); + colDataAppend(pDestGpCol, i, (const char*)&groupId, false); + colDataAppendNULL(pDestCalStartTsCol, i); + colDataAppendNULL(pDestCalEndTsCol, i); + pDestBlock->info.rows++; + } + return TSDB_CODE_SUCCESS; +} + static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { int32_t code = TSDB_CODE_SUCCESS; if (isIntervalWindow(pInfo)) { code = generateIntervalScanRange(pInfo, pSrcBlock, pDestBlock); - } else { + } else if (isSessionWindow(pInfo) || isStateWindow(pInfo)) { code = generateSessionScanRange(pInfo, pSrcBlock, pDestBlock); } pDestBlock->info.type = STREAM_CLEAR; @@ -1510,14 +1556,23 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); } break; case STREAM_DELETE_DATA: { - pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; - pInfo->updateResIndex = 0; - generateScanRange(pInfo, pBlock, pInfo->pUpdateRes); - prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); - copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); - pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; - return pInfo->pDeleteDataRes; + printDataBlock(pBlock, "stream scan delete recv"); + if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) { + generateDeleteResultBlock(pInfo, pBlock, pInfo->pDeleteDataRes); + pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; + printDataBlock(pBlock, "stream scan delete result"); + return pInfo->pDeleteDataRes; + } else { + pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; + pInfo->updateResIndex = 0; + generateScanRange(pInfo, pBlock, pInfo->pUpdateRes); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); + pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + printDataBlock(pBlock, "stream scan delete data"); + return pInfo->pDeleteDataRes; + } } break; default: break; @@ -1532,7 +1587,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; return pInfo->pRes; } break; - case STREAM_SCAN_FROM_DELETERES: { + case STREAM_SCAN_FROM_DELETE_DATA: { generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; @@ -1646,7 +1701,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->scanMode = STREAM_SCAN_FROM_RES; return pInfo->pUpdateDataRes; } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) { - pInfo->scanMode = STREAM_SCAN_FROM_DELETERES; + pInfo->scanMode = STREAM_SCAN_FROM_DELETE_DATA; } } } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index d773f8a629..b3a85807ae 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -955,8 +955,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder); - int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -983,7 +983,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, - numOfOutput); + numOfOutput); doCloseWindow(pResultRowInfo, pInfo, pResult); @@ -1406,20 +1406,25 @@ void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SHashObj* pUpdatedMap) { SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* tsStarts = (TSKEY*)pStartCol->pData; + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + TSKEY* tsEnds = (TSKEY*)pEndCol->pData; SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* groupIds = (uint64_t*)pGroupCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, TSDB_ORDER_ASC); - doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]); - SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]}; - if (pDelWins) { - taosArrayPush(pDelWins, &winRes); - } - if (pUpdatedMap) { - taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); - } + do { + doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]); + SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]}; + if (pDelWins) { + taosArrayPush(pDelWins, &winRes); + } + if (pUpdatedMap) { + taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); + } + getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + } while (win.skey <= tsEnds[i]); } } @@ -2775,7 +2780,7 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true); compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); } - if (num > 1 && pUpdatedMap) { + if (num > 0 && pUpdatedMap) { saveWinResultRow(pCurResult, pWinRes->groupId, pUpdatedMap); setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur); } @@ -2807,15 +2812,14 @@ void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) { static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } -static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, - SHashObj* pUpdatedMap) { +static void doHashIntervalAgg(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, + SHashObj* pUpdatedMap) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; SExprSupp* pSup = &pOperatorInfo->exprSupp; int32_t numOfOutput = pSup->numOfExprs; int32_t step = 1; - bool ascScan = true; TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t forwardRows = 0; @@ -2824,7 +2828,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; - int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); + int32_t startPos = 0; TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); STimeWindow nextWin = {0}; if (IS_FINAL_OP(pInfo)) { @@ -3165,7 +3169,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true); - doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); + doHashIntervalAgg(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -3183,7 +3187,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); - doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL); + doHashIntervalAgg(pChildOp, pBlock, pBlock->info.groupId, NULL); } } @@ -3591,7 +3595,8 @@ SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) { // don't add new window SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex) { - SArray* pWinInfos = getWinInfos(pAggSup, groupId); + STimeWindow searchWin = {.skey = startTs, .ekey = endTs}; + SArray* pWinInfos = getWinInfos(pAggSup, groupId); pAggSup->pCurWins = pWinInfos; int32_t size = taosArrayGetSize(pWinInfos); @@ -3603,7 +3608,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start SResultWindowInfo* pWin = NULL; if (index >= 0) { pWin = taosArrayGet(pWinInfos, index); - if (isInWindow(pWin, startTs, gap)) { + if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) { *pIndex = index; return pWin; } @@ -3611,7 +3616,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start if (index + 1 < size) { pWin = taosArrayGet(pWinInfos, index + 1); - if (isInWindow(pWin, startTs, gap)) { + if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) { *pIndex = index + 1; return pWin; } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) { @@ -3789,7 +3794,7 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->win, true); compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition)); - if (pWinInfo->isOutput) { + if (pWinInfo->isOutput && pStDeleted) { SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId}; taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); pWinInfo->isOutput = false; @@ -3882,19 +3887,24 @@ static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { - int32_t winIndex = 0; - while (1) { - SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex); - if (!pCurWin) { - break; - } + int32_t winIndex = 0; + SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex); + if (!pCurWin) { + continue; + } + + do { SResultWindowInfo delWin = *pCurWin; deleteWindow(pAggSup->pCurWins, winIndex, fp); if (result) { delWin.groupId = gpDatas[i]; taosArrayPush(result, &delWin); } - } + if (winIndex >= taosArrayGetSize(pAggSup->pCurWins)) { + break; + } + pCurWin = taosArrayGet(pAggSup->pCurWins, winIndex); + } while (pCurWin->win.skey <= endDatas[i]); } } @@ -3975,26 +3985,16 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It } static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t numOfOutput, - SOperatorInfo* pOperator, SHashObj* pStUpdated, bool needCreate) { + SOperatorInfo* pOperator, SHashObj* pStUpdated) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - int32_t size = taosArrayGetSize(pWinArray); + int32_t size = taosArrayGetSize(pWinArray); ASSERT(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i); - SResultRow* pCurResult = NULL; uint64_t groupId = pParentWin->groupId; - int32_t winIndex = 0; - if (needCreate) { - pParentWin = - getSessionTimeWindow(&pInfo->streamAggSup, pParentWin->win.skey, pParentWin->win.ekey, groupId, 0, &winIndex); - } - setWindowOutputBuf(pParentWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, - &pInfo->streamAggSup, pTaskInfo); - int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); - int32_t num = 0; + int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); for (int32_t j = 0; j < numOfChildren; j++) { SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; @@ -4007,31 +4007,36 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin for (int32_t k = index; k < chWinSize; k++) { SResultWindowInfo* pChWin = taosArrayGet(pChWins, k); if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) { + int32_t winIndex = 0; + SResultWindowInfo* pNewParWin = + getSessionTimeWindow(&pInfo->streamAggSup, pChWin->win.skey, pChWin->win.ekey, groupId, 0, &winIndex); + SResultRow* pPareResult = NULL; + setWindowOutputBuf(pNewParWin, &pPareResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, + &pInfo->streamAggSup, pTaskInfo); SResultRow* pChResult = NULL; setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput, pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo); - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pChWin->win, true); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pNewParWin->win, true); compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); + + int32_t winNum = getNumCompactWindow(pInfo->streamAggSup.pCurWins, winIndex, pInfo->gap); + if (winNum > 0) { + compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, NULL, pOperator); + } + SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId); releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage); - num++; - continue; + + bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pNewParWin->pos.pageId); + setBufPageDirty(bufPage, true); + releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage); + SWinKey value = {.ts = pNewParWin->win.skey, .groupId = groupId}; + taosHashPut(pStUpdated, &pNewParWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey)); } else if (!pChWin->isClosed) { break; } } } - if (num == 0 && needCreate) { - deleteWindow(pInfo->streamAggSup.pCurWins, winIndex, NULL); - } - if (pStUpdated && num > 0) { - SWinKey value = {.ts = pParentWin->win.skey, .groupId = groupId}; - taosHashPut(pStUpdated, &pParentWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey)); - } - SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId); - ASSERT(size > 0); - setBufPageDirty(bufPage, true); - releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage); } } @@ -4192,7 +4197,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX, pChildOp->exprSupp.numOfExprs, 0, NULL); - rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, NULL, false); + rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated); } taosArrayDestroy(pWins); continue; @@ -4206,7 +4211,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; // gap must be 0 doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL, NULL); - rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated, true); + rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated); } copyDeleteWindowInfo(pWins, pInfo->pStDeleted); removeSessionResults(pStUpdated, pWins); @@ -4743,7 +4748,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_CLEAR) { doClearStateWindows(&pInfo->streamAggSup, pBlock, pSeUpdated, pInfo->pSeDeleted); continue; - } else if (pBlock->info.type == STREAM_DELETE_DATA) { + } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, destroyStateWinInfo); copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); @@ -5468,25 +5473,24 @@ _error: return NULL; } -static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, - SSDataBlock* pBlock, int32_t scanFlag, SHashObj* pUpdatedMap) { +static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, + int32_t scanFlag, SHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; SExprSupp* pSup = &pOperatorInfo->exprSupp; - int32_t startPos = 0; - int32_t numOfOutput = pSup->numOfExprs; + int32_t startPos = 0; + int32_t numOfOutput = pSup->numOfExprs; SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); - TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; - uint64_t tableGroupId = pBlock->info.groupId; - bool ascScan = true; - TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); - SResultRow* pResult = NULL; + TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; + uint64_t tableGroupId = pBlock->info.groupId; + bool ascScan = true; + TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); + SResultRow* pResult = NULL; - STimeWindow win = - getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); - int32_t ret = TSDB_CODE_SUCCESS; + STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + int32_t ret = TSDB_CODE_SUCCESS; if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, @@ -5547,11 +5551,88 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo } } +static void doStreamIntervalAggImpl2(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, + SHashObj* pUpdatedMap) { + SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; + + SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + SExprSupp* pSup = &pOperatorInfo->exprSupp; + int32_t numOfOutput = pSup->numOfExprs; + int32_t step = 1; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t forwardRows = 0; + int32_t aa = 4; + + ASSERT(pSDataBlock->pDataBlock != NULL); + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + + int32_t startPos = 0; + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); + STimeWindow nextWin = + getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + while (1) { + bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); + if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { + startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); + if (startPos < 0) { + break; + } + continue; + } + + int32_t code = setOutputBuf(&nextWin, &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, + &pInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, + TSDB_ORDER_ASC); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); + } + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + pSDataBlock->info.rows, numOfOutput); + SWinKey key = { + .ts = nextWin.skey, + .groupId = tableGroupId, + }; + saveOutput(pTaskInfo, &key, pResult, pInfo->aggSup.resultRowSize); + releaseOutputBuf(pTaskInfo, &key, pResult); + int32_t prevEndPos = (forwardRows - 1) * step + startPos; + ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); + startPos = + getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); + if (startPos < 0) { + break; + } + } +} + +void doBuildResult(SOperatorInfo* pOperator, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + // set output datablock version + pBlock->info.version = pTaskInfo->version; + + blockDataCleanup(pBlock); + if (!hasRemainResults(pGroupResInfo)) { + return; + } + + // clear the existed group id + pBlock->info.groupId = 0; + buildDataBlockFromGroupRes(pTaskInfo, pBlock, &pOperator->exprSupp, pGroupResInfo); +} + static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - int64_t maxTs = INT64_MIN; - SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int64_t maxTs = INT64_MIN; + SExprSupp* pSup = &pOperator->exprSupp; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -5594,7 +5675,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { NULL); qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); continue; - } else if (pBlock->info.type == STREAM_DELETE_DATA) { + } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval, pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { @@ -5622,6 +5703,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { maxTs = TMAX(maxTs, pBlock->info.window.ekey); doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap); + // new disc buf + // doStreamIntervalAggImpl2(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); @@ -5664,6 +5747,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } taosArraySort(pUpdated, resultrowComparAsc); + // new disc buf finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); @@ -5676,6 +5760,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + // new disc buf + // doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); printDataBlock(pInfo->binfo.pRes, "single interval"); return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; } @@ -5697,25 +5783,29 @@ void destroyStreamIntervalOperatorInfo(void* param) { SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode; - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); ASSERT(numOfCols > 0); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - SInterval interval = {.interval = pIntervalPhyNode->interval, - .sliding = pIntervalPhyNode->sliding, - .intervalUnit = pIntervalPhyNode->intervalUnit, - .slidingUnit = pIntervalPhyNode->slidingUnit, - .offset = pIntervalPhyNode->offset, - .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, }; - STimeWindowAggSupp twAggSupp = {.waterMark = pIntervalPhyNode->window.watermark, - .calTrigger = pIntervalPhyNode->window.triggerType, - .maxTs = INT64_MIN, }; + SInterval interval = { + .interval = pIntervalPhyNode->interval, + .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, + .offset = pIntervalPhyNode->offset, + .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, + }; + STimeWindowAggSupp twAggSupp = { + .waterMark = pIntervalPhyNode->window.watermark, + .calTrigger = pIntervalPhyNode->window.triggerType, + .maxTs = INT64_MIN, + }; ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY); pOperator->pTaskInfo = pTaskInfo; pInfo->interval = interval; @@ -5732,11 +5822,11 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys } } - pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;; + pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); SExprSupp* pSup = &pOperator->exprSupp; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -5758,8 +5848,9 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, - destroyStreamIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamIntervalOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 648ae5a538..5844784ea4 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2175,6 +2175,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_AVG_PARTIAL, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateAvgPartial, + .dataRequiredFunc = statisDataRequired, .getEnvFunc = getAvgFuncEnv, .initFunc = avgFunctionSetup, .processFunc = avgFunction, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index f396f6e609..0d7fd1a6da 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -4711,7 +4711,7 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { colDataAppendNULL(pOutput, i); // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, i); + appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); } continue; } @@ -4724,11 +4724,11 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { } else { pInfo->count = 0; } - colDataAppend(pOutput, i, (char*)&output, false); + colDataAppend(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false); // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, i); + appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); } } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 8976daadbd..a41462e3fd 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -362,18 +362,14 @@ static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) { int32_t code; tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags, code); - ; if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize, code); - ; } return code; @@ -406,14 +402,11 @@ static int32_t jsonToSchema(const SJson* pJson, void* pObj) { int32_t code; tjsonGetNumberValue(pJson, jkSchemaType, pNode->type, code); - ; if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name); @@ -466,26 +459,20 @@ static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) { int32_t code; tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId, code); - ; if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo); @@ -926,7 +913,6 @@ static int32_t jsonToLogicFillNode(const SJson* pJson, void* pObj) { int32_t code = jsonToLogicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillLogicPlanWStartTs, &pNode->pWStartTs); @@ -2815,7 +2801,6 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkColumnDbName, pNode->dbName); @@ -3118,7 +3103,6 @@ static int32_t jsonToOperatorNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkOperatorLeft, &pNode->pLeft); @@ -3153,7 +3137,6 @@ static int32_t jsonToLogicConditionNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkLogicCondParameters, &pNode->pParameterList); @@ -3442,11 +3425,9 @@ static int32_t jsonToOrderByExprNode(const SJson* pJson, void* pObj) { int32_t code = jsonToNodeObject(pJson, jkOrderByExprExpr, &pNode->pExpr); if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder, code); - ; } return code; @@ -3624,7 +3605,6 @@ static int32_t jsonToFillNode(const SJson* pJson, void* pObj) { int32_t code; tjsonGetNumberValue(pJson, jkFillMode, pNode->mode, code); - ; if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues); } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c new file mode 100644 index 0000000000..5fe31ed78e --- /dev/null +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -0,0 +1,3108 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodesUtil.h" +#include "plannodes.h" +#include "tdatablock.h" + +#define NODES_MSG_DEFAULT_LEN 1024 + +#define tlvForEach(pDecoder, pTlv, code) \ + while (TSDB_CODE_SUCCESS == code && TSDB_CODE_SUCCESS == (code = tlvGetNextTlv(pDecoder, &pTlv)) && NULL != pTlv) + +typedef struct STlv { + int16_t type; + int16_t len; + char value[0]; +} STlv; + +typedef struct STlvEncoder { + int32_t allocSize; + int32_t offset; + char* pBuf; + int32_t tlvCount; +} STlvEncoder; + +typedef struct STlvDecoder { + int32_t bufSize; + int32_t offset; + const char* pBuf; +} STlvDecoder; + +typedef int32_t (*FToMsg)(const void* pObj, STlvEncoder* pEncoder); +typedef int32_t (*FToObject)(STlvDecoder* pDecoder, void* pObj); +typedef void* (*FMakeObject)(int16_t type); +typedef int32_t (*FSetObject)(STlv* pTlv, void* pObj); + +static int32_t nodeToMsg(const void* pObj, STlvEncoder* pEncoder); +static int32_t nodeListToMsg(const void* pObj, STlvEncoder* pEncoder); +static int32_t msgToNode(STlvDecoder* pDecoder, void** pObj); +static int32_t msgToNodeFromTlv(STlv* pTlv, void** pObj); +static int32_t msgToNodeList(STlvDecoder* pDecoder, void** pObj); +static int32_t msgToNodeListFromTlv(STlv* pTlv, void** pObj); + +static int32_t initTlvEncoder(STlvEncoder* pEncoder) { + pEncoder->allocSize = NODES_MSG_DEFAULT_LEN; + pEncoder->offset = 0; + pEncoder->tlvCount = 0; + pEncoder->pBuf = taosMemoryMalloc(pEncoder->allocSize); + return NULL == pEncoder->pBuf ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS; +} + +static void clearTlvEncoder(STlvEncoder* pEncoder) { taosMemoryFree(pEncoder->pBuf); } + +static void endTlvEncode(STlvEncoder* pEncoder, char** pMsg, int32_t* pLen) { + *pMsg = pEncoder->pBuf; + pEncoder->pBuf = NULL; + *pLen = pEncoder->offset; + // nodesWarn("encode tlv count = %d, tl size = %d", pEncoder->tlvCount, sizeof(STlv) * pEncoder->tlvCount); +} + +static int32_t tlvEncodeImpl(STlvEncoder* pEncoder, int16_t type, const void* pValue, int16_t len) { + int32_t tlvLen = sizeof(STlv) + len; + if (pEncoder->offset + tlvLen > pEncoder->allocSize) { + void* pNewBuf = taosMemoryRealloc(pEncoder->pBuf, pEncoder->allocSize * 2); + if (NULL == pNewBuf) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pEncoder->pBuf = pNewBuf; + pEncoder->allocSize = pEncoder->allocSize * 2; + } + STlv* pTlv = (STlv*)(pEncoder->pBuf + pEncoder->offset); + pTlv->type = type; + pTlv->len = len; + memcpy(pTlv->value, pValue, len); + pEncoder->offset += tlvLen; + ++(pEncoder->tlvCount); + return TSDB_CODE_SUCCESS; +} + +static int32_t tlvEncodeI8(STlvEncoder* pEncoder, int16_t type, int8_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeI16(STlvEncoder* pEncoder, int16_t type, int16_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeI32(STlvEncoder* pEncoder, int16_t type, int32_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeI64(STlvEncoder* pEncoder, int16_t type, int64_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeU8(STlvEncoder* pEncoder, int16_t type, uint8_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeU16(STlvEncoder* pEncoder, int16_t type, uint16_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeU64(STlvEncoder* pEncoder, int16_t type, uint64_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeDouble(STlvEncoder* pEncoder, int16_t type, double value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeEnum(STlvEncoder* pEncoder, int16_t type, int32_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeBool(STlvEncoder* pEncoder, int16_t type, bool value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeCStr(STlvEncoder* pEncoder, int16_t type, const char* pValue) { + return tlvEncodeImpl(pEncoder, type, pValue, strlen(pValue)); +} + +static int32_t tlvEncodeBinary(STlvEncoder* pEncoder, int16_t type, const void* pValue, int32_t len) { + return tlvEncodeImpl(pEncoder, type, pValue, len); +} + +static int32_t tlvEncodeObj(STlvEncoder* pEncoder, int16_t type, FToMsg func, const void* pObj) { + if (NULL == pObj) { + return TSDB_CODE_SUCCESS; + } + + int32_t start = pEncoder->offset; + pEncoder->offset += sizeof(STlv); + int32_t code = func(pObj, pEncoder); + if (TSDB_CODE_SUCCESS == code) { + STlv* pTlv = (STlv*)(pEncoder->pBuf + start); + pTlv->type = type; + pTlv->len = pEncoder->offset - start - sizeof(STlv); + } + return code; +} + +static int32_t tlvEncodeObjArray(STlvEncoder* pEncoder, int16_t type, FToMsg func, const void* pArray, int32_t itemSize, + int32_t num) { + int32_t code = TSDB_CODE_SUCCESS; + if (num > 0) { + int32_t start = pEncoder->offset; + pEncoder->offset += sizeof(STlv); + for (size_t i = 0; TSDB_CODE_SUCCESS == code && i < num; ++i) { + code = tlvEncodeObj(pEncoder, 0, func, (const char*)pArray + i * itemSize); + } + if (TSDB_CODE_SUCCESS == code) { + STlv* pTlv = (STlv*)(pEncoder->pBuf + start); + pTlv->type = type; + pTlv->len = pEncoder->offset - start - sizeof(STlv); + } + } + return code; +} + +static int32_t tlvGetNextTlv(STlvDecoder* pDecoder, STlv** pTlv) { + if (pDecoder->offset == pDecoder->bufSize) { + *pTlv = NULL; + return TSDB_CODE_SUCCESS; + } + + *pTlv = (STlv*)(pDecoder->pBuf + pDecoder->offset); + if ((*pTlv)->len + pDecoder->offset > pDecoder->bufSize) { + return TSDB_CODE_FAILED; + } + pDecoder->offset += sizeof(STlv) + (*pTlv)->len; + return TSDB_CODE_SUCCESS; +} + +static bool tlvDecodeEnd(STlvDecoder* pDecoder) { return pDecoder->offset == pDecoder->bufSize; } + +static int32_t tlvDecodeImpl(STlv* pTlv, void* pValue, int16_t len) { + if (pTlv->len != len) { + return TSDB_CODE_FAILED; + } + memcpy(pValue, pTlv->value, len); + return TSDB_CODE_SUCCESS; +} + +static int32_t tlvDecodeI8(STlv* pTlv, int8_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeI16(STlv* pTlv, int16_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeI32(STlv* pTlv, int32_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeI64(STlv* pTlv, int64_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeU8(STlv* pTlv, uint8_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeU16(STlv* pTlv, uint16_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeU64(STlv* pTlv, uint64_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeDouble(STlv* pTlv, double* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeBool(STlv* pTlv, bool* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeEnum(STlv* pTlv, void* pValue, int16_t len) { + int32_t value = 0; + memcpy(&value, pTlv->value, pTlv->len); + switch (len) { + case 1: + *(int8_t*)pValue = value; + break; + case 2: + *(int16_t*)pValue = value; + break; + case 4: + *(int32_t*)pValue = value; + break; + default: + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue) { + memcpy(pValue, pTlv->value, pTlv->len); + return TSDB_CODE_SUCCESS; +} + +static int32_t tlvDecodeDynBinary(STlv* pTlv, void** pValue) { + *pValue = taosMemoryMalloc(pTlv->len); + if (NULL == *pValue) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(*pValue, pTlv->value, pTlv->len); + return TSDB_CODE_SUCCESS; +} + +static int32_t tlvDecodeObjFromTlv(STlv* pTlv, FToObject func, void* pObj) { + STlvDecoder decoder = {.bufSize = pTlv->len, .offset = 0, .pBuf = pTlv->value}; + return func(&decoder, pObj); +} + +static int32_t tlvDecodeObj(STlvDecoder* pDecoder, FToObject func, void* pObj) { + STlv* pTlv = NULL; + int32_t code = tlvGetNextTlv(pDecoder, &pTlv); + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeObjFromTlv(pTlv, func, pObj); + } + return code; +} + +static int32_t tlvDecodeObjArray(STlvDecoder* pDecoder, FToObject func, void* pArray, int32_t itemSize) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t i = 0; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { code = tlvDecodeObjFromTlv(pTlv, func, (char*)pArray + itemSize * i++); } + return code; +} + +static int32_t tlvDecodeObjArrayFromTlv(STlv* pTlv, FToObject func, void* pArray, int32_t itemSize) { + STlvDecoder decoder = {.bufSize = pTlv->len, .offset = 0, .pBuf = pTlv->value}; + return tlvDecodeObjArray(&decoder, func, pArray, itemSize); +} + +static int32_t tlvDecodeDynObjFromTlv(STlv* pTlv, FMakeObject makeFunc, FToObject toFunc, void** pObj) { + *pObj = makeFunc(pTlv->type); + if (NULL == *pObj) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return tlvDecodeObjFromTlv(pTlv, toFunc, *pObj); +} + +static int32_t tlvDecodeDynObj(STlvDecoder* pDecoder, FMakeObject makeFunc, FToObject toFunc, void** pObj) { + STlv* pTlv = NULL; + int32_t code = tlvGetNextTlv(pDecoder, &pTlv); + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeDynObjFromTlv(pTlv, makeFunc, toFunc, pObj); + } + return code; +} + +enum { DATA_TYPE_CODE_TYPE = 1, DATA_TYPE_CODE_PRECISION, DATA_TYPE_CODE_SCALE, DATA_TYPE_CODE_BYTES }; + +static int32_t dataTypeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SDataType* pNode = (const SDataType*)pObj; + + int32_t code = tlvEncodeI8(pEncoder, DATA_TYPE_CODE_TYPE, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU8(pEncoder, DATA_TYPE_CODE_PRECISION, pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU8(pEncoder, DATA_TYPE_CODE_SCALE, pNode->scale); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, DATA_TYPE_CODE_BYTES, pNode->bytes); + } + + return code; +} + +static int32_t msgToDataType(STlvDecoder* pDecoder, void* pObj) { + SDataType* pNode = (SDataType*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case DATA_TYPE_CODE_TYPE: + code = tlvDecodeI8(pTlv, &pNode->type); + break; + case DATA_TYPE_CODE_PRECISION: + code = tlvDecodeU8(pTlv, &pNode->precision); + break; + case DATA_TYPE_CODE_SCALE: + code = tlvDecodeU8(pTlv, &pNode->scale); + break; + case DATA_TYPE_CODE_BYTES: + code = tlvDecodeI32(pTlv, &pNode->bytes); + break; + default: + break; + } + } + + return code; +} + +enum { EXPR_CODE_RES_TYPE = 1 }; + +static int32_t exprNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SExprNode* pNode = (const SExprNode*)pObj; + return tlvEncodeObj(pEncoder, EXPR_CODE_RES_TYPE, dataTypeToMsg, &pNode->resType); +} + +static int32_t msgToExprNode(STlvDecoder* pDecoder, void* pObj) { + SExprNode* pNode = (SExprNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case EXPR_CODE_RES_TYPE: + code = tlvDecodeObjFromTlv(pTlv, msgToDataType, &pNode->resType); + break; + default: + break; + } + } + + return code; +} + +enum { + COLUMN_CODE_EXPR_BASE = 1, + COLUMN_CODE_TABLE_ID, + COLUMN_CODE_TABLE_TYPE, + COLUMN_CODE_COLUMN_ID, + COLUMN_CODE_COLUMN_TYPE, + COLUMN_CODE_DATABLOCK_ID, + COLUMN_CODE_SLOT_ID +}; + +static int32_t columnNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SColumnNode* pNode = (const SColumnNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, COLUMN_CODE_EXPR_BASE, exprNodeToMsg, pNode); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, COLUMN_CODE_TABLE_ID, pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, COLUMN_CODE_TABLE_TYPE, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI16(pEncoder, COLUMN_CODE_COLUMN_ID, pNode->colId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, COLUMN_CODE_COLUMN_TYPE, pNode->colType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI16(pEncoder, COLUMN_CODE_DATABLOCK_ID, pNode->dataBlockId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI16(pEncoder, COLUMN_CODE_SLOT_ID, pNode->slotId); + } + + return code; +} + +static int32_t msgToColumnNode(STlvDecoder* pDecoder, void* pObj) { + SColumnNode* pNode = (SColumnNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case COLUMN_CODE_EXPR_BASE: + code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); + break; + case COLUMN_CODE_TABLE_ID: + code = tlvDecodeU64(pTlv, &pNode->tableId); + break; + case COLUMN_CODE_TABLE_TYPE: + code = tlvDecodeI8(pTlv, &pNode->tableType); + break; + case COLUMN_CODE_COLUMN_ID: + code = tlvDecodeI16(pTlv, &pNode->colId); + break; + case COLUMN_CODE_COLUMN_TYPE: + code = tlvDecodeEnum(pTlv, &pNode->colType, sizeof(pNode->colType)); + break; + case COLUMN_CODE_DATABLOCK_ID: + code = tlvDecodeI16(pTlv, &pNode->dataBlockId); + break; + case COLUMN_CODE_SLOT_ID: + code = tlvDecodeI16(pTlv, &pNode->slotId); + break; + default: + break; + } + } + + return code; +} + +enum { VALUE_CODE_EXPR_BASE = 1, VALUE_CODE_IS_NULL, VALUE_CODE_DATUM }; + +static int32_t datumToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SValueNode* pNode = (const SValueNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + break; + case TSDB_DATA_TYPE_BOOL: + code = tlvEncodeBool(pEncoder, VALUE_CODE_DATUM, pNode->datum.b); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + code = tlvEncodeI64(pEncoder, VALUE_CODE_DATUM, pNode->datum.i); + break; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + code = tlvEncodeU64(pEncoder, VALUE_CODE_DATUM, pNode->datum.u); + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + code = tlvEncodeDouble(pEncoder, VALUE_CODE_DATUM, pNode->datum.d); + break; + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_NCHAR: + code = tlvEncodeBinary(pEncoder, VALUE_CODE_DATUM, pNode->datum.p, varDataTLen(pNode->datum.p)); + break; + case TSDB_DATA_TYPE_JSON: + code = tlvEncodeBinary(pEncoder, VALUE_CODE_DATUM, pNode->datum.p, getJsonValueLen(pNode->datum.p)); + break; + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + // todo + default: + break; + } + + return code; +} + +static int32_t valueNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SValueNode* pNode = (const SValueNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, VALUE_CODE_EXPR_BASE, exprNodeToMsg, pNode); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, VALUE_CODE_IS_NULL, pNode->isNull); + } + if (TSDB_CODE_SUCCESS == code) { + code = datumToMsg(pNode, pEncoder); + } + + return code; +} + +static int32_t msgToDatum(STlv* pTlv, void* pObj) { + SValueNode* pNode = (SValueNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + break; + case TSDB_DATA_TYPE_BOOL: + code = tlvDecodeBool(pTlv, &pNode->datum.b); + *(bool*)&pNode->typeData = pNode->datum.b; + break; + case TSDB_DATA_TYPE_TINYINT: + code = tlvDecodeI64(pTlv, &pNode->datum.i); + *(int8_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_SMALLINT: + code = tlvDecodeI64(pTlv, &pNode->datum.i); + *(int16_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_INT: + code = tlvDecodeI64(pTlv, &pNode->datum.i); + *(int32_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_BIGINT: + code = tlvDecodeI64(pTlv, &pNode->datum.i); + *(int64_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_TIMESTAMP: + code = tlvDecodeI64(pTlv, &pNode->datum.i); + *(int64_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_UTINYINT: + code = tlvDecodeU64(pTlv, &pNode->datum.u); + *(uint8_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_USMALLINT: + code = tlvDecodeU64(pTlv, &pNode->datum.u); + *(uint16_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_UINT: + code = tlvDecodeU64(pTlv, &pNode->datum.u); + *(uint32_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_UBIGINT: + code = tlvDecodeU64(pTlv, &pNode->datum.u); + *(uint64_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_FLOAT: + code = tlvDecodeDouble(pTlv, &pNode->datum.d); + *(float*)&pNode->typeData = pNode->datum.d; + break; + case TSDB_DATA_TYPE_DOUBLE: + code = tlvDecodeDouble(pTlv, &pNode->datum.d); + *(double*)&pNode->typeData = pNode->datum.d; + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + code = tlvDecodeDynBinary(pTlv, (void**)&pNode->datum.p); + if (TSDB_CODE_SUCCESS == code) { + varDataSetLen(pNode->datum.p, pNode->node.resType.bytes - VARSTR_HEADER_SIZE); + } + break; + case TSDB_DATA_TYPE_JSON: + code = tlvDecodeDynBinary(pTlv, (void**)&pNode->datum.p); + break; + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + // todo + default: + break; + } + + return code; +} + +static int32_t msgToValueNode(STlvDecoder* pDecoder, void* pObj) { + SValueNode* pNode = (SValueNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case VALUE_CODE_EXPR_BASE: + code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); + break; + case VALUE_CODE_IS_NULL: + code = tlvDecodeBool(pTlv, &pNode->isNull); + break; + case VALUE_CODE_DATUM: + code = msgToDatum(pTlv, pNode); + break; + default: + break; + } + } + + return code; +} + +enum { OPERATOR_CODE_EXPR_BASE = 1, OPERATOR_CODE_OP_TYPE, OPERATOR_CODE_LEFT, OPERATOR_CODE_RIGHT }; + +static int32_t operatorNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SOperatorNode* pNode = (const SOperatorNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, OPERATOR_CODE_EXPR_BASE, exprNodeToMsg, pNode); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, OPERATOR_CODE_OP_TYPE, pNode->opType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, OPERATOR_CODE_LEFT, nodeToMsg, pNode->pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, OPERATOR_CODE_RIGHT, nodeToMsg, pNode->pRight); + } + + return code; +} + +static int32_t msgToOperatorNode(STlvDecoder* pDecoder, void* pObj) { + SOperatorNode* pNode = (SOperatorNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case OPERATOR_CODE_EXPR_BASE: + code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); + break; + case OPERATOR_CODE_OP_TYPE: + code = tlvDecodeEnum(pTlv, &pNode->opType, sizeof(pNode->opType)); + break; + case OPERATOR_CODE_LEFT: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pLeft); + break; + case OPERATOR_CODE_RIGHT: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pRight); + break; + default: + break; + } + } + + return code; +} + +enum { LOGIC_COND_CODE_EXPR_BASE = 1, LOGIC_COND_CODE_COND_TYPE, LOGIC_COND_CODE_PARAMETERS }; + +static int32_t logicConditionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SLogicConditionNode* pNode = (const SLogicConditionNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, LOGIC_COND_CODE_EXPR_BASE, exprNodeToMsg, pNode); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, LOGIC_COND_CODE_COND_TYPE, pNode->condType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, LOGIC_COND_CODE_PARAMETERS, nodeListToMsg, pNode->pParameterList); + } + + return code; +} + +static int32_t msgToLogicConditionNode(STlvDecoder* pDecoder, void* pObj) { + SLogicConditionNode* pNode = (SLogicConditionNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case LOGIC_COND_CODE_EXPR_BASE: + code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); + break; + case LOGIC_COND_CODE_COND_TYPE: + code = tlvDecodeEnum(pTlv, &pNode->condType, sizeof(pNode->condType)); + break; + case LOGIC_COND_CODE_PARAMETERS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pParameterList); + break; + default: + break; + } + } + + return code; +} + +enum { + FUNCTION_CODE_EXPR_BASE = 1, + FUNCTION_CODE_FUNCTION_ID, + FUNCTION_CODE_FUNCTION_TYPE, + FUNCTION_CODE_PARAMETERS, + FUNCTION_CODE_UDF_BUF_SIZE +}; + +static int32_t functionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SFunctionNode* pNode = (const SFunctionNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, FUNCTION_CODE_EXPR_BASE, exprNodeToMsg, pNode); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, FUNCTION_CODE_FUNCTION_ID, pNode->funcId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, FUNCTION_CODE_FUNCTION_TYPE, pNode->funcType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, FUNCTION_CODE_PARAMETERS, nodeListToMsg, pNode->pParameterList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, FUNCTION_CODE_UDF_BUF_SIZE, pNode->udfBufSize); + } + + return code; +} + +static int32_t msgToFunctionNode(STlvDecoder* pDecoder, void* pObj) { + SFunctionNode* pNode = (SFunctionNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case FUNCTION_CODE_EXPR_BASE: + code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); + break; + case FUNCTION_CODE_FUNCTION_ID: + code = tlvDecodeI32(pTlv, &pNode->funcId); + break; + case FUNCTION_CODE_FUNCTION_TYPE: + code = tlvDecodeI32(pTlv, &pNode->funcType); + break; + case FUNCTION_CODE_PARAMETERS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pParameterList); + break; + case FUNCTION_CODE_UDF_BUF_SIZE: + code = tlvDecodeI32(pTlv, &pNode->udfBufSize); + break; + default: + break; + } + } + + return code; +} + +enum { ORDER_BY_EXPR_CODE_EXPR = 1, ORDER_BY_EXPR_CODE_ORDER, ORDER_BY_EXPR_CODE_NULL_ORDER }; + +static int32_t orderByExprNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SOrderByExprNode* pNode = (const SOrderByExprNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, ORDER_BY_EXPR_CODE_EXPR, nodeToMsg, pNode->pExpr); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, ORDER_BY_EXPR_CODE_ORDER, pNode->order); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, ORDER_BY_EXPR_CODE_NULL_ORDER, pNode->nullOrder); + } + + return code; +} + +static int32_t msgToOrderByExprNode(STlvDecoder* pDecoder, void* pObj) { + SOrderByExprNode* pNode = (SOrderByExprNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case ORDER_BY_EXPR_CODE_EXPR: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pExpr); + break; + case ORDER_BY_EXPR_CODE_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->order, sizeof(pNode->order)); + break; + case ORDER_BY_EXPR_CODE_NULL_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->nullOrder, sizeof(pNode->nullOrder)); + break; + default: + break; + } + } + + return code; +} + +enum { LIMIT_CODE_LIMIT = 1, LIMIT_CODE_OFFSET }; + +static int32_t limitNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SLimitNode* pNode = (const SLimitNode*)pObj; + + int32_t code = tlvEncodeI64(pEncoder, LIMIT_CODE_LIMIT, pNode->limit); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, LIMIT_CODE_OFFSET, pNode->offset); + } + + return code; +} + +static int32_t msgToLimitNode(STlvDecoder* pDecoder, void* pObj) { + SLimitNode* pNode = (SLimitNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case LIMIT_CODE_LIMIT: + code = tlvDecodeI64(pTlv, &pNode->limit); + break; + case LIMIT_CODE_OFFSET: + code = tlvDecodeI64(pTlv, &pNode->offset); + break; + default: + break; + } + } + + return code; +} + +enum { NAME_CODE_TYPE = 1, NAME_CODE_ACCT_ID, NAME_CODE_DB_NAME, NAME_CODE_TABLE_NAME }; + +static int32_t nameToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SName* pNode = (const SName*)pObj; + + int32_t code = tlvEncodeU8(pEncoder, NAME_CODE_TYPE, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, NAME_CODE_ACCT_ID, pNode->acctId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, NAME_CODE_DB_NAME, pNode->dbname); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, NAME_CODE_TABLE_NAME, pNode->tname); + } + + return code; +} + +static int32_t msgToName(STlvDecoder* pDecoder, void* pObj) { + SName* pNode = (SName*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case NAME_CODE_TYPE: + code = tlvDecodeU8(pTlv, &pNode->type); + break; + case NAME_CODE_ACCT_ID: + code = tlvDecodeI32(pTlv, &pNode->acctId); + break; + case NAME_CODE_DB_NAME: + code = tlvDecodeCStr(pTlv, pNode->dbname); + break; + case NAME_CODE_TABLE_NAME: + code = tlvDecodeCStr(pTlv, pNode->tname); + break; + default: + break; + } + } + + return code; +} + +enum { TIME_WINDOW_CODE_START_KEY = 1, TIME_WINDOW_CODE_END_KEY }; + +static int32_t timeWindowToMsg(const void* pObj, STlvEncoder* pEncoder) { + const STimeWindow* pNode = (const STimeWindow*)pObj; + + int32_t code = tlvEncodeI64(pEncoder, TIME_WINDOW_CODE_START_KEY, pNode->skey); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, TIME_WINDOW_CODE_END_KEY, pNode->ekey); + } + + return code; +} + +static int32_t msgToTimeWindow(STlvDecoder* pDecoder, void* pObj) { + STimeWindow* pNode = (STimeWindow*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case TIME_WINDOW_CODE_START_KEY: + code = tlvDecodeI64(pTlv, &pNode->skey); + break; + case TIME_WINDOW_CODE_END_KEY: + code = tlvDecodeI64(pTlv, &pNode->ekey); + break; + default: + break; + } + } + + return code; +} + +enum { NODE_LIST_CODE_DATA_TYPE = 1, NODE_LIST_CODE_NODE_LIST }; + +static int32_t nodeListNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SNodeListNode* pNode = (const SNodeListNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, NODE_LIST_CODE_DATA_TYPE, dataTypeToMsg, &pNode->dataType); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, NODE_LIST_CODE_NODE_LIST, nodeListToMsg, pNode->pNodeList); + } + + return code; +} + +static int32_t msgToNodeListNode(STlvDecoder* pDecoder, void* pObj) { + SNodeListNode* pNode = (SNodeListNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case NODE_LIST_CODE_DATA_TYPE: + code = tlvDecodeObjFromTlv(pTlv, msgToDataType, &pNode->dataType); + break; + case NODE_LIST_CODE_NODE_LIST: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pNodeList); + break; + default: + break; + } + } + + return code; +} + +enum { TARGET_CODE_DATA_BLOCK_ID = 1, TARGET_CODE_SLOT_ID, TARGET_CODE_EXPR }; + +static int32_t targetNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const STargetNode* pNode = (const STargetNode*)pObj; + + int32_t code = tlvEncodeI16(pEncoder, TARGET_CODE_DATA_BLOCK_ID, pNode->dataBlockId); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI16(pEncoder, TARGET_CODE_SLOT_ID, pNode->slotId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, TARGET_CODE_EXPR, nodeToMsg, pNode->pExpr); + } + + return code; +} + +static int32_t msgToTargetNode(STlvDecoder* pDecoder, void* pObj) { + STargetNode* pNode = (STargetNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case TARGET_CODE_DATA_BLOCK_ID: + code = tlvDecodeI16(pTlv, &pNode->dataBlockId); + break; + case TARGET_CODE_SLOT_ID: + code = tlvDecodeI16(pTlv, &pNode->slotId); + break; + case TARGET_CODE_EXPR: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pExpr); + break; + default: + break; + } + } + + return code; +} + +enum { + DATA_BLOCK_DESC_CODE_DATA_BLOCK_ID = 1, + DATA_BLOCK_DESC_CODE_SLOTS, + DATA_BLOCK_DESC_CODE_TOTAL_ROW_SIZE, + DATA_BLOCK_DESC_CODE_OUTPUT_ROW_SIZE, + DATA_BLOCK_DESC_CODE_PRECISION +}; + +static int32_t dataBlockDescNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SDataBlockDescNode* pNode = (const SDataBlockDescNode*)pObj; + + int32_t code = tlvEncodeI16(pEncoder, DATA_BLOCK_DESC_CODE_DATA_BLOCK_ID, pNode->dataBlockId); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, DATA_BLOCK_DESC_CODE_SLOTS, nodeListToMsg, pNode->pSlots); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, DATA_BLOCK_DESC_CODE_TOTAL_ROW_SIZE, pNode->totalRowSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, DATA_BLOCK_DESC_CODE_OUTPUT_ROW_SIZE, pNode->outputRowSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU8(pEncoder, DATA_BLOCK_DESC_CODE_PRECISION, pNode->precision); + } + + return code; +} + +static int32_t msgToDataBlockDescNode(STlvDecoder* pDecoder, void* pObj) { + SDataBlockDescNode* pNode = (SDataBlockDescNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case DATA_BLOCK_DESC_CODE_DATA_BLOCK_ID: + code = tlvDecodeI16(pTlv, &pNode->dataBlockId); + break; + case DATA_BLOCK_DESC_CODE_SLOTS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pSlots); + break; + case DATA_BLOCK_DESC_CODE_TOTAL_ROW_SIZE: + code = tlvDecodeI32(pTlv, &pNode->totalRowSize); + break; + case DATA_BLOCK_DESC_CODE_OUTPUT_ROW_SIZE: + code = tlvDecodeI32(pTlv, &pNode->outputRowSize); + break; + case DATA_BLOCK_DESC_CODE_PRECISION: + code = tlvDecodeU8(pTlv, &pNode->precision); + break; + default: + break; + } + } + + return code; +} + +enum { + SLOT_DESC_CODE_SLOT_ID = 1, + SLOT_DESC_CODE_DATA_TYPE, + SLOT_DESC_CODE_RESERVE, + SLOT_DESC_CODE_OUTPUT, + SLOT_DESC_CODE_TAG +}; + +static int32_t slotDescNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSlotDescNode* pNode = (const SSlotDescNode*)pObj; + + int32_t code = tlvEncodeI16(pEncoder, SLOT_DESC_CODE_SLOT_ID, pNode->slotId); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, SLOT_DESC_CODE_DATA_TYPE, dataTypeToMsg, &pNode->dataType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, SLOT_DESC_CODE_RESERVE, pNode->reserve); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, SLOT_DESC_CODE_OUTPUT, pNode->output); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, SLOT_DESC_CODE_TAG, pNode->tag); + } + + return code; +} + +static int32_t msgToSlotDescNode(STlvDecoder* pDecoder, void* pObj) { + SSlotDescNode* pNode = (SSlotDescNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case SLOT_DESC_CODE_SLOT_ID: + code = tlvDecodeI16(pTlv, &pNode->slotId); + break; + case SLOT_DESC_CODE_DATA_TYPE: + code = tlvDecodeObjFromTlv(pTlv, msgToDataType, &pNode->dataType); + break; + case SLOT_DESC_CODE_RESERVE: + code = tlvDecodeBool(pTlv, &pNode->reserve); + break; + case SLOT_DESC_CODE_OUTPUT: + code = tlvDecodeBool(pTlv, &pNode->output); + break; + case SLOT_DESC_CODE_TAG: + code = tlvDecodeBool(pTlv, &pNode->tag); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_NODE_CODE_OUTPUT_DESC = 1, + PHY_NODE_CODE_CONDITIONS, + PHY_NODE_CODE_CHILDREN, + PHY_NODE_CODE_LIMIT, + PHY_NODE_CODE_SLIMIT +}; + +static int32_t physiNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SPhysiNode* pNode = (const SPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_NODE_CODE_OUTPUT_DESC, nodeToMsg, pNode->pOutputDataBlockDesc); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_NODE_CODE_CONDITIONS, nodeToMsg, pNode->pConditions); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_NODE_CODE_CHILDREN, nodeListToMsg, pNode->pChildren); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_NODE_CODE_LIMIT, nodeToMsg, pNode->pLimit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_NODE_CODE_SLIMIT, nodeToMsg, pNode->pSlimit); + } + + return code; +} + +static int32_t msgToPhysiNode(STlvDecoder* pDecoder, void* pObj) { + SPhysiNode* pNode = (SPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_NODE_CODE_OUTPUT_DESC: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pOutputDataBlockDesc); + break; + case PHY_NODE_CODE_CONDITIONS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pConditions); + break; + case PHY_NODE_CODE_CHILDREN: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pChildren); + break; + case PHY_NODE_CODE_LIMIT: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pLimit); + break; + case PHY_NODE_CODE_SLIMIT: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pSlimit); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_SCAN_CODE_BASE_NODE = 1, + PHY_SCAN_CODE_SCAN_COLS, + PHY_SCAN_CODE_SCAN_PSEUDO_COLS, + PHY_SCAN_CODE_BASE_UID, + PHY_SCAN_CODE_BASE_SUID, + PHY_SCAN_CODE_BASE_TABLE_TYPE, + PHY_SCAN_CODE_BASE_TABLE_NAME +}; + +static int32_t physiScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SScanPhysiNode* pNode = (const SScanPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_SCAN_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SCAN_CODE_SCAN_COLS, nodeListToMsg, pNode->pScanCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SCAN_CODE_SCAN_PSEUDO_COLS, nodeListToMsg, pNode->pScanPseudoCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, PHY_SCAN_CODE_BASE_UID, pNode->uid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, PHY_SCAN_CODE_BASE_SUID, pNode->suid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_SCAN_CODE_BASE_TABLE_TYPE, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SCAN_CODE_BASE_TABLE_NAME, nameToMsg, &pNode->tableName); + } + + return code; +} + +static int32_t msgToPhysiScanNode(STlvDecoder* pDecoder, void* pObj) { + SScanPhysiNode* pNode = (SScanPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_SCAN_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_SCAN_CODE_SCAN_COLS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pScanCols); + break; + case PHY_SCAN_CODE_SCAN_PSEUDO_COLS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pScanPseudoCols); + break; + case PHY_SCAN_CODE_BASE_UID: + code = tlvDecodeU64(pTlv, &pNode->uid); + break; + case PHY_SCAN_CODE_BASE_SUID: + code = tlvDecodeU64(pTlv, &pNode->suid); + break; + case PHY_SCAN_CODE_BASE_TABLE_TYPE: + code = tlvDecodeI8(pTlv, &pNode->tableType); + break; + case PHY_SCAN_CODE_BASE_TABLE_NAME: + code = tlvDecodeObjFromTlv(pTlv, msgToName, &pNode->tableName); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_LAST_ROW_SCAN_CODE_SCAN = 1, PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS, PHY_LAST_ROW_SCAN_CODE_GROUP_SORT }; + +static int32_t physiLastRowScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_LAST_ROW_SCAN_CODE_SCAN, physiScanNodeToMsg, &pNode->scan); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS, nodeListToMsg, pNode->pGroupTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_LAST_ROW_SCAN_CODE_GROUP_SORT, pNode->groupSort); + } + + return code; +} + +static int32_t msgToPhysiLastRowScanNode(STlvDecoder* pDecoder, void* pObj) { + SLastRowScanPhysiNode* pNode = (SLastRowScanPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_LAST_ROW_SCAN_CODE_SCAN: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiScanNode, &pNode->scan); + break; + case PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pGroupTags); + break; + case PHY_LAST_ROW_SCAN_CODE_GROUP_SORT: + code = tlvDecodeBool(pTlv, &pNode->groupSort); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_TABLE_SCAN_CODE_SCAN = 1, + PHY_TABLE_SCAN_CODE_SCAN_COUNT, + PHY_TABLE_SCAN_CODE_REVERSE_SCAN_COUNT, + PHY_TABLE_SCAN_CODE_SCAN_RANGE, + PHY_TABLE_SCAN_CODE_RATIO, + PHY_TABLE_SCAN_CODE_DATA_REQUIRED, + PHY_TABLE_SCAN_CODE_DYN_SCAN_FUNCS, + PHY_TABLE_SCAN_CODE_GROUP_TAGS, + PHY_TABLE_SCAN_CODE_GROUP_SORT, + PHY_TABLE_SCAN_CODE_INTERVAL, + PHY_TABLE_SCAN_CODE_OFFSET, + PHY_TABLE_SCAN_CODE_SLIDING, + PHY_TABLE_SCAN_CODE_INTERVAL_UNIT, + PHY_TABLE_SCAN_CODE_SLIDING_UNIT, + PHY_TABLE_SCAN_CODE_TRIGGER_TYPE, + PHY_TABLE_SCAN_CODE_WATERMARK, + PHY_TABLE_SCAN_CODE_IG_EXPIRED, + PHY_TABLE_SCAN_CODE_ASSIGN_BLOCK_UID, +}; + +static int32_t physiTableScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_TABLE_SCAN_CODE_SCAN, physiScanNodeToMsg, &pNode->scan); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU8(pEncoder, PHY_TABLE_SCAN_CODE_SCAN_COUNT, pNode->scanSeq[0]); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU8(pEncoder, PHY_TABLE_SCAN_CODE_REVERSE_SCAN_COUNT, pNode->scanSeq[1]); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_TABLE_SCAN_CODE_SCAN_RANGE, timeWindowToMsg, &pNode->scanRange); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeDouble(pEncoder, PHY_TABLE_SCAN_CODE_RATIO, pNode->ratio); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_TABLE_SCAN_CODE_DATA_REQUIRED, pNode->dataRequired); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_TABLE_SCAN_CODE_DYN_SCAN_FUNCS, nodeListToMsg, pNode->pDynamicScanFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_TABLE_SCAN_CODE_GROUP_TAGS, nodeListToMsg, pNode->pGroupTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_TABLE_SCAN_CODE_GROUP_SORT, pNode->groupSort); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_TABLE_SCAN_CODE_INTERVAL, pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_TABLE_SCAN_CODE_OFFSET, pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_TABLE_SCAN_CODE_SLIDING, pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_TABLE_SCAN_CODE_INTERVAL_UNIT, pNode->intervalUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_TABLE_SCAN_CODE_SLIDING_UNIT, pNode->slidingUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_TABLE_SCAN_CODE_TRIGGER_TYPE, pNode->triggerType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_TABLE_SCAN_CODE_WATERMARK, pNode->watermark); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_TABLE_SCAN_CODE_IG_EXPIRED, pNode->igExpired); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_TABLE_SCAN_CODE_ASSIGN_BLOCK_UID, pNode->assignBlockUid); + } + + return code; +} + +static int32_t msgToPhysiTableScanNode(STlvDecoder* pDecoder, void* pObj) { + STableScanPhysiNode* pNode = (STableScanPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_TABLE_SCAN_CODE_SCAN: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiScanNode, &pNode->scan); + break; + case PHY_TABLE_SCAN_CODE_SCAN_COUNT: + code = tlvDecodeU8(pTlv, pNode->scanSeq); + break; + case PHY_TABLE_SCAN_CODE_REVERSE_SCAN_COUNT: + code = tlvDecodeU8(pTlv, pNode->scanSeq + 1); + break; + case PHY_TABLE_SCAN_CODE_SCAN_RANGE: + code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, &pNode->scanRange); + break; + case PHY_TABLE_SCAN_CODE_RATIO: + code = tlvDecodeDouble(pTlv, &pNode->ratio); + break; + case PHY_TABLE_SCAN_CODE_DATA_REQUIRED: + code = tlvDecodeI32(pTlv, &pNode->dataRequired); + break; + case PHY_TABLE_SCAN_CODE_DYN_SCAN_FUNCS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pDynamicScanFuncs); + break; + case PHY_TABLE_SCAN_CODE_GROUP_TAGS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pGroupTags); + break; + case PHY_TABLE_SCAN_CODE_GROUP_SORT: + code = tlvDecodeBool(pTlv, &pNode->groupSort); + break; + case PHY_TABLE_SCAN_CODE_INTERVAL: + code = tlvDecodeI64(pTlv, &pNode->interval); + break; + case PHY_TABLE_SCAN_CODE_OFFSET: + code = tlvDecodeI64(pTlv, &pNode->offset); + break; + case PHY_TABLE_SCAN_CODE_SLIDING: + code = tlvDecodeI64(pTlv, &pNode->sliding); + break; + case PHY_TABLE_SCAN_CODE_INTERVAL_UNIT: + code = tlvDecodeI8(pTlv, &pNode->intervalUnit); + break; + case PHY_TABLE_SCAN_CODE_SLIDING_UNIT: + code = tlvDecodeI8(pTlv, &pNode->slidingUnit); + break; + case PHY_TABLE_SCAN_CODE_TRIGGER_TYPE: + code = tlvDecodeI8(pTlv, &pNode->triggerType); + break; + case PHY_TABLE_SCAN_CODE_WATERMARK: + code = tlvDecodeI64(pTlv, &pNode->watermark); + break; + case PHY_TABLE_SCAN_CODE_IG_EXPIRED: + code = tlvDecodeI8(pTlv, &pNode->igExpired); + break; + case PHY_TABLE_SCAN_CODE_ASSIGN_BLOCK_UID: + code = tlvDecodeBool(pTlv, &pNode->assignBlockUid); + break; + default: + break; + } + } + + return code; +} + +enum { EP_CODE_FQDN = 1, EP_CODE_port }; + +static int32_t epToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SEp* pNode = (const SEp*)pObj; + + int32_t code = tlvEncodeCStr(pEncoder, EP_CODE_FQDN, pNode->fqdn); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU16(pEncoder, EP_CODE_port, pNode->port); + } + + return code; +} + +static int32_t msgToEp(STlvDecoder* pDecoder, void* pObj) { + SEp* pNode = (SEp*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case EP_CODE_FQDN: + code = tlvDecodeCStr(pTlv, pNode->fqdn); + break; + case EP_CODE_port: + code = tlvDecodeU16(pTlv, &pNode->port); + break; + default: + break; + } + } + + return code; +} + +enum { EP_SET_CODE_IN_USE = 1, EP_SET_CODE_NUM_OF_EPS, EP_SET_CODE_EPS }; + +static int32_t epSetToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SEpSet* pNode = (const SEpSet*)pObj; + + int32_t code = tlvEncodeI8(pEncoder, EP_SET_CODE_IN_USE, pNode->inUse); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, EP_SET_CODE_NUM_OF_EPS, pNode->numOfEps); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObjArray(pEncoder, EP_SET_CODE_EPS, epToMsg, pNode->eps, sizeof(SEp), pNode->numOfEps); + } + + return code; +} + +static int32_t msgToEpSet(STlvDecoder* pDecoder, void* pObj) { + SEpSet* pNode = (SEpSet*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case EP_SET_CODE_IN_USE: + code = tlvDecodeI8(pTlv, &pNode->inUse); + break; + case EP_SET_CODE_NUM_OF_EPS: + code = tlvDecodeI8(pTlv, &pNode->numOfEps); + break; + case EP_SET_CODE_EPS: + code = tlvDecodeObjArrayFromTlv(pTlv, msgToEp, pNode->eps, sizeof(SEp)); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_SYSTABLE_SCAN_CODE_SCAN = 1, + PHY_SYSTABLE_SCAN_CODE_MGMT_EP_SET, + PHY_SYSTABLE_SCAN_CODE_SHOW_REWRITE, + PHY_SYSTABLE_SCAN_CODE_ACCOUNT_ID, + PHY_SYSTABLE_SCAN_CODE_SYS_INFO +}; + +static int32_t physiSysTableScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSystemTableScanPhysiNode* pNode = (const SSystemTableScanPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_SYSTABLE_SCAN_CODE_SCAN, physiScanNodeToMsg, &pNode->scan); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SYSTABLE_SCAN_CODE_MGMT_EP_SET, epSetToMsg, &pNode->mgmtEpSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_SYSTABLE_SCAN_CODE_SHOW_REWRITE, pNode->showRewrite); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_SYSTABLE_SCAN_CODE_ACCOUNT_ID, pNode->accountId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_SYSTABLE_SCAN_CODE_SYS_INFO, pNode->sysInfo); + } + + return code; +} + +static int32_t msgToPhysiSysTableScanNode(STlvDecoder* pDecoder, void* pObj) { + SSystemTableScanPhysiNode* pNode = (SSystemTableScanPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_SYSTABLE_SCAN_CODE_SCAN: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiScanNode, &pNode->scan); + break; + case PHY_SYSTABLE_SCAN_CODE_MGMT_EP_SET: + code = tlvDecodeObjFromTlv(pTlv, msgToEpSet, &pNode->mgmtEpSet); + break; + case PHY_SYSTABLE_SCAN_CODE_SHOW_REWRITE: + code = tlvDecodeBool(pTlv, &pNode->showRewrite); + break; + case PHY_SYSTABLE_SCAN_CODE_ACCOUNT_ID: + code = tlvDecodeI32(pTlv, &pNode->accountId); + break; + case PHY_SYSTABLE_SCAN_CODE_SYS_INFO: + code = tlvDecodeBool(pTlv, &pNode->sysInfo); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_PROJECT_CODE_BASE_NODE = 1, + PHY_PROJECT_CODE_PROJECTIONS, + PHY_PROJECT_CODE_MERGE_DATA_BLOCK, + PHY_PROJECT_CODE_IGNORE_GROUP_ID +}; + +static int32_t physiProjectNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_PROJECT_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_PROJECT_CODE_PROJECTIONS, nodeListToMsg, pNode->pProjections); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_PROJECT_CODE_MERGE_DATA_BLOCK, pNode->mergeDataBlock); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_PROJECT_CODE_IGNORE_GROUP_ID, pNode->ignoreGroupId); + } + + return code; +} + +static int32_t msgToPhysiProjectNode(STlvDecoder* pDecoder, void* pObj) { + SProjectPhysiNode* pNode = (SProjectPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_PROJECT_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_PROJECT_CODE_PROJECTIONS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pProjections); + break; + case PHY_PROJECT_CODE_MERGE_DATA_BLOCK: + code = tlvDecodeBool(pTlv, &pNode->mergeDataBlock); + break; + case PHY_PROJECT_CODE_IGNORE_GROUP_ID: + code = tlvDecodeBool(pTlv, &pNode->ignoreGroupId); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_SORT_MERGE_JOIN_CODE_BASE_NODE = 1, + PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE, + PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION, + PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, + PHY_SORT_MERGE_JOIN_CODE_TARGETS, + PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER +}; + +static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE, pNode->joinType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION, nodeToMsg, pNode->pMergeCondition); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, nodeToMsg, pNode->pOnConditions); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TARGETS, nodeListToMsg, pNode->pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); + } + + return code; +} + +static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { + SSortMergeJoinPhysiNode* pNode = (SSortMergeJoinPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_SORT_MERGE_JOIN_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE: + code = tlvDecodeEnum(pTlv, &pNode->joinType, sizeof(pNode->joinType)); + break; + case PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pMergeCondition); + break; + case PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pOnConditions); + break; + case PHY_SORT_MERGE_JOIN_CODE_TARGETS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); + break; + case PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_AGG_CODE_BASE_NODE = 1, + PHY_AGG_CODE_EXPR, + PHY_AGG_CODE_GROUP_KEYS, + PHY_AGG_CODE_AGG_FUNCS, + PHY_AGG_CODE_MERGE_DATA_BLOCK +}; + +static int32_t physiAggNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SAggPhysiNode* pNode = (const SAggPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_AGG_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_AGG_CODE_EXPR, nodeListToMsg, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_AGG_CODE_GROUP_KEYS, nodeListToMsg, pNode->pGroupKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_AGG_CODE_AGG_FUNCS, nodeListToMsg, pNode->pAggFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_AGG_CODE_MERGE_DATA_BLOCK, pNode->mergeDataBlock); + } + + return code; +} + +static int32_t msgToPhysiAggNode(STlvDecoder* pDecoder, void* pObj) { + SAggPhysiNode* pNode = (SAggPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_AGG_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_AGG_CODE_EXPR: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs); + break; + case PHY_AGG_CODE_GROUP_KEYS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pGroupKeys); + break; + case PHY_AGG_CODE_AGG_FUNCS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pAggFuncs); + break; + case PHY_AGG_CODE_MERGE_DATA_BLOCK: + code = tlvDecodeBool(pTlv, &pNode->mergeDataBlock); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_EXCHANGE_CODE_BASE_NODE = 1, + PHY_EXCHANGE_CODE_SRC_GROUP_ID, + PHY_EXCHANGE_CODE_SINGLE_CHANNEL, + PHY_EXCHANGE_CODE_SRC_ENDPOINTS +}; + +static int32_t physiExchangeNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SExchangePhysiNode* pNode = (const SExchangePhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_EXCHANGE_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_EXCHANGE_CODE_SRC_GROUP_ID, pNode->srcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_EXCHANGE_CODE_SINGLE_CHANNEL, pNode->singleChannel); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_EXCHANGE_CODE_SRC_ENDPOINTS, nodeListToMsg, pNode->pSrcEndPoints); + } + + return code; +} + +static int32_t msgToPhysiExchangeNode(STlvDecoder* pDecoder, void* pObj) { + SExchangePhysiNode* pNode = (SExchangePhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_EXCHANGE_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_EXCHANGE_CODE_SRC_GROUP_ID: + code = tlvDecodeI32(pTlv, &pNode->srcGroupId); + break; + case PHY_EXCHANGE_CODE_SINGLE_CHANNEL: + code = tlvDecodeBool(pTlv, &pNode->singleChannel); + break; + case PHY_EXCHANGE_CODE_SRC_ENDPOINTS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pSrcEndPoints); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_MERGE_CODE_BASE_NODE = 1, + PHY_MERGE_CODE_MERGE_KEYS, + PHY_MERGE_CODE_TARGETS, + PHY_MERGE_CODE_NUM_OF_CHANNELS, + PHY_MERGE_CODE_SRC_GROUP_ID, + PHY_MERGE_CODE_GROUP_SORT +}; + +static int32_t physiMergeNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SMergePhysiNode* pNode = (const SMergePhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_MERGE_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_MERGE_CODE_MERGE_KEYS, nodeListToMsg, pNode->pMergeKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_MERGE_CODE_TARGETS, nodeListToMsg, pNode->pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_MERGE_CODE_NUM_OF_CHANNELS, pNode->numOfChannels); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_MERGE_CODE_SRC_GROUP_ID, pNode->srcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_MERGE_CODE_GROUP_SORT, pNode->groupSort); + } + + return code; +} + +static int32_t msgToPhysiMergeNode(STlvDecoder* pDecoder, void* pObj) { + SMergePhysiNode* pNode = (SMergePhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_MERGE_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_MERGE_CODE_MERGE_KEYS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pMergeKeys); + break; + case PHY_MERGE_CODE_TARGETS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); + break; + case PHY_MERGE_CODE_NUM_OF_CHANNELS: + code = tlvDecodeI32(pTlv, &pNode->numOfChannels); + break; + case PHY_MERGE_CODE_SRC_GROUP_ID: + code = tlvDecodeI32(pTlv, &pNode->srcGroupId); + break; + case PHY_MERGE_CODE_GROUP_SORT: + code = tlvDecodeBool(pTlv, &pNode->groupSort); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_SORT_CODE_BASE_NODE = 1, PHY_SORT_CODE_EXPR, PHY_SORT_CODE_SORT_KEYS, PHY_SORT_CODE_TARGETS }; + +static int32_t physiSortNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSortPhysiNode* pNode = (const SSortPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_SORT_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_CODE_EXPR, nodeListToMsg, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_CODE_SORT_KEYS, nodeListToMsg, pNode->pSortKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_CODE_TARGETS, nodeListToMsg, pNode->pTargets); + } + + return code; +} + +static int32_t msgToPhysiSortNode(STlvDecoder* pDecoder, void* pObj) { + SSortPhysiNode* pNode = (SSortPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_SORT_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_SORT_CODE_EXPR: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs); + break; + case PHY_SORT_CODE_SORT_KEYS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pSortKeys); + break; + case PHY_SORT_CODE_TARGETS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_WINDOW_CODE_BASE_NODE = 1, + PHY_WINDOW_CODE_EXPR, + PHY_WINDOW_CODE_FUNCS, + PHY_WINDOW_CODE_TS_PK, + PHY_WINDOW_CODE_TS_END, + PHY_WINDOW_CODE_TRIGGER_TYPE, + PHY_WINDOW_CODE_WATERMARK, + PHY_WINDOW_CODE_IG_EXPIRED, + PHY_WINDOW_CODE_INPUT_TS_ORDER, + PHY_WINDOW_CODE_OUTPUT_TS_ORDER, + PHY_WINDOW_CODE_MERGE_DATA_BLOCK +}; + +static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_EXPR, nodeListToMsg, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_FUNCS, nodeListToMsg, pNode->pFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_TS_PK, nodeToMsg, pNode->pTspk); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_TS_END, nodeToMsg, pNode->pTsEnd); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_WINDOW_CODE_TRIGGER_TYPE, pNode->triggerType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_WINDOW_CODE_WATERMARK, pNode->watermark); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_WINDOW_CODE_IG_EXPIRED, pNode->igExpired); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_WINDOW_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_WINDOW_CODE_OUTPUT_TS_ORDER, pNode->outputTsOrder); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_WINDOW_CODE_MERGE_DATA_BLOCK, pNode->mergeDataBlock); + } + + return code; +} + +static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) { + SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_WINDOW_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_WINDOW_CODE_EXPR: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs); + break; + case PHY_WINDOW_CODE_FUNCS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pFuncs); + break; + case PHY_WINDOW_CODE_TS_PK: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTspk); + break; + case PHY_WINDOW_CODE_TS_END: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTsEnd); + break; + case PHY_WINDOW_CODE_TRIGGER_TYPE: + code = tlvDecodeI8(pTlv, &pNode->triggerType); + break; + case PHY_WINDOW_CODE_WATERMARK: + code = tlvDecodeI64(pTlv, &pNode->watermark); + break; + case PHY_WINDOW_CODE_IG_EXPIRED: + code = tlvDecodeI8(pTlv, &pNode->igExpired); + break; + case PHY_WINDOW_CODE_INPUT_TS_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); + break; + case PHY_WINDOW_CODE_OUTPUT_TS_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->outputTsOrder, sizeof(pNode->outputTsOrder)); + break; + case PHY_WINDOW_CODE_MERGE_DATA_BLOCK: + code = tlvDecodeBool(pTlv, &pNode->mergeDataBlock); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_INTERVAL_CODE_WINDOW = 1, + PHY_INTERVAL_CODE_INTERVAL, + PHY_INTERVAL_CODE_OFFSET, + PHY_INTERVAL_CODE_SLIDING, + PHY_INTERVAL_CODE_INTERVAL_UNIT, + PHY_INTERVAL_CODE_SLIDING_UNIT +}; + +static int32_t physiIntervalNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_INTERVAL_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_INTERVAL_CODE_INTERVAL, pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_INTERVAL_CODE_OFFSET, pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_INTERVAL_CODE_SLIDING, pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_INTERVAL_CODE_INTERVAL_UNIT, pNode->intervalUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_INTERVAL_CODE_SLIDING_UNIT, pNode->slidingUnit); + } + + return code; +} + +static int32_t msgToPhysiIntervalNode(STlvDecoder* pDecoder, void* pObj) { + SIntervalPhysiNode* pNode = (SIntervalPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_INTERVAL_CODE_WINDOW: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window); + break; + case PHY_INTERVAL_CODE_INTERVAL: + code = tlvDecodeI64(pTlv, &pNode->interval); + break; + case PHY_INTERVAL_CODE_OFFSET: + code = tlvDecodeI64(pTlv, &pNode->offset); + break; + case PHY_INTERVAL_CODE_SLIDING: + code = tlvDecodeI64(pTlv, &pNode->sliding); + break; + case PHY_INTERVAL_CODE_INTERVAL_UNIT: + code = tlvDecodeI8(pTlv, &pNode->intervalUnit); + break; + case PHY_INTERVAL_CODE_SLIDING_UNIT: + code = tlvDecodeI8(pTlv, &pNode->slidingUnit); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_FILL_CODE_BASE_NODE = 1, + PHY_FILL_CODE_MODE, + PHY_FILL_CODE_FILL_EXPRS, + PHY_FILL_CODE_NOT_FILL_EXPRS, + PHY_FILL_CODE_WSTART, + PHY_FILL_CODE_VALUES, + PHY_FILL_CODE_TIME_RANGE, + PHY_FILL_CODE_INPUT_TS_ORDER +}; + +static int32_t physiFillNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SFillPhysiNode* pNode = (const SFillPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_FILL_CODE_MODE, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_FILL_EXPRS, nodeListToMsg, pNode->pFillExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_NOT_FILL_EXPRS, nodeListToMsg, pNode->pNotFillExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_WSTART, nodeToMsg, pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_VALUES, nodeToMsg, pNode->pValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_TIME_RANGE, timeWindowToMsg, &pNode->timeRange); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_FILL_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); + } + + return code; +} + +static int32_t msgToPhysiFillNode(STlvDecoder* pDecoder, void* pObj) { + SFillPhysiNode* pNode = (SFillPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_FILL_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_FILL_CODE_MODE: + code = tlvDecodeEnum(pTlv, &pNode->mode, sizeof(pNode->mode)); + break; + case PHY_FILL_CODE_FILL_EXPRS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pFillExprs); + break; + case PHY_FILL_CODE_NOT_FILL_EXPRS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pNotFillExprs); + break; + case PHY_FILL_CODE_WSTART: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pWStartTs); + break; + case PHY_FILL_CODE_VALUES: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pValues); + break; + case PHY_FILL_CODE_TIME_RANGE: + code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, (void**)&pNode->timeRange); + break; + case PHY_FILL_CODE_INPUT_TS_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_SESSION_CODE_WINDOW = 1, PHY_SESSION_CODE_GAP }; + +static int32_t physiSessionWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSessionWinodwPhysiNode* pNode = (const SSessionWinodwPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_SESSION_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_SESSION_CODE_GAP, pNode->gap); + } + + return code; +} + +static int32_t msgToPhysiSessionWindowNode(STlvDecoder* pDecoder, void* pObj) { + SSessionWinodwPhysiNode* pNode = (SSessionWinodwPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_SESSION_CODE_WINDOW: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window); + break; + case PHY_SESSION_CODE_GAP: + code = tlvDecodeI64(pTlv, &pNode->gap); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_STATE_CODE_WINDOW = 1, PHY_STATE_CODE_KEY }; + +static int32_t physiStateWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SStateWinodwPhysiNode* pNode = (const SStateWinodwPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_STATE_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_STATE_CODE_KEY, nodeToMsg, pNode->pStateKey); + } + + return code; +} + +static int32_t msgToPhysiStateWindowNode(STlvDecoder* pDecoder, void* pObj) { + SStateWinodwPhysiNode* pNode = (SStateWinodwPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_STATE_CODE_WINDOW: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window); + break; + case PHY_STATE_CODE_KEY: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pStateKey); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_PARTITION_CODE_BASE_NODE = 1, PHY_PARTITION_CODE_EXPR, PHY_PARTITION_CODE_KEYS, PHY_PARTITION_CODE_TARGETS }; + +static int32_t physiPartitionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SPartitionPhysiNode* pNode = (const SPartitionPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_PARTITION_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_PARTITION_CODE_EXPR, nodeListToMsg, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_PARTITION_CODE_KEYS, nodeListToMsg, pNode->pPartitionKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_PARTITION_CODE_TARGETS, nodeListToMsg, pNode->pTargets); + } + + return code; +} + +static int32_t msgToPhysiPartitionNode(STlvDecoder* pDecoder, void* pObj) { + SPartitionPhysiNode* pNode = (SPartitionPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_PARTITION_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_PARTITION_CODE_EXPR: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs); + break; + case PHY_PARTITION_CODE_KEYS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pPartitionKeys); + break; + case PHY_PARTITION_CODE_TARGETS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_INDEF_ROWS_FUNC_CODE_BASE_NODE = 1, PHY_INDEF_ROWS_FUNC_CODE_EXPRS, PHY_INDEF_ROWS_FUNC_CODE_FUNCS }; + +static int32_t physiIndefRowsFuncNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SIndefRowsFuncPhysiNode* pNode = (const SIndefRowsFuncPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_INDEF_ROWS_FUNC_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INDEF_ROWS_FUNC_CODE_EXPRS, nodeListToMsg, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INDEF_ROWS_FUNC_CODE_FUNCS, nodeListToMsg, pNode->pFuncs); + } + + return code; +} + +static int32_t msgToPhysiIndefRowsFuncNode(STlvDecoder* pDecoder, void* pObj) { + SIndefRowsFuncPhysiNode* pNode = (SIndefRowsFuncPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_INDEF_ROWS_FUNC_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_INDEF_ROWS_FUNC_CODE_EXPRS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs); + break; + case PHY_INDEF_ROWS_FUNC_CODE_FUNCS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pFuncs); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_INERP_FUNC_CODE_BASE_NODE = 1, + PHY_INERP_FUNC_CODE_EXPR, + PHY_INERP_FUNC_CODE_FUNCS, + PHY_INERP_FUNC_CODE_TIME_RANGE, + PHY_INERP_FUNC_CODE_INTERVAL, + PHY_INERP_FUNC_CODE_INTERVAL_UNIT, + PHY_INERP_FUNC_CODE_FILL_MODE, + PHY_INERP_FUNC_CODE_FILL_VALUES, + PHY_INERP_FUNC_CODE_TIME_SERIES +}; + +static int32_t physiInterpFuncNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SInterpFuncPhysiNode* pNode = (const SInterpFuncPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_EXPR, nodeListToMsg, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_FUNCS, nodeListToMsg, pNode->pFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_TIME_RANGE, timeWindowToMsg, &pNode->timeRange); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_INERP_FUNC_CODE_INTERVAL, pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_INERP_FUNC_CODE_INTERVAL_UNIT, pNode->intervalUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_INERP_FUNC_CODE_FILL_MODE, pNode->fillMode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_FILL_VALUES, nodeToMsg, pNode->pFillValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_TIME_SERIES, nodeToMsg, pNode->pTimeSeries); + } + + return code; +} + +static int32_t msgToPhysiInterpFuncNode(STlvDecoder* pDecoder, void* pObj) { + SInterpFuncPhysiNode* pNode = (SInterpFuncPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_INERP_FUNC_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_INERP_FUNC_CODE_EXPR: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs); + break; + case PHY_INERP_FUNC_CODE_FUNCS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pFuncs); + break; + case PHY_INERP_FUNC_CODE_TIME_RANGE: + code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, &pNode->timeRange); + break; + case PHY_INERP_FUNC_CODE_INTERVAL: + code = tlvDecodeI64(pTlv, &pNode->interval); + break; + case PHY_INERP_FUNC_CODE_INTERVAL_UNIT: + code = tlvDecodeI8(pTlv, &pNode->intervalUnit); + break; + case PHY_INERP_FUNC_CODE_FILL_MODE: + code = tlvDecodeEnum(pTlv, &pNode->fillMode, sizeof(pNode->fillMode)); + break; + case PHY_INERP_FUNC_CODE_FILL_VALUES: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pFillValues); + break; + case PHY_INERP_FUNC_CODE_TIME_SERIES: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTimeSeries); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_DATA_SINK_CODE_INPUT_DESC = 1 }; + +static int32_t physicDataSinkNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SDataSinkNode* pNode = (const SDataSinkNode*)pObj; + return tlvEncodeObj(pEncoder, PHY_DATA_SINK_CODE_INPUT_DESC, nodeToMsg, pNode->pInputDataBlockDesc); +} + +static int32_t msgToPhysicDataSinkNode(STlvDecoder* pDecoder, void* pObj) { + SDataSinkNode* pNode = (SDataSinkNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_DATA_SINK_CODE_INPUT_DESC: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pInputDataBlockDesc); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_DISPATCH_CODE_SINK = 1 }; + +static int32_t physiDispatchNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SDataDispatcherNode* pNode = (const SDataDispatcherNode*)pObj; + return tlvEncodeObj(pEncoder, PHY_DISPATCH_CODE_SINK, physicDataSinkNodeToMsg, &pNode->sink); +} + +static int32_t msgToPhysiDispatchNode(STlvDecoder* pDecoder, void* pObj) { + SDataDispatcherNode* pNode = (SDataDispatcherNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_DISPATCH_CODE_SINK: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysicDataSinkNode, &pNode->sink); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_QUERY_INSERT_CODE_SINK = 1, + PHY_QUERY_INSERT_CODE_COLS, + PHY_QUERY_INSERT_CODE_TABLE_ID, + PHY_QUERY_INSERT_CODE_STABLE_ID, + PHY_QUERY_INSERT_CODE_TABLE_TYPE, + PHY_QUERY_INSERT_CODE_TABLE_NAME, + PHY_QUERY_INSERT_CODE_VG_ID, + PHY_QUERY_INSERT_CODE_EP_SET +}; + +static int32_t physiQueryInsertNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SQueryInserterNode* pNode = (const SQueryInserterNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_QUERY_INSERT_CODE_SINK, physicDataSinkNodeToMsg, &pNode->sink); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_QUERY_INSERT_CODE_COLS, nodeListToMsg, pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, PHY_QUERY_INSERT_CODE_TABLE_ID, pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, PHY_QUERY_INSERT_CODE_STABLE_ID, pNode->stableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_QUERY_INSERT_CODE_TABLE_TYPE, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, PHY_QUERY_INSERT_CODE_TABLE_NAME, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_QUERY_INSERT_CODE_VG_ID, pNode->vgId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_QUERY_INSERT_CODE_EP_SET, epSetToMsg, &pNode->epSet); + } + + return code; +} + +static int32_t msgToPhysiQueryInsertNode(STlvDecoder* pDecoder, void* pObj) { + SQueryInserterNode* pNode = (SQueryInserterNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_QUERY_INSERT_CODE_SINK: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysicDataSinkNode, &pNode->sink); + break; + case PHY_QUERY_INSERT_CODE_COLS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pCols); + break; + case PHY_QUERY_INSERT_CODE_TABLE_ID: + code = tlvDecodeU64(pTlv, &pNode->tableId); + break; + case PHY_QUERY_INSERT_CODE_STABLE_ID: + code = tlvDecodeU64(pTlv, &pNode->stableId); + break; + case PHY_QUERY_INSERT_CODE_TABLE_TYPE: + code = tlvDecodeI8(pTlv, &pNode->tableType); + break; + case PHY_QUERY_INSERT_CODE_TABLE_NAME: + code = tlvDecodeCStr(pTlv, pNode->tableName); + break; + case PHY_QUERY_INSERT_CODE_VG_ID: + code = tlvDecodeI32(pTlv, &pNode->vgId); + break; + case PHY_QUERY_INSERT_CODE_EP_SET: + code = tlvDecodeObjFromTlv(pTlv, msgToEpSet, &pNode->epSet); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_DELETER_CODE_SINK = 1, + PHY_DELETER_CODE_TABLE_ID, + PHY_DELETER_CODE_TABLE_TYPE, + PHY_DELETER_CODE_TABLE_FNAME, + PHY_DELETER_CODE_TS_COL_NAME, + PHY_DELETER_CODE_DELETE_TIME_RANGE, + PHY_DELETER_CODE_AFFECTED_ROWS +}; + +static int32_t physiDeleteNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SDataDeleterNode* pNode = (const SDataDeleterNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_DELETER_CODE_SINK, physicDataSinkNodeToMsg, &pNode->sink); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, PHY_DELETER_CODE_TABLE_ID, pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_DELETER_CODE_TABLE_TYPE, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, PHY_DELETER_CODE_TABLE_FNAME, pNode->tableFName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, PHY_DELETER_CODE_TS_COL_NAME, pNode->tsColName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_DELETER_CODE_DELETE_TIME_RANGE, timeWindowToMsg, &pNode->deleteTimeRange); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_DELETER_CODE_AFFECTED_ROWS, nodeToMsg, pNode->pAffectedRows); + } + + return code; +} + +static int32_t msgToPhysiDeleteNode(STlvDecoder* pDecoder, void* pObj) { + SDataDeleterNode* pNode = (SDataDeleterNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_DELETER_CODE_SINK: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysicDataSinkNode, &pNode->sink); + break; + case PHY_DELETER_CODE_TABLE_ID: + code = tlvDecodeU64(pTlv, &pNode->tableId); + break; + case PHY_DELETER_CODE_TABLE_TYPE: + code = tlvDecodeI8(pTlv, &pNode->tableType); + break; + case PHY_DELETER_CODE_TABLE_FNAME: + code = tlvDecodeCStr(pTlv, pNode->tableFName); + break; + case PHY_DELETER_CODE_TS_COL_NAME: + code = tlvDecodeCStr(pTlv, pNode->tsColName); + break; + case PHY_DELETER_CODE_DELETE_TIME_RANGE: + code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, &pNode->deleteTimeRange); + break; + case PHY_DELETER_CODE_AFFECTED_ROWS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pAffectedRows); + break; + default: + break; + } + } + + return code; +} + +enum { SUBPLAN_ID_CODE_QUERY_ID = 1, SUBPLAN_ID_CODE_GROUP_ID, SUBPLAN_ID_CODE_SUBPLAN_ID }; + +static int32_t subplanIdToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSubplanId* pNode = (const SSubplanId*)pObj; + + int32_t code = tlvEncodeU64(pEncoder, SUBPLAN_ID_CODE_QUERY_ID, pNode->queryId); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, SUBPLAN_ID_CODE_GROUP_ID, pNode->groupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, SUBPLAN_ID_CODE_SUBPLAN_ID, pNode->subplanId); + } + + return code; +} + +static int32_t msgToSubplanId(STlvDecoder* pDecoder, void* pObj) { + SSubplanId* pNode = (SSubplanId*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case SUBPLAN_ID_CODE_QUERY_ID: + code = tlvDecodeU64(pTlv, &pNode->queryId); + break; + case SUBPLAN_ID_CODE_GROUP_ID: + code = tlvDecodeI32(pTlv, &pNode->groupId); + break; + case SUBPLAN_ID_CODE_SUBPLAN_ID: + code = tlvDecodeI32(pTlv, &pNode->subplanId); + break; + default: + break; + } + } + + return code; +} + +enum { QUERY_NODE_ADDR_CODE_NODE_ID = 1, QUERY_NODE_ADDR_CODE_EP_SET }; + +static int32_t queryNodeAddrToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SQueryNodeAddr* pNode = (const SQueryNodeAddr*)pObj; + + int32_t code = tlvEncodeI32(pEncoder, QUERY_NODE_ADDR_CODE_NODE_ID, pNode->nodeId); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, QUERY_NODE_ADDR_CODE_EP_SET, epSetToMsg, &pNode->epSet); + } + + return code; +} + +static int32_t msgToQueryNodeAddr(STlvDecoder* pDecoder, void* pObj) { + SQueryNodeAddr* pNode = (SQueryNodeAddr*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case QUERY_NODE_ADDR_CODE_NODE_ID: + code = tlvDecodeI32(pTlv, &pNode->nodeId); + break; + case QUERY_NODE_ADDR_CODE_EP_SET: + code = tlvDecodeObjFromTlv(pTlv, msgToEpSet, &pNode->epSet); + break; + } + } + + return code; +} + +enum { + SUBPLAN_CODE_SUBPLAN_ID = 1, + SUBPLAN_CODE_SUBPLAN_TYPE, + SUBPLAN_CODE_MSG_TYPE, + SUBPLAN_CODE_LEVEL, + SUBPLAN_CODE_DBFNAME, + SUBPLAN_CODE_USER, + SUBPLAN_CODE_EXECNODE, + SUBPLAN_CODE_ROOT_NODE, + SUBPLAN_CODE_DATA_SINK, + SUBPLAN_CODE_TAG_COND, + SUBPLAN_CODE_TAG_INDEX_COND +}; + +static int32_t subplanToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSubplan* pNode = (const SSubplan*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, SUBPLAN_CODE_SUBPLAN_ID, subplanIdToMsg, &pNode->id); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, SUBPLAN_CODE_SUBPLAN_TYPE, pNode->subplanType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, SUBPLAN_CODE_MSG_TYPE, pNode->msgType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, SUBPLAN_CODE_LEVEL, pNode->level); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, SUBPLAN_CODE_DBFNAME, pNode->dbFName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, SUBPLAN_CODE_USER, pNode->user); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, SUBPLAN_CODE_EXECNODE, queryNodeAddrToMsg, &pNode->execNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, SUBPLAN_CODE_ROOT_NODE, nodeToMsg, pNode->pNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, SUBPLAN_CODE_DATA_SINK, nodeToMsg, pNode->pDataSink); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, SUBPLAN_CODE_TAG_COND, nodeToMsg, pNode->pTagCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, SUBPLAN_CODE_TAG_INDEX_COND, nodeToMsg, pNode->pTagIndexCond); + } + + return code; +} + +static int32_t msgToSubplan(STlvDecoder* pDecoder, void* pObj) { + SSubplan* pNode = (SSubplan*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case SUBPLAN_CODE_SUBPLAN_ID: + code = tlvDecodeObjFromTlv(pTlv, msgToSubplanId, &pNode->id); + break; + case SUBPLAN_CODE_SUBPLAN_TYPE: + code = tlvDecodeEnum(pTlv, &pNode->subplanType, sizeof(pNode->subplanType)); + break; + case SUBPLAN_CODE_MSG_TYPE: + code = tlvDecodeI32(pTlv, &pNode->msgType); + break; + case SUBPLAN_CODE_LEVEL: + code = tlvDecodeI32(pTlv, &pNode->level); + break; + case SUBPLAN_CODE_DBFNAME: + code = tlvDecodeCStr(pTlv, pNode->dbFName); + break; + case SUBPLAN_CODE_USER: + code = tlvDecodeCStr(pTlv, pNode->user); + break; + case SUBPLAN_CODE_EXECNODE: + code = tlvDecodeObjFromTlv(pTlv, msgToQueryNodeAddr, &pNode->execNode); + break; + case SUBPLAN_CODE_ROOT_NODE: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pNode); + break; + case SUBPLAN_CODE_DATA_SINK: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pDataSink); + break; + case SUBPLAN_CODE_TAG_COND: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTagCond); + break; + case SUBPLAN_CODE_TAG_INDEX_COND: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTagIndexCond); + break; + default: + break; + } + } + + return code; +} + +enum { QUERY_PLAN_CODE_QUERY_ID = 1, QUERY_PLAN_CODE_NUM_OF_SUBPLANS, QUERY_PLAN_CODE_SUBPLANS }; + +static int32_t queryPlanToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SQueryPlan* pNode = (const SQueryPlan*)pObj; + + int32_t code = tlvEncodeU64(pEncoder, QUERY_PLAN_CODE_QUERY_ID, pNode->queryId); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, QUERY_PLAN_CODE_NUM_OF_SUBPLANS, pNode->numOfSubplans); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, QUERY_PLAN_CODE_SUBPLANS, nodeListToMsg, pNode->pSubplans); + } + + return code; +} + +static int32_t msgToQueryPlan(STlvDecoder* pDecoder, void* pObj) { + SQueryPlan* pNode = (SQueryPlan*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case QUERY_PLAN_CODE_QUERY_ID: + code = tlvDecodeU64(pTlv, &pNode->queryId); + break; + case QUERY_PLAN_CODE_NUM_OF_SUBPLANS: + code = tlvDecodeI32(pTlv, &pNode->numOfSubplans); + break; + case QUERY_PLAN_CODE_SUBPLANS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pSubplans); + break; + default: + break; + } + } + + return code; +} + +static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pObj)) { + case QUERY_NODE_COLUMN: + code = columnNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_VALUE: + code = valueNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_OPERATOR: + code = operatorNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_LOGIC_CONDITION: + code = logicConditionNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_FUNCTION: + code = functionNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_ORDER_BY_EXPR: + code = orderByExprNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_LIMIT: + code = limitNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_NODE_LIST: + code = nodeListNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_TARGET: + code = targetNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_DATABLOCK_DESC: + code = dataBlockDescNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_SLOT_DESC: + code = slotDescNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_LEFT_VALUE: + break; + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + code = physiScanNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + code = physiLastRowScanNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + code = physiTableScanNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + code = physiSysTableScanNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + code = physiProjectNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: + code = physiJoinNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: + code = physiAggNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + code = physiExchangeNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE: + code = physiMergeNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: + code = physiSortNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: + code = physiIntervalNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_FILL: + code = physiFillNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: + code = physiSessionWindowNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: + code = physiStateWindowNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: + code = physiPartitionNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: + code = physiIndefRowsFuncNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: + code = physiInterpFuncNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: + code = physiDispatchNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: + code = physiQueryInsertNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + code = physiDeleteNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_SUBPLAN: + code = subplanToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN: + code = queryPlanToMsg(pObj, pEncoder); + break; + default: + nodesWarn("specificNodeToMsg unknown node = %s", nodesNodeName(nodeType(pObj))); + break; + } + if (TSDB_CODE_SUCCESS != code) { + nodesError("specificNodeToMsg error node = %s", nodesNodeName(nodeType(pObj))); + } + return code; +} + +static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pObj)) { + case QUERY_NODE_COLUMN: + code = msgToColumnNode(pDecoder, pObj); + break; + case QUERY_NODE_VALUE: + code = msgToValueNode(pDecoder, pObj); + break; + case QUERY_NODE_OPERATOR: + code = msgToOperatorNode(pDecoder, pObj); + break; + case QUERY_NODE_LOGIC_CONDITION: + code = msgToLogicConditionNode(pDecoder, pObj); + break; + case QUERY_NODE_FUNCTION: + code = msgToFunctionNode(pDecoder, pObj); + break; + case QUERY_NODE_ORDER_BY_EXPR: + code = msgToOrderByExprNode(pDecoder, pObj); + break; + case QUERY_NODE_LIMIT: + code = msgToLimitNode(pDecoder, pObj); + break; + case QUERY_NODE_NODE_LIST: + code = msgToNodeListNode(pDecoder, pObj); + break; + case QUERY_NODE_TARGET: + code = msgToTargetNode(pDecoder, pObj); + break; + case QUERY_NODE_DATABLOCK_DESC: + code = msgToDataBlockDescNode(pDecoder, pObj); + break; + case QUERY_NODE_SLOT_DESC: + code = msgToSlotDescNode(pDecoder, pObj); + break; + case QUERY_NODE_LEFT_VALUE: + break; + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + code = msgToPhysiScanNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + code = msgToPhysiLastRowScanNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + code = msgToPhysiTableScanNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + code = msgToPhysiSysTableScanNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + code = msgToPhysiProjectNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: + code = msgToPhysiJoinNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: + code = msgToPhysiAggNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + code = msgToPhysiExchangeNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE: + code = msgToPhysiMergeNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: + code = msgToPhysiSortNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: + code = msgToPhysiIntervalNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_FILL: + code = msgToPhysiFillNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: + code = msgToPhysiSessionWindowNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: + code = msgToPhysiStateWindowNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: + code = msgToPhysiPartitionNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: + code = msgToPhysiIndefRowsFuncNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: + code = msgToPhysiInterpFuncNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: + code = msgToPhysiDispatchNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: + code = msgToPhysiQueryInsertNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + code = msgToPhysiDeleteNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_SUBPLAN: + code = msgToSubplan(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN: + code = msgToQueryPlan(pDecoder, pObj); + break; + default: + nodesWarn("msgToSpecificNode unknown node = %s", nodesNodeName(nodeType(pObj))); + break; + } + if (TSDB_CODE_SUCCESS != code) { + nodesError("msgToSpecificNode error node = %s", nodesNodeName(nodeType(pObj))); + } + return code; +} + +static int32_t nodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + return tlvEncodeObj(pEncoder, nodeType(pObj), specificNodeToMsg, pObj); +} + +static int32_t msgToNode(STlvDecoder* pDecoder, void** pObj) { + return tlvDecodeDynObj(pDecoder, (FMakeObject)nodesMakeNode, msgToSpecificNode, pObj); +} + +static int32_t msgToNodeFromTlv(STlv* pTlv, void** pObj) { + STlvDecoder decoder = {.bufSize = pTlv->len, .offset = 0, .pBuf = pTlv->value}; + return msgToNode(&decoder, pObj); +} + +static int32_t nodeListToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SNodeList* pList = (const SNodeList*)pObj; + + SNode* pNode = NULL; + FOREACH(pNode, pList) { + int32_t code = nodeToMsg(pNode, pEncoder); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t msgToNodeList(STlvDecoder* pDecoder, void** pObj) { + SNodeList* pList = nodesMakeList(); + + int32_t code = TSDB_CODE_SUCCESS; + while (TSDB_CODE_SUCCESS == code && !tlvDecodeEnd(pDecoder)) { + SNode* pNode = NULL; + code = msgToNode(pDecoder, (void**)&pNode); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListAppend(pList, pNode); + } + } + if (TSDB_CODE_SUCCESS == code) { + *pObj = pList; + } else { + nodesDestroyList(pList); + } + return code; +} + +static int32_t msgToNodeListFromTlv(STlv* pTlv, void** pObj) { + STlvDecoder decoder = {.bufSize = pTlv->len, .offset = 0, .pBuf = pTlv->value}; + return msgToNodeList(&decoder, pObj); +} + +int32_t nodesNodeToMsg(const SNode* pNode, char** pMsg, int32_t* pLen) { + if (NULL == pNode || NULL == pMsg || NULL == pLen) { + terrno = TSDB_CODE_FAILED; + return TSDB_CODE_FAILED; + } + + STlvEncoder encoder; + int32_t code = initTlvEncoder(&encoder); + if (TSDB_CODE_SUCCESS == code) { + code = nodeToMsg(pNode, &encoder); + } + if (TSDB_CODE_SUCCESS == code) { + endTlvEncode(&encoder, pMsg, pLen); + } + clearTlvEncoder(&encoder); + + terrno = code; + return code; +} + +int32_t nodesMsgToNode(const char* pMsg, int32_t len, SNode** pNode) { + if (NULL == pMsg || NULL == pNode) { + return TSDB_CODE_SUCCESS; + } + + STlvDecoder decoder = {.bufSize = len, .offset = 0, .pBuf = pMsg}; + int32_t code = msgToNode(&decoder, (void**)pNode); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(*pNode); + *pNode = NULL; + } + + terrno = code; + return code; +} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 240aa0d6c0..91787f5a72 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3770,7 +3770,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName checkDbRangeOption(pCxt, "walSegmentSize", pOptions->walSegmentSize, TSDB_DB_MIN_WAL_SEGMENT_SIZE, INT32_MAX); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_SST_TRIGGER, TSDB_MAX_SST_TRIGGER); + code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "tablePrefix", pOptions->tablePrefix, TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX); @@ -6604,7 +6604,17 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS pReq->colId = pSchema->colId; SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema); - if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) { + + if (QUERY_NODE_VALUE != pStmt->pVal->node.type) { + SValueNode* pVal = NULL; + pCxt->errCode = createTagValFromExpr(pCxt, targetDt, (SNode*)pStmt->pVal, &pVal); + if (pCxt->errCode) { + return pCxt->errCode; + } + + nodesDestroyNode((SNode*)pStmt->pVal); + pStmt->pVal = pVal; + } else if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) { return pCxt->errCode; } diff --git a/source/libs/parser/test/parAlterToBalanceTest.cpp b/source/libs/parser/test/parAlterToBalanceTest.cpp index 8eb64719b2..871f28b9db 100644 --- a/source/libs/parser/test/parAlterToBalanceTest.cpp +++ b/source/libs/parser/test/parAlterToBalanceTest.cpp @@ -88,7 +88,7 @@ TEST_F(ParserInitialATest, alterDnode) { * | REPLICA int_value -- todo: enum 1, 3, default 1, unit replica * | STRICT {'off' | 'on'} -- todo: default 'off' * | WAL_LEVEL int_value -- enum 1, 2, default 1 - * | SST_TRIGGER int_value -- rang [1, 128], default 8 + * | SST_TRIGGER int_value -- rang [1, 16], default 8 * } */ TEST_F(ParserInitialATest, alterDatabase) { @@ -161,8 +161,8 @@ TEST_F(ParserInitialATest, alterDatabase) { setAlterDbFsync(200); setAlterDbWal(1); setAlterDbCacheModel(TSDB_CACHE_MODEL_LAST_ROW); - setAlterDbSstTrigger(20); - run("ALTER DATABASE test CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 WAL_LEVEL 1 SST_TRIGGER 20"); + setAlterDbSstTrigger(16); + run("ALTER DATABASE test CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 WAL_LEVEL 1 SST_TRIGGER 16"); clearAlterDbReq(); initAlterDb("test"); @@ -237,7 +237,7 @@ TEST_F(ParserInitialATest, alterDatabaseSemanticCheck) { run("ALTER DATABASE test WAL_LEVEL 0", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test WAL_LEVEL 3", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test SST_TRIGGER 0", TSDB_CODE_PAR_INVALID_DB_OPTION); - run("ALTER DATABASE test SST_TRIGGER 129", TSDB_CODE_PAR_INVALID_DB_OPTION); + run("ALTER DATABASE test SST_TRIGGER 17", TSDB_CODE_PAR_INVALID_DB_OPTION); // Regardless of the specific sentence run("ALTER DATABASE db WAL_LEVEL 0 # td-14436", TSDB_CODE_PAR_SYNTAX_ERROR, PARSER_STAGE_PARSE); } diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 96f7d29230..b280b32a94 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -19,6 +19,7 @@ #include #include +#include #include "cmdnodes.h" #include "mockCatalogService.h" @@ -251,6 +252,7 @@ class PlannerTestBaseImpl { string splitLogicPlan_; string scaledLogicPlan_; string physiPlan_; + string physiPlanMsg_; vector physiSubplans_; }; @@ -274,6 +276,7 @@ class PlannerTestBaseImpl { res_.splitLogicPlan_.clear(); res_.scaledLogicPlan_.clear(); res_.physiPlan_.clear(); + res_.physiPlanMsg_.clear(); res_.physiSubplans_.clear(); } @@ -408,6 +411,8 @@ class PlannerTestBaseImpl { SNode* pSubplan; FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { res_.physiSubplans_.push_back(toString(pSubplan)); } } + res_.physiPlanMsg_ = toMsg((SNode*)(*pPlan)); + cout << "json len: " << res_.physiPlan_.length() << ", msg len: " << res_.physiPlanMsg_.length() << endl; } void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { @@ -446,12 +451,45 @@ class PlannerTestBaseImpl { string toString(const SNode* pRoot) { char* pStr = NULL; int32_t len = 0; + + auto start = chrono::steady_clock::now(); DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len) + if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pRoot)) { + cout << "nodesNodeToString: " + << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl; + } + string str(pStr); taosMemoryFreeClear(pStr); return str; } + string toMsg(const SNode* pRoot) { + char* pStr = NULL; + int32_t len = 0; + + auto start = chrono::steady_clock::now(); + DO_WITH_THROW(nodesNodeToMsg, pRoot, &pStr, &len) + cout << "nodesNodeToMsg: " + << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl; + + SNode* pNode = NULL; + char* pNewStr = NULL; + int32_t newlen = 0; + DO_WITH_THROW(nodesMsgToNode, pStr, len, &pNode) + DO_WITH_THROW(nodesNodeToMsg, pNode, &pNewStr, &newlen) + if (newlen != len || 0 != memcmp(pStr, pNewStr, len)) { + cout << "nodesNodeToMsg error!!!!!!!!!!!!!! len = " << len << ", newlen = " << newlen << endl; + DO_WITH_THROW(nodesNodeToString, pNode, false, &pNewStr, &newlen) + cout << "nodesNodeToString " << pNewStr << endl; + } + taosMemoryFreeClear(pNewStr); + + string str(pStr, len); + taosMemoryFreeClear(pStr); + return str; + } + caseEnv caseEnv_; stmtEnv stmtEnv_; stmtRes res_; diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index dfd6f012cc..5efdbb4679 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -112,6 +112,29 @@ int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { return tdbTbDelete(pState->pStateDb, key, sizeof(SWinKey), &pState->txn); } +int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { + // todo refactor + int32_t size = *pVLen; + if (streamStateGet(pState, key, pVal, pVLen) == 0) { + return 0; + } + void* tmp = taosMemoryCalloc(1, size); + if (streamStatePut(pState, key, &tmp, size) == 0) { + taosMemoryFree(tmp); + int32_t code = streamStateGet(pState, key, pVal, pVLen); + ASSERT(code == 0); + return code; + } + taosMemoryFree(tmp); + return -1; +} + +int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal) { + // todo refactor + streamFreeVal(pVal); + return 0; +} + SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) return NULL; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 51098374b0..6f29b54f80 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2766,8 +2766,6 @@ const char* syncStr(ESyncState state) { } int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { - SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); - if (ths->state != TAOS_SYNC_STATE_FOLLOWER) { syncNodeEventLog(ths, "I am not follower, can not do leader transfer"); return 0; @@ -2799,6 +2797,8 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p } */ + SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); + do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), "do leader transfer, index:%ld", pEntry->index); diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index b42aba560f..faebe5bbec 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1992,6 +1992,313 @@ void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { } } +// ---- message process SyncHeartbeat---- +SyncHeartbeat* syncHeartbeatBuild(int32_t vgId) { + uint32_t bytes = sizeof(SyncHeartbeat); + SyncHeartbeat* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_SYNC_HEARTBEAT; + return pMsg; +} + +void syncHeartbeatDestroy(SyncHeartbeat* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncHeartbeatSerialize(const SyncHeartbeat* pMsg, char* buf, uint32_t bufLen) { + ASSERT(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncHeartbeatDeserialize(const char* buf, uint32_t len, SyncHeartbeat* pMsg) { + memcpy(pMsg, buf, len); + ASSERT(len == pMsg->bytes); +} + +char* syncHeartbeatSerialize2(const SyncHeartbeat* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + ASSERT(buf != NULL); + syncHeartbeatSerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncHeartbeat* syncHeartbeatDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncHeartbeat* pMsg = taosMemoryMalloc(bytes); + ASSERT(pMsg != NULL); + syncHeartbeatDeserialize(buf, len, pMsg); + ASSERT(len == pMsg->bytes); + return pMsg; +} + +void syncHeartbeat2RpcMsg(const SyncHeartbeat* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncHeartbeatSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncHeartbeatFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeat* pMsg) { + syncHeartbeatDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncHeartbeat* syncHeartbeatFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncHeartbeat* pMsg = syncHeartbeatDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + ASSERT(pMsg != NULL); + return pMsg; +} + +cJSON* syncHeartbeat2Json(const SyncHeartbeat* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncHeartbeat", pRoot); + return pJson; +} + +char* syncHeartbeat2Str(const SyncHeartbeat* pMsg) { + cJSON* pJson = syncHeartbeat2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void syncHeartbeatPrint(const SyncHeartbeat* pMsg) { + char* serialized = syncHeartbeat2Str(pMsg); + printf("syncHeartbeatPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncHeartbeatPrint2(char* s, const SyncHeartbeat* pMsg) { + char* serialized = syncHeartbeat2Str(pMsg); + printf("syncHeartbeatPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncHeartbeatLog(const SyncHeartbeat* pMsg) { + char* serialized = syncHeartbeat2Str(pMsg); + sTrace("syncHeartbeatLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncHeartbeatLog2(char* s, const SyncHeartbeat* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncHeartbeat2Str(pMsg); + sTrace("syncHeartbeatLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +// ---- message process SyncHeartbeatReply---- +SyncHeartbeatReply* syncHeartbeatReplyBuild(int32_t vgId) { + uint32_t bytes = sizeof(SyncHeartbeatReply); + SyncHeartbeatReply* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_SYNC_HEARTBEAT_REPLY; + return pMsg; +} + +void syncHeartbeatReplyDestroy(SyncHeartbeatReply* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncHeartbeatReplySerialize(const SyncHeartbeatReply* pMsg, char* buf, uint32_t bufLen) { + ASSERT(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncHeartbeatReplyDeserialize(const char* buf, uint32_t len, SyncHeartbeatReply* pMsg) { + memcpy(pMsg, buf, len); + ASSERT(len == pMsg->bytes); +} + +char* syncHeartbeatReplySerialize2(const SyncHeartbeatReply* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + ASSERT(buf != NULL); + syncHeartbeatReplySerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncHeartbeatReply* syncHeartbeatReplyDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncHeartbeatReply* pMsg = taosMemoryMalloc(bytes); + ASSERT(pMsg != NULL); + syncHeartbeatReplyDeserialize(buf, len, pMsg); + ASSERT(len == pMsg->bytes); + return pMsg; +} + +void syncHeartbeatReply2RpcMsg(const SyncHeartbeatReply* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncHeartbeatReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncHeartbeatReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeatReply* pMsg) { + syncHeartbeatReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncHeartbeatReply* syncHeartbeatReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncHeartbeatReply* pMsg = syncHeartbeatReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + ASSERT(pMsg != NULL); + return pMsg; +} + +cJSON* syncHeartbeatReply2Json(const SyncHeartbeatReply* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + cJSON_AddStringToObject(pRoot, "matchIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncHeartbeatReply", pRoot); + return pJson; +} + +char* syncHeartbeatReply2Str(const SyncHeartbeatReply* pMsg) { + cJSON* pJson = syncHeartbeatReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void syncHeartbeatReplyPrint(const SyncHeartbeatReply* pMsg) { + char* serialized = syncHeartbeatReply2Str(pMsg); + printf("syncHeartbeatReplyPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg) { + char* serialized = syncHeartbeatReply2Str(pMsg); + printf("syncHeartbeatReplyPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg) { + char* serialized = syncHeartbeatReply2Str(pMsg); + sTrace("syncHeartbeatReplyLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncHeartbeatReply2Str(pMsg); + sTrace("syncHeartbeatReplyLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + // ---- message process SyncApplyMsg---- SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncApplyMsg) + dataLen; diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 72845d0c1d..b9cc7a391d 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -57,6 +57,8 @@ add_executable(syncLeaderTransferTest "") add_executable(syncReconfigFinishTest "") add_executable(syncRestoreFromSnapshot "") add_executable(syncRaftCfgIndexTest "") +add_executable(syncHeartbeatTest "") +add_executable(syncHeartbeatReplyTest "") target_sources(syncTest @@ -295,6 +297,14 @@ target_sources(syncRaftCfgIndexTest PRIVATE "syncRaftCfgIndexTest.cpp" ) +target_sources(syncHeartbeatTest + PRIVATE + "syncHeartbeatTest.cpp" +) +target_sources(syncHeartbeatReplyTest + PRIVATE + "syncHeartbeatReplyTest.cpp" +) target_include_directories(syncTest @@ -592,6 +602,16 @@ target_include_directories(syncRaftCfgIndexTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncHeartbeatTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncHeartbeatReplyTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -830,6 +850,14 @@ target_link_libraries(syncRaftCfgIndexTest sync gtest_main ) +target_link_libraries(syncHeartbeatTest + sync + gtest_main +) +target_link_libraries(syncHeartbeatReplyTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncHeartbeatReplyTest.cpp b/source/libs/sync/test/syncHeartbeatReplyTest.cpp new file mode 100644 index 0000000000..0ccd7b70bb --- /dev/null +++ b/source/libs/sync/test/syncHeartbeatReplyTest.cpp @@ -0,0 +1,105 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncHeartbeatReply *createMsg() { + SyncHeartbeatReply *pMsg = syncHeartbeatReplyBuild(1000); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + + pMsg->term = 33; + pMsg->privateTerm = 44; + pMsg->startTime = taosGetTimestampMs(); + return pMsg; +} + +void test1() { + SyncHeartbeatReply *pMsg = createMsg(); + syncHeartbeatReplyLog2((char *)"test1:", pMsg); + syncHeartbeatReplyDestroy(pMsg); +} + +void test2() { + SyncHeartbeatReply *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncHeartbeatReplySerialize(pMsg, serialized, len); + SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyBuild(1000); + syncHeartbeatReplyDeserialize(serialized, len, pMsg2); + syncHeartbeatReplyLog2((char *)"test2: syncHeartbeatReplySerialize -> syncHeartbeatReplyDeserialize ", + pMsg2); + + taosMemoryFree(serialized); + syncHeartbeatReplyDestroy(pMsg); + syncHeartbeatReplyDestroy(pMsg2); +} + +void test3() { + SyncHeartbeatReply *pMsg = createMsg(); + uint32_t len; + char * serialized = syncHeartbeatReplySerialize2(pMsg, &len); + SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyDeserialize2(serialized, len); + syncHeartbeatReplyLog2((char *)"test3: syncHeartbeatReplySerialize3 -> syncHeartbeatReplyDeserialize2 ", + pMsg2); + + taosMemoryFree(serialized); + syncHeartbeatReplyDestroy(pMsg); + syncHeartbeatReplyDestroy(pMsg2); +} + +void test4() { + SyncHeartbeatReply *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncHeartbeatReply2RpcMsg(pMsg, &rpcMsg); + SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyBuild(1000); + syncHeartbeatReplyFromRpcMsg(&rpcMsg, pMsg2); + syncHeartbeatReplyLog2((char *)"test4: syncHeartbeatReply2RpcMsg -> syncHeartbeatReplyFromRpcMsg ", + pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncHeartbeatReplyDestroy(pMsg); + syncHeartbeatReplyDestroy(pMsg2); +} + +void test5() { + SyncHeartbeatReply *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncHeartbeatReply2RpcMsg(pMsg, &rpcMsg); + SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyFromRpcMsg2(&rpcMsg); + syncHeartbeatReplyLog2((char *)"test5: syncHeartbeatReply2RpcMsg -> syncHeartbeatReplyFromRpcMsg2 ", + pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncHeartbeatReplyDestroy(pMsg); + syncHeartbeatReplyDestroy(pMsg2); +} + +int main() { + gRaftDetailLog = true; + + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/sync/test/syncHeartbeatTest.cpp b/source/libs/sync/test/syncHeartbeatTest.cpp new file mode 100644 index 0000000000..d910c828f1 --- /dev/null +++ b/source/libs/sync/test/syncHeartbeatTest.cpp @@ -0,0 +1,99 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncHeartbeat *createMsg() { + SyncHeartbeat *pMsg = syncHeartbeatBuild(789); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 8; + pMsg->commitIndex = 33; + pMsg->privateTerm = 44; + return pMsg; +} + +void test1() { + SyncHeartbeat *pMsg = createMsg(); + syncHeartbeatLog2((char *)"test1:", pMsg); + syncHeartbeatDestroy(pMsg); +} + +void test2() { + SyncHeartbeat *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncHeartbeatSerialize(pMsg, serialized, len); + SyncHeartbeat *pMsg2 = syncHeartbeatBuild(789); + syncHeartbeatDeserialize(serialized, len, pMsg2); + syncHeartbeatLog2((char *)"test2: syncHeartbeatSerialize -> syncHeartbeatDeserialize ", pMsg2); + + taosMemoryFree(serialized); + syncHeartbeatDestroy(pMsg); + syncHeartbeatDestroy(pMsg2); +} + +void test3() { + SyncHeartbeat *pMsg = createMsg(); + uint32_t len; + char * serialized = syncHeartbeatSerialize2(pMsg, &len); + SyncHeartbeat *pMsg2 = syncHeartbeatDeserialize2(serialized, len); + syncHeartbeatLog2((char *)"test3: syncHeartbeatSerialize2 -> syncHeartbeatDeserialize2 ", pMsg2); + + taosMemoryFree(serialized); + syncHeartbeatDestroy(pMsg); + syncHeartbeatDestroy(pMsg2); +} + +void test4() { + SyncHeartbeat *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncHeartbeat2RpcMsg(pMsg, &rpcMsg); + SyncHeartbeat *pMsg2 = (SyncHeartbeat *)taosMemoryMalloc(rpcMsg.contLen); + syncHeartbeatFromRpcMsg(&rpcMsg, pMsg2); + syncHeartbeatLog2((char *)"test4: syncHeartbeat2RpcMsg -> syncHeartbeatFromRpcMsg ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncHeartbeatDestroy(pMsg); + syncHeartbeatDestroy(pMsg2); +} + +void test5() { + SyncHeartbeat *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncHeartbeat2RpcMsg(pMsg, &rpcMsg); + SyncHeartbeat *pMsg2 =syncHeartbeatFromRpcMsg2(&rpcMsg); + syncHeartbeatLog2((char *)"test5: syncHeartbeat2RpcMsg -> syncHeartbeatFromRpcMsg2 ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncHeartbeatDestroy(pMsg); + syncHeartbeatDestroy(pMsg2); +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + gRaftDetailLog = true; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index dc2d937c49..62cdf4e784 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -151,8 +151,8 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, static FORCE_INLINE void destroyUserdata(STransMsg* userdata); static FORCE_INLINE void destroyCmsg(void* cmsg); static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst); +static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx); -static void transDestroyConnCtx(STransConnCtx* ctx); // thread obj static SCliThrd* createThrdObj(); static void destroyThrdObj(SCliThrd* pThrd); @@ -1124,7 +1124,7 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, return cli; } -FORCE_INLINE void destroyUserdata(STransMsg* userdata) { +static FORCE_INLINE void destroyUserdata(STransMsg* userdata) { if (userdata->pCont == NULL) { return; } @@ -1132,7 +1132,7 @@ FORCE_INLINE void destroyUserdata(STransMsg* userdata) { userdata->pCont = NULL; } -FORCE_INLINE void destroyCmsg(void* arg) { +static FORCE_INLINE void destroyCmsg(void* arg) { SCliMsg* pMsg = arg; if (pMsg == NULL) { return; @@ -1198,7 +1198,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosMemoryFree(pThrd); } -static void transDestroyConnCtx(STransConnCtx* ctx) { +static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx) { // taosMemoryFree(ctx); } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 9db7d6c455..5c437e6f7a 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -168,6 +168,9 @@ static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) { } pReader->pIdxFile = pIdxFile; + + pReader->curFileFirstVer = fileFirstVer; + return 0; } @@ -372,7 +375,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) { int64_t code; -// ASSERT(pRead->curVersion == pHead->head.version); + // ASSERT(pRead->curVersion == pHead->head.version); code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR); if (code < 0) { @@ -415,7 +418,8 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { } if (walValidBodyCksum(*ppHead) != 0) { - wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); + wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, + ver); pRead->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; diff --git a/source/util/src/tuuid.c b/source/util/src/tuuid.c index 9101aec949..7460ccbc82 100644 --- a/source/util/src/tuuid.c +++ b/source/util/src/tuuid.c @@ -51,11 +51,11 @@ int64_t tGenIdPI64(void) { int64_t id; while (true) { - int64_t ts = taosGetTimestampMs(); + int64_t ts = taosGetTimestampMs() >> 8; uint64_t pid = taosGetPId(); int32_t val = atomic_add_fetch_32(&tUUIDSerialNo, 1); - id = ((tUUIDHashId & 0x07FF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); + id = ((tUUIDHashId & 0x07FF) << 52) | ((pid & 0x0F) << 48) | ((ts & 0x3FFFFFF) << 20) | (val & 0xFFFFF); if (id) { break; } diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index 1a057c5875..534c177587 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -23,9 +23,9 @@ void simpleTest() { ASSERT_EQ(getTotalBufSize(pBuf), 1024); - SIDList list = getDataBufPagesIdList(pBuf, groupId); + SIDList list = getDataBufPagesIdList(pBuf); ASSERT_EQ(taosArrayGetSize(list), 1); - ASSERT_EQ(getNumOfBufGroupId(pBuf), 1); + //ASSERT_EQ(getNumOfBufGroupId(pBuf), 1); releaseBufPage(pBuf, pBufPage); @@ -98,7 +98,7 @@ void writeDownTest() { SFilePage* pBufPagex = static_cast(getBufPage(pBuf, writePageId)); ASSERT_EQ(*(int32_t*)pBufPagex->data, nx); - SArray* pa = getDataBufPagesIdList(pBuf, groupId); + SArray* pa = getDataBufPagesIdList(pBuf); ASSERT_EQ(taosArrayGetSize(pa), 5); destroyDiskbasedBuf(pBuf); @@ -152,7 +152,7 @@ void recyclePageTest() { SFilePage* pBufPagex1 = static_cast(getBufPage(pBuf, 1)); - SArray* pa = getDataBufPagesIdList(pBuf, groupId); + SArray* pa = getDataBufPagesIdList(pBuf); ASSERT_EQ(taosArrayGetSize(pa), 6); destroyDiskbasedBuf(pBuf); diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 46bae734ea..161c878440 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -221,6 +221,7 @@ ./test.sh -f tsim/table/describe.sim ./test.sh -f tsim/table/double.sim ./test.sh -f tsim/table/float.sim +./test.sh -f tsim/table/hash.sim ./test.sh -f tsim/table/int.sim ./test.sh -f tsim/table/limit.sim ./test.sh -f tsim/table/smallint.sim @@ -248,6 +249,12 @@ ./test.sh -f tsim/stream/windowClose.sim ./test.sh -f tsim/stream/ignoreExpiredData.sim ./test.sh -f tsim/stream/sliding.sim +#./test.sh -f tsim/stream/partitionbyColumnInterval.sim +#./test.sh -f tsim/stream/partitionbyColumnSession.sim +#./test.sh -f tsim/stream/partitionbyColumnState.sim +#./test.sh -f tsim/stream/deleteInterval.sim +#./test.sh -f tsim/stream/deleteSession.sim +#./test.sh -f tsim/stream/deleteState.sim # ---- transaction ---- ./test.sh -f tsim/trans/lossdata1.sim diff --git a/tests/script/sh/abs_max.c b/tests/script/sh/abs_max.c deleted file mode 100644 index d623adacf9..0000000000 --- a/tests/script/sh/abs_max.c +++ /dev/null @@ -1,88 +0,0 @@ -#include -#include -#include - -typedef struct SUdfInit{ - int maybe_null; /* 1 if function can return NULL */ - int decimals; /* for real functions */ - long long length; /* For string functions */ - char *ptr; /* free pointer for function data */ - int const_item; /* 0 if result is independent of arguments */ -} SUdfInit; - - -#define TSDB_DATA_INT_NULL 0x80000000LL -#define TSDB_DATA_BIGINT_NULL 0x8000000000000000LL - -void abs_max(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput, - int* numOfOutput, short otype, short obytes, SUdfInit* buf) { - int i; - int r = 0; - printf("abs_max input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); - if (itype == 5) { - r=*(long *)dataOutput; - *numOfOutput=0; - - for(i=0;i r) { - r = v; - } - } - - *(long *)dataOutput=r; - - printf("abs_max out, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput); - } -} - - - -void abs_max_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) { - int i; - int r = 0; - printf("abs_max_finalize dataoutput:%p:%d, numOfOutput:%d, buf:%p\n", dataOutput, *dataOutput, *numOfOutput, buf); - *numOfOutput=1; - printf("abs_max finalize, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput); -} - -void abs_max_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) { - int r = 0; - - if (numOfRows > 0) { - r = *((long *)data); - } - printf("abs_max_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf); - for (int i = 1; i < numOfRows; ++i) { - printf("abs_max_merge %d - %ld\n", i, *((long *)data + i)); - if (*((long*)data + i) > r) { - r= *((long*)data + i); - } - } - - *(long*)dataOutput=r; - if (numOfRows > 0) { - *numOfOutput=1; - } else { - *numOfOutput=0; - } - - printf("abs_max_merge, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput); -} - - -int abs_max_init(SUdfInit* buf) { - printf("abs_max init\n"); - return 0; -} - - -void abs_max_destroy(SUdfInit* buf) { - printf("abs_max destroy\n"); -} - diff --git a/tests/script/sh/add_one.c b/tests/script/sh/add_one.c deleted file mode 100644 index e12cf8f26f..0000000000 --- a/tests/script/sh/add_one.c +++ /dev/null @@ -1,33 +0,0 @@ -#include -#include -#include - -typedef struct SUdfInit{ - int maybe_null; /* 1 if function can return NULL */ - int decimals; /* for real functions */ - long long length; /* For string functions */ - char *ptr; /* free pointer for function data */ - int const_item; /* 0 if result is independent of arguments */ -} SUdfInit; - -void add_one(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBUf, char* tsOutput, - int* numOfOutput, short otype, short obytes, SUdfInit* buf) { - int i; - int r = 0; - printf("add_one input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); - if (itype == 4) { - for(i=0;i -#include -#include - -typedef struct SUdfInit{ - int maybe_null; /* 1 if function can return NULL */ - int decimals; /* for real functions */ - long long length; /* For string functions */ - char *ptr; /* free pointer for function data */ - int const_item; /* 0 if result is independent of arguments */ -} SUdfInit; - -typedef struct SDemo{ - double sum; - int num; - short otype; -}SDemo; - -#define FLOAT_NULL 0x7FF00000 // it is an NAN -#define DOUBLE_NULL 0x7FFFFF0000000000LL // it is an NAN - - -void demo(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput, - int* numOfOutput, short otype, short obytes, SUdfInit* buf) { - int i; - double r = 0; - SDemo *p = (SDemo *)interBuf; - SDemo *q = (SDemo *)dataOutput; - printf("demo input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, interBUf:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, interBuf, tsOutput, numOfOutput, buf); - - for(i=0;isum += r*r; - } - - p->otype = otype; - p->num += numOfRows; - - q->sum = p->sum; - q->num = p->num; - q->otype = p->otype; - - *numOfOutput=1; - - printf("demo out, sum:%f, num:%d, numOfOutput:%d\n", p->sum, p->num, *numOfOutput); -} - - -void demo_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) { - int i; - SDemo *p = (SDemo *)data; - SDemo res = {0}; - printf("demo_merge input data:%p, rows:%d, dataoutput:%p, numOfOutput:%p, buf:%p\n", data, numOfRows, dataOutput, numOfOutput, buf); - - for(i=0;isum * p->sum; - res.num += p->num; - p++; - } - - p->sum = res.sum; - p->num = res.num; - - *numOfOutput=1; - - printf("demo out, sum:%f, num:%d, numOfOutput:%d\n", p->sum, p->num, *numOfOutput); -} - - - -void demo_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) { - SDemo *p = (SDemo *)interBuf; - printf("demo_finalize interbuf:%p, numOfOutput:%p, buf:%p, sum:%f, num:%d\n", interBuf, numOfOutput, buf, p->sum, p->num); - if (p->otype == 6) { - if (p->num != 30000) { - *(unsigned int *)dataOutput = FLOAT_NULL; - } else { - *(float *)dataOutput = (float)(p->sum / p->num); - } - printf("finalize values:%f\n", *(float *)dataOutput); - } else if (p->otype == 7) { - if (p->num != 30000) { - *(unsigned long long *)dataOutput = DOUBLE_NULL; - } else { - *(double *)dataOutput = (double)(p->sum / p->num); - } - printf("finalize values:%f\n", *(double *)dataOutput); - } - - *numOfOutput=1; - - printf("demo finalize, numOfOutput:%d\n", *numOfOutput); -} - - -int demo_init(SUdfInit* buf) { - printf("demo init\n"); - return 0; -} - - -void demo_destroy(SUdfInit* buf) { - printf("demo destroy\n"); -} - diff --git a/tests/script/sh/demo.lua b/tests/script/sh/demo.lua deleted file mode 100644 index c5e5582fc3..0000000000 --- a/tests/script/sh/demo.lua +++ /dev/null @@ -1,43 +0,0 @@ -funcName = "test" - -global = {} - -function test_init() - return global -end - -function test_add(rows, ans, key) - t = {} - t["sum"] = 0.0 - t["num"] = 0 - for i=1, #rows do - t["sum"] = t["sum"] + rows[i] * rows[i] - end - t["num"] = #rows - - - if (ans[key] ~= nil) - then - ans[key]["sum"] = ans[key]["sum"] + t["sum"] - ans[key]["num"] = ans[key]["num"] + t["num"] - else - ans[key] = t - end - - return ans; -end - -function test_finalize(ans, key) - local ret = 0.0 - - if (ans[key] ~= nil and ans[key]["num"] == 30000) - then - ret = ans[key]["sum"]/ans[key]["num"] - ans[key]["sum"] = 0.0 - ans[key]["num"] = 0 - else - ret = inf - end - - return ret, ans -end diff --git a/tests/script/sh/sum_double.c b/tests/script/sh/sum_double.c deleted file mode 100644 index d6eea5d291..0000000000 --- a/tests/script/sh/sum_double.c +++ /dev/null @@ -1,84 +0,0 @@ -#include -#include -#include - -typedef struct SUdfInit{ - int maybe_null; /* 1 if function can return NULL */ - int decimals; /* for real functions */ - long long length; /* For string functions */ - char *ptr; /* free pointer for function data */ - int const_item; /* 0 if result is independent of arguments */ -} SUdfInit; - -#define TSDB_DATA_INT_NULL 0x80000000LL - - -void sum_double(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput, - int* numOfOutput, short otype, short obytes, SUdfInit* buf) { - int i; - int r = 0; - printf("sum_double input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); - if (itype == 4) { - r=*(int *)dataOutput; - *numOfOutput=0; - - for(i=0;iptr)=*(int*)dataOutput*2; - *(int*)dataOutput=*(int*)(buf->ptr); - printf("sum_double finalize, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput); -} - -void sum_double_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) { - int r = 0; - int sum = 0; - - printf("sum_double_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf); - for (int i = 0; i < numOfRows; ++i) { - printf("sum_double_merge %d - %d\n", i, *((int*)data + i)); - sum +=*((int*)data + i); - } - - *(int*)dataOutput+=sum; - if (numOfRows > 0) { - *numOfOutput=1; - } else { - *numOfOutput=0; - } - - printf("sum_double_merge, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput); -} - - -int sum_double_init(SUdfInit* buf) { - buf->maybe_null=1; - buf->ptr = taosMemoryMalloc(sizeof(int)); - printf("sum_double init\n"); - return 0; -} - - -void sum_double_destroy(SUdfInit* buf) { - taosMemoryFree(buf->ptr); - printf("sum_double destroy\n"); -} - diff --git a/tests/script/tsim/column/table.sim b/tests/script/tsim/column/table.sim index 4f1d32c373..03c4799681 100644 --- a/tests/script/tsim/column/table.sim +++ b/tests/script/tsim/column/table.sim @@ -159,6 +159,7 @@ if $data01 != 10 then return -1 endi if $data02 != 4.500000000 then + print expect 4.500000000, actual: $data02 return -1 endi if $data03 != 4.500000000 then diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index 70ca1179b0..8942f7f702 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -5,7 +5,7 @@ sleep 50 sql connect print =============== create database -sql create database test vgroups 1 +sql create database test vgroups 1; sql select * from information_schema.ins_databases if $rows != 3 then return -1 @@ -13,7 +13,7 @@ endi print $data00 $data01 $data02 -sql use test +sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); diff --git a/tests/script/tsim/stream/deleteInterval.sim b/tests/script/tsim/stream/deleteInterval.sim new file mode 100644 index 0000000000..dfd0ddc9d3 --- /dev/null +++ b/tests/script/tsim/stream/deleteInterval.sim @@ -0,0 +1,419 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 interval(10s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sleep 200 +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop0: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print =====rows=$rows + goto loop0 +endi + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop1: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop1 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213001,2,2,2,2.0); +sql insert into t1 values(1648791213002,3,3,3,3.0); +sql insert into t1 values(1648791213003,4,4,4,4.0); + +sleep 200 +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002; + +$loop_count = 0 + +loop3: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop3 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); + +$loop_count = 0 + +loop4: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop4 +endi + +sleep 200 + +sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop5: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop5 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213005,2,2,2,2.0); +sql insert into t1 values(1648791213006,3,3,3,3.0); +sql insert into t1 values(1648791213007,4,4,4,4.0); + +sql insert into t1 values(1648791223000,1,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,2.0); +sql insert into t1 values(1648791223002,3,3,3,3.0); +sql insert into t1 values(1648791223003,4,4,4,4.0); + +sql insert into t1 values(1648791233000,1,1,1,1.0); +sql insert into t1 values(1648791233001,2,2,2,2.0); +sql insert into t1 values(1648791233008,3,3,3,3.0); +sql insert into t1 values(1648791233009,4,4,4,4.0); + +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop6: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop6 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop6 +endi + +if $data12 != 4 then + print =====data12=$data12 + goto loop6 +endi + +sql drop stream if exists streams2; +sql drop database if exists test2; +sql create database test2 vgroups 4; +sql use test2; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop7: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop7 +endi + +sleep 200 + +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop8: +sleep 200 + +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop8 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); +sql insert into t2 values(1648791223000,1,2,3,1.0); +sql insert into t2 values(1648791223001,1,2,3,1.0); +sql insert into t2 values(1648791223002,3,2,3,1.0); +sql insert into t2 values(1648791223003,3,2,3,1.0); + +sleep 200 + +sql delete from t2 where ts >= 1648791223000 and ts <= 1648791223001; + +$loop_count = 0 + +loop11: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop11 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop11 +endi + +if $data11 != 6 then + print =====data11=$data11 + goto loop11 +endi + +if $data12 != 3 then + print =====data12=$data12 + goto loop11 +endi + +sleep 200 + +sql delete from st where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop12: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop12 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop12 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop12 +endi + +sql insert into t1 values(1648791213004,3,2,3,1.0); +sql insert into t1 values(1648791213005,3,2,3,1.0); +sql insert into t1 values(1648791213006,3,2,3,1.0); +sql insert into t1 values(1648791223004,1,2,3,1.0); +sql insert into t2 values(1648791213004,3,2,3,1.0); +sql insert into t2 values(1648791213005,3,2,3,1.0); +sql insert into t2 values(1648791213006,3,2,3,1.0); +sql insert into t2 values(1648791223004,1,2,3,1.0); + +sleep 200 + +sql delete from t2 where ts >= 1648791213004 and ts <= 1648791213006; + +$loop_count = 0 + +loop13: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop13 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop13 +endi + +if $data02 != 3 then + print =====data02=$data02 + goto loop13 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop13 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop13 +endi + +sql insert into t1 values(1648791223005,1,2,3,1.0); +sql insert into t1 values(1648791223006,1,2,3,1.0); +sql insert into t2 values(1648791223005,1,2,3,1.0); +sql insert into t2 values(1648791223006,1,2,3,1.0); + +sql insert into t1 values(1648791233005,4,2,3,1.0); +sql insert into t1 values(1648791233006,2,2,3,1.0); +sql insert into t2 values(1648791233005,5,2,3,1.0); +sql insert into t2 values(1648791233006,3,2,3,1.0); + +sleep 200 + +sql delete from st where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop14: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop14 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop14 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop14 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop14 +endi + +if $data12 != 3 then + print =====data12=$data12 + goto loop14 +endi + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/deleteSession.sim b/tests/script/tsim/stream/deleteSession.sim new file mode 100644 index 0000000000..541609633b --- /dev/null +++ b/tests/script/tsim/stream/deleteSession.sim @@ -0,0 +1,532 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 session(ts, 5s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sleep 200 +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop0: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print =====rows=$rows + goto loop0 +endi + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop1: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop1 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213001,2,2,2,2.0); +sql insert into t1 values(1648791213002,3,3,3,3.0); +sql insert into t1 values(1648791213003,4,4,4,4.0); + +sleep 200 +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002; + +$loop_count = 0 + +loop3: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop3 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); + +$loop_count = 0 + +loop4: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop4 +endi + +sleep 200 + +sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop5: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop5 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213005,2,2,2,2.0); +sql insert into t1 values(1648791213006,3,3,3,3.0); +sql insert into t1 values(1648791213007,4,4,4,4.0); + +sql insert into t1 values(1648791223000,1,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,2.0); +sql insert into t1 values(1648791223002,3,3,3,3.0); +sql insert into t1 values(1648791223003,4,4,4,4.0); + +sql insert into t1 values(1648791233000,1,1,1,1.0); +sql insert into t1 values(1648791233001,2,2,2,2.0); +sql insert into t1 values(1648791233008,3,3,3,3.0); +sql insert into t1 values(1648791233009,4,4,4,4.0); + +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop6: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop6 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop6 +endi + +if $data12 != 4 then + print =====data12=$data12 + goto loop6 +endi + +sql drop stream if exists streams2; +sql drop database if exists test2; +sql create database test2 vgroups 4; +sql use test2; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop7: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop7 +endi + +sleep 200 + +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop8: +sleep 200 + +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop8 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); +sql insert into t2 values(1648791223000,1,2,3,1.0); +sql insert into t2 values(1648791223001,1,2,3,1.0); +sql insert into t2 values(1648791223002,3,2,3,1.0); +sql insert into t2 values(1648791223003,3,2,3,1.0); + +sleep 200 + +sql delete from t2 where ts >= 1648791223000 and ts <= 1648791223001; + +$loop_count = 0 + +loop11: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop11 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop11 +endi + +if $data11 != 6 then + print =====data11=$data11 + goto loop11 +endi + +if $data12 != 3 then + print =====data12=$data12 + goto loop11 +endi + +sleep 200 + +sql delete from st where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop12: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop12 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop12 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop12 +endi + +sql insert into t1 values(1648791213004,3,2,3,1.0); +sql insert into t1 values(1648791213005,3,2,3,1.0); +sql insert into t1 values(1648791213006,3,2,3,1.0); +sql insert into t1 values(1648791223004,1,2,3,1.0); +sql insert into t2 values(1648791213004,3,2,3,1.0); +sql insert into t2 values(1648791213005,3,2,3,1.0); +sql insert into t2 values(1648791213006,3,2,3,1.0); +sql insert into t2 values(1648791223004,1,2,3,1.0); + +sleep 200 + +sql delete from t2 where ts >= 1648791213004 and ts <= 1648791213006; + +$loop_count = 0 + +loop13: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop13 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop13 +endi + +if $data02 != 3 then + print =====data02=$data02 + goto loop13 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop13 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop13 +endi + +sql insert into t1 values(1648791223005,1,2,3,1.0); +sql insert into t1 values(1648791223006,1,2,3,1.0); +sql insert into t2 values(1648791223005,1,2,3,1.0); +sql insert into t2 values(1648791223006,1,2,3,1.0); + +sql insert into t1 values(1648791233005,4,2,3,1.0); +sql insert into t1 values(1648791233006,2,2,3,1.0); +sql insert into t2 values(1648791233005,5,2,3,1.0); +sql insert into t2 values(1648791233006,3,2,3,1.0); + +sleep 200 + +sql delete from st where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop14: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop14 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop14 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop14 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop14 +endi + +if $data12 != 3 then + print =====data12=$data12 + goto loop14 +endi + +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop database if exists test3; +sql drop database if exists test; +sql create database test3 vgroups 4; +sql create database test vgroups 1; +sql use test3; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams3 trigger at_once into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); + +sql insert into t1 values(1648791210000,1,1,1,NULL); +sql insert into t1 values(1648791210001,2,2,2,NULL); +sql insert into t2 values(1648791213001,3,3,3,NULL); +sql insert into t2 values(1648791213003,4,4,4,NULL); +sql insert into t1 values(1648791216000,5,5,5,NULL); +sql insert into t1 values(1648791216002,6,6,6,NULL); +sql insert into t1 values(1648791216004,7,7,7,NULL); +sql insert into t2 values(1648791218001,8,8,8,NULL); +sql insert into t2 values(1648791218003,9,9,9,NULL); +sql insert into t1 values(1648791222000,10,10,10,NULL); +sql insert into t1 values(1648791222003,11,11,11,NULL); +sql insert into t1 values(1648791222005,12,12,12,NULL); + +sql insert into t1 values(1648791232005,13,13,13,NULL); +sql insert into t2 values(1648791242005,14,14,14,NULL); + +$loop_count = 0 + +loop19: +sleep 200 +sql select * from test.streamt3 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 3 then + print =====rows=$rows + goto loop19 +endi + +sql delete from t2 where ts >= 1648791213001 and ts <= 1648791218003; + +$loop_count = 0 + +loop20: +sleep 200 +sql select * from test.streamt3 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 5 then + print =====rows=$rows + goto loop20 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop20 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop20 +endi + +if $data11 != 3 then + print =====data11=$data11 + goto loop20 +endi + +if $data12 != 7 then + print =====data12=$data12 + goto loop20 +endi + +if $data21 != 3 then + print =====data21=$data21 + goto loop20 +endi + +if $data22 != 12 then + print =====data22=$data22 + goto loop20 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop20 +endi + +if $data32 != 13 then + print =====data32=$data32 + goto loop20 +endi + +if $data41 != 1 then + print =====data41=$data41 + goto loop20 +endi + +if $data42 != 14 then + print =====data42=$data42 + goto loop20 +endi + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/deleteState.sim b/tests/script/tsim/stream/deleteState.sim new file mode 100644 index 0000000000..ecd9f55340 --- /dev/null +++ b/tests/script/tsim/stream/deleteState.sim @@ -0,0 +1,198 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 state_window(a); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sleep 200 +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop0: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print =====rows=$rows + goto loop0 +endi + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop1: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop1 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213001,1,2,2,2.0); +sql insert into t1 values(1648791213002,1,3,3,3.0); +sql insert into t1 values(1648791213003,1,4,4,4.0); + +sleep 200 +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002; + +$loop_count = 0 + +loop3: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop3 +endi + +sql insert into t1 values(1648791223000,2,2,3,1.0); +sql insert into t1 values(1648791223001,2,2,3,1.0); +sql insert into t1 values(1648791223002,2,2,3,1.0); +sql insert into t1 values(1648791223003,2,2,3,1.0); + +$loop_count = 0 + +loop4: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop4 +endi + +sleep 200 + +sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop5: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop5 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213005,1,2,2,2.0); +sql insert into t1 values(1648791213006,1,3,3,3.0); +sql insert into t1 values(1648791213007,1,4,4,4.0); + +sql insert into t1 values(1648791223000,2,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,2.0); +sql insert into t1 values(1648791223002,2,3,3,3.0); +sql insert into t1 values(1648791223003,2,4,4,4.0); + +sql insert into t1 values(1648791233000,3,1,1,1.0); +sql insert into t1 values(1648791233001,3,2,2,2.0); +sql insert into t1 values(1648791233008,3,3,3,3.0); +sql insert into t1 values(1648791233009,3,4,4,4.0); + +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop6: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop6 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop6 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop6 +endi + +if $data12 != 4 then + print =====data12=$data12 + goto loop6 +endi + + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/partitionbyColumn0.sim b/tests/script/tsim/stream/partitionbyColumnInterval.sim similarity index 100% rename from tests/script/tsim/stream/partitionbyColumn0.sim rename to tests/script/tsim/stream/partitionbyColumnInterval.sim diff --git a/tests/script/tsim/stream/partitionbyColumn1.sim b/tests/script/tsim/stream/partitionbyColumnSession.sim similarity index 100% rename from tests/script/tsim/stream/partitionbyColumn1.sim rename to tests/script/tsim/stream/partitionbyColumnSession.sim diff --git a/tests/script/tsim/stream/partitionbyColumn2.sim b/tests/script/tsim/stream/partitionbyColumnState.sim similarity index 100% rename from tests/script/tsim/stream/partitionbyColumn2.sim rename to tests/script/tsim/stream/partitionbyColumnState.sim diff --git a/tests/script/tsim/table/hash.sim b/tests/script/tsim/table/hash.sim new file mode 100644 index 0000000000..664f867137 --- /dev/null +++ b/tests/script/tsim/table/hash.sim @@ -0,0 +1,84 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +#=========== prepare +#sql create database d1 vgroups 2 +sql create database d1 vgroups 2 table_prefix 3 table_suffix 2 +sql select * from information_schema.ins_databases +print $data(d1)[27] $data(d1)[28] +if $data(d1)[27] != 3 then + return -1 +endi +if $data(d1)[28] != 2 then + return -1 +endi + +sql use d1; +sql create table st (ts timestamp, i int) tags (j int); +sql create table st_ct_1 using st tags(3) st_ct_2 using st tags(4) st_ct_3 using st tags(5) st_ct_4 using st tags(6) st_ct_5 using st tags(7) +sql insert into st_ct_1 values(now+1s, 1) +sql insert into st_ct_1 values(now+2s, 2) +sql insert into st_ct_1 values(now+3s, 3) +sql insert into st_ct_2 values(now+1s, 1) +sql insert into st_ct_2 values(now+2s, 2) +sql insert into st_ct_2 values(now+3s, 3) +sql insert into st_ct_3 values(now+1s, 1) +sql insert into st_ct_3 values(now+2s, 2) +sql insert into st_ct_3 values(now+3s, 2) +sql insert into st_ct_4 values(now+1s, 1) +sql insert into st_ct_4 values(now+2s, 2) +sql insert into st_ct_4 values(now+3s, 2) +sql insert into st_ct_5 values(now+1s, 1) +sql insert into st_ct_5 values(now+2s, 2) +sql insert into st_ct_5 values(now+3s, 2) + +# check query +sql select * from st +if $rows != 15 then + return -1 +endi + +# check table vgroup +sql select * from information_schema.ins_tables where db_name = 'd1' +if $data(st_ct_1)[6] != 2 then + return -1 +endi +if $data(st_ct_2)[6] != 2 then + return -1 +endi +if $data(st_ct_3)[6] != 2 then + return -1 +endi +if $data(st_ct_4)[6] != 2 then + return -1 +endi +if $data(st_ct_5)[6] != 2 then + return -1 +endi + +# check invalid table name +sql create table c1 using st tags(3) +sql create table c12 using st tags(3) +sql create table c123 using st tags(3) +sql create table c1234 using st tags(3) +sql create table c12345 using st tags(3) +sql select * from information_schema.ins_tables where db_name = 'd1' +if $data(c1)[6] != 2 then + return -1 +endi +if $data(c12)[6] != 3 then + return -1 +endi +if $data(c123)[6] != 2 then + return -1 +endi +if $data(c1234)[6] != 3 then + return -1 +endi +if $data(c12345)[6] != 3 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 11df13d451..8987ba3bbd 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -333,7 +333,7 @@ python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py python3 ./test.py -f 7-tmq/tmq_taosx.py -# python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py #------------querPolicy 2-----------