xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into fix/3.0_bugfix_wxy

This commit is contained in:
Xiaoyu Wang 2022-10-28 10:46:31 +08:00
parent e8c0515dce f25387018d
commit 0840e79f90
148 changed files with 3068 additions and 1712 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/cmake.define
View File

@ -100,6 +100,8 @@ IF (TD_WINDOWS)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}")
SET(JEMALLOC_ENABLED OFF)
ELSE ()
IF (${TD_DARWIN})
set(CMAKE_MACOSX_RPATH 0)

2
cmake/taosadapter_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taosadapter
ExternalProject_Add(taosadapter
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
GIT_TAG cc43ef0
GIT_TAG a11131c
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG f9c1d32
GIT_TAG 7321fbb
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

2
contrib/CMakeLists.txt
View File

@ -270,7 +270,7 @@ if(${JEMALLOC_ENABLED})
PREFIX "jemalloc"
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls --with-malloc-conf='background_thread:true,metadata_thp:auto'
CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls
BUILD_COMMAND ${MAKE}
)
INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include)

10
docs/en/28-releases/index.md Normal file
View File

@ -0,0 +1,10 @@
---
title: Releases
---
```mdx-code-block
import DocCardList from '@theme/DocCardList';
import {useCurrentSidebarCategory} from '@docusaurus/theme-common';
<DocCardList items={useCurrentSidebarCategory().items}/>
```

4
examples/c/tmq.c
View File

@ -47,7 +47,7 @@ static int32_t msg_process(TAOS_RES* msg) {
int32_t precision = taos_result_precision(msg);
rows++;
taos_print_row(buf, row, fields, numOfFields);
printf("row content: %s\n", buf);
printf("precision: %d, row content: %s\n", precision, buf);
}
return rows;
@ -70,7 +70,7 @@ static int32_t init_env() {
taos_free_result(pRes);
// create database
pRes = taos_query(pConn, "create database tmqdb");
pRes = taos_query(pConn, "create database tmqdb precision 'ns'");
if (taos_errno(pRes) != 0) {
printf("error in create tmqdb, reason:%s\n", taos_errstr(pRes));
return -1;

1
include/common/tglobal.h
View File

@ -91,6 +91,7 @@ extern bool tsQueryPlannerTrace;
extern int32_t tsQueryNodeChunkSize;
extern bool tsQueryUseNodeAllocator;
extern bool tsKeepColumnName;
extern bool tsEnableQueryHb;
// client
extern int32_t tsMinSlidingTime;

10
include/libs/executor/executor.h
View File

@ -89,6 +89,16 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n
*/
int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type);
/**
* Set block for sma
* @param tinfo
* @param pBlocks
* @param numOfInputBlock
* @param type
* @return
*/
int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type);
/**
* Update the table id list, add or remove.
*

2
include/libs/sync/sync.h
View File

@ -38,7 +38,7 @@ extern bool gRaftDetailLog;
#define SYNC_DEL_WAL_MS (1000 * 60)
#define SYNC_ADD_QUORUM_COUNT 3
#define SYNC_MNODE_LOG_RETENTION 10000
#define SYNC_VNODE_LOG_RETENTION 500
#define SYNC_VNODE_LOG_RETENTION 100
#define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000

16
include/util/tdef.h
View File

@ -234,9 +234,9 @@ typedef enum ELogicConditionType {
* - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus
* the final value is 65531-(4096-1)*4 = 49151.
*/
#define TSDB_MAX_BYTES_PER_ROW 49151
#define TSDB_MAX_TAGS_LEN 16384
#define TSDB_MAX_TAGS 128
#define TSDB_MAX_BYTES_PER_ROW 49151
#define TSDB_MAX_TAGS_LEN 16384
#define TSDB_MAX_TAGS 128
#define TSDB_MAX_COL_TAG_NUM (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS)
#define TSDB_MAX_JSON_TAG_LEN 16384
@ -370,13 +370,13 @@ typedef enum ELogicConditionType {
#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1
#define TSDB_REP_DEF_DB_WAL_RET_PERIOD 0
#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD (24 * 60 * 60 * 4)
#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD 0
#define TSDB_DB_MIN_WAL_RETENTION_SIZE -1
#define TSDB_REP_DEF_DB_WAL_RET_SIZE 0
#define TSDB_REPS_DEF_DB_WAL_RET_SIZE -1
#define TSDB_REPS_DEF_DB_WAL_RET_SIZE 0
#define TSDB_DB_MIN_WAL_ROLL_PERIOD 0
#define TSDB_REP_DEF_DB_WAL_ROLL_PERIOD 0
#define TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD (24 * 60 * 60 * 1)
#define TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD 0
#define TSDB_DB_MIN_WAL_SEGMENT_SIZE 0
#define TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE 0
@ -409,7 +409,7 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_RPC_THREADS 10
#endif
#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type
#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type
#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default
@ -486,7 +486,7 @@ enum {
#define MAX_NUM_STR_SIZE 40
#define MAX_META_MSG_IN_BATCH 1048576
#define MAX_META_MSG_IN_BATCH 1048576
#define MAX_META_BATCH_RSP_SIZE (1 * 1048576 * 1024)
#ifdef __cplusplus

56
include/util/tfunctional.h
View File

@ -1,56 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_FUNCTIONAL_H_
#define _TD_UTIL_FUNCTIONAL_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
// TODO: hard to use, trying to rewrite it using va_list
typedef void* (*GenericVaFunc)(void* args[]);
typedef int32_t (*I32VaFunc)(void* args[]);
typedef void (*VoidVaFunc)(void* args[]);
typedef struct GenericSavedFunc {
GenericVaFunc func;
void* args[];
} tGenericSavedFunc;
typedef struct I32SavedFunc {
I32VaFunc func;
void* args[];
} tI32SavedFunc;
typedef struct VoidSavedFunc {
VoidVaFunc func;
void* args[];
} tVoidSavedFunc;
tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int32_t numOfArgs);
tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int32_t numOfArgs);
tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int32_t numOfArgs);
void* genericInvoke(tGenericSavedFunc* const pSavedFunc);
int32_t i32Invoke(tI32SavedFunc* const pSavedFunc);
void voidInvoke(tVoidSavedFunc* const pSavedFunc);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_FUNCTIONAL_H_*/

2
include/util/tskiplist.h
View File

@ -19,7 +19,6 @@
#include "os.h"
#include "taos.h"
#include "tarray.h"
#include "tfunctional.h"
#ifdef __cplusplus
extern "C" {
@ -67,7 +66,6 @@ typedef struct SSkipList {
uint32_t size;
SSkipListNode *pHead; // point to the first element
SSkipListNode *pTail; // point to the last element
tGenericSavedFunc *insertHandleFn;
} SSkipList;
typedef struct SSkipListIterator {

26
packaging/docker/DockerfileCloud Normal file
View File

@ -0,0 +1,26 @@
FROM ubuntu:18.04
WORKDIR /root
ARG pkgFile
ARG dirName
ARG cpuType
RUN echo ${pkgFile} && echo ${dirName}
COPY ${pkgFile} /root/
ENV TINI_VERSION v0.19.0
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${cpuType} /tini
ENV DEBIAN_FRONTEND=noninteractive
WORKDIR /root/
RUN tar -zxf ${pkgFile} && cd /root/${dirName}/ && /bin/bash install.sh -e no && cd /root && rm /root/${pkgFile} && rm -rf /root/${dirName} && apt-get update && apt-get install -y locales tzdata netcat && locale-gen en_US.UTF-8 && apt-get clean && rm -rf /var/lib/apt/lists/ && chmod +x /tini
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" \
LC_CTYPE=en_US.UTF-8 \
LANG=en_US.UTF-8 \
LC_ALL=en_US.UTF-8
COPY ./run.sh /usr/bin/
COPY ./bin/* /usr/bin/
ENTRYPOINT ["/tini", "--", "/usr/bin/entrypoint.sh"]
CMD ["bash", "-c", "/usr/bin/run.sh"]
VOLUME [ "/var/lib/taos", "/var/log/taos", "/corefile" ]

59
packaging/docker/dockerbuild.sh
View File

@ -19,8 +19,9 @@ passWord=""
pkgFile=""
verType="stable"
dockerLatest="n"
cloudBuild="n"
while getopts "hc:n:p:f:V:a:b:" arg
while getopts "hc:n:p:f:V:a:b:d:" arg
do
case $arg in
c)
@ -47,6 +48,10 @@ do
#echo "verType=$OPTARG"
verType=$(echo $OPTARG)
;;
d)
#echo "cloudBuild=$OPTARG"
cloudBuild=$(echo $OPTARG)
;;
a)
#echo "dockerLatest=$OPTARG"
dockerLatest=$(echo $OPTARG)
@ -58,6 +63,7 @@ do
echo " -V [stable | beta] "
echo " -f [pkg file] "
echo " -a [y | n ] "
echo " -d [cloud build ] "
exit 0
;;
?) #unknow option
@ -83,6 +89,9 @@ else
echo "unknow verType, nor stabel or beta"
exit 1
fi
if [ "$cloudBuild" == "y" ]; then
dockername=cloud-${dockername}
fi
echo "cpuType=${cpuType} version=${version} pkgFile=${pkgFile} verType=${verType} "
@ -90,10 +99,15 @@ echo "$(pwd)"
echo "====NOTES: ${pkgFile} must be in the same directory as dockerbuild.sh===="
scriptDir=$(dirname $(readlink -f $0))
comunityArchiveDir=/nas/TDengine/v$version/community # community version’package directory
communityDir=${scriptDir}/../../../community
DockerfilePath=${communityDir}/packaging/docker/
Dockerfile=${communityDir}/packaging/docker/Dockerfile
if [ "$cloudBuild" == "y" ]; then
comunityArchiveDir=/nas/TDengine/v$version/cloud
Dockerfile=${communityDir}/packaging/docker/DockerfileCloud
else
comunityArchiveDir=/nas/TDengine/v$version/community
Dockerfile=${communityDir}/packaging/docker/Dockerfile
fi
cd ${scriptDir}
cp -f ${comunityArchiveDir}/${pkgFile} .
@ -111,42 +125,15 @@ else
fi
docker build --rm -f "${Dockerfile}" --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} --build-arg cpuType=${cpuTypeAlias}
docker login -u tdengine -p ${passWord} #replace the docker registry username and password
docker push tdengine/tdengine-${dockername}:${version}
if [ -n "$(docker ps -aq)" ] ;then
echo "delete docker process"
docker stop $(docker ps -aq)
docker rm $(docker ps -aq)
if [ "$cloudBuild" != "y" ]; then
docker login -u tdengine -p ${passWord} #replace the docker registry username and password
docker push tdengine/tdengine-${dockername}:${version}
fi
if [ -n "$(pidof taosd)" ] ;then
echo "kill taosd "
kill -9 $(pidof taosd)
fi
if [ -n "$(pidof power)" ] ;then
echo "kill power "
kill -9 $(pidof power)
fi
echo ">>>>>>>>>>>>> check whether tdengine/tdengine-${dockername}:${version} has been published"
docker run -d --name doctest -p 6030-6049:6030-6049 -p 6030-6049:6030-6049/udp tdengine/tdengine-${dockername}:${version}
sleep 2
curl -u root:taosdata -d 'show variables;' 127.0.0.1:6041/rest/sql > temp1.data
data_version=$( cat temp1.data |jq .data| jq '.[]' |grep "version" -A 2 -B 1 | jq ".[1]")
echo "${data_version}"
if [ "${data_version}" == "\"${version}\"" ] ; then
echo "docker version is right "
else
echo "docker version is wrong "
exit 1
fi
rm -rf temp1.data
# set this version to latest version
if [ ${dockerLatest} == 'y' ] ;then
if [ "$cloudBuild" != "y" ] && [ ${dockerLatest} == 'y' ] ;then
docker tag tdengine/tdengine-${dockername}:${version} tdengine/tdengine-${dockername}:latest
docker push tdengine/tdengine-${dockername}:latest
fi
rm -f ${pkgFile}

36
packaging/docker/run.sh Normal file
View File

@ -0,0 +1,36 @@
#!/bin/bash
TAOS_RUN_TAOSBENCHMARK_TEST_ONCE=0
while ((1))
do
# echo "outer loop: $a"
sleep 10
output=`taos -k`
status=${output:0:1}
# echo $output
# echo $status
if [ "$status"x = "0"x ]
then
taosd &
fi
# echo "$status"x "$TAOS_RUN_TAOSBENCHMARK_TEST"x "$TAOS_RUN_TAOSBENCHMARK_TEST_ONCE"x
if [ "$status"x = "2"x ] && [ "$TAOS_RUN_TAOSBENCHMARK_TEST"x = "1"x ] && [ "$TAOS_RUN_TAOSBENCHMARK_TEST_ONCE"x = "0"x ]
then
TAOS_RUN_TAOSBENCHMARK_TEST_ONCE=1
# result=`taos -s "show databases;" | grep " test "`
# if [ "${result:0:5}"x != " test"x ]
# then
# taosBenchmark -y -t 1000 -n 1000 -S 900000
# fi
taos -s "select stable_name from information_schema.ins_stables where db_name = 'test';"|grep -q -w meters
if [ $? -ne 0 ]; then
taosBenchmark -y -t 1000 -n 1000 -S 900000
taos -s "create user admin_user pass 'NDS65R6t' sysinfo 0;"
taos -s "GRANT ALL on test.* to admin_user;"
fi
fi
# check taosadapter
nc -z localhost 6041
if [ $? -ne 0 ]; then
taosadapter &
fi
done

14
packaging/release.sh
View File

@ -17,7 +17,7 @@ set -e
# -H [ false | true]
# set parameters by default value
verMode=edge # [cluster, edge]
verMode=edge # [cluster, edge, cloud]
verType=stable # [stable, beta]
cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 ...]
osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...]
@ -169,7 +169,7 @@ build_time=$(date +"%F %R")
# get commint id from git
gitinfo=$(git rev-parse --verify HEAD)
if [[ "$verMode" == "cluster" ]]; then
if [[ "$verMode" == "cluster" ]] || [[ "$verMode" == "cloud" ]]; then
enterprise_dir="${top_dir}/../enterprise"
cd ${enterprise_dir}
gitinfoOfInternal=$(git rev-parse --verify HEAD)
@ -205,7 +205,7 @@ else
BUILD_HTTP=false
fi
if [[ "$verMode" == "cluster" ]]; then
if [[ "$verMode" == "cluster" ]] || [[ "$verMode" == "cloud" ]]; then
BUILD_HTTP=internal
fi
@ -217,10 +217,12 @@ fi
# check support cpu type
if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "arm64" ]] || [[ "$cpuType" == "arm32" ]] || [[ "$cpuType" == "mips64" ]]; then
if [ "$verMode" != "cluster" ]; then
if [ "$verMode" == "edge" ]; then
# community-version compile
cmake ../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro}
else
elif [ "$verMode" == "cloud" ]; then
cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_CLOUD=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro}
elif [ "$verMode" == "cluster" ]; then
if [[ "$dbName" != "taos" ]]; then
replace_enterprise_$dbName
fi
@ -244,7 +246,7 @@ cd ${curr_dir}
# 3. Call the corresponding script for packaging
if [ "$osType" != "Darwin" ]; then
if [[ "$verMode" != "cluster" ]] && [[ "$pagMode" == "full" ]] && [[ "$cpuType" == "x64" ]] && [[ "$dbName" == "taos" ]]; then
if [[ "$verMode" != "cluster" ]] && [[ "$verMode" != "cloud" ]] && [[ "$pagMode" == "full" ]] && [[ "$cpuType" == "x64" ]] && [[ "$dbName" == "taos" ]]; then
ret='0'
command -v dpkg >/dev/null 2>&1 || { ret='1'; }
if [ "$ret" -eq 0 ]; then

12
packaging/tools/make_install.bat
View File

@ -47,6 +47,15 @@ copy %binary_dir%\\build\\bin\\udfd.exe %tagert_dir% > nul
if exist %binary_dir%\\build\\bin\\taosBenchmark.exe (
copy %binary_dir%\\build\\bin\\taosBenchmark.exe %tagert_dir% > nul
)
if exist %binary_dir%\\build\\lib\\taosws.dll.lib (
copy %binary_dir%\\build\\lib\\taosws.dll.lib %tagert_dir%\\driver > nul
)
if exist %binary_dir%\\build\\lib\\taosws.dll (
copy %binary_dir%\\build\\lib\\taosws.dll %tagert_dir%\\driver > nul
)
if exist %binary_dir%\\build\\bin\\taosdump.exe (
copy %binary_dir%\\build\\bin\\taosdump.exe %tagert_dir% > nul
)
if exist %binary_dir%\\build\\bin\\taosadapter.exe (
copy %binary_dir%\\build\\bin\\taosadapter.exe %tagert_dir% > nul
)
@ -54,4 +63,7 @@ if exist %binary_dir%\\build\\bin\\taosadapter.exe (
mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)&& echo To start/stop TDengine with administrator privileges: sc start/stop taosd &goto :eof
:hasAdmin
copy /y C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 > nul
if exist C:\\TDengine\\driver\\taosws.dll (
copy /y C:\\TDengine\\driver\\taosws.dll C:\\Windows\\System32 > nul
)
sc query "taosd" >nul || sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND

8
packaging/tools/makeclient.sh
View File

@ -39,6 +39,8 @@ release_dir="${top_dir}/release"
if [ "$verMode" == "cluster" ]; then
install_dir="${release_dir}/${productName}-enterprise-client-${version}"
elif [ "$verMode" == "cloud" ]; then
install_dir="${release_dir}/${productName}-cloud-client-${version}"
else
install_dir="${release_dir}/${productName}-client-${version}"
fi
@ -138,6 +140,10 @@ if [ "$verMode" == "cluster" ]; then
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install_client.sh >>install_client_temp.sh
mv install_client_temp.sh ${install_dir}/install_client.sh
fi
if [ "$verMode" == "cloud" ]; then
sed 's/verMode=edge/verMode=cloud/g' ${install_dir}/install_client.sh >>install_client_temp.sh
mv install_client_temp.sh ${install_dir}/install_client.sh
fi
if [ "$pagMode" == "lite" ]; then
sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install_client.sh >>install_client_temp.sh
@ -161,7 +167,7 @@ if [[ $productName == "TDengine" ]]; then
mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json
fi
if [ "$verMode" == "cluster" ]; then
if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then
# Copy connector
connector_dir="${code_dir}/connector"
mkdir -p ${install_dir}/connector

13
packaging/tools/makepkg.sh
View File

@ -39,6 +39,8 @@ release_dir="${top_dir}/release"
#package_name='linux'
if [ "$verMode" == "cluster" ]; then
install_dir="${release_dir}/${productName}-enterprise-server-${version}"
elif [ "$verMode" == "cloud" ]; then
install_dir="${release_dir}/${productName}-cloud-server-${version}"
else
install_dir="${release_dir}/${productName}-server-${version}"
fi
@ -217,7 +219,10 @@ fi
if [ "$verMode" == "cluster" ]; then
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh
mv remove_temp.sh ${install_dir}/bin/remove.sh
fi
if [ "$verMode" == "cloud" ]; then
sed 's/verMode=edge/verMode=cloud/g' ${install_dir}/bin/remove.sh >>remove_temp.sh
mv remove_temp.sh ${install_dir}/bin/remove.sh
fi
cd ${install_dir}
@ -234,6 +239,10 @@ if [ "$verMode" == "cluster" ]; then
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install.sh >>install_temp.sh
mv install_temp.sh ${install_dir}/install.sh
fi
if [ "$verMode" == "cloud" ]; then
sed 's/verMode=edge/verMode=cloud/g' ${install_dir}/install.sh >>install_temp.sh
mv install_temp.sh ${install_dir}/install.sh
fi
if [ "$pagMode" == "lite" ]; then
sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install.sh >>install_temp.sh
mv install_temp.sh ${install_dir}/install.sh
@ -288,7 +297,7 @@ mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo
[ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || :
# Copy connector
if [ "$verMode" == "cluster" ]; then
if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then
connector_dir="${code_dir}/connector"
mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then

2
source/client/inc/clientLog.h
View File

@ -30,7 +30,7 @@ extern "C" {
#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscPerf(...) do { taosPrintLog("TSC ", 0, cDebugFlag, __VA_ARGS__); } while(0)
//#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on
#ifdef __cplusplus

22
source/client/src/clientEnv.c
View File

@ -77,19 +77,19 @@ static void deregisterRequest(SRequestObj *pRequest) {
pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst);
if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) {
tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
"us, exec:%" PRId64 "us",
duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart,
pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd,
pRequest->metric.execEnd - pRequest->metric.semanticEnd);
// tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
// "us, exec:%" PRId64 "us",
// duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart,
// pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd,
// pRequest->metric.execEnd - pRequest->metric.semanticEnd);
atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration);
} else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
"us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%" PRIx64,
duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart,
pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd,
pRequest->metric.planEnd - pRequest->metric.semanticEnd,
pRequest->metric.resultReady - pRequest->metric.planEnd, pRequest->requestId);
// tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
// "us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%" PRIx64,
// duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart,
// pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd,
// pRequest->metric.planEnd - pRequest->metric.semanticEnd,
// pRequest->metric.resultReady - pRequest->metric.planEnd, pRequest->requestId);
atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration);
}

2
source/client/src/clientTmq.c
View File

@ -1619,7 +1619,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
taosGetQitem(tmq->qall, (void**)&rspWrapper);
if (rspWrapper == NULL) {
tscDebug("consumer %" PRId64 " mqueue empty", tmq->consumerId);
/*tscDebug("consumer %" PRId64 " mqueue empty", tmq->consumerId);*/
return NULL;
}
}

2
source/common/src/systable.c
View File

@ -270,7 +270,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
{TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema), true},
{TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema), true},
// {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)},
{TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)},
{TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema), true},
{TSDB_INS_TABLE_DATABASES, userDBSchema, tListLen(userDBSchema), false},
{TSDB_INS_TABLE_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema), false},

6
source/common/src/tdatablock.c
View File

@ -1707,8 +1707,6 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
if (IS_VAR_DATA_TYPE(data.info.type)) {
buf = taosDecodeBinary(buf, (void**)&data.varmeta.offset, pBlock->info.rows * sizeof(int32_t));
data.varmeta.length = pBlock->info.rows * sizeof(int32_t);
data.varmeta.allocLen = data.varmeta.length;
} else {
buf = taosDecodeBinary(buf, (void**)&data.nullbitmap, BitmapLen(pBlock->info.rows));
}
@ -1716,6 +1714,10 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
int32_t len = 0;
buf = taosDecodeFixedI32(buf, &len);
buf = taosDecodeBinary(buf, (void**)&data.pData, len);
if (IS_VAR_DATA_TYPE(data.info.type)) {
data.varmeta.length = len;
data.varmeta.allocLen = len;
}
taosArrayPush(pBlock->pDataBlock, &data);
}
return (void*)buf;

5
source/common/src/tglobal.c
View File

@ -82,6 +82,7 @@ bool tsSmlDataFormat = false;
// query
int32_t tsQueryPolicy = 1;
int32_t tsQueryRspPolicy = 0;
bool tsEnableQueryHb = false;
int32_t tsQuerySmaOptimize = 0;
int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data.
bool tsQueryPlannerTrace = false;
@ -284,6 +285,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, 1) != 0) return -1;
if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, false) != 0) return -1;
if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1;
if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1;
if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, true) != 0) return -1;
@ -644,6 +646,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32;
tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval;
tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32;
tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval;
tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32;
@ -780,6 +783,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
if (strcasecmp("enableCoreFile", name) == 0) {
bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
taosSetCoreDump(enableCore);
} else if (strcasecmp("enableQueryHb", name) == 0) {
tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval;
}
break;
}

11
source/common/src/tmsg.c
View File

@ -5989,7 +5989,11 @@ int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
if (pRsp->withSchema) {
SSchemaWrapper *pSW = (SSchemaWrapper *)taosMemoryCalloc(1, sizeof(SSchemaWrapper));
if (pSW == NULL) return -1;
if (tDecodeSSchemaWrapper(pDecoder, pSW) < 0) return -1;
if (tDecodeSSchemaWrapper(pDecoder, pSW) < 0) {
taosMemoryFree(pSW);
return -1;
}
taosArrayPush(pRsp->blockSchema, &pSW);
}
@ -6070,7 +6074,10 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) {
if (pRsp->withSchema) {
SSchemaWrapper *pSW = (SSchemaWrapper *)taosMemoryCalloc(1, sizeof(SSchemaWrapper));
if (pSW == NULL) return -1;
if (tDecodeSSchemaWrapper(pDecoder, pSW) < 0) return -1;
if (tDecodeSSchemaWrapper(pDecoder, pSW) < 0) {
taosMemoryFree(pSW);
return -1;
}
taosArrayPush(pRsp->blockSchema, &pSW);
}

16
source/common/src/ttime.c
View File

@ -507,29 +507,29 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
double tmp = time;
switch (toUnit) {
case 's': {
tmp /= (NANOSECOND_PER_SEC / factors[fromPrecision]); // the result of division is an integer
time /= (NANOSECOND_PER_SEC / factors[fromPrecision]);
tmp = (double)time;
break;
}
case 'm':
tmp /= (NANOSECOND_PER_MINUTE / factors[fromPrecision]); // the result of division is an integer
time /= (NANOSECOND_PER_MINUTE / factors[fromPrecision]);
tmp = (double)time;
break;
case 'h':
tmp /= (NANOSECOND_PER_HOUR / factors[fromPrecision]); // the result of division is an integer
time /= (NANOSECOND_PER_HOUR / factors[fromPrecision]);
tmp = (double)time;
break;
case 'd':
tmp /= (NANOSECOND_PER_DAY / factors[fromPrecision]); // the result of division is an integer
time /= (NANOSECOND_PER_DAY / factors[fromPrecision]);
tmp = (double)time;
break;
case 'w':
tmp /= (NANOSECOND_PER_WEEK / factors[fromPrecision]); // the result of division is an integer
time /= (NANOSECOND_PER_WEEK / factors[fromPrecision]);
tmp = (double)time;
break;
case 'a':
tmp /= (NANOSECOND_PER_MSEC / factors[fromPrecision]); // the result of division is an integer
time /= (NANOSECOND_PER_MSEC / factors[fromPrecision]);
tmp = (double)time;
break;
case 'u':
// the result of (NANOSECOND_PER_USEC/(double)factors[fromPrecision]) maybe a double
@ -540,13 +540,13 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
break;
}
case TSDB_TIME_PRECISION_MICRO: {
tmp /= 1;
time /= 1;
tmp = (double)time;
break;
}
case TSDB_TIME_PRECISION_NANO: {
tmp /= 1000;
time /= 1000;
tmp = (double)time;
break;
}
}

1
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -332,7 +332,6 @@ SArray *dmGetMsgHandles() {
// Requests handled by MNODE
if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
// if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_RSP, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH_RSP, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
code = 0;

50
source/dnode/mnode/impl/inc/mndOffset.h
View File

@ -1,50 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_OFFSET_H_
#define _TD_MND_OFFSET_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitOffset(SMnode *pMnode);
void mndCleanupOffset(SMnode *pMnode);
SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key);
void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset);
SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset);
SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw);
int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs);
static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) {
return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName);
}
int32_t mndDropOffsetByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndDropOffsetByTopic(SMnode *pMnode, STrans *pTrans, const char *topic);
int32_t mndDropOffsetBySubKey(SMnode *pMnode, STrans *pTrans, const char *subKey);
bool mndOffsetFromTopic(SMqOffsetObj *pOffset, const char *topic);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_OFFSET_H_*/

4
source/dnode/mnode/impl/inc/mndSubscribe.h
View File

@ -31,6 +31,10 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub);
int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) {
return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName);
}
int32_t mndDropSubByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topic);
int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub);

7
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -18,7 +18,6 @@
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndOffset.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndStb.h"
@ -408,12 +407,6 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
.offset = -1,
};
// 2.2.2 fetch vg offset
SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey);
if (pOffsetObj != NULL) {
vgEp.offset = atomic_load_64(&pOffsetObj->offset);
mndReleaseOffset(pMnode, pOffsetObj);
}
taosArrayPush(topicEp.vgs, &vgEp);
}
taosArrayPush(rsp.topics, &topicEp);

5
source/dnode/mnode/impl/src/mndDb.c
View File

@ -17,7 +17,6 @@
#include "mndDb.h"
#include "mndCluster.h"
#include "mndDnode.h"
#include "mndOffset.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndSma.h"
@ -641,12 +640,8 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
}
if (pAlter->pageSize > 0 && pAlter->pageSize != pDb->cfg.pageSize) {
#if 1
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
#else
pDb->cfg.pageSize = pAlter->pageSize;
terrno = 0;
#endif
}
if (pAlter->daysPerFile > 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) {

5
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -776,11 +776,6 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
}
}
if (numOfVnodes > 0) {
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
goto _OVER;
}
code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;

4
source/dnode/mnode/impl/src/mndMain.c
View File

@ -23,7 +23,6 @@
#include "mndGrant.h"
#include "mndInfoSchema.h"
#include "mndMnode.h"
#include "mndOffset.h"
#include "mndPerfSchema.h"
#include "mndPrivilege.h"
#include "mndProfile.h"
@ -305,7 +304,6 @@ static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma) != 0) return -1;
@ -424,7 +422,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
return NULL;
}
mInfo("mnode open successfully ");
mInfo("mnode open successfully");
return pMnode;
}

370
source/dnode/mnode/impl/src/mndOffset.c
View File

@ -1,370 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "mndOffset.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndStb.h"
#include "mndTopic.h"
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
#include "tname.h"
#define MND_OFFSET_VER_NUMBER 1
#define MND_OFFSET_RESERVE_SIZE 64
static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset);
static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset);
static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffsetObj *pNewOffset);
static int32_t mndProcessCommitOffsetReq(SRpcMsg *pReq);
int32_t mndInitOffset(SMnode *pMnode) {
SSdbTable table = {
.sdbType = SDB_OFFSET,
.keyType = SDB_KEY_BINARY,
.encodeFp = (SdbEncodeFp)mndOffsetActionEncode,
.decodeFp = (SdbDecodeFp)mndOffsetActionDecode,
.insertFp = (SdbInsertFp)mndOffsetActionInsert,
.updateFp = (SdbUpdateFp)mndOffsetActionUpdate,
.deleteFp = (SdbDeleteFp)mndOffsetActionDelete,
};
mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq);
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupOffset(SMnode *pMnode) {}
bool mndOffsetFromTopic(SMqOffsetObj *pOffset, const char *topic) {
int32_t i = 0;
while (pOffset->key[i] != ':') i++;
while (pOffset->key[i] != ':') i++;
if (strcmp(&pOffset->key[i + 1], topic) == 0) return true;
return false;
}
bool mndOffsetFromSubKey(SMqOffsetObj *pOffset, const char *subKey) {
int32_t i = 0;
while (pOffset->key[i] != ':') i++;
if (strcmp(&pOffset->key[i + 1], subKey) == 0) return true;
return false;
}
SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
void *buf = NULL;
int32_t tlen = tEncodeSMqOffsetObj(NULL, pOffset);
int32_t size = sizeof(int32_t) + tlen + MND_OFFSET_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_OFFSET, MND_OFFSET_VER_NUMBER, size);
if (pRaw == NULL) goto OFFSET_ENCODE_OVER;
buf = taosMemoryMalloc(tlen);
if (buf == NULL) goto OFFSET_ENCODE_OVER;
void *abuf = buf;
tEncodeSMqOffsetObj(&abuf, pOffset);
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, OFFSET_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_ENCODE_OVER);
SDB_SET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_ENCODE_OVER);
SDB_SET_DATALEN(pRaw, dataPos, OFFSET_ENCODE_OVER);
terrno = TSDB_CODE_SUCCESS;
OFFSET_ENCODE_OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("offset:%s, failed to encode to raw:%p since %s", pOffset->key, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("offset:%s, encode to raw:%p, row:%p", pOffset->key, pRaw, pOffset);
return pRaw;
}
SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
void *buf = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto OFFSET_DECODE_OVER;
if (sver != MND_OFFSET_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto OFFSET_DECODE_OVER;
}
int32_t size = sizeof(SMqOffsetObj);
SSdbRow *pRow = sdbAllocRow(size);
if (pRow == NULL) goto OFFSET_DECODE_OVER;
SMqOffsetObj *pOffset = sdbGetRowObj(pRow);
if (pOffset == NULL) goto OFFSET_DECODE_OVER;
int32_t dataPos = 0;
int32_t tlen;
SDB_GET_INT32(pRaw, dataPos, &tlen, OFFSET_DECODE_OVER);
buf = taosMemoryMalloc(tlen + 1);
if (buf == NULL) goto OFFSET_DECODE_OVER;
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_DECODE_OVER);
SDB_GET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_DECODE_OVER);
if (tDecodeSMqOffsetObj(buf, pOffset) == NULL) {
goto OFFSET_DECODE_OVER;
}
terrno = TSDB_CODE_SUCCESS;
OFFSET_DECODE_OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("offset:%s, failed to decode from raw:%p since %s", pOffset->key, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
mTrace("offset:%s, decode from raw:%p, row:%p", pOffset->key, pRaw, pOffset);
return pRow;
}
int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) {
int32_t sz = taosArrayGetSize(vgs);
for (int32_t i = 0; i < sz; i++) {
int32_t vgId = *(int32_t *)taosArrayGet(vgs, i);
SMqOffsetObj offsetObj = {0};
if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, vgId) < 0) {
return -1;
}
// TODO assign db
offsetObj.offset = -1;
SSdbRaw *pOffsetRaw = mndOffsetActionEncode(&offsetObj);
if (pOffsetRaw == NULL) {
return -1;
}
(void)sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
// commit log or redo log?
if (mndTransAppendRedolog(pTrans, pOffsetRaw) < 0) {
return -1;
}
}
return 0;
}
static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) {
char key[TSDB_PARTITION_KEY_LEN];
SMnode *pMnode = pMsg->info.node;
char *msgStr = pMsg->pCont;
SMqCMCommitOffsetReq commitOffsetReq;
SDecoder decoder;
tDecoderInit(&decoder, msgStr, pMsg->contLen);
tDecodeSMqCMCommitOffsetReq(&decoder, &commitOffsetReq);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "commit-offset");
if (pTrans == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tDecoderClear(&decoder);
return -1;
}
for (int32_t i = 0; i < commitOffsetReq.num; i++) {
SMqOffset *pOffset = &commitOffsetReq.offsets[i];
mInfo("commit offset %" PRId64 " to vgId:%d of consumer group %s on topic %s", pOffset->offset, pOffset->vgId,
pOffset->cgroup, pOffset->topicName);
if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) {
mError("submit offset to topic %s failed", pOffset->topicName);
return -1;
}
bool create = false;
SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key);
if (pOffsetObj == NULL) {
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pOffset->topicName);
if (pTopic == NULL) {
terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
mError("submit offset to topic %s failed since %s", pOffset->topicName, terrstr());
continue;
}
pOffsetObj = taosMemoryMalloc(sizeof(SMqOffsetObj));
pOffsetObj->dbUid = pTopic->dbUid;
mndReleaseTopic(pMnode, pTopic);
memcpy(pOffsetObj->key, key, TSDB_PARTITION_KEY_LEN);
create = true;
}
pOffsetObj->offset = pOffset->offset;
SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj);
(void)sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
mndTransAppendCommitlog(pTrans, pOffsetRaw);
if (create) {
taosMemoryFree(pOffsetObj);
} else {
mndReleaseOffset(pMnode, pOffsetObj);
}
}
tDecoderClear(&decoder);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("mq-commit-offset-trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
return TSDB_CODE_ACTION_IN_PROGRESS;
}
static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset) {
mTrace("offset:%s, perform insert action", pOffset->key);
return 0;
}
static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) {
mTrace("offset:%s, perform delete action", pOffset->key);
return 0;
}
static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) {
mTrace("offset:%s, perform update action", pOldOffset->key);
atomic_store_64(&pOldOffset->offset, pNewOffset->offset);
return 0;
}
SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key) {
SSdb *pSdb = pMnode->pSdb;
SMqOffsetObj *pOffset = sdbAcquire(pSdb, SDB_OFFSET, key);
if (pOffset == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_OFFSET_NOT_EXIST;
}
return pOffset;
}
void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pOffset);
}
static void mndCancelGetNextOffset(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter);
}
static int32_t mndSetDropOffsetCommitLogs(SMnode *pMnode, STrans *pTrans, SMqOffsetObj *pOffset) {
SSdbRaw *pCommitRaw = mndOffsetActionEncode(pOffset);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
}
static int32_t mndSetDropOffsetRedoLogs(SMnode *pMnode, STrans *pTrans, SMqOffsetObj *pOffset) {
SSdbRaw *pRedoRaw = mndOffsetActionEncode(pOffset);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
}
int32_t mndDropOffsetByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SMqOffsetObj *pOffset = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_OFFSET, pIter, (void **)&pOffset);
if (pIter == NULL) break;
if (pOffset->dbUid != pDb->uid) {
sdbRelease(pSdb, pOffset);
continue;
}
if (mndSetDropOffsetCommitLogs(pMnode, pTrans, pOffset) < 0) {
sdbRelease(pSdb, pOffset);
sdbCancelFetch(pSdb, pIter);
code = -1;
break;
}
sdbRelease(pSdb, pOffset);
}
return code;
}
int32_t mndDropOffsetByTopic(SMnode *pMnode, STrans *pTrans, const char *topic) {
int32_t code = -1;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SMqOffsetObj *pOffset = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_OFFSET, pIter, (void **)&pOffset);
if (pIter == NULL) break;
if (!mndOffsetFromTopic(pOffset, topic)) {
sdbRelease(pSdb, pOffset);
continue;
}
if (mndSetDropOffsetCommitLogs(pMnode, pTrans, pOffset) < 0) {
sdbRelease(pSdb, pOffset);
goto END;
}
sdbRelease(pSdb, pOffset);
}
code = 0;
END:
return code;
}
int32_t mndDropOffsetBySubKey(SMnode *pMnode, STrans *pTrans, const char *subKey) {
int32_t code = -1;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SMqOffsetObj *pOffset = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_OFFSET, pIter, (void **)&pOffset);
if (pIter == NULL) break;
if (!mndOffsetFromSubKey(pOffset, subKey)) {
sdbRelease(pSdb, pOffset);
continue;
}
if (mndSetDropOffsetCommitLogs(pMnode, pTrans, pOffset) < 0) {
sdbRelease(pSdb, pOffset);
goto END;
}
sdbRelease(pSdb, pOffset);
}
code = 0;
END:
return code;
}

1
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -18,7 +18,6 @@
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndOffset.h"
#include "mndShow.h"
#include "mndSnode.h"
#include "mndStb.h"

8
source/dnode/mnode/impl/src/mndSnode.c
View File

@ -273,9 +273,6 @@ _OVER:
}
static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
#if 1
return TSDB_CODE_OPS_NOT_SUPPORT;
#else
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SSnodeObj *pObj = NULL;
@ -318,7 +315,6 @@ _OVER:
mndReleaseSnode(pMnode, pObj);
mndReleaseDnode(pMnode, pDnode);
return code;
#endif
}
static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) {
@ -390,9 +386,6 @@ _OVER:
}
static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
#if 1
return TSDB_CODE_OPS_NOT_SUPPORT;
#else
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SSnodeObj *pObj = NULL;
@ -429,7 +422,6 @@ _OVER:
mndReleaseSnode(pMnode, pObj);
return code;
#endif
}
static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

8
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -19,7 +19,6 @@
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndOffset.h"
#include "mndScheduler.h"
#include "mndShow.h"
#include "mndStb.h"
@ -77,7 +76,6 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_DELETE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP, mndProcessDropCgroupReq);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP_RSP, mndTransProcessRsp);
@ -684,12 +682,6 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) {
mInfo("trans:%d, used to drop cgroup:%s on topic %s", pTrans->id, dropReq.cgroup, dropReq.topic);
if (mndDropOffsetBySubKey(pMnode, pTrans, pSub->key) < 0) {
mndReleaseSubscribe(pMnode, pSub);
mndTransDrop(pTrans);
return -1;
}
if (mndSetDropSubCommitLogs(pMnode, pTrans, pSub) < 0) {
mError("cgroup %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr());
mndReleaseSubscribe(pMnode, pSub);

1
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -18,7 +18,6 @@
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndOffset.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndStb.h"

48
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -1112,7 +1112,9 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
mInfo("vgId:%d, will add 1 vnodes", pVgroup->vgId);
if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1]) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, -1) != 0) return -1;
for (int32_t i = 0; i < newVg.replica - 1; ++i) {
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
}
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
mInfo("vgId:%d, will remove 1 vnodes", pVgroup->vgId);
@ -1120,8 +1122,10 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
SVnodeGid del = newVg.vnodeGid[vnIndex];
newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica];
memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid));
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, -1) != 0) return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1;
for (int32_t i = 0; i < newVg.replica; ++i) {
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
}
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
{
@ -1193,21 +1197,11 @@ static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDb
pGid->dnodeId = newDnodeId;
pGid->syncState = TAOS_SYNC_STATE_ERROR;
if (pVgroup->replica == 2) {
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pGid) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[0].dnodeId) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1;
} else if (pVgroup->replica == 4) {
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pGid) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[0].dnodeId) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[1].dnodeId) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[2].dnodeId) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1;
} else {
mError("vgId:%d, failed to add 1 vnode since invalid replica:%d", pVgroup->vgId, pVgroup->replica);
terrno = TSDB_CODE_MND_APP_ERROR;
return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pGid) != 0) return -1;
for (int32_t i = 0; i < pVgroup->replica - 1; ++i) {
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1;
}
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1;
return 0;
}
@ -1232,21 +1226,11 @@ static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, S
memcpy(pGid, &pVgroup->vnodeGid[pVgroup->replica], sizeof(SVnodeGid));
memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid));
if (pVgroup->replica == 1) {
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[0].dnodeId) != 0) return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1;
} else if (pVgroup->replica == 3) {
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[0].dnodeId) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[1].dnodeId) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[2].dnodeId) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1;
} else {
mError("vgId:%d, failed to remove 1 vnode since invalid replica:%d", pVgroup->vgId, pVgroup->replica);
terrno = TSDB_CODE_MND_APP_ERROR;
return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1;
for (int32_t i = 0; i < pVgroup->replica; ++i) {
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1;
}
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1;
return 0;
}
@ -1890,9 +1874,6 @@ _OVER:
}
static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) {
#if 1
return TSDB_CODE_OPS_NOT_SUPPORT;
#else
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SArray *pArray = NULL;
@ -1941,7 +1922,6 @@ _OVER:
taosArrayDestroy(pArray);
return code;
#endif
}
bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; }

2
source/dnode/qnode/src/qnode.c
View File

@ -90,7 +90,7 @@ int32_t qndProcessQueryMsg(SQnode *pQnode, int64_t ts, SRpcMsg *pMsg) {
code = qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg, ts);
break;
case TDMT_SCH_CANCEL_TASK:
code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg, ts);
//code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg, ts);
break;
case TDMT_SCH_DROP_TASK:
code = qWorkerProcessDropMsg(pQnode, pQnode->pQuery, pMsg, ts);

2
source/dnode/vnode/src/inc/tq.h
View File

@ -155,7 +155,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea
// tqExec
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp);
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols);
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision);
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp);
int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry);

14
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -15,11 +15,11 @@
#include "sma.h"
#define RSMA_QTASKEXEC_SMOOTH_SIZE (100) // cnt
#define RSMA_SUBMIT_BATCH_SIZE (1024) // cnt
#define RSMA_FETCH_DELAY_MAX (120000) // ms
#define RSMA_FETCH_ACTIVE_MAX (1000) // ms
#define RSMA_FETCH_INTERVAL (5000) // ms
#define RSMA_QTASKEXEC_SMOOTH_SIZE (100) // cnt
#define RSMA_SUBMIT_BATCH_SIZE (1024) // cnt
#define RSMA_FETCH_DELAY_MAX (120000) // ms
#define RSMA_FETCH_ACTIVE_MAX (1000) // ms
#define RSMA_FETCH_INTERVAL (5000) // ms
SSmaMgmt smaMgmt = {
.inited = 0,
@ -839,7 +839,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize,
tdRsmaPrintSubmitReq(pSma, pReq);
}
#endif
if (qSetMultiStreamInput(qTaskInfo, pMsg, msgSize, inputType) < 0) {
if (qSetSMAInput(qTaskInfo, pMsg, msgSize, inputType) < 0) {
smaError("vgId:%d, rsma %" PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno));
return TSDB_CODE_FAILED;
}
@ -1404,7 +1404,7 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo) {
pItem->nScanned = 0;
if ((terrno = qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) {
if ((terrno = qSetSMAInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) {
goto _err;
}
if (tdRSmaExecAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, pInfo->suid) < 0) {

44
source/dnode/vnode/src/tq/tq.c
View File

@ -433,16 +433,9 @@ static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t su
}
#endif
if (subType == TOPIC_SUB_TYPE__COLUMN) {
pRsp->withSchema = false;
} else {
pRsp->withSchema = true;
pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
if (pRsp->blockSchema == NULL) {
// TODO free
return -1;
}
}
ASSERT(subType == TOPIC_SUB_TYPE__COLUMN);
pRsp->withSchema = false;
return 0;
}
@ -523,17 +516,28 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
tqOffsetResetToLog(&fetchOffsetNew, walGetFirstVer(pTq->pVnode->pWal));
}
} else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, offset reset to %" PRId64, consumerId,
pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.rspOffset.version);
if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
code = -1;
tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, offset reset to %" PRId64, consumerId,
pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.rspOffset.version);
if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
code = -1;
}
tDeleteSMqDataRsp(&dataRsp);
return code;
} else {
STaosxRsp taosxRsp = {0};
tqInitTaosxRsp(&taosxRsp, pReq);
tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) {
code = -1;
}
tDeleteSTaosxRsp(&taosxRsp);
return code;
}
tDeleteSMqDataRsp(&dataRsp);
return code;
} else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
tqError("tmq poll: subkey %s, no offset committed for consumer %" PRId64
" in vg %d, subkey %s, reset none failed",
@ -667,8 +671,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
if (tqTaosxScanLog(pTq, pHandle, pCont, &taosxRsp) < 0) {
/*ASSERT(0);*/
}
// TODO batch optimization:
// TODO continue scan until meeting batch requirement
if (taosxRsp.blockNum > 0 /* threshold */) {
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) {

15
source/dnode/vnode/src/tq/tqExec.c
View File

@ -15,14 +15,14 @@
#include "tq.h"
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols) {
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) {
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
void* buf = taosMemoryCalloc(1, dataStrLen);
if (buf == NULL) return -1;
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
pRetrieve->useconds = 0;
pRetrieve->precision = TSDB_DEFAULT_PRECISION;
pRetrieve->precision = precision;
pRetrieve->compressed = 0;
pRetrieve->completed = 1;
pRetrieve->numOfRows = htonl(pBlock->info.rows);
@ -95,7 +95,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
break;
}
tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols);
tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
pRsp->blockNum++;
if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
@ -174,7 +174,8 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
}
}
tqAddBlockDataToRsp(pDataBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pDataBlock->pDataBlock));
tqAddBlockDataToRsp(pDataBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pDataBlock->pDataBlock),
pTq->pVnode->config.tsdbCfg.precision);
pRsp->blockNum++;
if (pOffset->type == TMQ_OFFSET__LOG) {
continue;
@ -256,7 +257,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp
pRsp->createTableNum++;
}
}
tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock));
tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock),
pTq->pVnode->config.tsdbCfg.precision);
blockDataFreeRes(&block);
tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);
pRsp->blockNum++;
@ -291,7 +293,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp
pRsp->createTableNum++;
}
}
tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock));
tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock),
pTq->pVnode->config.tsdbCfg.precision);
blockDataFreeRes(&block);
tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);
pRsp->blockNum++;

2
source/dnode/vnode/src/tq/tqPush.c
View File

@ -270,7 +270,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
break;
}
tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols);
tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
pRsp->blockNum++;
}

145
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -230,7 +230,21 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST
break;
}
} else {
taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = colVal});
SLastCol lastCol = {.ts = keyTs, .colVal = colVal};
if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) {
SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol);
taosMemoryFree(pLastCol->colVal.value.pData);
lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData);
if (lastCol.colVal.value.pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _invalidate;
}
memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData);
}
taosArraySet(pLast, iCol, &lastCol);
}
}
}
@ -342,7 +356,21 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb
break;
}
} else {
taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = colVal});
SLastCol lastCol = {.ts = keyTs, .colVal = colVal};
if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) {
SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol);
taosMemoryFree(pLastCol->colVal.value.pData);
lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData);
if (lastCol.colVal.value.pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _invalidate;
}
memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData);
}
taosArraySet(pLast, iCol, &lastCol);
}
}
}
@ -1065,72 +1093,71 @@ _err:
// iterate next row non deleted backward ts, version (from high to low)
static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) {
int code = 0;
for (;;) {
for (int i = 0; i < 4; ++i) {
if (pIter->input[i].next && !pIter->input[i].stop) {
code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow);
if (code) goto _err;
for (int i = 0; i < 4; ++i) {
if (pIter->input[i].next && !pIter->input[i].stop) {
code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow);
if (code) goto _err;
if (pIter->input[i].pRow == NULL) {
pIter->input[i].stop = true;
pIter->input[i].next = false;
}
}
}
if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop && pIter->input[3].stop) {
*ppRow = NULL;
return code;
}
// select maxpoint(s) from mem, imem, fs and last
TSDBROW *max[4] = {0};
int iMax[4] = {-1, -1, -1, -1};
int nMax = 0;
TSKEY maxKey = TSKEY_MIN;
for (int i = 0; i < 4; ++i) {
if (!pIter->input[i].stop && pIter->input[i].pRow != NULL) {
TSDBKEY key = TSDBROW_KEY(pIter->input[i].pRow);
// merging & deduplicating on client side
if (maxKey <= key.ts) {
if (maxKey < key.ts) {
nMax = 0;
maxKey = key.ts;
if (pIter->input[i].pRow == NULL) {
pIter->input[i].stop = true;
pIter->input[i].next = false;
}
iMax[nMax] = i;
max[nMax++] = pIter->input[i].pRow;
}
}
}
// delete detection
TSDBROW *merge[4] = {0};
int iMerge[4] = {-1, -1, -1, -1};
int nMerge = 0;
for (int i = 0; i < nMax; ++i) {
TSDBKEY maxKey1 = TSDBROW_KEY(max[i]);
bool deleted = tsdbKeyDeleted(&maxKey1, pIter->pSkyline, &pIter->iSkyline);
if (!deleted) {
iMerge[nMerge] = iMax[i];
merge[nMerge++] = max[i];
if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop && pIter->input[3].stop) {
*ppRow = NULL;
return code;
}
pIter->input[iMax[i]].next = deleted;
// select maxpoint(s) from mem, imem, fs and last
TSDBROW *max[4] = {0};
int iMax[4] = {-1, -1, -1, -1};
int nMax = 0;
TSKEY maxKey = TSKEY_MIN;
for (int i = 0; i < 4; ++i) {
if (!pIter->input[i].stop && pIter->input[i].pRow != NULL) {
TSDBKEY key = TSDBROW_KEY(pIter->input[i].pRow);
// merging & deduplicating on client side
if (maxKey <= key.ts) {
if (maxKey < key.ts) {
nMax = 0;
maxKey = key.ts;
}
iMax[nMax] = i;
max[nMax++] = pIter->input[i].pRow;
}
}
}
// delete detection
TSDBROW *merge[4] = {0};
int iMerge[4] = {-1, -1, -1, -1};
int nMerge = 0;
for (int i = 0; i < nMax; ++i) {
TSDBKEY maxKey1 = TSDBROW_KEY(max[i]);
bool deleted = tsdbKeyDeleted(&maxKey1, pIter->pSkyline, &pIter->iSkyline);
if (!deleted) {
iMerge[nMerge] = iMax[i];
merge[nMerge++] = max[i];
}
pIter->input[iMax[i]].next = deleted;
}
if (nMerge > 0) {
pIter->input[iMerge[0]].next = true;
*ppRow = merge[0];
return code;
}
}
if (nMerge > 0) {
pIter->input[iMerge[0]].next = true;
*ppRow = merge[0];
} else {
*ppRow = NULL;
}
return code;
_err:
return code;
}

2
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -290,7 +290,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
// only apply to the child tables, ordinary tables will not incur this filter procedure.
size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
if (size > 1) {
if (size >= 1) {
SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0);
SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1);

65
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -111,7 +111,7 @@ typedef struct SDataBlockIter {
int32_t index;
SArray* blockList; // SArray<SFileDataBlockInfo>
int32_t order;
SDataBlk block; // current SDataBlk data
SDataBlk block; // current SDataBlk data
SHashObj* pTableMap;
} SDataBlockIter;
@ -169,14 +169,14 @@ static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbRe
static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader,
SRowMerger* pMerger);
static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
SRowMerger* pMerger);
SRowMerger* pMerger, SVersionRange* pVerRange);
static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger,
STsdbReader* pReader);
static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid);
static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData,
int32_t rowIndex);
static void setComposedBlockFlag(STsdbReader* pReader, bool composed);
static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order);
static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pRange);
static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList,
STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow);
@ -1052,7 +1052,7 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr)
if (pBlockInfo != NULL) {
STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
if (pScanInfo == NULL) {
tsdbError("failed to locate the uid:%"PRIu64" in query table uid list, %s", pBlockInfo->uid, idStr);
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, %s", pBlockInfo->uid, idStr);
return TSDB_CODE_INVALID_PARA;
}
@ -1466,7 +1466,8 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB
return false;
}
static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) {
static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo,
SVersionRange* pVerRange) {
while (1) {
bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree);
if (!hasVal) {
@ -1475,7 +1476,8 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc
TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
TSDBKEY k = TSDBROW_KEY(&row);
if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) {
if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order,
pVerRange)) {
return true;
}
}
@ -1483,7 +1485,7 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc
static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader,
STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader) {
bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo);
bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo, &pReader->verRange);
if (hasVal) {
int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
if (next1 != ts) {
@ -1602,7 +1604,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
return code;
}
}
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge);
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange);
}
if (minKey == k.ts) {
@ -1647,7 +1649,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
return code;
}
}
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge);
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange);
}
if (minKey == key) {
@ -1699,7 +1701,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
tRowMerge(&merge, &fRow1);
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge);
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange);
code = tRowMergerGetRow(&merge, &pTSRow);
if (code != TSDB_CODE_SUCCESS) {
@ -1717,7 +1719,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
return code;
}
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge);
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange);
ASSERT(mergeBlockData);
// merge with block data if ts == key
@ -1771,7 +1773,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader
TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
tRowMerge(&merge, &fRow1);
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge);
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange);
code = tRowMergerGetRow(&merge, &pTSRow);
if (code != TSDB_CODE_SUCCESS) {
@ -1882,7 +1884,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
}
}
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge);
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange);
}
if (minKey == ik.ts) {
@ -1901,8 +1903,8 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
}
}
code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline,
&merge, pReader);
code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge,
pReader);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -1973,7 +1975,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
return code;
}
}
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge);
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange);
}
if (minKey == key) {
@ -1993,7 +1995,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
}
}
if (merge.pTSchema == NULL) {
if (merge.pTSchema == NULL) {
return code;
}
@ -2095,7 +2097,8 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum
}
TSDBKEY k = {.ts = ts, .version = ver};
if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, &k, pReader->order)) {
if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, &k, pReader->order,
&pReader->verRange)) {
return false;
}
@ -2130,7 +2133,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan
return false;
}
return nextRowFromLastBlocks(pLBlockReader, pScanInfo);
return nextRowFromLastBlocks(pLBlockReader, pScanInfo, &pReader->verRange);
}
static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) {
@ -2225,8 +2228,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
if (pBlockScanInfo == NULL) {
code = TSDB_CODE_INVALID_PARA;
tsdbError("failed to locate the uid:%"PRIu64" in query table uid list, total tables:%d, %s",
pBlockInfo->uid, taosHashGetSize(pReader->status.pTableMap), pReader->idStr);
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid,
taosHashGetSize(pReader->status.pTableMap), pReader->idStr);
goto _end;
}
@ -2290,7 +2293,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
}
_end:
pResBlock->info.uid = (pBlockScanInfo != NULL)? pBlockScanInfo->uid:0;
pResBlock->info.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
blockDataUpdateTsWindow(pResBlock, 0);
setComposedBlockFlag(pReader, true);
@ -2859,7 +2862,7 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_
return (SVersionRange){.minVer = startVer, .maxVer = endVer};
}
bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order) {
bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pVerRange) {
ASSERT(pKey != NULL);
if (pDelList == NULL) {
return false;
@ -2887,7 +2890,8 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32
return false;
}
if (pCurrent->ts <= pKey->ts && pNext->ts >= pKey->ts && pCurrent->version >= pKey->version) {
if (pCurrent->ts <= pKey->ts && pNext->ts >= pKey->ts && pCurrent->version >= pKey->version &&
pVerRange->maxVer >= pCurrent->version) {
return true;
}
@ -2903,7 +2907,8 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32
continue;
}
if (pCurrent->ts <= pKey->ts && pNext->ts >= pKey->ts && pCurrent->version >= pKey->version) {
if (pCurrent->ts <= pKey->ts && pNext->ts >= pKey->ts && pCurrent->version >= pKey->version &&
pVerRange->maxVer >= pCurrent->version) {
return true;
}
}
@ -2973,7 +2978,7 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p
// it is a valid data version
if ((key.version <= pReader->verRange.maxVer && key.version >= pReader->verRange.minVer) &&
(!hasBeenDropped(pDelList, &pIter->index, &key, pReader->order))) {
(!hasBeenDropped(pDelList, &pIter->index, &key, pReader->order, &pReader->verRange))) {
return pRow;
}
@ -2992,7 +2997,7 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p
}
if (key.version <= pReader->verRange.maxVer && key.version >= pReader->verRange.minVer &&
(!hasBeenDropped(pDelList, &pIter->index, &key, pReader->order))) {
(!hasBeenDropped(pDelList, &pIter->index, &key, pReader->order, &pReader->verRange))) {
return pRow;
}
}
@ -3130,9 +3135,9 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
}
int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
SRowMerger* pMerger) {
SRowMerger* pMerger, SVersionRange* pVerRange) {
pScanInfo->lastKey = ts;
while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo)) {
while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) {
int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
if (next1 == ts) {
TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
@ -3772,7 +3777,7 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) {
STableBlockScanInfo* pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid));
if (pBlockScanInfo == NULL) { // no data block for the table of given uid
if (pBlockScanInfo == NULL) { // no data block for the table of given uid
return false;
}

4
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -369,8 +369,8 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_FETCH_RSP:
return qWorkerProcessRspMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_CANCEL_TASK:
return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg, 0);
//case TDMT_SCH_CANCEL_TASK:
// return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_DROP_TASK:
return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_QUERY_HEARTBEAT:

2
source/libs/catalog/src/catalog.c
View File

@ -570,7 +570,7 @@ int32_t catalogInit(SCatalogCfg* cfg) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
atomic_store_8((int8_t*)&gCtgMgmt.exit, false);
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
if (cfg) {
memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg));

83
source/libs/catalog/src/ctgCache.c
View File

@ -72,7 +72,10 @@ void ctgRUnlockVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->vgC
void ctgWUnlockVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); }
void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->dbLock); }
void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) {
CTG_UNLOCK(CTG_READ, &dbCache->dbLock);
taosHashRelease(pCtg->dbCache, dbCache);
}
int32_t ctgAcquireDBCacheImpl(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) {
char *p = strchr(dbFName, '.');
@ -80,7 +83,14 @@ int32_t ctgAcquireDBCacheImpl(SCatalog *pCtg, const char *dbFName, SCtgDBCache *
dbFName = p + 1;
}
SCtgDBCache *dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName));
SCtgDBCache *dbCache = NULL;
if (acquire) {
dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName));
} else {
dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName));
}
if (NULL == dbCache) {
*pCache = NULL;
ctgDebug("db not in cache, dbFName:%s", dbFName);
@ -611,14 +621,18 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) {
node->op = operation;
CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
if (gCtgMgmt.queue.stopQueue) {
ctgFreeQNode(node);
CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
CTG_RET(TSDB_CODE_CTG_EXIT);
}
gCtgMgmt.queue.stopQueue = operation->stopQueue;
gCtgMgmt.queue.tail->next = node;
gCtgMgmt.queue.tail = node;
gCtgMgmt.queue.stopQueue = operation->stopQueue;
CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
ctgDebug("action [%s] added into queue", opName);
@ -1356,7 +1370,6 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam
SCtgTbCache *pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName));
STableMeta *orig = (pCache ? pCache->pMeta : NULL);
int8_t origType = 0;
uint64_t origSuid = 0;
if (orig) {
origType = orig->tableType;
@ -1375,8 +1388,6 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam
CTG_CACHE_STAT_DEC(numOfStb, 1);
ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:0x%" PRIx64, dbFName, tbName, orig->suid);
}
origSuid = orig->suid;
}
}
@ -1408,8 +1419,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam
return TSDB_CODE_SUCCESS;
}
if (origSuid != meta->suid &&
taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), tbName, strlen(tbName) + 1) != 0) {
if (taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), tbName, strlen(tbName) + 1) != 0) {
ctgError("taosHashPut to stable cache failed, suid:0x%" PRIx64, meta->suid);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
@ -1991,6 +2001,59 @@ _return:
CTG_RET(code);
}
void ctgFreeCacheOperationData(SCtgCacheOperation *op) {
if (NULL == op || NULL == op->data) {
return;
}
switch (op->opId) {
case CTG_OP_UPDATE_VGROUP: {
SCtgUpdateVgMsg *msg = op->data;
ctgFreeVgInfo(msg->dbInfo);
taosMemoryFreeClear(op->data);
break;
}
case CTG_OP_UPDATE_TB_META: {
SCtgUpdateTbMetaMsg *msg = op->data;
taosMemoryFreeClear(msg->pMeta->tbMeta);
taosMemoryFreeClear(msg->pMeta);
taosMemoryFreeClear(op->data);
break;
}
case CTG_OP_DROP_DB_CACHE:
case CTG_OP_DROP_DB_VGROUP:
case CTG_OP_DROP_STB_META:
case CTG_OP_DROP_TB_META:
case CTG_OP_UPDATE_VG_EPSET:
case CTG_OP_DROP_TB_INDEX:
case CTG_OP_CLEAR_CACHE: {
taosMemoryFreeClear(op->data);
break;
}
case CTG_OP_UPDATE_USER: {
SCtgUpdateUserMsg *msg = op->data;
taosHashCleanup(msg->userAuth.createdDbs);
taosHashCleanup(msg->userAuth.readDbs);
taosHashCleanup(msg->userAuth.writeDbs);
taosMemoryFreeClear(op->data);
break;
}
case CTG_OP_UPDATE_TB_INDEX: {
SCtgUpdateTbIndexMsg *msg = op->data;
if (msg->pIndex) {
taosArrayDestroyEx(msg->pIndex->pIndex, tFreeSTableIndexInfo);
taosMemoryFreeClear(msg->pIndex);
}
taosMemoryFreeClear(op->data);
break;
}
default: {
qError("invalid cache op id:%d", op->opId);
break;
}
}
}
void ctgCleanupCacheQueue(void) {
SCtgQNode *node = NULL;
SCtgQNode *nodeNext = NULL;
@ -2009,7 +2072,7 @@ void ctgCleanupCacheQueue(void) {
stopQueue = true;
CTG_RT_STAT_INC(numOfOpDequeue, 1);
} else {
taosMemoryFree(op->data);
ctgFreeCacheOperationData(op);
CTG_RT_STAT_INC(numOfOpAbort, 1);
}
@ -2047,7 +2110,7 @@ void *ctgUpdateThreadFunc(void *param) {
qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
}
if (atomic_load_8((int8_t *)&gCtgMgmt.exit)) {
if (atomic_load_8((int8_t *)&gCtgMgmt.queue.stopQueue)) {
ctgCleanupCacheQueue();
break;
}

8
source/libs/catalog/test/CMakeLists.txt
View File

@ -18,8 +18,8 @@ IF(NOT TD_DARWIN)
PRIVATE "${TD_SOURCE_DIR}/source/libs/catalog/inc"
)
# add_test(
# NAME catalogTest
# COMMAND catalogTest
# )
add_test(
NAME catalogTest
COMMAND catalogTest
)
ENDIF()

236
source/libs/catalog/test/catalogTests.cpp
View File

@ -27,8 +27,8 @@
#ifdef WINDOWS
#define TD_USE_WINSOCK
#endif
#include "catalog.h"
#include "catalogInt.h"
#include "catalog.h"
#include "os.h"
#include "stub.h"
#include "taos.h"
@ -49,14 +49,13 @@ void ctgTestSetRspCTableMeta();
void ctgTestSetRspSTableMeta();
void ctgTestSetRspMultiSTableMeta();
// extern "C" SCatalogMgmt gCtgMgmt;
enum {
CTGT_RSP_VGINFO = 1,
CTGT_RSP_TBMETA,
CTGT_RSP_CTBMETA,
CTGT_RSP_STBMETA,
CTGT_RSP_MSTBMETA,
CTGT_RSP_INDEXINFO_E,
CTGT_RSP_TBMETA_NOT_EXIST,
};
@ -142,6 +141,7 @@ void ctgTestInitLogFile() {
ctgdEnableDebug("api");
ctgdEnableDebug("meta");
ctgdEnableDebug("cache");
ctgdEnableDebug("lock");
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
@ -254,8 +254,8 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) {
rspMsg->tableType = TSDB_SUPER_TABLE;
rspMsg->sversion = ctgTestSVersion + 1;
rspMsg->tversion = ctgTestTVersion + 1;
rspMsg->suid = ctgTestSuid + 1;
rspMsg->tuid = ctgTestSuid + 1;
rspMsg->suid = ctgTestSuid;
rspMsg->tuid = ctgTestSuid;
rspMsg->vgId = 1;
rspMsg->pSchemas = (SSchema *)taosMemoryCalloc(rspMsg->numOfTags + rspMsg->numOfColumns, sizeof(SSchema));
@ -283,6 +283,8 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) {
}
void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
SUseDbRsp usedbRsp = {0};
strcpy(usedbRsp.db, ctgTestDbname);
usedbRsp.vgVersion = ctgTestVgVersion;
@ -320,9 +322,13 @@ void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *
pRsp->code = 0;
pRsp->contLen = contLen;
pRsp->pCont = pReq;
taosArrayDestroy(usedbRsp.pVgroupInfos);
}
void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
STableMetaRsp metaRsp = {0};
strcpy(metaRsp.dbFName, ctgTestDbname);
strcpy(metaRsp.tbName, ctgTestTablename);
@ -362,10 +368,14 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *
}
void ctgTestRspTableMetaNotExist(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
pRsp->code = CTG_ERR_CODE_TABLE_NOT_EXIST;
}
void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
STableMetaRsp metaRsp = {0};
strcpy(metaRsp.dbFName, ctgTestDbname);
strcpy(metaRsp.tbName, ctgTestCurrentCTableName ? ctgTestCurrentCTableName : ctgTestCTablename);
@ -412,6 +422,8 @@ void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg
}
void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
STableMetaRsp metaRsp = {0};
strcpy(metaRsp.dbFName, ctgTestDbname);
strcpy(metaRsp.tbName, ctgTestCurrentSTableName ? ctgTestCurrentSTableName : ctgTestSTablename);
@ -423,7 +435,7 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg
metaRsp.sversion = ctgTestSVersion;
metaRsp.tversion = ctgTestTVersion;
metaRsp.suid = ctgTestSuid;
metaRsp.tuid = ctgTestSuid++;
metaRsp.tuid = ctgTestSuid+1;
metaRsp.vgId = 0;
metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema));
@ -458,6 +470,8 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg
}
void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
static int32_t idx = 1;
STableMetaRsp metaRsp = {0};
@ -507,6 +521,16 @@ void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRp
tFreeSTableMetaRsp(&metaRsp);
}
void ctgTestRspErrIndexInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
pRsp->code = TSDB_CODE_MND_DB_INDEX_NOT_EXIST;
pRsp->contLen = 0;
pRsp->pCont = NULL;
}
void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
switch (ctgTestRspFunc[ctgTestRspIdx]) {
case CTGT_RSP_VGINFO:
@ -524,6 +548,9 @@ void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp
case CTGT_RSP_MSTBMETA:
ctgTestRspMultiSTableMeta(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_INDEXINFO_E:
ctgTestRspErrIndexInfo(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_TBMETA_NOT_EXIST:
ctgTestRspTableMetaNotExist(shandle, pEpSet, pMsg, pRsp);
break;
@ -773,13 +800,15 @@ void ctgTestSetRspDbVgroupsAndMultiSuperMeta() {
void *ctgTestGetDbVgroupThread(void *param) {
struct SCatalog *pCtg = (struct SCatalog *)param;
int32_t code = 0;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SArray *vgList = NULL;
int32_t n = 0;
while (!ctgTestStop) {
code = catalogGetDBVgList(pCtg, mockPointer, ctgTestDbname, &vgList);
if (code) {
printf("code:%x\n", code);
assert(0);
}
@ -863,7 +892,7 @@ void *ctgTestGetCtableMetaThread(void *param) {
while (!ctgTestStop) {
code = ctgReadTbMetaFromCache(pCtg, &ctx, &tbMeta);
if (code || !inCache) {
if (code || NULL == tbMeta) {
assert(0);
}
@ -917,12 +946,12 @@ void *ctgTestSetCtableMetaThread(void *param) {
return NULL;
}
#if 1
TEST(tableMeta, normalTable) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
@ -966,6 +995,8 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFree(tableMeta);
while (true) {
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
@ -987,6 +1018,8 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFree(tableMeta);
SDbVgVersion *dbs = NULL;
SSTableVersion *stb = NULL;
uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
@ -1023,12 +1056,12 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(allStbNum, 0);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(tableMeta, childTableCase) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
ctgTestInitLogFile();
@ -1060,6 +1093,8 @@ TEST(tableMeta, childTableCase) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFree(tableMeta);
while (true) {
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
@ -1095,6 +1130,8 @@ TEST(tableMeta, childTableCase) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFree(tableMeta);
SDbVgVersion *dbs = NULL;
SSTableVersion *stb = NULL;
uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
@ -1131,12 +1168,12 @@ TEST(tableMeta, childTableCase) {
ASSERT_EQ(allStbNum, 1);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(tableMeta, superTableCase) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
ctgTestSetRspDbVgroupsAndSuperMeta();
@ -1161,13 +1198,15 @@ TEST(tableMeta, superTableCase) {
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->uid, ctgTestSuid);
ASSERT_EQ(tableMeta->suid, ctgTestSuid);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFree(tableMeta);
while (true) {
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
@ -1194,6 +1233,8 @@ TEST(tableMeta, superTableCase) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFree(tableMeta);
while (true) {
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (2 != n) {
@ -1215,6 +1256,8 @@ TEST(tableMeta, superTableCase) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFree(tableMeta);
SDbVgVersion *dbs = NULL;
SSTableVersion *stb = NULL;
uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
@ -1252,12 +1295,12 @@ TEST(tableMeta, superTableCase) {
ASSERT_EQ(allStbNum, 1);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(tableMeta, rmStbMeta) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
ctgTestInitLogFile();
@ -1284,13 +1327,15 @@ TEST(tableMeta, rmStbMeta) {
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->uid, ctgTestSuid);
ASSERT_EQ(tableMeta->suid, ctgTestSuid);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFree(tableMeta);
while (true) {
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
@ -1300,7 +1345,7 @@ TEST(tableMeta, rmStbMeta) {
}
}
code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid - 1);
code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid);
ASSERT_EQ(code, 0);
while (true) {
@ -1320,12 +1365,12 @@ TEST(tableMeta, rmStbMeta) {
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(tableMeta, updateStbMeta) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
ctgTestInitLogFile();
@ -1352,8 +1397,8 @@ TEST(tableMeta, updateStbMeta) {
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->uid, ctgTestSuid);
ASSERT_EQ(tableMeta->suid, ctgTestSuid);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
@ -1399,8 +1444,8 @@ TEST(tableMeta, updateStbMeta) {
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion + 1);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion + 1);
ASSERT_EQ(tableMeta->uid, ctgTestSuid + 1);
ASSERT_EQ(tableMeta->suid, ctgTestSuid + 1);
ASSERT_EQ(tableMeta->uid, ctgTestSuid);
ASSERT_EQ(tableMeta->suid, ctgTestSuid);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
ASSERT_EQ(tableMeta->tableInfo.precision, 1 + 1);
@ -1409,12 +1454,42 @@ TEST(tableMeta, updateStbMeta) {
taosMemoryFreeClear(tableMeta);
catalogDestroy();
memset(&gCtgMgmt.stat, 0, sizeof(gCtgMgmt.stat));
}
TEST(getIndexInfo, notExists) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_INDEXINFO_E;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SIndexInfo info;
code = catalogGetIndexMeta(pCtg, mockPointer, "index1", &info);
ASSERT_TRUE(code != 0);
catalogDestroy();
}
TEST(refreshGetMeta, normal2normal) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -1488,12 +1563,12 @@ TEST(refreshGetMeta, normal2normal) {
taosMemoryFreeClear(tableMeta);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(refreshGetMeta, normal2notexist) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -1558,12 +1633,12 @@ TEST(refreshGetMeta, normal2notexist) {
ASSERT_TRUE(tableMeta == NULL);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(refreshGetMeta, normal2child) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -1639,14 +1714,15 @@ TEST(refreshGetMeta, normal2child) {
taosMemoryFreeClear(tableMeta);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
ctgTestCurrentCTableName = NULL;
ctgTestCurrentSTableName = NULL;
}
TEST(refreshGetMeta, stable2child) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -1699,8 +1775,8 @@ TEST(refreshGetMeta, stable2child) {
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->uid, ctgTestSuid);
ASSERT_EQ(tableMeta->suid, ctgTestSuid);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
@ -1725,14 +1801,14 @@ TEST(refreshGetMeta, stable2child) {
taosMemoryFreeClear(tableMeta);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
ctgTestCurrentCTableName = NULL;
ctgTestCurrentSTableName = NULL;
}
TEST(refreshGetMeta, stable2stable) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -1784,8 +1860,8 @@ TEST(refreshGetMeta, stable2stable) {
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->uid, ctgTestSuid);
ASSERT_EQ(tableMeta->suid, ctgTestSuid);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
@ -1802,8 +1878,8 @@ TEST(refreshGetMeta, stable2stable) {
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->uid, ctgTestSuid);
ASSERT_EQ(tableMeta->suid, ctgTestSuid);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
@ -1811,14 +1887,14 @@ TEST(refreshGetMeta, stable2stable) {
taosMemoryFreeClear(tableMeta);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
ctgTestCurrentCTableName = NULL;
ctgTestCurrentSTableName = NULL;
}
TEST(refreshGetMeta, child2stable) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
@ -1888,8 +1964,8 @@ TEST(refreshGetMeta, child2stable) {
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1);
ASSERT_EQ(tableMeta->uid, ctgTestSuid);
ASSERT_EQ(tableMeta->suid, ctgTestSuid);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
@ -1897,14 +1973,14 @@ TEST(refreshGetMeta, child2stable) {
taosMemoryFreeClear(tableMeta);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
ctgTestCurrentCTableName = NULL;
ctgTestCurrentSTableName = NULL;
}
TEST(tableDistVgroup, normalTable) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
@ -1933,19 +2009,15 @@ TEST(tableDistVgroup, normalTable) {
strcpy(n.tname, ctgTestTablename);
code = catalogGetTableDistVgInfo(pCtg, mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(vgInfo->vgId, 8);
ASSERT_EQ(vgInfo->epSet.numOfEps, 3);
ASSERT_TRUE(code != 0);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(tableDistVgroup, childTableCase) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
@ -1975,19 +2047,15 @@ TEST(tableDistVgroup, childTableCase) {
strcpy(n.tname, ctgTestCTablename);
code = catalogGetTableDistVgInfo(pCtg, mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(vgInfo->vgId, 9);
ASSERT_EQ(vgInfo->epSet.numOfEps, 4);
ASSERT_TRUE(code != 0);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(tableDistVgroup, superTableCase) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
@ -2028,13 +2096,15 @@ TEST(tableDistVgroup, superTableCase) {
ASSERT_EQ(vgInfo->vgId, 3);
ASSERT_EQ(vgInfo->epSet.numOfEps, 3);
taosArrayDestroy(vgList);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(dbVgroup, getSetDbVgroupCase) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo *dbVgroup = NULL;
@ -2067,6 +2137,8 @@ TEST(dbVgroup, getSetDbVgroupCase) {
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum);
taosArrayDestroy(vgList);
while (true) {
uint64_t n = 0;
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
@ -2082,12 +2154,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
code = catalogGetTableDistVgInfo(pCtg, mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(pvgInfo->vgId, 8);
ASSERT_EQ(pvgInfo->epSet.numOfEps, 3);
taosArrayDestroy(vgList);
ASSERT_TRUE(code != 0);
ctgTestBuildDBVgroup(&dbVgroup);
code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup);
@ -2109,20 +2176,15 @@ TEST(dbVgroup, getSetDbVgroupCase) {
ASSERT_EQ(vgInfo.epSet.numOfEps, 2);
code = catalogGetTableDistVgInfo(pCtg, mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(pvgInfo->vgId, 8);
ASSERT_EQ(pvgInfo->epSet.numOfEps, 3);
taosArrayDestroy(vgList);
ASSERT_TRUE(code != 0);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(multiThread, getSetRmSameDbVgroup) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo dbVgroup = {0};
@ -2169,12 +2231,12 @@ TEST(multiThread, getSetRmSameDbVgroup) {
taosSsleep(1);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(multiThread, getSetRmDiffDbVgroup) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo dbVgroup = {0};
@ -2221,12 +2283,12 @@ TEST(multiThread, getSetRmDiffDbVgroup) {
taosSsleep(1);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(multiThread, ctableMeta) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo dbVgroup = {0};
@ -2272,12 +2334,12 @@ TEST(multiThread, ctableMeta) {
taosSsleep(2);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(rentTest, allRent) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo dbVgroup = {0};
@ -2319,6 +2381,8 @@ TEST(rentTest, allRent) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFree(tableMeta);
while (ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) {
taosMsleep(50);
}
@ -2349,10 +2413,8 @@ TEST(rentTest, allRent) {
}
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
#endif
int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv);

36
source/libs/executor/inc/executorimpl.h
View File

@ -319,27 +319,25 @@ typedef struct {
} SAggOptrPushDownInfo;
typedef struct STableScanInfo {
STsdbReader* dataReader;
SReadHandle readHandle;
STsdbReader* dataReader;
SReadHandle readHandle;
SLimitInfo limitInfo;
SFileBlockLoadRecorder readRecorder;
SScanInfo scanInfo;
int32_t scanTimes;
SNode* pFilterNode; // filter info, which is push down by optimizer
SSDataBlock* pResBlock;
SColMatchInfo matchInfo;
SExprSupp pseudoSup;
SQueryTableDataCond cond;
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
int32_t dataBlockLoadFlag;
SSampleExecInfo sample; // sample execution info
int32_t currentGroupId;
int32_t currentTable;
int8_t scanMode;
int8_t noTable;
SAggOptrPushDownInfo pdInfo;
int8_t assignBlockUid;
SSDataBlock* pResBlock;
SColMatchInfo matchInfo;
SExprSupp pseudoSup;
SQueryTableDataCond cond;
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
int32_t dataBlockLoadFlag;
SSampleExecInfo sample; // sample execution info
int32_t currentGroupId;
int32_t currentTable;
int8_t scanMode;
SAggOptrPushDownInfo pdInfo;
int8_t assignBlockUid;
} STableScanInfo;
typedef struct STableMergeScanInfo {
@ -357,7 +355,7 @@ typedef struct STableMergeScanInfo {
SSDataBlock* pSortInputBlock;
int64_t startTs; // sort start time
SArray* sortSourceParams;
SLimitInfo limitInfo;
SFileBlockLoadRecorder readRecorder;
int64_t numOfRows;
SScanInfo scanInfo;
@ -374,6 +372,7 @@ typedef struct STableMergeScanInfo {
SQueryTableDataCond cond;
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
int32_t dataBlockLoadFlag;
// if the upstream is an interval operator, the interval info is also kept here to get the time
// window to check if current data block needs to be loaded.
SInterval interval;
@ -903,6 +902,7 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG
int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf);
bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo);
void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator);
void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset,
int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput);

61
source/libs/executor/src/executor.c
View File

@ -30,6 +30,46 @@ static void cleanupRefPool() {
taosCloseRef(ref);
}
static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) {
ASSERT(pOperator != NULL);
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
if (pOperator->numOfDownstream == 0) {
qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id);
return TSDB_CODE_QRY_APP_ERROR;
}
if (pOperator->numOfDownstream > 1) { // not handle this in join query
qError("join not supported for stream block scan, %s" PRIx64, id);
return TSDB_CODE_QRY_APP_ERROR;
}
pOperator->status = OP_NOT_OPENED;
return doSetSMABlock(pOperator->pDownstream[0], input, numOfBlocks, type, id);
} else {
pOperator->status = OP_NOT_OPENED;
SStreamScanInfo* pInfo = pOperator->info;
if (type == STREAM_INPUT__MERGED_SUBMIT) {
for (int32_t i = 0; i < numOfBlocks; i++) {
SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*));
taosArrayPush(pInfo->pBlockLists, &pReq);
}
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
} else if (type == STREAM_INPUT__DATA_SUBMIT) {
taosArrayPush(pInfo->pBlockLists, &input);
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
} else if (type == STREAM_INPUT__DATA_BLOCK) {
for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i];
taosArrayPush(pInfo->pBlockLists, &pDataBlock);
}
pInfo->blockType = STREAM_INPUT__DATA_BLOCK;
}
return TSDB_CODE_SUCCESS;
}
}
static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) {
ASSERT(pOperator != NULL);
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
@ -100,6 +140,27 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO
return code;
}
int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) {
if (tinfo == NULL) {
return TSDB_CODE_QRY_APP_ERROR;
}
if (pBlocks == NULL || numOfBlocks == 0) {
return TSDB_CODE_SUCCESS;
}
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
int32_t code = doSetSMABlock(pTaskInfo->pRoot, (void*)pBlocks, numOfBlocks, type, GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed to set the sma block data", GET_TASKID(pTaskInfo));
} else {
qDebug("%s set the sma block successfully", GET_TASKID(pTaskInfo));
}
return code;
}
qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema) {
if (msg == NULL) {
// create raw scan

2
source/libs/executor/src/projectoperator.c
View File

@ -210,8 +210,6 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
pOperator->status = OP_OPENED;
}
qDebug("enter project");
if (pOperator->status == OP_EXEC_DONE) {
if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) {
pOperator->status = OP_OPENED;

74
source/libs/executor/src/scanoperator.c
View File

@ -355,6 +355,34 @@ static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlo
}
}
// todo handle the slimit info
void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator) {
SLimit* pLimit = &pLimitInfo->limit;
if (pLimit->offset > 0 && pLimitInfo->remainOffset > 0) {
if (pLimitInfo->remainOffset >= pBlock->info.rows) {
pLimitInfo->remainOffset -= pBlock->info.rows;
pBlock->info.rows = 0;
qDebug("current block ignore due to offset, current:%"PRId64", %s", pLimitInfo->remainOffset, GET_TASKID(pTaskInfo));
} else {
blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset);
pLimitInfo->remainOffset = 0;
}
}
if (pLimit->limit != -1 && pLimit->limit <= (pLimitInfo->numOfOutputRows + pBlock->info.rows)) {
// limit the output rows
int32_t overflowRows = pLimitInfo->numOfOutputRows + pBlock->info.rows - pLimit->limit;
int32_t keep = pBlock->info.rows - overflowRows;
blockDataKeepFirstNRows(pBlock, keep);
qDebug("output limit %"PRId64" has reached, %s", pLimit->limit, GET_TASKID(pTaskInfo));
// setTaskStatus(pTaskInfo, TASK_COMPLETED);
pOperator->status = OP_EXEC_DONE;
}
}
static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
uint32_t* status) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -364,6 +392,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
pCost->totalBlocks += 1;
pCost->totalRows += pBlock->info.rows;
bool loadSMA = false;
*status = pInfo->dataBlockLoadFlag;
@ -379,6 +408,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pCost->filterOutBlocks += 1;
pCost->totalRows += pBlock->info.rows;
return TSDB_CODE_SUCCESS;
} else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
@ -446,6 +476,9 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
// restore the previous value
pCost->totalRows -= pBlock->info.rows;
if (pTableScanInfo->pFilterNode != NULL) {
int64_t st = taosGetTimestampUs();
doFilter(pTableScanInfo->pFilterNode, pBlock, &pTableScanInfo->matchInfo, pOperator->exprSupp.pFilterInfo);
@ -462,6 +495,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
}
}
applyLimitOffset(&pInfo->limitInfo, pBlock, pTaskInfo, pOperator);
pCost->totalRows += pBlock->info.rows;
pInfo->limitInfo.numOfOutputRows = pCost->totalRows;
return TSDB_CODE_SUCCESS;
}
@ -691,10 +728,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
// if scan table by table
if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) {
if (pInfo->noTable) {
return NULL;
}
int32_t numOfTables = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
while (1) {
@ -727,7 +760,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
}
SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
tsdbReaderClose(pInfo->dataReader);
int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader,
@ -749,9 +781,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
return NULL;
}
SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
// tsdbSetTableList(pInfo->dataReader, tableList);
tsdbReaderReset(pInfo->dataReader, &pInfo->cond);
pInfo->scanTimes = 0;
@ -798,9 +827,15 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
}
SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc;
int32_t numOfCols = 0;
int32_t numOfCols = 0;
int32_t code = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID,
&pInfo->matchInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo);
code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
if (code != TSDB_CODE_SUCCESS) {
@ -825,6 +860,9 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
if (pInfo->pFilterNode != NULL) {
code = filterInitFromNode((SNode*)pInfo->pFilterNode, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
}
pInfo->scanFlag = MAIN_SCAN;
@ -847,10 +885,12 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
return pOperator;
_error:
taosMemoryFreeClear(pInfo);
taosMemoryFreeClear(pOperator);
if (pInfo != NULL) {
destroyTableScanOperatorInfo(pInfo);
}
pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
return NULL;
}
@ -1698,8 +1738,10 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S
j++;
}
}
uint32_t cap = pDst->info.capacity;
pDst->info = pSrc->info;
pDst->info.rows = j;
pDst->info.capacity = cap;
return 0;
}
@ -3484,7 +3526,6 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
qError("%p buildUserTable", pTaskInfo);
SSysTableScanInfo* pInfo = pOperator->info;
if (pInfo->pCur == NULL) {
pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta);
@ -4440,6 +4481,9 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock*
}
qDebug("%s get sorted row blocks, rows:%d", GET_TASKID(pTaskInfo), pResBlock->info.rows);
applyLimitOffset(&pInfo->limitInfo, pResBlock, pTaskInfo, pOperator);
pInfo->limitInfo.numOfOutputRows += pResBlock->info.rows;
return (pResBlock->info.rows > 0) ? pResBlock : NULL;
}
@ -4455,6 +4499,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
if (!pInfo->hasGroupId) {
pInfo->hasGroupId = true;
@ -4467,6 +4512,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId;
startGroupTableMergeScan(pOperator);
}
SSDataBlock* pBlock = NULL;
while (pInfo->tableStartIndex < tableListSize) {
pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pInfo->pResBlock, pOperator->resultInfo.capacity,
@ -4554,6 +4600,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
if (pTableScanNode->pGroupTags) {
taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid);
}
@ -4592,6 +4639,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
pInfo->pSortInfo = generateSortByTsInfo(pInfo->matchInfo.pList, pInfo->cond.order);
pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo);
int32_t rowSize = pInfo->pResBlock->info.rowSize;
pInfo->bufPageSize = getProperSortPageSize(rowSize);

18
source/libs/executor/src/sortoperator.c
View File

@ -531,7 +531,7 @@ typedef struct SMultiwayMergeOperatorInfo {
SOptrBasicInfo binfo;
int32_t bufPageSize;
uint32_t sortBufSize; // max buffer size for in-memory sort
SLimitInfo limitInfo;
SArray* pSortInfo;
SSortHandle* pSortHandle;
SColMatchInfo matchInfo;
@ -592,6 +592,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
blockDataEnsureCapacity(p, capacity);
_retry:
while (1) {
STupleHandle* pTupleHandle = NULL;
if (pInfo->groupSort) {
@ -626,14 +627,22 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
} else {
appendOneRowToDataBlock(p, pTupleHandle);
}
if (p->info.rows >= capacity) {
break;
}
}
if (pInfo->groupSort) {
pInfo->hasGroupId = false;
}
if (p->info.rows > 0) { // todo extract method
applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo, pOperator);
if (p->info.rows == 0) {
goto _retry;
}
blockDataEnsureCapacity(pDataBlock, p->info.rows);
int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
for (int32_t i = 0; i < numOfCols; ++i) {
@ -650,9 +659,9 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
}
blockDataDestroy(p);
qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
pDataBlock->info.rows);
return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
}
@ -717,6 +726,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
goto _error;
}
initLimitInfo(pMergePhyNode->node.pLimit, pMergePhyNode->node.pSlimit, &pInfo->limitInfo);
pInfo->binfo.pRes = createResDataBlock(pDescNode);
int32_t rowSize = pInfo->binfo.pRes->info.rowSize;
ASSERT(rowSize < 100 * 1024 * 1024);
@ -725,6 +735,10 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
int32_t numOfOutputCols = 0;
code = extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
SSDataBlock* pInputBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc);
initResultSizeInfo(&pOperator->resultInfo, 1024);

88
source/libs/executor/src/timewindowoperator.c
View File

@ -809,23 +809,6 @@ static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {
return TSDB_CODE_SUCCESS;
}
int32_t compareResKey(void* pKey, void* data, int32_t index) {
SArray* res = (SArray*)data;
SResKeyPos* pos = taosArrayGetP(res, index);
SWinKey* pData = (SWinKey*)pKey;
if (pData->ts == *(int64_t*)pos->key) {
if (pData->groupId > pos->groupId) {
return 1;
} else if (pData->groupId < pos->groupId) {
return -1;
}
return 0;
} else if (pData->ts > *(int64_t*)pos->key) {
return 1;
}
return -1;
}
static int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated) {
winInfo.sessionWin.win.ekey = winInfo.sessionWin.win.skey;
return tSimpleHashPut(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));
@ -863,12 +846,6 @@ static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) {
}
}
int64_t getWinReskey(void* data, int32_t index) {
SArray* res = (SArray*)data;
SWinKey* pos = taosArrayGet(res, index);
return pos->ts;
}
int32_t compareWinRes(void* pKey, void* data, int32_t index) {
SArray* res = (SArray*)data;
SWinKey* pos = taosArrayGet(res, index);
@ -1307,27 +1284,6 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
}
}
// todo merged with the build group result.
static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList,
int32_t* rowEntryInfoOffset) {
size_t num = taosArrayGetSize(pUpdateList);
for (int32_t i = 0; i < num; ++i) {
SResKeyPos* pPos = taosArrayGetP(pUpdateList, i);
SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId);
SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
for (int32_t j = 0; j < numOfOutput; ++j) {
SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, j, rowEntryInfoOffset);
if (pRow->numOfRows < pEntry->numOfRes) {
pRow->numOfRows = pEntry->numOfRes;
}
}
releaseBufPage(pBuf, bufPage);
}
}
static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) {
for (int i = 0; i < num; i++) {
if (type == STREAM_INVERT) {
@ -1578,16 +1534,6 @@ static void closeChildIntervalWindow(SOperatorInfo* pOperator, SArray* pChildren
}
}
static void freeAllPages(SArray* pageIds, SDiskbasedBuf* pDiskBuf) {
int32_t size = taosArrayGetSize(pageIds);
for (int32_t i = 0; i < size; i++) {
int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i);
// SFilePage* bufPage = getBufPage(pDiskBuf, pageId);
// dBufSetBufPageRecycled(pDiskBuf, bufPage);
}
taosArrayClear(pageIds);
}
static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWins, int32_t* index,
SSDataBlock* pBlock) {
blockDataCleanup(pBlock);
@ -2325,9 +2271,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
pSliceInfo->current =
taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
if (pResBlock->info.rows >= pResBlock->info.capacity) {
break;
}
}
}
@ -2337,6 +2280,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
}
if (ts == pSliceInfo->current) {
blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1);
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
@ -2377,9 +2321,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
pInterval->precision);
if (pResBlock->info.rows >= pResBlock->info.capacity) {
break;
}
}
if (pSliceInfo->current > pSliceInfo->win.ekey) {
@ -2398,10 +2339,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
doSetOperatorCompleted(pOperator);
break;
}
if (pResBlock->info.rows >= pResBlock->info.capacity) {
break;
}
}
} else if (ts < pSliceInfo->current) {
// in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate
@ -2419,9 +2356,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
pInterval->precision);
if (pResBlock->info.rows >= pResBlock->info.capacity) {
break;
}
}
if (pSliceInfo->current > pSliceInfo->win.ekey) {
@ -2443,9 +2377,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
pInterval->precision);
if (pResBlock->info.rows >= pResBlock->info.capacity) {
break;
}
}
if (pSliceInfo->current > pSliceInfo->win.ekey) {
@ -2467,13 +2398,11 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
pSliceInfo->current =
taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
if (pResBlock->info.rows >= pResBlock->info.capacity) {
break;
}
}
// add current row if timestamp match
if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1);
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
@ -2510,9 +2439,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
pInterval->precision);
if (pResBlock->info.rows >= pResBlock->info.capacity) {
break;
}
}
if (pSliceInfo->current > pSliceInfo->win.ekey) {
@ -2528,9 +2454,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
pSliceInfo->current =
taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
if (pResBlock->info.rows >= pResBlock->info.capacity) {
break;
}
}
}
@ -2549,9 +2472,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
pSliceInfo->current =
taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
if (pResBlock->info.rows >= pResBlock->info.capacity) {
break;
}
}
// restore the value
@ -3379,7 +3299,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
.calTrigger = pIntervalPhyNode->window.triggerType,
.maxTs = INT64_MIN,
.minTs = INT64_MAX,
.deleteMark = INT64_MAX,
// for test 315360000000
.deleteMark = 1000LL * 60LL * 60LL * 24LL * 365LL * 10LL,
// .deleteMark = INT64_MAX,
};
ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY);
pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;

8
source/libs/function/src/builtins.c
View File

@ -842,11 +842,17 @@ static int32_t translateElapsedImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t
}
static int32_t translateElapsedPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
#if 0
return translateElapsedImpl(pFunc, pErrBuf, len, true);
#endif
return 0;
}
static int32_t translateElapsedMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
#if 0
return translateElapsedImpl(pFunc, pErrBuf, len, false);
#endif
return 0;
}
static int32_t translateLeastSQR(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
@ -2348,8 +2354,6 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.finalizeFunc = elapsedFinalize,
.invertFunc = NULL,
.combineFunc = elapsedCombine,
.pPartialFunc = "_elapsed_partial",
.pMergeFunc = "_elapsed_merge"
},
{
.name = "_elapsed_partial",

24
source/libs/function/src/builtinsimpl.c
View File

@ -5060,15 +5060,19 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return pInfo->numSampled;
}
bool getTailFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
#if 0
SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0);
SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);
int32_t numOfPoints = pVal->datum.i;
pEnv->calcMemSize = sizeof(STailInfo) + numOfPoints * (POINTER_BYTES + sizeof(STailItem) + pCol->node.resType.bytes);
#endif
return true;
}
bool tailFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
#if 0
if (!functionSetup(pCtx, pResultInfo)) {
return false;
}
@ -5096,11 +5100,13 @@ bool tailFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
pInfo->pItems[i] = (STailItem*)(pItem + i * unitSize);
pInfo->pItems[i]->isNull = false;
}
#endif
return true;
}
static void tailAssignResult(STailItem* pItem, char* data, int32_t colBytes, TSKEY ts, bool isNull) {
#if 0
pItem->timestamp = ts;
if (isNull) {
pItem->isNull = true;
@ -5108,8 +5114,10 @@ static void tailAssignResult(STailItem* pItem, char* data, int32_t colBytes, TSK
pItem->isNull = false;
memcpy(pItem->data, data, colBytes);
}
#endif
}
#if 0
static int32_t tailCompFn(const void* p1, const void* p2, const void* param) {
STailItem* d1 = *(STailItem**)p1;
STailItem* d2 = *(STailItem**)p2;
@ -5127,8 +5135,10 @@ static void doTailAdd(STailInfo* pInfo, char* data, TSKEY ts, bool isNull) {
taosheapadjust((void*)pList, sizeof(STailItem**), 0, pInfo->numOfPoints - 1, NULL, tailCompFn, NULL, 0);
}
}
#endif
int32_t tailFunction(SqlFunctionCtx* pCtx) {
#if 0
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
STailInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
@ -5162,9 +5172,12 @@ int32_t tailFunction(SqlFunctionCtx* pCtx) {
}
return pInfo->numOfPoints;
#endif
return 0;
}
int32_t tailFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
#if 0
SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx);
STailInfo* pInfo = GET_ROWCELL_INTERBUF(pEntryInfo);
pEntryInfo->complete = true;
@ -5183,14 +5196,19 @@ int32_t tailFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
}
return pEntryInfo->numOfRes;
#endif
return 0;
}
bool getUniqueFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
#if 0
pEnv->calcMemSize = sizeof(SUniqueInfo) + UNIQUE_MAX_RESULT_SIZE;
#endif
return true;
}
bool uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) {
#if 0
if (!functionSetup(pCtx, pResInfo)) {
return false;
}
@ -5204,9 +5222,11 @@ bool uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) {
} else {
pInfo->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
}
#endif
return true;
}
#if 0
static void doUniqueAdd(SUniqueInfo* pInfo, char* data, TSKEY ts, bool isNull) {
// handle null elements
if (isNull == true) {
@ -5237,8 +5257,10 @@ static void doUniqueAdd(SUniqueInfo* pInfo, char* data, TSKEY ts, bool isNull) {
pHashItem->timestamp = ts;
}
}
#endif
int32_t uniqueFunction(SqlFunctionCtx* pCtx) {
#if 0
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SUniqueInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
@ -5273,6 +5295,8 @@ int32_t uniqueFunction(SqlFunctionCtx* pCtx) {
}
return pInfo->numOfPoints;
#endif
return 0;
}
bool getModeFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {

11
source/libs/function/src/udfd.c
View File

@ -888,10 +888,11 @@ static int32_t udfdUvInit() {
}
global.loop = loop;
uv_pipe_init(global.loop, &global.ctrlPipe, 1);
uv_pipe_open(&global.ctrlPipe, 0);
uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb);
if (tsStartUdfd) { // udfd is started by taosd, which shall exit when taosd exit
uv_pipe_init(global.loop, &global.ctrlPipe, 1);
uv_pipe_open(&global.ctrlPipe, 0);
uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb);
}
getUdfdPipeName(global.listenPipeName, sizeof(global.listenPipeName));
removeListeningPipe();
@ -979,13 +980,13 @@ int32_t udfdDeinitResidentFuncs() {
char* funcName = taosArrayGet(global.residentFuncs, i);
SUdf** udfInHash = taosHashGet(global.udfsHash, funcName, strlen(funcName));
if (udfInHash) {
taosHashRemove(global.udfsHash, funcName, strlen(funcName));
SUdf* udf = *udfInHash;
if (udf->destroyFunc) {
(udf->destroyFunc)();
}
uv_dlclose(&udf->lib);
taosMemoryFree(udf);
taosHashRemove(global.udfsHash, funcName, strlen(funcName));
}
}
taosArrayDestroy(global.residentFuncs);

2
source/libs/planner/src/planOptimizer.c
View File

@ -2452,7 +2452,7 @@ static const SOptimizeRule optimizeRuleSet[] = {
{.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize},
{.pName = "LastRowScan", .optimizeFunc = lastRowScanOptimize},
{.pName = "TagScan", .optimizeFunc = tagScanOptimize},
// {.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize}
{.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize}
};
// clang-format on

12
source/libs/planner/src/planSpliter.c
View File

@ -97,6 +97,7 @@ static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pChild, SE
if (NULL == pExchange->node.pLimit) {
return TSDB_CODE_OUT_OF_MEMORY;
}
((SLimitNode*)pChild->pLimit)->limit += ((SLimitNode*)pChild->pLimit)->offset;
((SLimitNode*)pChild->pLimit)->offset = 0;
}
@ -470,6 +471,12 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla
if (NULL == pMerge->node.pTargets || NULL == pMerge->pInputs) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
if (TSDB_CODE_SUCCESS == code && NULL != pSplitNode->pLimit) {
pMerge->node.pLimit = nodesCloneNode(pSplitNode->pLimit);
if (NULL == pMerge->node.pLimit) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
if (NULL == pSubplan) {
code = nodesListMakeAppend(&pSplitNode->pChildren, (SNode*)pMerge);
@ -934,6 +941,7 @@ static int32_t stbSplSplitScanNodeWithoutPartTags(SSplitContext* pCxt, SStableSp
if (NULL == pSplitNode->pLimit) {
return TSDB_CODE_OUT_OF_MEMORY;
}
((SLimitNode*)pInfo->pSplitNode->pLimit)->limit += ((SLimitNode*)pInfo->pSplitNode->pLimit)->offset;
((SLimitNode*)pInfo->pSplitNode->pLimit)->offset = 0;
}
}
@ -1021,6 +1029,10 @@ static int32_t stbSplSplitMergeScanNode(SSplitContext* pCxt, SLogicSubplan* pSub
SNodeList* pMergeKeys = NULL;
int32_t code = stbSplCreateMergeScanNode(pScan, &pMergeScan, &pMergeKeys);
if (TSDB_CODE_SUCCESS == code) {
if (NULL != pMergeScan->pLimit) {
((SLimitNode*)pMergeScan->pLimit)->limit += ((SLimitNode*)pMergeScan->pLimit)->offset;
((SLimitNode*)pMergeScan->pLimit)->offset = 0;
}
code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, pMergeScan, groupSort);
}
if (TSDB_CODE_SUCCESS == code) {

2
source/libs/qcom/src/queryUtil.c
View File

@ -202,6 +202,7 @@ char* jobTaskStatusStr(int32_t status) {
return "UNKNOWN";
}
#if 0
SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name) {
SSchema s = {0};
s.type = type;
@ -211,6 +212,7 @@ SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* nam
tstrncpy(s.name, name, tListLen(s.name));
return s;
}
#endif
void freeSTableMetaRspPointer(void *p) {
tFreeSTableMetaRsp(*(void**)p);

2
source/libs/qworker/src/qwDbg.c
View File

@ -253,7 +253,7 @@ int32_t qwDbgEnableDebug(char *option) {
}
if (0 == strcasecmp(option, "dead")) {
gQWDebug.sleepSimulate = true;
gQWDebug.deadSimulate = true;
qError("qw dead debug enabled");
return TSDB_CODE_SUCCESS;
}

4
source/libs/qworker/src/qwMsg.c
View File

@ -146,6 +146,7 @@ int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieve
return TSDB_CODE_SUCCESS;
}
#if 0
int32_t qwBuildAndSendCancelRsp(SRpcHandleInfo *pConn, int32_t code) {
STaskCancelRsp *pRsp = (STaskCancelRsp *)rpcMallocCont(sizeof(STaskCancelRsp));
pRsp->code = code;
@ -177,6 +178,7 @@ int32_t qwBuildAndSendDropRsp(SRpcHandleInfo *pConn, int32_t code) {
tmsgSendRsp(&rpcRsp);
return TSDB_CODE_SUCCESS;
}
#endif
int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) {
STaskDropReq *req = (STaskDropReq *)rpcMallocCont(sizeof(STaskDropReq));
@ -490,6 +492,7 @@ int32_t qWorkerProcessRspMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64
return TSDB_CODE_SUCCESS;
}
#if 0
int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) {
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;
@ -530,6 +533,7 @@ _return:
return TSDB_CODE_SUCCESS;
}
#endif
int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) {
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {

2
source/libs/qworker/test/qworkerTests.cpp
View File

@ -796,7 +796,7 @@ void *fetchQueueThread(void *param) {
qWorkerProcessFetchMsg(mockPointer, mgmt, fetchRpc, 0);
break;
case TDMT_SCH_CANCEL_TASK:
qWorkerProcessCancelMsg(mockPointer, mgmt, fetchRpc, 0);
//qWorkerProcessCancelMsg(mockPointer, mgmt, fetchRpc, 0);
break;
case TDMT_SCH_DROP_TASK:
qWorkerProcessDropMsg(mockPointer, mgmt, fetchRpc, 0);

6
source/libs/scalar/src/filter.c
View File

@ -248,9 +248,9 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
}
}
if (optr == OP_TYPE_JSON_CONTAINS && type == TSDB_DATA_TYPE_JSON) {
return 28;
}
// if (optr == OP_TYPE_JSON_CONTAINS && type == TSDB_DATA_TYPE_JSON) {
// return 28;
// }
switch (type) {
case TSDB_DATA_TYPE_BOOL:

3
source/libs/scalar/src/scalar.c
View File

@ -1085,7 +1085,8 @@ EDealRes sclRewriteNonConstOperator(SNode **pNode, SScalarCtx *ctx) {
EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) {
SFunctionNode *node = (SFunctionNode *)*pNode;
SNode *tnode = NULL;
if (!fmIsScalarFunc(node->funcId) && (!ctx->dual)) {
if ((!fmIsScalarFunc(node->funcId) && (!ctx->dual)) ||
fmIsUserDefinedFunc(node->funcId)) {
return DEAL_RES_CONTINUE;
}

2
source/libs/scalar/src/sclvector.c
View File

@ -505,7 +505,7 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
(*pLeftData)++;
}
if (typeRight == TSDB_DATA_TYPE_JSON) {
if (tTagIsJson(*pLeftData)) {
if (tTagIsJson(*pRightData)) {
terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
return false;
}

4
source/libs/scheduler/inc/schInt.h
View File

@ -57,8 +57,8 @@ typedef enum {
#define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ
#define SCHEDULE_DEFAULT_MAX_NODE_NUM 20
#define SCH_DEFAULT_TASK_TIMEOUT_USEC 10000000
#define SCH_MAX_TASK_TIMEOUT_USEC 60000000
#define SCH_DEFAULT_TASK_TIMEOUT_USEC 60000000
#define SCH_MAX_TASK_TIMEOUT_USEC 300000000
#define SCH_DEFAULT_MAX_RETRY_NUM 6
#define SCH_MIN_AYSNC_EXEC_NUM 3

4
source/libs/scheduler/src/schRemote.c
View File

@ -145,7 +145,9 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
int32_t msgType = pMsg->msgType;
bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode));
SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, execId));
if (SCH_IS_QUERY_JOB(pJob)) {
SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, execId));
}
SCH_ERR_JRET(schValidateRspMsgType(pJob, pTask, msgType));

3
source/libs/scheduler/src/schTask.c
View File

@ -127,6 +127,7 @@ int32_t schDropTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int32_
if (taosHashRemove(pTask->execNodes, &execId, sizeof(execId))) {
SCH_TASK_DLOG("execId %d already not in execNodeList", execId);
SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR);
} else {
SCH_TASK_DLOG("execId %d removed from execNodeList", execId);
}
@ -914,7 +915,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) {
SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask));
if (SCH_IS_QUERY_JOB(pJob)) {
// SCH_ERR_RET(schEnsureHbConnection(pJob, pTask));
SCH_ERR_RET(schEnsureHbConnection(pJob, pTask));
}
SCH_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType));

5
source/libs/scheduler/src/schUtil.c
View File

@ -17,6 +17,7 @@
#include "command.h"
#include "query.h"
#include "schInt.h"
#include "tglobal.h"
#include "tmsg.h"
#include "tref.h"
#include "trpc.h"
@ -184,6 +185,10 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) {
}
int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) {
if (!tsEnableQueryHb) {
return TSDB_CODE_SUCCESS;
}
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx);
SQueryNodeEpId epId = {0};

182
source/libs/sync/src/syncAppendEntries.c
View File

@ -89,188 +89,6 @@
// /\ UNCHANGED <<candidateVars, leaderVars>>
//
static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) {
int32_t code;
SyncIndex delBegin = pMsg->prevLogIndex + 1;
SyncIndex delEnd = ths->pLogStore->syncLogLastIndex(ths->pLogStore);
// invert roll back!
for (SyncIndex index = delEnd; index >= delBegin; --index) {
if (ths->pFsm->FpRollBackCb != NULL) {
SSyncRaftEntry* pRollBackEntry;
code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, index, &pRollBackEntry);
ASSERT(code == 0);
ASSERT(pRollBackEntry != NULL);
if (syncUtilUserRollback(pRollBackEntry->msgType)) {
SRpcMsg rpcMsg;
syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
SFsmCbMeta cbMeta = {0};
cbMeta.index = pRollBackEntry->index;
cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index);
cbMeta.isWeak = pRollBackEntry->isWeak;
cbMeta.code = 0;
cbMeta.state = ths->state;
cbMeta.seqNum = pRollBackEntry->seqNum;
ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta);
rpcFreeCont(rpcMsg.pCont);
}
syncEntryDestory(pRollBackEntry);
}
}
// delete confict entries
code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin);
ASSERT(code == 0);
return code;
}
// if FromIndex > walCommitVer, return 0
// else return num of pass entries
static int32_t syncNodeDoMakeLogSame(SSyncNode* ths, SyncIndex FromIndex) {
int32_t code = 0;
int32_t pass = 0;
SyncIndex delBegin = FromIndex;
SyncIndex delEnd = ths->pLogStore->syncLogLastIndex(ths->pLogStore);
// invert roll back!
for (SyncIndex index = delEnd; index >= delBegin; --index) {
if (ths->pFsm->FpRollBackCb != NULL) {
SSyncRaftEntry* pRollBackEntry;
code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, index, &pRollBackEntry);
ASSERT(code == 0);
ASSERT(pRollBackEntry != NULL);
if (syncUtilUserRollback(pRollBackEntry->msgType)) {
SRpcMsg rpcMsg;
syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
SFsmCbMeta cbMeta = {0};
cbMeta.index = pRollBackEntry->index;
cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index);
cbMeta.isWeak = pRollBackEntry->isWeak;
cbMeta.code = 0;
cbMeta.state = ths->state;
cbMeta.seqNum = pRollBackEntry->seqNum;
ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta);
rpcFreeCont(rpcMsg.pCont);
}
syncEntryDestory(pRollBackEntry);
}
}
// update delete begin
SyncIndex walCommitVer = logStoreWalCommitVer(ths->pLogStore);
if (delBegin <= walCommitVer) {
delBegin = walCommitVer + 1;
pass = walCommitVer - delBegin + 1;
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "update delete begin to %" PRId64, delBegin);
syncNodeEventLog(ths, logBuf);
} while (0);
}
// delete confict entries
code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin);
ASSERT(code == 0);
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "make log same from:%" PRId64 ", delbegin:%" PRId64 ", pass:%d", FromIndex,
delBegin, pass);
syncNodeEventLog(ths, logBuf);
} while (0);
return pass;
}
int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry, int32_t code) {
SRpcMsg rpcMsg;
syncEntry2OriginalRpc(pEntry, &rpcMsg);
// leader transfer
if (pEntry->originalRpcType == TDMT_SYNC_LEADER_TRANSFER) {
int32_t code = syncDoLeaderTransfer(ths, &rpcMsg, pEntry);
ASSERT(code == 0);
}
if (ths->pFsm != NULL) {
if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) {
SFsmCbMeta cbMeta = {0};
cbMeta.index = pEntry->index;
cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index);
cbMeta.isWeak = pEntry->isWeak;
cbMeta.code = code;
cbMeta.state = ths->state;
cbMeta.seqNum = pEntry->seqNum;
ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta);
}
}
rpcFreeCont(rpcMsg.pCont);
return 0;
}
static bool syncNodeOnAppendEntriesBatchLogOK(SSyncNode* pSyncNode, SyncAppendEntriesBatch* pMsg) {
if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) {
return true;
}
SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode);
if (pMsg->prevLogIndex > myLastIndex) {
sDebug("vgId:%d, sync log not ok, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex);
return false;
}
SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1);
if (myPreLogTerm == SYNC_TERM_INVALID) {
sDebug("vgId:%d, sync log not ok2, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex);
return false;
}
if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) {
return true;
}
sDebug("vgId:%d, sync log not ok3, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex);
return false;
}
// really pre log match
// prevLogIndex == -1
static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) {
if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) {
return true;
}
SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode);
if (pMsg->prevLogIndex > myLastIndex) {
sDebug("vgId:%d, sync log not ok, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex);
return false;
}
SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1);
if (myPreLogTerm == SYNC_TERM_INVALID) {
sDebug("vgId:%d, sync log not ok2, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex);
return false;
}
if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) {
return true;
}
sDebug("vgId:%d, sync log not ok3, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex);
return false;
}
int32_t syncNodeFollowerCommit(SSyncNode* ths, SyncIndex newCommitIndex) {
// maybe update commit index, leader notice me
if (newCommitIndex > ths->commitIndex) {

14
source/libs/sync/src/syncMain.c
View File

@ -1148,6 +1148,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
pSyncNode->pRaftCfg = NULL;
}
// init by SSyncInfo
pSyncNode->vgId = pSyncInfo->vgId;
SSyncCfg* pCfg = &pSyncInfo->syncCfg;
sDebug("vgId:%d, replica:%d selfIndex:%d", pSyncNode->vgId, pCfg->replicaNum, pCfg->myIndex);
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
@ -1155,8 +1157,6 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
sDebug("vgId:%d, index:%d ep:%s:%u", pSyncNode->vgId, i, pNode->nodeFqdn, pNode->nodePort);
}
// init by SSyncInfo
pSyncNode->vgId = pSyncInfo->vgId;
memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path));
snprintf(pSyncNode->raftStorePath, sizeof(pSyncNode->raftStorePath), "%s%sraft_store.json", pSyncInfo->path,
TD_DIRSEP);
@ -1645,7 +1645,9 @@ int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) {
for (int i = 0; i < pSyncNode->peersNum; ++i) {
SSyncTimer* pSyncTimer = syncNodeGetHbTimer(pSyncNode, &(pSyncNode->peersId[i]));
syncHbTimerStart(pSyncNode, pSyncTimer);
if (pSyncTimer != NULL) {
syncHbTimerStart(pSyncNode, pSyncTimer);
}
}
return ret;
@ -1662,7 +1664,9 @@ int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) {
for (int i = 0; i < pSyncNode->peersNum; ++i) {
SSyncTimer* pSyncTimer = syncNodeGetHbTimer(pSyncNode, &(pSyncNode->peersId[i]));
syncHbTimerStop(pSyncNode, pSyncTimer);
if (pSyncTimer != NULL) {
syncHbTimerStop(pSyncNode, pSyncTimer);
}
}
return ret;
@ -3424,7 +3428,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde
// config change finish
if (pEntry->originalRpcType == TDMT_SYNC_CONFIG_CHANGE_FINISH) {
if (rpcMsg.pCont != NULL) {
if (rpcMsg.pCont != NULL && rpcMsg.contLen > 0) {
code = syncNodeConfigChangeFinish(ths, &rpcMsg, pEntry);
ASSERT(code == 0);
}

2
source/libs/sync/src/syncMessage.c
View File

@ -3026,7 +3026,7 @@ void syncReconfigFinishFromRpcMsg(const SRpcMsg* pRpcMsg, SyncReconfigFinish* pM
}
SyncReconfigFinish* syncReconfigFinishFromRpcMsg2(const SRpcMsg* pRpcMsg) {
SyncReconfigFinish* pMsg = syncReconfigFinishDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
SyncReconfigFinish* pMsg = syncReconfigFinishDeserialize2(pRpcMsg->pCont, (uint32_t)(pRpcMsg->contLen));
ASSERT(pMsg != NULL);
return pMsg;
}

4
source/libs/transport/src/transCli.c
View File

@ -1612,8 +1612,8 @@ int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) {
SCvtAddr cvtAddr = {0};
if (ip != NULL && fqdn != NULL) {
if (strlen(ip) <= sizeof(cvtAddr.ip)) memcpy(cvtAddr.ip, ip, strlen(ip));
if (strlen(fqdn) <= sizeof(cvtAddr.fqdn)) memcpy(cvtAddr.fqdn, fqdn, strlen(fqdn));
tstrncpy(cvtAddr.ip, ip, sizeof(cvtAddr.ip));
tstrncpy(cvtAddr.fqdn, fqdn, sizeof(cvtAddr.fqdn));
cvtAddr.cvt = true;
}
for (int i = 0; i < pTransInst->numOfThreads; i++) {

4
source/libs/transport/src/transComm.c
View File

@ -249,8 +249,8 @@ int transAsyncSend(SAsyncPool* pool, queue* q) {
if (atomic_load_8(&pool->stop) == 1) {
return -1;
}
int idx = pool->index;
idx = idx % pool->nAsync;
int idx = pool->index % pool->nAsync;
// no need mutex here
if (pool->index++ > pool->nAsync) {
pool->index = 0;

6
source/libs/wal/src/walMeta.c
View File

@ -27,7 +27,7 @@ bool FORCE_INLINE walIsEmpty(SWal* pWal) { return pWal->vers.firstVer == -1; }
int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; }
int64_t FORCE_INLINE walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; }
int64_t FORCE_INLINE walGetSnapshotVer(SWal* pWal) { return pWal->vers.snapshotVer; }
int64_t FORCE_INLINE walGetLastVer(SWal* pWal) { return pWal->vers.lastVer; }
@ -124,7 +124,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
SWalCkHead* logContent = (SWalCkHead*)candidate;
if (walValidHeadCksum(logContent) != 0) {
wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
offset + ((char*)(logContent)-buf), fnameStr);
offset + ((char*)(logContent)-buf), fnameStr);
haystack = candidate + 1;
if (firstTrial) {
break;
@ -163,7 +163,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
if (walValidBodyCksum(logContent) != 0) {
terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
offset + ((char*)(logContent)-buf), fnameStr);
offset + ((char*)(logContent)-buf), fnameStr);
haystack = candidate + 1;
if (firstTrial) {
break;

4
source/libs/wal/src/walRef.c
View File

@ -32,6 +32,7 @@ SWalRef *walOpenRef(SWal *pWal) {
return pRef;
}
#if 0
void walCloseRef(SWal *pWal, int64_t refId) {
SWalRef **ppRef = taosHashGet(pWal->pRefHash, &refId, sizeof(int64_t));
if (ppRef == NULL) return;
@ -39,6 +40,7 @@ void walCloseRef(SWal *pWal, int64_t refId) {
taosHashRemove(pWal->pRefHash, &refId, sizeof(int64_t));
taosMemoryFree(pRef);
}
#endif
int32_t walRefVer(SWalRef *pRef, int64_t ver) {
SWal *pWal = pRef->pWal;
@ -65,10 +67,12 @@ int32_t walRefVer(SWalRef *pRef, int64_t ver) {
return 0;
}
#if 0
void walUnrefVer(SWalRef *pRef) {
pRef->refId = -1;
pRef->refFile = -1;
}
#endif
SWalRef *walRefCommittedVer(SWal *pWal) {
SWalRef *pRef = walOpenRef(pWal);

4
source/libs/wal/src/walSeek.c
View File

@ -19,6 +19,7 @@
#include "tref.h"
#include "walInt.h"
#if 0
static int64_t walSeekWritePos(SWal* pWal, int64_t ver) {
int64_t code = 0;
@ -47,6 +48,7 @@ static int64_t walSeekWritePos(SWal* pWal, int64_t ver) {
}
return 0;
}
#endif
int walInitWriteFile(SWal* pWal) {
TdFilePtr pIdxTFile, pLogTFile;
@ -134,6 +136,7 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) {
return fileFirstVer;
}
#if 0
int walSeekWriteVer(SWal* pWal, int64_t ver) {
int64_t code;
if (ver == pWal->vers.lastVer) {
@ -158,3 +161,4 @@ int walSeekWriteVer(SWal* pWal, int64_t ver) {
return 0;
}
#endif

33
source/libs/wal/test/walMetaTest.cpp
View File

@ -257,6 +257,39 @@ TEST_F(WalCleanEnv, rollback) {
ASSERT_EQ(code, 0);
}
TEST_F(WalCleanEnv, rollbackMultiFile) {
int code;
for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i);
if (i == 5) {
walBeginSnapshot(pWal, i);
walEndSnapshot(pWal);
}
}
code = walRollback(pWal, 12);
ASSERT_NE(code, 0);
ASSERT_EQ(pWal->vers.lastVer, 9);
code = walRollback(pWal, 9);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, 8);
code = walRollback(pWal, 5);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, 4);
code = walRollback(pWal, 3);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, 2);
code = walWrite(pWal, 3, 3, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, 3);
code = walSaveMeta(pWal);
ASSERT_EQ(code, 0);
}
TEST_F(WalCleanDeleteEnv, roll) {
int code;
int i;

48
source/util/src/tfunctional.c
View File

@ -1,48 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "tfunctional.h"
FORCE_INLINE void* genericInvoke(tGenericSavedFunc* const pSavedFunc) { return pSavedFunc->func(pSavedFunc->args); }
#if 0
tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int32_t numOfArgs) {
tGenericSavedFunc* pSavedFunc = taosMemoryMalloc(sizeof(tGenericSavedFunc) + numOfArgs * (sizeof(void*)));
if (pSavedFunc == NULL) return NULL;
pSavedFunc->func = func;
return pSavedFunc;
}
tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int32_t numOfArgs) {
tI32SavedFunc* pSavedFunc = taosMemoryMalloc(sizeof(tI32SavedFunc) + numOfArgs * sizeof(void*));
if (pSavedFunc == NULL) return NULL;
pSavedFunc->func = func;
return pSavedFunc;
}
tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int32_t numOfArgs) {
tVoidSavedFunc* pSavedFunc = taosMemoryMalloc(sizeof(tVoidSavedFunc) + numOfArgs * sizeof(void*));
if (pSavedFunc == NULL) return NULL;
pSavedFunc->func = func;
return pSavedFunc;
}
FORCE_INLINE int32_t i32Invoke(tI32SavedFunc* const pSavedFunc) { return pSavedFunc->func(pSavedFunc->args); }
FORCE_INLINE void voidInvoke(tVoidSavedFunc* const pSavedFunc) {
if (pSavedFunc) pSavedFunc->func(pSavedFunc->args);
}
#endif

2
source/util/src/thash.c
View File

@ -798,7 +798,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
}
void *taosHashIterate(SHashObj *pHashObj, void *p) {
if (pHashObj == NULL) return NULL;
if (pHashObj == NULL || pHashObj->size == 0) return NULL;
int slot = 0;
char *data = NULL;

24
source/util/src/tskiplist.c
View File

@ -87,7 +87,6 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _
#if SKIP_LIST_RECORD_PERFORMANCE
pSkipList->state.nTotalMemSize += sizeof(SSkipList);
#endif
pSkipList->insertHandleFn = NULL;
return pSkipList;
}
@ -105,8 +104,6 @@ void tSkipListDestroy(SSkipList *pSkipList) {
tSkipListFreeNode(pTemp);
}
taosMemoryFreeClear(pSkipList->insertHandleFn);
tSkipListUnlock(pSkipList);
if (pSkipList->lock != NULL) {
taosThreadRwlockDestroy(pSkipList->lock);
@ -684,35 +681,14 @@ static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipL
} else {
pNode = SL_NODE_GET_BACKWARD_POINTER(direction[0], 0);
}
if (pSkipList->insertHandleFn) {
pSkipList->insertHandleFn->args[0] = pData;
pSkipList->insertHandleFn->args[1] = pNode->pData;
pData = genericInvoke(pSkipList->insertHandleFn);
}
if (pData) {
atomic_store_ptr(&(pNode->pData), pData);
}
} else {
// for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows!
if (pSkipList->insertHandleFn) {
pSkipList->insertHandleFn->args[0] = NULL;
pSkipList->insertHandleFn->args[1] = NULL;
genericInvoke(pSkipList->insertHandleFn);
}
}
} else {
pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList));
if (pNode != NULL) {
// insertHandleFn will be assigned only for timeseries data,
// in which case, pData is pointed to an memory to be freed later;
// while for metadata, the mem alloc will not be called.
if (pSkipList->insertHandleFn) {
pSkipList->insertHandleFn->args[0] = pData;
pSkipList->insertHandleFn->args[1] = NULL;
pData = genericInvoke(pSkipList->insertHandleFn);
}
pNode->pData = pData;
tSkipListDoInsert(pSkipList, direction, pNode, isForward);
}
}

1
source/util/src/tstrbuild.c
View File

@ -58,6 +58,7 @@ void taosStringBuilderAppendChar(SStringBuilder* sb, char c) {
void taosStringBuilderAppendStringLen(SStringBuilder* sb, const char* str, size_t len) {
taosStringBuilderEnsureCapacity(sb, len);
if(!sb->buf) return;
memcpy(sb->buf + sb->pos, str, len);
sb->pos += len;
}

58
source/util/test/arrayTest.cpp
View File

@ -5,52 +5,52 @@
#include "tarray.h"
#include "tcompare.h"
/*
namespace {
static void remove_batch_test() {
SArray* pa = (SArray*)taosArrayInit(4, sizeof(int32_t));
SArray* pa = (SArray*)taosArrayInit(4, sizeof(int32_t));
for (int32_t i = 0; i < 20; ++i) {
int32_t a = i;
taosArrayPush(pa, &a);
}
for (int32_t i = 0; i < 20; ++i) {
int32_t a = i;
taosArrayPush(pa, &a);
}
SArray* delList = (SArray*)taosArrayInit(4, sizeof(int32_t));
taosArrayRemoveBatch(pa, (const int32_t*)TARRAY_GET_START(delList), taosArrayGetSize(delList));
EXPECT_EQ(taosArrayGetSize(pa), 20);
SArray* delList = (SArray*)taosArrayInit(4, sizeof(int32_t));
taosArrayRemoveBatch(pa, (const int32_t*)TARRAY_GET_START(delList), taosArrayGetSize(delList));
EXPECT_EQ(taosArrayGetSize(pa), 20);
int32_t a = 5;
taosArrayPush(delList, &a);
int32_t a = 5;
taosArrayPush(delList, &a);
taosArrayRemoveBatch(pa, (const int32_t*)TARRAY_GET_START(delList), taosArrayGetSize(delList));
EXPECT_EQ(taosArrayGetSize(pa), 19);
EXPECT_EQ(*(int*)taosArrayGet(pa, 5), 6);
taosArrayRemoveBatch(pa, (const int32_t*)TARRAY_GET_START(delList), taosArrayGetSize(delList));
EXPECT_EQ(taosArrayGetSize(pa), 19);
EXPECT_EQ(*(int*)taosArrayGet(pa, 5), 6);
taosArrayInsert(pa, 5, &a);
EXPECT_EQ(taosArrayGetSize(pa), 20);
EXPECT_EQ(*(int*)taosArrayGet(pa, 5), 5);
taosArrayInsert(pa, 5, &a);
EXPECT_EQ(taosArrayGetSize(pa), 20);
EXPECT_EQ(*(int*)taosArrayGet(pa, 5), 5);
taosArrayClear(delList);
taosArrayClear(delList);
a = 6;
taosArrayPush(delList, &a);
a = 6;
taosArrayPush(delList, &a);
a = 9;
taosArrayPush(delList, &a);
a = 9;
taosArrayPush(delList, &a);
a = 14;
taosArrayPush(delList, &a);
taosArrayRemoveBatch(pa, (const int32_t*)TARRAY_GET_START(delList), taosArrayGetSize(delList));
EXPECT_EQ(taosArrayGetSize(pa), 17);
a = 14;
taosArrayPush(delList, &a);
taosArrayRemoveBatch(pa, (const int32_t*)TARRAY_GET_START(delList), taosArrayGetSize(delList));
EXPECT_EQ(taosArrayGetSize(pa), 17);
taosArrayDestroy(pa);
taosArrayDestroy(delList);
taosArrayDestroy(pa);
taosArrayDestroy(delList);
}
} // namespace
TEST(arrayTest, array_list_test) { remove_batch_test(); }
*/
TEST(arrayTest, array_search_test) {
SArray* pa = (SArray*)taosArrayInit(4, sizeof(int32_t));

2
tests/pytest/util/dnodes.py
View File

@ -40,6 +40,7 @@ class TDSimClient:
"jniDebugFlag": "143",
"qDebugFlag": "143",
"supportVnodes": "1024",
"enableQueryHb": "1",
"telemetryReporting": "0",
}
@ -136,6 +137,7 @@ class TDDnode:
"wDebugFlag": "143",
"numOfLogLines": "100000000",
"statusInterval": "1",
"enableQueryHb": "1",
"supportVnodes": "1024",
"telemetryReporting": "0"
}

5
tests/script/api/makefile
View File

@ -17,6 +17,5 @@ exe:
clean:
rm $(ROOT)batchprepare
rm $(ROOT)stmtBatchTest
rm $(ROOT)stmtTest
rm $(ROOT)stmt_function
rm $(ROOT)stopquery
rm $(ROOT)dbTableRoute

11
tests/script/api/stopquery.c
View File

@ -36,7 +36,7 @@ int64_t st, et;
char hostName[128];
char dbName[128];
char tbName[128];
int32_t runTimes = 1000;
int32_t runTimes = 10;
typedef struct {
int id;
@ -657,7 +657,6 @@ int sqConCleanupAsyncQuery(bool fetch) {
void sqRunAllCase(void) {
#if 1
sqStopSyncQuery(false);
sqStopSyncQuery(true);
sqStopAsyncQuery(false);
@ -678,33 +677,33 @@ void sqRunAllCase(void) {
sqConCloseAsyncQuery(false);
sqConCloseAsyncQuery(true);
sqKillSyncQuery(false);
sqKillSyncQuery(true);
sqKillAsyncQuery(false);
sqKillAsyncQuery(true);
#if 0
/*
sqConKillSyncQuery(false);
sqConKillSyncQuery(true);
sqConKillAsyncQuery(false);
sqConKillAsyncQuery(true);
#endif
/*
sqConCleanupSyncQuery(false);
sqConCleanupSyncQuery(true);
sqConCleanupAsyncQuery(false);
sqConCleanupAsyncQuery(true);
*/
#endif
int32_t l = 5;
while (l) {
printf("%d\n", l--);
sleep(1);
}
printf("test done\n");
}
int main(int argc, char *argv[]) {
if (argc != 4) {
printf("usage: %s server-ip dbname tablename\n", argv[0]);

44
tests/script/jenkins/basic.txt
View File

@ -25,7 +25,6 @@
./test.sh -f tsim/db/delete_reusevnode2.sim
./test.sh -f tsim/db/delete_writing1.sim
./test.sh -f tsim/db/delete_writing2.sim
# unsupport ./test.sh -f tsim/db/dropdnodes.sim
./test.sh -f tsim/db/error1.sim
./test.sh -f tsim/db/keep.sim
./test.sh -f tsim/db/len.sim
@ -36,26 +35,26 @@
./test.sh -f tsim/db/taosdlog.sim
# ---- dnode
# unsupport ./test.sh -f tsim/dnode/balance_replica1.sim
# unsupport ./test.sh -f tsim/dnode/balance_replica3.sim
# unsupport ./test.sh -f tsim/dnode/balance1.sim
# unsupport ./test.sh -f tsim/dnode/balance2.sim
# unsupport ./test.sh -f tsim/dnode/balance3.sim
# unsupport ./test.sh -f tsim/dnode/balancex.sim
./test.sh -f tsim/dnode/balance_replica1.sim
./test.sh -f tsim/dnode/balance_replica3.sim
./test.sh -f tsim/dnode/balance1.sim
./test.sh -f tsim/dnode/balance2.sim
./test.sh -f tsim/dnode/balance3.sim
./test.sh -f tsim/dnode/balancex.sim
./test.sh -f tsim/dnode/create_dnode.sim
./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim
# unsupport ./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
# unsupport ./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim
# unsupport ./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
# unsupport ./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
# unsupport ./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim
./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
./test.sh -f tsim/dnode/offline_reason.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim
# unsupport ./test.sh -f tsim/dnode/vnode_clean.sim
./test.sh -f tsim/dnode/vnode_clean.sim
./test.sh -f tsim/dnode/use_dropped_dnode.sim
# ---- import ----
@ -155,7 +154,7 @@
./test.sh -f tsim/parser/select_with_tags.sim
./test.sh -f tsim/parser/selectResNum.sim
./test.sh -f tsim/parser/set_tag_vals.sim
# TD-19572 ./test.sh -f tsim/parser/single_row_in_tb.sim
./test.sh -f tsim/parser/single_row_in_tb.sim
./test.sh -f tsim/parser/sliding.sim
./test.sh -f tsim/parser/slimit_alter_tags.sim
./test.sh -f tsim/parser/slimit.sim
@ -185,7 +184,7 @@
./test.sh -f tsim/qnode/basic1.sim
# ---- snode ----
# unsupport ./test.sh -f tsim/snode/basic1.sim
./test.sh -f tsim/snode/basic1.sim
# ---- mnode
./test.sh -f tsim/mnode/basic1.sim
@ -303,7 +302,6 @@
# --- sma
./test.sh -f tsim/sma/drop_sma.sim
./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
# temp disable
./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
@ -319,13 +317,13 @@
./test.sh -f tsim/valgrind/checkUdf.sim
# --- vnode ----
# unsupport ./test.sh -f tsim/vnode/replica3_basic.sim
# unsupport ./test.sh -f tsim/vnode/replica3_repeat.sim
# unsupport ./test.sh -f tsim/vnode/replica3_vgroup.sim
# unsupport ./test.sh -f tsim/vnode/replica3_many.sim
# unsupport ./test.sh -f tsim/vnode/replica3_import.sim
# unsupport ./test.sh -f tsim/vnode/stable_balance_replica1.sim
# unsupport ./test.sh -f tsim/vnode/stable_dnode2_stop.sim
./test.sh -f tsim/vnode/replica3_basic.sim
./test.sh -f tsim/vnode/replica3_repeat.sim
./test.sh -f tsim/vnode/replica3_vgroup.sim
./test.sh -f tsim/vnode/replica3_many.sim
./test.sh -f tsim/vnode/replica3_import.sim
./test.sh -f tsim/vnode/stable_balance_replica1.sim
./test.sh -f tsim/vnode/stable_dnode2_stop.sim
./test.sh -f tsim/vnode/stable_dnode2.sim
./test.sh -f tsim/vnode/stable_dnode3.sim
./test.sh -f tsim/vnode/stable_replica3_dnode6.sim

6
tests/script/tsim/db/alter_option.sim
View File

@ -111,13 +111,13 @@ endi
if $data21_db != 1000 then # wal_level fsyncperiod
return -1
endi
if $data22_db != 345600 then # wal_retention_period
if $data22_db != 0 then # wal_retention_period
return -1
endi
if $data23_db != -1 then # wal_retention_size
if $data23_db != 0 then # wal_retention_size
return -1
endi
if $data24_db != 86400 then # wal_roll_period
if $data24_db != 0 then # wal_roll_period
return -1
endi
if $data25_db != 0 then # wal_segment_size

107
tests/script/tsim/db/dropdnodes.sim
View File

@ -1,107 +0,0 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/deploy.sh -n dnode2 -i 2
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4
print ========== prepare data
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
sleep 2000
sql connect
sql create dnode $hostname2
sleep 2000
sql create database db
sql use db
print ========== step1
sql create table mt (ts timestamp, tbcol int) TAGS(tgcol int)
sql create table db.t1 using db.mt tags(1)
sql create table db.t2 using db.mt tags(2)
sql create table db.t3 using db.mt tags(3)
sql create table db.t4 using db.mt tags(4)
sql create table db.t5 using db.mt tags(5)
sql create table db.t6 using db.mt tags(6)
sql create table db.t7 using db.mt tags(7)
sql create table db.t8 using db.mt tags(8)
sql create table db.t9 using db.mt tags(9)
sql create table db.t10 using db.mt tags(10)
sql create table db.t11 using db.mt tags(11)
sql create table db.t12 using db.mt tags(12)
sql create table db.t13 using db.mt tags(13)
sql create table db.t14 using db.mt tags(14)
sql create table db.t15 using db.mt tags(15)
sql create table db.t16 using db.mt tags(16)
sql insert into db.t1 values(now, 1)
sql insert into db.t2 values(now, 1)
sql insert into db.t3 values(now, 1)
sql insert into db.t4 values(now, 1)
sql insert into db.t5 values(now, 1)
sql insert into db.t6 values(now, 1)
sql insert into db.t7 values(now, 1)
sql insert into db.t8 values(now, 1)
sql insert into db.t9 values(now, 1)
sql insert into db.t10 values(now, 1)
sql insert into db.t11 values(now, 1)
sql insert into db.t12 values(now, 1)
sql insert into db.t13 values(now, 1)
sql insert into db.t14 values(now, 1)
sql insert into db.t15 values(now, 1)
sql insert into db.t16 values(now, 1)
print ========== step2
sql show tables
print $rows
if $rows != 16 then
return -1
endi
sql select * from mt
print $rows
if $rows != 16 then
return -1
endi
print ========== step3
system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep 3000
sql drop dnode $hostname2
sleep 2000
print ========== step3
sql show tables
print $rows
if $rows != 8 then
return -1
endi
sql select * from mt
print $rows
if $rows != 8 then
return -1
endi
sql select * from db.t5
if $rows != 1 then
return -1
endi
sql select * from db.t13
if $rows != 1 then
return -1
endi
sql_error select * from db.t1
sql_error select * from db.t9
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT

108
tests/script/tsim/dnode/balance1.sim
View File

@ -81,6 +81,42 @@ if $data(2)[2] != 2 then
return -1
endi
sql select * from d1.t1 order by t desc
print $data01 $data11 $data21 $data31 $data41
if $data01 != 11 then
return -1
endi
if $data11 != 12 then
return -1
endi
if $data21 != 13 then
return -1
endi
if $data31 != 14 then
return -1
endi
if $data41 != 15 then
return -1
endi
sql select * from d2.t2 order by t desc
print $data01 $data11 $data21 $data31 $data41
if $data01 != 21 then
return -1
endi
if $data11 != 22 then
return -1
endi
if $data21 != 23 then
return -1
endi
if $data31 != 24 then
return -1
endi
if $data41 != 25 then
return -1
endi
print ========== step4
sql drop dnode 2
sql select * from information_schema.ins_dnodes
@ -93,6 +129,42 @@ if $data(2)[2] != null then
return -1
endi
sql select * from d1.t1 order by t desc
print $data01 $data11 $data21 $data31 $data41
if $data01 != 11 then
return -1
endi
if $data11 != 12 then
return -1
endi
if $data21 != 13 then
return -1
endi
if $data31 != 14 then
return -1
endi
if $data41 != 15 then
return -1
endi
sql select * from d2.t2 order by t desc
print $data01 $data11 $data21 $data31 $data41
if $data01 != 21 then
return -1
endi
if $data11 != 22 then
return -1
endi
if $data21 != 23 then
return -1
endi
if $data31 != 24 then
return -1
endi
if $data41 != 25 then
return -1
endi
system sh/exec.sh -n dnode2 -s stop -x SIGINT
print ========== step5
@ -131,6 +203,42 @@ if $data(3)[2] != 1 then
return -1
endi
sql select * from d1.t1 order by t desc
print $data01 $data11 $data21 $data31 $data41
if $data01 != 11 then
return -1
endi
if $data11 != 12 then
return -1
endi
if $data21 != 13 then
return -1
endi
if $data31 != 14 then
return -1
endi
if $data41 != 15 then
return -1
endi
sql select * from d2.t2 order by t desc
print $data01 $data11 $data21 $data31 $data41
if $data01 != 21 then
return -1
endi
if $data11 != 22 then
return -1
endi
if $data21 != 23 then
return -1
endi
if $data31 != 24 then
return -1
endi
if $data41 != 25 then
return -1
endi
print ========== step6
sql create database d3 vgroups 1
sql create table d3.t3 (t timestamp, i int)

6
tests/script/tsim/parser/limit_stb.sim
View File

@ -39,9 +39,9 @@ endi
if $data01 != 1 then
return -1
endi
if $data41 != 5 then
return -1
endi
#if $data41 != 5 then
# return -1
#endi
sql select * from $stb order by ts desc limit 5
if $rows != 5 then

36
tests/script/tsim/stream/distributeIntervalRetrive0.sim
View File

@ -236,7 +236,43 @@ endi
print loop3 over
sql drop stream if exists streams1;
sql drop database if exists test1;
sql create database test1 vgroups 4 keep 7000;
sql use test1;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams1 trigger at_once into streamt1 as select _wstart as c0, count(*) c1, count(a) c2 from st interval(10s) ;
sql insert into t1 values(1648791211000,1,2,3);
sql insert into t1 values(1262275200000,2,2,3);
sql insert into t2 values(1262275200000,1,2,3);
$loop_count = 0
loop4:
sleep 300
sql select * from streamt1 order by c0;
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 2 then
print =====loop4=rows=$rows
goto loop4
endi
if $data01 != 2 then
print =====loop4=data11=$data11
goto loop4
endi
print loop4 over
#==system sh/exec.sh -n dnode1 -s stop -x SIGINT

26
tests/script/tsim/vnode/replica3_many.sim
View File

@ -59,7 +59,7 @@ step12:
print ====> db not ready!
return -1
endi
sql show d1.vgroups
sql show db1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
$leaderExist = 0
if $rows != 1 then
@ -87,19 +87,19 @@ step13:
print ====> db not ready!
return -1
endi
sql show d1.vgroups
sql show db1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
$leaderExist = 0
if $rows != 1 then
return -1
endi
if $data(3)[4] == leader then
if $data(2)[4] == leader then
$leaderExist = 1
endi
if $data(3)[6] == leader then
if $data(2)[6] == leader then
$leaderExist = 1
endi
if $data(3)[8] == leader then
if $data(2)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@ -115,16 +115,16 @@ step14:
print ====> db not ready!
return -1
endi
sql show d1.vgroups
sql show db1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
$leaderExist = 0
if $data(4)[4] == leader then
if $data(2)[4] == leader then
$leaderExist = 1
endi
if $data(4)[6] == leader then
if $data(2)[6] == leader then
$leaderExist = 1
endi
if $data(4)[8] == leader then
if $data(2)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@ -140,16 +140,16 @@ step15:
print ====> db not ready!
return -1
endi
sql show d1.vgroups
sql show db1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
$leaderExist = 0
if $data(4)[4] == leader then
if $data(2)[4] == leader then
$leaderExist = 1
endi
if $data(4)[6] == leader then
if $data(2)[6] == leader then
$leaderExist = 1
endi
if $data(4)[8] == leader then
if $data(2)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then

Some files were not shown because too many files have changed in this diff Show More