From bfb6bce0073e2461a7be08e0b7242d069cc46872 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 7 Aug 2022 19:39:24 +0800 Subject: [PATCH 01/17] fix read error --- source/libs/transport/src/transCli.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index dc7ffa9b13..0b8829a0c0 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -482,6 +482,7 @@ void cliReadTimeoutCb(uv_timer_t* handle) { // set up timeout cb SCliConn* conn = handle->data; tTrace("%s conn %p timeout, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); + uv_read_stop(conn->stream); cliHandleExceptImpl(conn, TSDB_CODE_RPC_TIMEOUT); } From 9c38e704d9f6550d0cc9e8b39e03604bb4336bbb Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 8 Aug 2022 09:34:12 +0800 Subject: [PATCH 02/17] build: add win package --- cmake/cmake.install | 4 +- contrib/CMakeLists.txt | 17 ++++ include/os/osDef.h | 4 +- packaging/release.bat | 109 ++++++++++++--------- packaging/release.sh | 2 +- packaging/tools/favicon.ico | Bin 0 -> 4286 bytes packaging/tools/install.sh | 1 - packaging/tools/makepkg.sh | 3 +- packaging/tools/taos.bat | 6 ++ packaging/tools/tdengine.iss | 81 +++++++++++++++ packaging/tools/windows_before_install.txt | 3 + source/os/src/osAtomic.c | 22 ++++- source/os/src/osString.c | 4 +- 13 files changed, 198 insertions(+), 58 deletions(-) create mode 100644 packaging/tools/favicon.ico create mode 100644 packaging/tools/taos.bat create mode 100644 packaging/tools/tdengine.iss create mode 100644 packaging/tools/windows_before_install.txt diff --git a/cmake/cmake.install b/cmake/cmake.install index 27edd8e27a..d9f759217a 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -10,7 +10,9 @@ ELSEIF (TD_WINDOWS) # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector) # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector) # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .) - INSTALL(FILES ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) + INSTALL(CODE "IF (NOT EXISTS ${CMAKE_INSTALL_PREFIX}/cfg/taos.cfg) + execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg ${CMAKE_INSTALL_PREFIX}/cfg/taos.cfg) + ENDIF ()") INSTALL(FILES ${TD_SOURCE_DIR}/include/client/taos.h DESTINATION include) INSTALL(FILES ${TD_SOURCE_DIR}/include/util/taoserror.h DESTINATION include) INSTALL(FILES ${TD_SOURCE_DIR}/include/libs/function/taosudf.h DESTINATION include) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b4e8825431..de7b75a245 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -27,6 +27,10 @@ else () cat("${TD_SUPPORT_DIR}/taosadapter_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() +if(TD_LINUX_64 AND JEMALLOC_ENABLED) + cat("${TD_SUPPORT_DIR}/jemalloc_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + # pthread if(${BUILD_PTHREAD}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -392,6 +396,19 @@ if(${BUILD_WITH_SQLITE}) endif(NOT TD_WINDOWS) endif(${BUILD_WITH_SQLITE}) +# jemalloc +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + include(ExternalProject) + ExternalProject_Add(jemalloc + PREFIX "jemalloc" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ + BUILD_COMMAND ${MAKE} + ) + INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include) +ENDIF () + # addr2line if(${BUILD_ADDR2LINE}) if(NOT ${TD_WINDOWS}) diff --git a/include/os/osDef.h b/include/os/osDef.h index 14f38eb7ff..be8689398d 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -57,7 +57,7 @@ extern "C" { #if defined(WINDOWS) char *stpcpy (char *dest, const char *src); - char *stpncpy (char *dest, const char *src, size_t n); + char *stpncpy (char *dest, const char *src, int n); // specific #ifndef __COMPAR_FN_T @@ -77,7 +77,7 @@ extern "C" { char * strsep(char **stringp, const char *delim); char * getpass(const char *prefix); - char * strndup(const char *s, size_t n); + char * strndup(const char *s, int n); // for send function in tsocket.c #define MSG_NOSIGNAL 0 diff --git a/packaging/release.bat b/packaging/release.bat index c1cf7875a5..d58e19cece 100644 --- a/packaging/release.bat +++ b/packaging/release.bat @@ -2,61 +2,78 @@ set internal_dir=%~dp0\..\..\ set community_dir=%~dp0\.. -cd %community_dir% -git checkout -- . -cd %community_dir%\packaging +set package_dir=%cd% :: %1 name %2 version if !%1==! GOTO USAGE if !%2==! GOTO USAGE -if %1 == taos GOTO TAOS -if %1 == power GOTO POWER -if %1 == tq GOTO TQ -if %1 == pro GOTO PRO -if %1 == kh GOTO KH -if %1 == jh GOTO JH -GOTO USAGE -:TAOS -goto RELEASE - -:POWER -call sed_power.bat %community_dir% -goto RELEASE - -:TQ -call sed_tq.bat %community_dir% -goto RELEASE - -:PRO -call sed_pro.bat %community_dir% -goto RELEASE - -:KH -call sed_kh.bat %community_dir% -goto RELEASE - -:JH -call sed_jh.bat %community_dir% -goto RELEASE - -:RELEASE -echo release windows-client-64 for %1, version: %2 -if not exist %internal_dir%\debug\ver-%2-64bit-%1 ( - md %internal_dir%\debug\ver-%2-64bit-%1 +if "%1" == "cluster" ( + set work_dir=%internal_dir% + set packagServerName_x64=TDengine-enterprise-server-%2-beta-Windows-x64 + set packagServerName_x86=TDengine-enterprise-server-%2-beta-Windows-x86 + set packagClientName_x64=TDengine-enterprise-client-%2-beta-Windows-x64 + set packagClientName_x86=TDengine-enterprise-client-%2-beta-Windows-x86 ) else ( - rd /S /Q %internal_dir%\debug\ver-%2-64bit-%1 - md %internal_dir%\debug\ver-%2-64bit-%1 + set work_dir=%community_dir% + set packagServerName_x64=TDengine-server-%2-Windows-x64 + set packagServerName_x86=TDengine-server-%2-Windows-x86 + set packagClientName_x64=TDengine-client-%2-Windows-x64 + set packagClientName_x86=TDengine-client-%2-Windows-x86 ) -cd %internal_dir%\debug\ver-%2-64bit-%1 -call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 -cmake ../../ -G "NMake Makefiles" -DVERNUMBER=%2 -DCPUTYPE=x64 -set CL=/MP4 -nmake install + +echo release windows-client for %1, version: %2 +if not exist %work_dir%\debug ( + md %work_dir%\debug +) +if not exist %work_dir%\debug\ver-%2-x64 ( + md %work_dir%\debug\ver-%2-x64 +) else ( + rd /S /Q %work_dir%\debug\ver-%2-x64 + md %work_dir%\debug\ver-%2-x64 +) +if not exist %work_dir%\debug\ver-%2-x86 ( + md %work_dir%\debug\ver-%2-x86 +) else ( + rd /S /Q %work_dir%\debug\ver-%2-x86 + md %work_dir%\debug\ver-%2-x86 +) +cd %work_dir%\debug\ver-%2-x64 +call vcvarsall.bat x64 +cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DBUILD_HTTP=false -DVERNUMBER=%2 -DCPUTYPE=x64 +cmake --build . +rd /s /Q C:\TDengine +cmake --install . +if not %errorlevel% == 0 ( call :RUNFAILED build x64 failed & exit /b 1) +cd %package_dir% +iscc /DMyAppInstallName="%packagServerName_x64%" /DMyAppVersion="%2" /DMyAppExcludeSource="" tools\tdengine.iss /O..\release +if not %errorlevel% == 0 ( call :RUNFAILED package %packagServerName_x64% failed & exit /b 1) +iscc /DMyAppInstallName="%packagClientName_x64%" /DMyAppVersion="%2" /DMyAppExcludeSource="taosd.exe" tools\tdengine.iss /O..\release +if not %errorlevel% == 0 ( call :RUNFAILED package %packagClientName_x64% failed & exit /b 1) + +cd %work_dir%\debug\ver-%2-x86 +call vcvarsall.bat x86 +cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DBUILD_HTTP=false -DVERNUMBER=%2 -DCPUTYPE=x86 +cmake --build . +rd /s /Q C:\TDengine +cmake --install . +if not %errorlevel% == 0 ( call :RUNFAILED build x86 failed & exit /b 1) +cd %package_dir% +iscc /DMyAppInstallName="%packagServerName_x86%" /DMyAppVersion="%2" /DMyAppExcludeSource="" tools\tdengine.iss /O..\release +if not %errorlevel% == 0 ( call :RUNFAILED package %packagServerName_x86% failed & exit /b 1) +iscc /DMyAppInstallName="%packagClientName_x86%" /DMyAppVersion="%2" /DMyAppExcludeSource="taosd.exe" tools\tdengine.iss /O..\release +if not %errorlevel% == 0 ( call :RUNFAILED package %packagClientName_x86% failed & exit /b 1) + goto EXIT0 :USAGE -echo Usage: release.bat $productName $version +echo Usage: release.bat $verMode $version goto EXIT0 -:EXIT0 \ No newline at end of file +:EXIT0 +exit /b + +:RUNFAILED +echo %* +cd %package_dir% +goto :eof \ No newline at end of file diff --git a/packaging/release.sh b/packaging/release.sh index 3426c2856d..09781dbe8e 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -26,7 +26,7 @@ soMode=dynamic # [static | dynamic] dbName=taos # [taos | ...] allocator=glibc # [glibc | jemalloc] verNumber="" -verNumberComp="2.0.0.0" +verNumberComp="3.0.0.0" httpdBuild=false while getopts "hv:V:c:o:l:s:d:a:n:m:H:" arg; do diff --git a/packaging/tools/favicon.ico b/packaging/tools/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..20b8026d1d2efd6d77b013a1b94cee7aaed11772 GIT binary patch literal 4286 zcmd6qPiPfa6vn3^1PLL8AR&}HB}fP&5~N5e5;TM$B0+qiiy{(Sgdh@ZDWR31h9VM* zNRS{wL?j6kL8Q1ST?8w|m2}~%i!RDTq?97|8M1iD^S#^OH<>%mjFZe*vT(xZx%d3J z_uO;NJ@@H(AMw}IK~uym zIKbvh@b_}byns#=G>g=*`4~K2F8QyaKZ5p1m@`0EXZ{OVVQJK_KzD*}@JRs>H}eB} zfqh6%=WH8^(QjqXe`5A)tk=c-1o*lDboMzA-+waNpuk|MWG1m^Uw<8(2OZ?1xqzvM zZ72TsVe>nvzG_GD7Mj3e_P++*x!EG2-Y8UOSM5@JM)_*n%w{t_Hsjdt zf$jw-fd_VjTVNn<2d`~OIWPGWjODv<87Lm&7mA%*7rOMZvfv4DCcRW$-ndj#KUHLvwtwrZyuKRdR z`-k#D_fI}+&zqnh z$QKh|1AX~=F3!suqoW=(l$OEc4wzNWFw8hD4(9MXkzWBnGj541VXHVP2MK;$M`Pr= zumFB{q!82&!~qLFwZz$v+A9D(l&06tcyCCjm_KOUFH7j{>c9;__5_H ziq16W--TWub~gC zYA@As`-aTO=lei!mEPH-K=Oo-ZcS|DSLRy|e~Q@ar_VbdGWo9kRh!!DB{&EM!nu1Z z^;W6Jj)T(~SPA-hw(M8*Co^ocj(RhFD)A=&6!Li^Bfk98d)Et8Luu@Xr&y->irb|O zzSGnBm52CV_u$bRjozte;5TYvA9N6Oz7xcdflPNaQ5&pF5A^%9Y~brWL+I!?jPmy! zJfX&dTrpOSw6k~68yKthZ!^z)vyhE}gKjUz&EzDrzavvF)eq$#wP&cww|luY(iGh` zFbkB=L@hGcIieoTxCcz9*WkSRy|74p)tl4e+&MPH%^ZWO_a%Nm!C&AU8}TYpr={fk zPCOJN^}sFrRjv~#u!im<`S6E9V8Z^oADoM~b=1EBT+dORY2TBazsT~1bXbqv`BSWU z9;%#ngCn2?@MZ;l<^XY!pP4z zDd@&sF8P0Oe&W7;@y>IMJUnH8*E69ec(?NGwoA_5`t9fT*Z)!awrlMo>u`R(CF(g3 zfO>Bcvky+k@-=1}hu(yL8rw?E+UmcL>Z@g2|K%YH7s|Ci%eGd2VQpo-QhoZ`R=;Z4 zhOK^AuiEll)vnF1S6 Date: Mon, 8 Aug 2022 09:57:34 +0800 Subject: [PATCH 03/17] build: add win package --- source/os/src/osAtomic.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index a65801df49..a9eb23abba 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -385,7 +385,8 @@ int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) { return _InterlockedExchange((int32_t volatile*)(ptr), (int32_t)(val)); #else return _InterlockedExchange64((int64_t volatile*)(ptr), (int64_t)(val)); -#endif#elif defined(_TD_NINGSI_60) +#endif +#elif defined(_TD_NINGSI_60) return atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val); #else return __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST); @@ -542,7 +543,8 @@ int64_t atomic_fetch_add_64(int64_t volatile* ptr, int64_t val) { return _InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)(val)); #else return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), (int64_t)(val)); -#endif#elif defined(_TD_NINGSI_60) +#endif +#elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); #else return __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST); From 43026ac31dec1ad19083e047db29c9287aed34d3 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 8 Aug 2022 11:09:39 +0800 Subject: [PATCH 04/17] refactor(stream) --- docs/zh/07-develop/07-tmq.md | 6 +- include/common/tcommon.h | 1 + include/common/tmsg.h | 2 +- include/libs/stream/tstream.h | 29 +++-- source/client/src/tmq.c | 3 +- source/dnode/mnode/impl/inc/mndDef.h | 3 +- source/dnode/mnode/impl/src/mndDef.c | 4 +- source/dnode/mnode/impl/src/mndScheduler.c | 21 ++-- source/dnode/mnode/impl/src/mndStream.c | 8 +- source/libs/stream/CMakeLists.txt | 3 +- source/libs/stream/src/streamMeta.c | 16 +-- source/libs/stream/src/streamRecover.c | 121 +++++++++++++-------- source/libs/stream/src/streamTask.c | 4 +- 13 files changed, 123 insertions(+), 98 deletions(-) diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.md index 7faccdcec1..459b88085f 100644 --- a/docs/zh/07-develop/07-tmq.md +++ b/docs/zh/07-develop/07-tmq.md @@ -6,11 +6,11 @@ title: 数据订阅 为了帮助应用实时获取写入 TDengine 的数据，或者以事件到达顺序处理数据，TDengine提供了类似消息队列产品的数据订阅、消费接口。这样在很多场景下，采用 TDengine 的时序数据处理系统不再需要集成消息队列产品，比如 kafka, 从而简化系统设计的复杂度，降低运营维护成本。 -与 kafka 一样，你需要定义 topic, 但 TDengine 的 topic 可以是一张超级表，或一张子表。不仅如此，你可以通过标签、表名、列、表达式等多种方法过滤所需数据，并且支持对数据进行函数变换、预处理（包括标量udf计算）。与其他消息队列软件相比，这是 TDengine 数据订阅功能的最大的优势，它提供了更大的灵活性，数据的颗粒度可以由应用随时调整，而且数据的过滤交给 TDengine，而不是应用完成，有效的减少传输的数据量。 +与 kafka 一样，你需要定义 topic, 但 TDengine 的 topic 是基于一个已经存在的超级表、子表或普通表的查询条件，即一个 SELECT 语句。你可以使用 SQL 对标签、表名、列、表达式等条件进行过滤，以及对数据进行标量函数与 UDF 计算（不包括数据聚合）。与其他消息队列软件相比，这是 TDengine 数据订阅功能的最大的优势，它提供了更大的灵活性，数据的颗粒度可以由应用随时调整，而且数据的过滤与预处理交给 TDengine，而不是应用完成，有效的减少传输的数据量与应用的复杂度。 -消费者订阅 topic 后，可以实时获得最新的数据。多个消费者可以组成一个消费者组 (consumer group), 一个消费者组里的多个消费者共享消费进度，便于多线程分布式的消费数据，提高数据通吐率。但不同消费者组即使消费同一个topic, 并不共享消费进度。一个消费者组可以订阅多个 topic。如果订阅的是超级表，数据可能会分布在多个不同的vnode上，也就是多个shard上，这样一个消费组里有多个消费者可以提高消费效率。TDengine 的消息队列提供了消息的ACK机制，在宕机、重启等复杂环境下确保at least once消费。 +消费者订阅 topic 后，可以实时获得最新的数据。多个消费者可以组成一个消费者组 (consumer group), 一个消费者组里的多个消费者共享消费进度，便于多线程、分布式地消费数据，提高消费速度。但不同消费者组中的消费者即使消费同一个topic, 并不共享消费进度。一个消费者可以订阅多个 topic。如果订阅的是超级表，数据可能会分布在多个不同的 vnode 上，也就是多个 shard 上，这样一个消费组里有多个消费者可以提高消费效率。TDengine 的消息队列提供了消息的ACK机制，在宕机、重启等复杂环境下确保 at least once 消费。 -为了实现上述功能，TDengine 采用了灵活的 WAL (Write-Ahead-Log) 文件切换与保留机制：可以按照时间或文件大小来保留WAL文件（详见create database语句）。在消费时，TDengine 从 WAL 中获取数据，并经过过滤、变换等操作，将数据推送给消费者。 +为了实现上述功能，TDengine 会为 WAL (Write-Ahead-Log) 文件自动创建索引以支持快速随机访问，并提供了灵活可配置的文件切换与保留机制：用户可以按需指定 WAL 文件保留的时间以及大小（详见 create database 语句）。通过以上方式将 WAL 改造成了一个保留事件到达顺序的、可持久化的存储引擎（但由于 TSDB 具有远比 WAL 更高的压缩率，我们不推荐保留太长时间，一般来说，不超过几天）。 对于以 topic 形式创建的查询，TDengine 将对接 WAL 而不是 TSDB 作为其存储引擎。在消费时，TDengine 根据当前消费进度从 WAL 直接读取数据，并使用统一的查询引擎实现过滤、变换等操作，将数据推送给消费者。 本文档不对消息队列本身的基础知识做介绍，如果需要了解，请自行搜索。 diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 7d78c2dc2f..be18ef1fc0 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -104,6 +104,7 @@ typedef struct SDataBlockInfo { uint32_t capacity; // TODO: optimize and remove following int64_t version; // used for stream, and need serialization + int64_t ts; // used for stream, and need serialization int32_t childId; // used for stream, do not serialize EStreamType type; // used for stream, do not serialize STimeWindow calWin; // used for stream, do not serialize diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3c3071c8df..9eabeb4f58 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3075,7 +3075,7 @@ typedef struct { void* msg; } SBatchRsp; -static FORCE_INLINE void tFreeSBatchRsp(void *p) { +static FORCE_INLINE void tFreeSBatchRsp(void* p) { if (NULL == p) { return; } diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 103ca6a4f0..239fcdad8d 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -17,6 +17,7 @@ #include "os.h" #include "query.h" #include "tdatablock.h" +#include "tdbInt.h" #include "tmsg.h" #include "tmsgcb.h" #include "tqueue.h" @@ -85,6 +86,12 @@ enum { TASK_OUTPUT__FETCH, }; +enum { + STREAM_QUEUE__SUCESS = 1, + STREAM_QUEUE__FAILED, + STREAM_QUEUE__PROCESSING, +}; + typedef struct { int8_t type; } SStreamQueueItem; @@ -123,12 +130,6 @@ typedef struct { SSDataBlock* pBlock; } SStreamTrigger; -enum { - STREAM_QUEUE__SUCESS = 1, - STREAM_QUEUE__FAILED, - STREAM_QUEUE__PROCESSING, -}; - typedef struct { STaosQueue* queue; STaosQall* qall; @@ -233,6 +234,7 @@ typedef struct { typedef struct SStreamTask { int64_t streamId; int32_t taskId; + int32_t totalLevel; int8_t taskLevel; int8_t outputType; int16_t dispatchMsgType; @@ -458,9 +460,20 @@ int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); int32_t streamTryExec(SStreamTask* pTask); int32_t streamSchedExec(SStreamTask* pTask); -typedef struct SStreamMeta SStreamMeta; +typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask); -SStreamMeta* streamMetaOpen(); +typedef struct SStreamMeta { + char* path; + TDB* db; + TTB* pTaskDb; + TTB* pStateDb; + SHashObj* pTasks; + void* ahandle; + TXN txn; + FTaskExpand* expandFunc; +} SStreamMeta; + +SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc); void streamMetaClose(SStreamMeta* streamMeta); int32_t streamMetaAddTask(SStreamMeta* pMeta, SStreamTask* pTask); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index f7d45dc6ff..073f772ee4 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1699,7 +1699,8 @@ int32_t tmq_consumer_close(tmq_t* tmq) { tmq_list_destroy(lst); - return rsp; + /*return rsp;*/ + return 0; } // TODO: free resources return 0; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 06c64dcea6..c4da9b5c3d 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -604,11 +604,11 @@ typedef struct { int64_t createTime; int64_t updateTime; int32_t version; + int32_t totalLevel; int64_t smaId; // 0 for unused // info int64_t uid; int8_t status; - int8_t isDistributed; // config int8_t igExpired; int8_t trigger; @@ -647,7 +647,6 @@ typedef struct { typedef struct { int64_t uid; int64_t streamId; - int8_t isDistributed; int8_t status; int8_t stage; } SStreamRecoverObj; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index abac0573da..08ce161409 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -23,11 +23,11 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1; if (tEncodeI32(pEncoder, pObj->version) < 0) return -1; + if (tEncodeI32(pEncoder, pObj->totalLevel) < 0) return -1; if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1; if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->isDistributed) < 0) return -1; if (tEncodeI8(pEncoder, pObj->igExpired) < 0) return -1; if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; @@ -69,11 +69,11 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1; if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; + if (tDecodeI32(pDecoder, &pObj->totalLevel) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->isDistributed) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->igExpired) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 218f82df18..a24b7ef459 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -307,10 +307,9 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); - ASSERT(totLevel <= 2); - pStream->tasks = taosArrayInit(totLevel, sizeof(void*)); - pStream->isDistributed = totLevel == 2; + int32_t planTotLevel = LIST_LENGTH(pPlan->pSubplans); + ASSERT(planTotLevel <= 2); + pStream->tasks = taosArrayInit(planTotLevel, sizeof(void*)); bool hasExtraSink = false; bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; @@ -320,7 +319,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { bool multiTarget = pDbObj->cfg.numOfVgroups > 1; - if (totLevel == 2 || externalTargetDB || multiTarget) { + if (planTotLevel == 2 || externalTargetDB || multiTarget) { /*if (true) {*/ SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); taosArrayPush(pStream->tasks, &taskOneLevel); @@ -338,8 +337,9 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } } } + pStream->totalLevel = planTotLevel + hasExtraSink; - if (totLevel > 1) { + if (planTotLevel > 1) { SStreamTask* pInnerTask; // inner level { @@ -371,13 +371,6 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } -#if 0 - SDbObj* pSourceDb = mndAcquireDb(pMnode, pStream->sourceDb); - ASSERT(pDbObj != NULL); - sdbRelease(pSdb, pSourceDb); - pInnerTask->numOfVgroups = pSourceDb->cfg.numOfVgroups; -#endif - if (tsSchedStreamToSnode) { SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode); if (pSnode == NULL) { @@ -464,7 +457,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } } - if (totLevel == 1) { + if (planTotLevel == 1) { SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); taosArrayPush(pStream->tasks, &taskOneLevel); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 902879701c..ba9bb2982f 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -36,7 +36,7 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); -static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq); +/*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/ static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -55,7 +55,7 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq); - mndSetMsgHandle(pMnode, TDMT_MND_RECOVER_STREAM, mndProcessRecoverStreamReq); + /*mndSetMsgHandle(pMnode, TDMT_MND_RECOVER_STREAM, mndProcessRecoverStreamReq);*/ mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp); @@ -540,6 +540,7 @@ static int32_t mndPersistTaskRecoverReq(STrans *pTrans, SStreamTask *pTask) { return 0; } +#if 0 int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { if (pStream->isDistributed) { int32_t lv = taosArrayGetSize(pStream->tasks); @@ -573,6 +574,7 @@ int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStrea } return 0; } +#endif int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t lv = taosArrayGetSize(pStream->tasks); @@ -755,6 +757,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } +#if 0 static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; @@ -817,6 +820,7 @@ static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } +#endif int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { SSdb *pSdb = pMnode->pSdb; diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 33e864158a..ceddf4f215 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -8,7 +8,8 @@ target_include_directories( target_link_libraries( stream - PRIVATE os util transport qcom executor tdb + PUBLIC tdb + PRIVATE os util transport qcom executor ) if(${BUILD_TEST}) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 7dfeefb261..085a0e4ce7 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -14,22 +14,8 @@ */ #include "executor.h" -#include "tdbInt.h" #include "tstream.h" -typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask); - -typedef struct SStreamMeta { - char* path; - TDB* db; - TTB* pTaskDb; - TTB* pStateDb; - SHashObj* pTasks; - void* ahandle; - TXN txn; - FTaskExpand* expandFunc; -} SStreamMeta; - SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc) { SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { @@ -150,7 +136,7 @@ int32_t streamMetaAbort(SStreamMeta* pMeta) { return 0; } -int32_t streamRestoreTask(SStreamMeta* pMeta) { +int32_t streamLoadTasks(SStreamMeta* pMeta) { TBC* pCur = NULL; if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { ASSERT(0); diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 3530c05688..2a77ce8a91 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -87,53 +87,80 @@ int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp return 0; } -int32_t streamProcessFailRecoverReq(SStreamTask* pTask, SMStreamTaskRecoverReq* pReq, SRpcMsg* pRsp) { -#if 0 - if (pTask->taskStatus != TASK_STATUS__FAIL) { - return 0; - } +typedef struct { + int32_t vgId; + int32_t childId; + int64_t ver; +} SStreamVgVerCheckpoint; - if (pTask->isStreamDistributed) { - if (pTask->taskType == TASK_TYPE__SOURCE) { - pTask->taskStatus = TASK_STATUS__PREPARE_RECOVER; - } else if (pTask->taskType != TASK_TYPE__SINK) { - pTask->taskStatus = TASK_STATUS__PREPARE_RECOVER; - bool hasCheckpoint = false; - int32_t childSz = taosArrayGetSize(pTask->childEpInfo); - for (int32_t i = 0; i < childSz; i++) { - SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i); - if (pEpInfo->checkpointVer == -1) { - hasCheckpoint = true; - break; - } - } - if (hasCheckpoint) { - // load from checkpoint - } else { - // recover child - } - } - } else { - if (pTask->taskType == TASK_TYPE__SOURCE) { - if (pTask->checkpointVer != -1) { - // load from checkpoint - } else { - // reset stream query task info - // TODO get snapshot ver - pTask->recoverSnapVer = -1; - qStreamPrepareRecover(pTask->exec.executor, pTask->startVer, pTask->recoverSnapVer); - pTask->taskStatus = TASK_STATUS__RECOVERING; - } - } - } - - if (pTask->taskStatus == TASK_STATUS__RECOVERING) { - if (streamPipelineExec(pTask, 100) < 0) { - // set fail - return -1; - } - } - -#endif +int32_t tEncodeSStreamVgVerCheckpoint(SEncoder* pEncoder, const SStreamVgVerCheckpoint* pCheckpoint) { + if (tEncodeI32(pEncoder, pCheckpoint->vgId) < 0) return -1; + if (tEncodeI32(pEncoder, pCheckpoint->childId) < 0) return -1; + if (tEncodeI64(pEncoder, pCheckpoint->ver) < 0) return -1; + return 0; +} + +int32_t tDecodeSStreamVgVerCheckpoint(SDecoder* pDecoder, SStreamVgVerCheckpoint* pCheckpoint) { + if (tDecodeI32(pDecoder, &pCheckpoint->vgId) < 0) return -1; + if (tDecodeI32(pDecoder, &pCheckpoint->childId) < 0) return -1; + if (tDecodeI64(pDecoder, &pCheckpoint->ver) < 0) return -1; + return 0; +} + +typedef struct { + int64_t streamId; + int64_t checkTs; + int64_t checkpointId; + int32_t taskId; + SArray* checkpointVer; // SArray +} SStreamAggVerCheckpoint; + +int32_t tEncodeSStreamAggVerCheckpoint(SEncoder* pEncoder, const SStreamAggVerCheckpoint* pCheckpoint) { + if (tEncodeI64(pEncoder, pCheckpoint->streamId) < 0) return -1; + if (tEncodeI64(pEncoder, pCheckpoint->checkTs) < 0) return -1; + if (tEncodeI64(pEncoder, pCheckpoint->checkpointId) < 0) return -1; + if (tEncodeI32(pEncoder, pCheckpoint->taskId) < 0) return -1; + int32_t sz = taosArrayGetSize(pCheckpoint->checkpointVer); + if (tEncodeI32(pEncoder, sz) < 0) return -1; + for (int32_t i = 0; i < sz; i++) { + SStreamVgVerCheckpoint* pOneVgCkpoint = taosArrayGet(pCheckpoint->checkpointVer, i); + if (tEncodeSStreamVgVerCheckpoint(pEncoder, pOneVgCkpoint) < 0) return -1; + } + return 0; +} + +int32_t tDecodeSStreamAggVerCheckpoint(SDecoder* pDecoder, SStreamAggVerCheckpoint* pCheckpoint) { + if (tDecodeI64(pDecoder, &pCheckpoint->streamId) < 0) return -1; + if (tDecodeI64(pDecoder, &pCheckpoint->checkTs) < 0) return -1; + if (tDecodeI64(pDecoder, &pCheckpoint->checkpointId) < 0) return -1; + if (tDecodeI32(pDecoder, &pCheckpoint->taskId) < 0) return -1; + int32_t sz; + if (tDecodeI32(pDecoder, &sz) < 0) return -1; + for (int32_t i = 0; i < sz; i++) { + SStreamVgVerCheckpoint oneVgCheckpoint; + if (tDecodeSStreamVgVerCheckpoint(pDecoder, &oneVgCheckpoint) < 0) return -1; + taosArrayPush(pCheckpoint->checkpointVer, &oneVgCheckpoint); + } + return 0; +} + +int32_t streamRecoverSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { + ASSERT(pTask->taskLevel == TASK_LEVEL__SINK); + // load status + void* pVal = NULL; + int32_t vLen = 0; + if (tdbTbGet(pMeta->pStateDb, &pTask->taskId, sizeof(void*), &pVal, &vLen) < 0) { + return -1; + } + SDecoder decoder; + tDecoderInit(&decoder, pVal, vLen); + SStreamAggVerCheckpoint aggCheckpoint; + tDecodeSStreamAggVerCheckpoint(&decoder, &aggCheckpoint); + /*pTask->*/ + return 0; +} + +int32_t streamRecoverTask(SStreamTask* pTask) { + // return 0; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 3a54981989..8b5bd849f6 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -52,6 +52,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { /*if (tStartEncode(pEncoder) < 0) return -1;*/ if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->totalLevel) < 0) return -1; if (tEncodeI8(pEncoder, pTask->taskLevel) < 0) return -1; if (tEncodeI8(pEncoder, pTask->outputType) < 0) return -1; if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; @@ -62,7 +63,6 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; - /*if (tEncodeI32(pEncoder, pTask->numOfVgroups) < 0) return -1;*/ int32_t epSz = taosArrayGetSize(pTask->childEpInfo); if (tEncodeI32(pEncoder, epSz) < 0) return -1; @@ -101,6 +101,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { /*if (tStartDecode(pDecoder) < 0) return -1;*/ if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->totalLevel) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->taskLevel) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->outputType) < 0) return -1; if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; @@ -111,7 +112,6 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; - /*if (tDecodeI32(pDecoder, &pTask->numOfVgroups) < 0) return -1;*/ int32_t epSz; if (tDecodeI32(pDecoder, &epSz) < 0) return -1; From e8ead35bd81dda99df2c7163c880c9ff6dda4688 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 8 Aug 2022 11:34:29 +0800 Subject: [PATCH 05/17] fix multi process problem --- source/libs/transport/src/transCli.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 0b8829a0c0..e1df181329 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -994,6 +994,8 @@ static void cliAsyncCb(uv_async_t* handle) { if (count >= 2) { tTrace("cli process batch size:%d", count); } + // if (!uv_is_active((uv_handle_t*)pThrd->prepare)) uv_prepare_start(pThrd->prepare, cliPrepareCb); + if (pThrd->stopMsg != NULL) cliHandleQuit(pThrd->stopMsg, pThrd); } static void cliPrepareCb(uv_prepare_t* handle) { @@ -1089,7 +1091,7 @@ static SCliThrd* createThrdObj() { pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); uv_prepare_init(pThrd->loop, pThrd->prepare); pThrd->prepare->data = pThrd; - uv_prepare_start(pThrd->prepare, cliPrepareCb); + // uv_prepare_start(pThrd->prepare, cliPrepareCb); int32_t timerSize = 512; pThrd->timerList = taosArrayInit(timerSize, sizeof(void*)); @@ -1126,7 +1128,6 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosMemoryFree(timer); } taosArrayDestroy(pThrd->timerList); - taosMemoryFree(pThrd->prepare); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); From 536c6bdbc5e732ffba754e322e12c587ae251762 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 8 Aug 2022 11:40:48 +0800 Subject: [PATCH 06/17] fix: add checks for stream query --- source/common/src/systable.c | 8 ++++---- source/libs/parser/src/parTranslater.c | 23 +++++++++++++++++------ source/libs/planner/src/planSpliter.c | 2 +- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index a79082ab23..b779f21cf5 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -16,8 +16,8 @@ #include "systable.h" #include "taos.h" #include "tdef.h" -#include "types.h" #include "tgrant.h" +#include "types.h" #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) @@ -97,7 +97,7 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "wal_retention_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "wal_retention_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "wal_roll_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "wal_seg_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "wal_segment_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, }; static const SSysDbTableSchema userFuncSchema[] = { @@ -243,8 +243,8 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)}, {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)}, {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)}, -// {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, -// {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, + // {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, + // {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema)}, {TSDB_INS_TABLE_DATABASES, userDBSchema, tListLen(userDBSchema)}, {TSDB_INS_TABLE_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9971f20d3d..be16289595 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4708,6 +4708,12 @@ static int32_t translateKillTransaction(STranslateContext* pCxt, SKillStmt* pStm return buildCmdMsg(pCxt, TDMT_MND_KILL_TRANS, (FSerializeFunc)tSerializeSKillTransReq, &killReq); } +static bool crossTableWithoutAggOper(SSelectStmt* pSelect) { + return NULL == pSelect->pWindow && !pSelect->hasAggFuncs && !pSelect->hasIndefiniteRowsFunc && + !pSelect->hasInterpFunc && TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType && + !isPartitionByTbname(pSelect->pPartitionByList); +} + static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { if (NULL != pStmt->pOptions->pWatermark && (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pWatermark))) { @@ -4723,14 +4729,19 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt return TSDB_CODE_SUCCESS; } - if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) { - SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; - if (QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable)) { - return TSDB_CODE_SUCCESS; - } + if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); } - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + + if (QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable) || + TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || + !pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); + } + + return TSDB_CODE_SUCCESS; } static void getSourceDatabase(SNode* pStmt, int32_t acctId, char* pDbFName) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index e10f0586ca..97977878ad 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -268,7 +268,7 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { case QUERY_NODE_LOGIC_PLAN_JOIN: return stbSplNeedSplitJoin(streamQuery, (SJoinLogicNode*)pNode); case QUERY_NODE_LOGIC_PLAN_PARTITION: - return stbSplIsMultiTbScanChild(streamQuery, pNode); + return streamQuery ? false : stbSplIsMultiTbScanChild(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_AGG: return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: From 819a48a4646f8136065de66a9ab2ea179a818db4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 8 Aug 2022 11:46:31 +0800 Subject: [PATCH 07/17] fix: skip 0 row blocks to dismiss crashing --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a9f07cbf24..c40fb98d62 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -115,12 +115,19 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { TSDBROW row = tsdbRowFromBlockData(&pReader->oBlockData, iRow); int64_t version = TSDBROW_VERSION(&row); + tsdbTrace("vgId:%d, vnode snapshot tsdb read for %s, %" PRId64 "(%" PRId64 " , %" PRId64 ")", + TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path, version, pReader->sver, pReader->ever); + if (version < pReader->sver || version > pReader->ever) continue; code = tBlockDataAppendRow(&pReader->nBlockData, &row, NULL); if (code) goto _err; } + if (pReader->nBlockData.nRow <= 0) { + continue; + } + // org data // compress data (todo) int32_t size = sizeof(TABLEID) + tPutBlockData(NULL, &pReader->nBlockData); @@ -808,7 +815,8 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { if (code) goto _err; _exit: - tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); + tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode), + pWriter->pTsdb->path); return code; _err: From cdd40e2863159650e4edd77bb3dd323fb11960be Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 8 Aug 2022 12:57:09 +0800 Subject: [PATCH 08/17] fix multi process problem --- source/libs/transport/src/.transSvr.c.swo | Bin 53248 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 source/libs/transport/src/.transSvr.c.swo diff --git a/source/libs/transport/src/.transSvr.c.swo b/source/libs/transport/src/.transSvr.c.swo deleted file mode 100644 index c9486c50867994926ea395577110ee4d15de5632..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53248 zcmeI53wT^tb?-%a7AT~=pY4fLyOJWyj|7L=0wbx#I@3q!md!Mh)P)+$?C_2p){vSqP4*V@u< zmTIj=z1iNJmtv`m@8|cE|nugzLw{-#a?)|8&Rw zrBI=m|9v~|f34&G{lfjl@b~cD{Y!lQ{^9y_!{3i|+z&td&I$LQ6c`qt50}^ftaD(U z1M3`E=fFA#);X}wfpre7b6}kV>l|3;z&Z#1OE^%P@9Wz@$vb2Zv;H6L|NnSkU*8A8 zGB_9f!vp&Iz6M?n_JLmnzX1N~XZreH0$vPW1Wtmdg9?}j=Ym1-K=1(Y9~jbK0q+5C z0j~tV2~L9Rz>~ox;9M{W9tysLasDOn2jJ!4Bv=3yFbM|10JuN+E{6W6!Kc9c!Mng~ zK?Pg_?gxI1v*1VI@4+X)AA`4mmw=nVb3qkc0nP_M2kr~Lj}zjH;5P6!@O$8O;8(zn zpazz}72qQ9i{K&P!Qek|aC{B?Irts$Yv5Nw3(SK3;6m_y93}q*z6L%IJ_k;K$AVAd zWO)O)4r~K=;A40*Xn{+?eZalJJ;0}NNPHaJ0-g%0;5=|2aBuJ#+WVD2ev(CS0gwzl z4y@tRYM0v8a&oL*ollmJ9jLdf3n%tfYu8=ctY25D&D2L4js9tZH%9BV+F(-V|AAc) zT6k)@U23+cE4BHlO3PF4Zygy-mRd(hTTD9HtTal^iW2WBwM&+DOgsbuvvbP}v+cp; z_*~bcg;K_A?Mky&TFf$Nd2t=(UDm8rM(6s;_-Ls%zgSV;{;ezjL#4&*dM6z+_6#Pa zrYh7?!XctyMi!uErqsGl1t}~>#WX9|4<*%FI}rz#TixZ6Zcn|(Q>07=w}7NwU8-z` z=dIL&pVftY zC+*b)mbcpVjO5*SGMS#PG>=u96LU|cTO@P!`XZuF)6CNo6Ujp=myMjr?n;s^Tau?P zBQkW8>zAwTWT9DK^8ZRE+OgA+34Q6j?T8v5MXjlz{IzPst{j_fAR@HVV4^RDYkHDv z`>HUiW|U29V3;}DoL6%GBE+E%8WZBDw!Ad4pp=#76e&(9)2{pebgT28dVZxjK7WI5 zP}|%{KpqSAz@No>y&=)k-#~M_%oRvz8EbWMX5I_oV%uol)z?1JsLWRu7`~dz<#w(e zk{g-1leB(%+?vJMtY4Z;RB1&4j?-&(?fT`)a>eEk<4!DZ|8l!>gJz2TZL5YSg9`g_ zUC&P2pfX9*4|3VsTR(J*_K)@BR<4Tm*6hN(H|YYRVktxOkucwY3@jRils0~4=++~V zNIkGJUt&84wlG&Rp&8l60DjEC$wchb)nW~;oava*%88XuCtY1_s7v)|6Duq$*;7%G zqot-ED_>VRad>8W&!w}IBh%A|CZ_hx?i)LBt@n;{wQ;o4)WkxnzU-k^rQEEvdp(v* zw2bU{l7k1vbD8RTGQKBSs5g6RFQ!oa+n#E(zICVeihc4N-;oWKwsO2P&5 zklHg)iq+hs z{BlW;(*vxEx~{|;Fz9@ZnX-)`GqY1;m(A`O8{IcDHP%1SJyy8Fmj7>=>(VOw}NDgh4$|`sCtdLumu`mn5L` z#_l~)E7LtSSnaRRFE3WcRqDmbEgvDt%HGVZ$iR$8pnxsF%cN9hXHR?=wJk2Fh5!^!x9NMeSqRFaF1 zw%d)JTeciOeth^yZF#ufJVFq4xl(IYT3dDxOAWlLzMQm<)|VIO(Or*KXzX&OdJG+~ z#E8{6k<=Hgt6hHJU^1c!y`EfNsa2Y##bk1Mj=Yk6o*#KByUWIB4vbAtCznl3B_qk? z$kfdE=s_AXnLIc(IWawEWsV%UDmgSVH8pZz=Biy*2b7hha;#FbvZ_ms#VQmWFEwFY z+i2K7HZ{7J^dpy!?;D@FiiT5ZGr5{4Fkzl4vRR#D+?Nh(nH{Zbd^}oeCoukab#XD7 zt0W903(Jc`$ZQYt8%8dl+_zz1h&dr?sH&}_Ls;f%_Ik~#M?w514~X7$L?cG6!SNVad?c40CzGBKUVRvJpiYvtkO zqNg5hm+F^*7WF9&m+MQrrTEMKe+72)ZtQH?|K|kb?=#r^?*K0V3!nn_fo)(bcp$g~ z`~L^v1K^$DRp6DN2JQ#Gg1!Gva0_@7cqMoW7zGak_XPh&xqk}e3wRZfFW?p6GB5^2 z*TrD{v(ABa4y@xaKp+6?{~xs^5yC6ybF^5m=+uzhD| zSyq#&%3`I|s*K&>6!*vKO;r~5A8GevJ(|bKDEP?{w{54~v4Im?t+w(woDOkDxVb{L z;LtCp^8DuAsXKHi*|fBI_w@x%Zfu!PQ>?`4(KKgx4^CpAE-ucM%3`UNvad8(S)_#M z!G%e%dh>2?87so2An)eIXA)@29yD(gWXVCpecV9SimE?XX@4%ZgCj`(w*9tnsEgRQ z`zuTJ=7~{gtC#z?`GQ1!Aujn1KnB^2OkZ_ibarfNYW9g!V>8YbCono@-TyvMrd9`| z(H>R7!}3TDn*@*`_#W}CS=FJ&sN>F{5t)`Zx_4~!Nwau6_Us$$C)>Ob(TB{DnW(+- zW%4u+Z{D4iWPY$Hkb@$tkjA;ZE9;DrZP3$Swso+SK3`_V(I)$!CGo!RX(++|kMaF~ zg?;~4@Juia{uO)vU%)58$HB|Mlfln}zr&8d9y|m*7H#7 z8@>Ua0UiY20s%XL*5c>D9juqX5d0jtl{N9pz?WG6u7RHduVC$a5c~=2+*9Co*0Y}m z9su6N`nA@v|AN5#rvTOKyQ*9IfxmoC`k$ZZ|JU@z?3XG_txCIp6Rl9&yqm6sTX-nh zn#yMWIszIWJB`0tS^a}qKia&zywDiP^+?7|+*&o9h?^u5DKW!{8ol+v!F~HqU3^wn z&))3$W<=bX>ZCQ#%8J9hve(Ge^Z|M(x@mn?Dk(J@3`a_I>WZYL_?K8#bpbzGo;h+T zYmPdDpIFw(GnnT}S;eyAL~@rJWTisF^;nj8c(zRnwjeOM+&UV?KP-PB{I(7me6w4D`7)Xm1xMt~U;-5#v4Ns9dobF?@?R!qOP|gr~ONaN*;&UvP3)63a?w zyf`DD(O&G;k7b<}Gwx*KSeA7yX~de`?Z+tk@MQ!m>}46Rzu#7{29EOm&dnzxuTa(~ zg5LCH!`P9->fvdRv%RVw%kpv87hP#qDn$&-8SZ1TEXjt+Rgfym6T?tD{rZ{c)v>Hz zbw1PAVp-iYWmxzNQzlaxX7;B+(Yk^Y)={o0}!2zYS2mQYO#Yf=nMu-H9bF|2lX zrj~VNY12TSU?-t-*`m3kWe${fvKZLf zKWHPj8Rwq-@U;WcP&E?`w@GG?QlWW-JGn8-ywy--oj-d8;cD5hFlr9Pum!O(O&X~> zMOj5^V>w;ny4^&s^q}?HJ5{JEcghE2UFDAI7?@TMMz-di&m;r>V4+!g5|lK8Idlr+ zW#=;;sR=<)&|z8AX0m0_{~k5(SBOm++(I7-Ip)ZNaDquCo-X-Ek@tD;P35DN@^#7a zqZPsrnn`I<@_j-zok!;-w4&0kBn|vAm3GY6C=C^xMvu%`7y_TwK?ptR`@A-)WB~zK3t&GvHm|CEzMB4JN=j;79lzJ_LRX zycs+Wd<_3X30w)jkN@EX;341(_#kcr9{}$MzYP>?zX{wMh~J+R&wnBxoT3_9(G6X> z?$VPRdri@ap=Es*H+9~yZ0TgFdV@lA6o1rE$Vy95RE)6Ohm*Quo?5sO6NU9Ckc44> zvFhH!Fe#hgwT%1{6sxG)r6qz_5a~q)3zE&Y;dHES5Diqe=$aU)r1CBQCVK)XE=(T& zWFWd$*9gQoxNS(DJh_T3LIRh;ME;1cp66jPOj>1LSjUEr+VQjLR%M$5@frvvro67hU!$OVy?g1i;C3mHcn zh+QL+2YS_}kko!Fq&B*>6-AwS^+*|t%&sY|cSrvps`pgpmXGvru<-z=U8zh^J<)86 zwu>5@IqWR2v=Ob#u25O1h$Y9$nwdD_S&IgOKE)9qiHKtIB$A zp{MymfX3K@T^t_LjEt#Fbs)y-qD~q-9w8(B60Pa&kaLfsa*0%cqD<3F*w92YYd)Va z%8|&j)&mTB@(e&kWD14JQqqIbHysJd`=#SGEt*6zAeWSAt6(qV#TXF=T4w_zze0j7 zTG%!__4oH@y6r&VT#SJp*t}a-#!#Y=Vkx!Kf_>Xl{FUyWGTM~AQM-cFgyPjQv#mC^ zj+B2a0pBnw^T?yjzo;7!Dtaq_>Gi^^lFB*k&aXt5av$eqVPA_Gg;XIS3Uq z6MH85=Q;iOMD=K?KCp95yS3cpku=d-<;IEF+Z8sSPG|muIxSXev7L3zi_YYihKJ&# z?Rt&1x>Ty8IHZzg?zuv$bb3skYI+*`Pk+d=zj%EBAu|EVZ;H{HmJ`sRhtSH}_u`uf zG``q}B;vT;pCdZG5786*xj>jQC zDh-o6gCLqjiT!2As_hf0k3>YIdn@Q;#zRx*T}FBrKN7N0v^@Ds=t2a2x}gwi5DPRH zbgiV-sFbS5}Kh`y#hZ+&fxYhwSaVl<40MjPx`kG@Jqi>{}cUCN9u4ZtmqR$u!c{3GJ4Aj zUW3%Q(jG8&03NB;i3Ue7H{jj!CpC-6B4_cOGziri%e-^mrOa2c5}($7}kZv3?f0sur5gZ;&33TbIMC z*_vehRE)XC*8eft`#v7*|7x&Dwf6s9a5cCJd|Bj9H6dhl#84}J;Ef^qOL@HsMj zJva=m1{=WH;CAf)Uj;7$F9er@3&A76_ptqM1HTVm1l7zY4q<41rDHQQ%?V zr@$T9{NDrL1+NCr2Q_dx=m-B3oBtN@WRQSIfCqseVC#Pud>gz9{0h+i`zL`Xg0sOR zz#Z89p9Y@W^0TicjKDZnB1N;NW z!FAwLa1Pi2ZpSb1QE)T(4e)GmHJAi@!KL6F@Ccyx`pZuLfttHL)@;_BnZ}*PLfJ0k zV6<;+g^5kt)NG~MoUP%r?RSOR6^4QZXR6gf_EhKW4^0*}(a;O~T+-s-?24|hB*_e? z;d!Mhl&1^DOxBu>>;6(9&c4{=8O+Y{4AGxVw@BS!16DCxq9$}LE^K)hYoQ54UtfF_ z-lkffR^AkcmZLxDpO)7o3T6oQ2zeC}W^7X4Q2~!r^C^Nua`Yq^IWtlMv|aYj#d;KS zbNx*Y{g^3gmXnoT^7%uAJFES^?|*dD$?v%~5u;t?q98N=w`H1qCi!~fuCiTp+g42A zoAPWEANg5GoR_Eh*s}Mf`7CNo=C79#?L{|^gTa`m*@-+I>B!2?MLUcwHQFaS1t2R2 zb8 zG;&~P${*(@Q(h}pNf~3!tw3#b)$+sDU{OVAxC-!8yLRc(R-dV`cXW#Iy?*1s zbk8*%mXNCT9V4K0X89{2+V%e;4R#Htx^F%z^JeJP8|{pOpu+b)c_EgK?!9^U9C4#+ zrF69Dibd2?*#H~&EIB5SHFW&JqubM(o7O!_!(4jT4_$yrjXXa3xSBgRYydpKPs#A!c_Qk{-!Au`Np%x-4gK`qxY z&L~4<#5RyyUtTspke+6K#gM)z`sUh)O3hlqLu>b$t?NxZvB>jkL>Fgv%UTmVqeLTsW zDV|W(;EttAy&dGKyUJ~`y7B;dSI(M=<0n&N>LQ^RHBrfem&XqrkWl+-cZX*MMou-# zL`$>RKqEDS>$a_YT!%wZT4PkKwLepBv|rUd3SKxsR`nb;48G}vqZsdr-XhH<|9~>EIrW@g%zdZ`h*%UmzO^_t_8FH=!qjF z*a=9V^e?meV?A*iZpNY!lK7QOHf76BS`xyo9R^5KYuF&)aB0!aY_y4dEe6iB+XG@H z(828(*Y;T4Q5HCzz!#hi?GIGwLbcdAgA zH1yH(2y@NarBv(xn7V!EVpD4U|JlL*{|9XTKL@t~odfuM&;+}H_W%Dow*EhZkAhpl zyMWFBXo5+w704g(O>F@;7ZodpA(&_C9Pck*Of8xq9kLvSIZ0ke##x;pw zoOr6eoTt*i>A3Gv3NGWb{nq&O?AZRvnXBw}7%`-G`+|ub6LqkLgZ#SwO&tEEy+iFA zY<%;_t@Lo`dg{fth)_&NCi{A1!FV@UYL|(UXx7V>R!fRb$qW|0sLUfRL%}e+uDW3 zB}S7DBBL{tiejTvEPDWA@%jfx-cc8R8}WDgf( zrGO|ewHm6PJqHOg0OIkOAsvRC>XTdZ45gu9Om+zxJ=$R%->gY(UxTu6>mcdG%J4-J1_bNQ+cJOTz3;0@GCN#I?3~VQ<;d0*ZPFk&dDmh# zWZ2D4KIY}TgL~=vNtlZUi!y(F&WtcJ3@g&QFz1dO2savo-o{T~!S==R=)Ot9M`Cqm zg2F_KF)L=BMKxultUc~)HJd>pTX{vftO1Qd4(ADnZ{e-9J;1BQgF&mse!2McaHLp+ zsT>)r(uDINNBh&Au1J+`s!wEAZDzmHKtU|BYt}f}&UP3lbVL~@(|sQKqQSDiVTDjL z#lsk1Jz_mw6!1MW#@c*)hq#>(3K231fr6nO()|dbTW2n4->2{dwkhnCvzl7@K;{24 zsiy%LYDg&W->QMzX}2A6%Drs5`V`Rk2YVK~o|+Lgy$VN!0SUTHSz zOUXW3YK=9fD;9EPEpH}oysA0|+Je|hJhOB;Sq{l^fw;28wc&dp_dqTl*M*7=mbqsLZ zHDTqZnSNAKsh; z{glz@a4j-rV$G*Y!xD`oKAQCE&#H!z9hc*xc4ErOE?fUkFz~ddEc-vcUr*=%eF(e{ zyaqf2=={Hnz!q>{@YCSS*!bETumCOscL#roU4JWh1JId&$HArGEO1|NFK`d=aqRoI zgO`FB8!*fFtAXAD5YPU*Ki_p`z@LCu0-aa*a&R>m1NQ`X10TcQe>YIv!0&_Sfd!y_ z1K+{cza4xHd=>mIcp11BTm^Q5v%!zB^*;vQ4Q>U$51tORpWtdBpTNVx!@%dT`TrYu zH+U9!BKSJC{D;7+!HdE3z)e6l{{Z*~cKp}DZ-cjh5ukX1Phzj%23`kV1%4Ae2@HWp zf*)b8e+>K?xEVCTG#CcjZ=iGi?*SA;@FDPe@GS65Fb7ItH&8o29GvRoBf#18DLekS zp@Uv(M`p`9Jhfk!f=fImKzr0x4Jz5C0nA&ZGiSt_fkdLvCGB=ccc#C)=4E0V+<)oC z!&|S_{OGH|{KzNr&=eQ)%ORoCRu?aFO~>W zy$Qi2ELkdXW-Wefj({D?rWj_Ijq~iXaSArOYe8NFBQd2bmeO%qU9IyHis3-AF=Swe z#JeMmv!;zuSVQfwy7-^x!%r<@#vS9{XE>X|!-(@;2zOP}XHp)89hwX?`_qG=MO5TJ zR|c{^)EyV?Z%PPjl5q}8W9%&R6?vdAVs&0<7&?;F!gJ&);gi_FR zmpbK^`r5YU7+yho9jS3;xsY22$diKPe)lD$BIk)0~3%~3HC zWL`PJ&gw{KmJXpY2{!^dN7SOuv)NX6(gDM_0C&EpB9?%jj5ylMP)?x%#(t>bN!DwO zVF?ae8k4q5^+BVP!PO4gURbV`?K~?I(O0d*ily+9BrJ<@+%iI9$?OST?ici#}dd>rRS=PVjz((*WEgRT9IIN8^i^eXqJ+dL*+^Ejy(6>6=nQnbaTs@UC3+c&C{l)uC+K#c&Qq4+ymQq@6SxX~O z{UHO?(+-po`e*o#-RfZ0-3`MtKu<1p5(yDHeB&-cvNpV-IJ2M@+7tFPIzw4Sm& zqmj&UZrkEfd2zam^qBEx(KPW+>YJG+la5H#J4Fkk6hZh)iSdiN3|q zEGUimYZ&6I+Lj%AOg9-((^;;ChOC!ukoI?CnJh_Z*ZTisFa{qYn=ohpe+AqAv*1(U zli-!$M({K+4W_^Vco6toY<|51;Qipu;1=*Ap!Wox1MUUx3BG{M|9PPO0eV;9m0%WdFY%`(A7RQ(zC+4YdCMEo}R5g0BGC{4WK+3SIP#{0Rx4~Dy{{lCI7lLPir-2>dtJwB`3-oTkYry$n1JHSdj|Jbs&Q~13d%?}% z_24<+N^lTd4$cLGK=A;I0T=>$hu@>Yqrf+@`M(T42>uk@3XX#v;1@sweja=s+h6Pa zPX*&(FX#t9#MXZUSOSN^As`>XXJ|*AHSluqQlLEq@&OzMJAvBy!C%{tnH=Hsumq^?xD4{ix#uj?kkAr&P$TwExzjokY($GA4x znRjtwZzdX@*fTaeH90yvIWaXeON79lsqq86NhPEWR&X|v^Og@nn7e0jGIbAu`e8r6 zD1ZsZlXvVA-DJ~gwBsJPZM(0gMfDYzNYvI=sYMrMbJZH5gPq#N3UoZ8GdO-gG~T?&ND)2B(aalYN^->`Mq{^b=08N%^d8|ypr)@zm01U7uS zu8~t4Qe%-jY8Hxg9m32OOYT-n*OTIFE~DVF?MkM>JYB@?3bIa%YkLFRSQ?H4=EW*T zik%iAH?Q{aMfHid?UeNv8IRZSIk<;7b~U7`70mvS)oC?rCs*<0=#TlMLaiYywu96`fi1k%X`bIajO&Jgla<~JhblVOE@mEkw#Vbc`yR&otGo}oN4U73dT*2;SAn`A}t zDyGxv!YI{fSN!DPs9x~)lx&r?tES7q8$B-vcpAGtar^Kpd zlqfePIhzPNFP=dbCYYaq%=YkU)BKrxB*<>=Xe^6}w^s%t&(;!o-l}E9RaKd{&&7Ep zb&+(+xOf@LkQJ2{S2X=qLg>OCQO0jEgH64;;kR>VFNwN`6rxY!8#M=4RRBun_w=!_N?f(+OITx#h_?A1tJ~5 z@^3u#@x}w)y??|W68Wrl)aO)HAVruz)O1D1&=J;~wLHP$NyJbyDVvFCBI805yPlK* z5^K7*?VFYk=k`n)gH0Z~4rL(Rx4cJ)^k$A%z&9Zip}!Jt?Z4@*`KWY_kIQsplXt3q=E35OTM*fThu8#=$c0hFYoehQ37f3V#ak%e zE107vn`ZQEm@@QSj~PQLT;QzMxm?0{AIi(rw61jkOOa}5DnmYNMI1S|Y!dWx;^{Et zU6=yXV(~3DhkKmZ0te-!)w@8MhYOQlaes-`Vhe$^MuvK(J*O8l#u%I|?kJKW2E+}s zHfeT8skyYM7GAnDJuGg9#Z}q<48=2}lSTzJM{!YPP?|^sxJ}@3O;*81z9icG%H-~yoc0DceK{=4AA z;LpHo!E3+~Z~$m8z!vaB?0mfw@GroxgKL1!3D^eo?tp*B&e!<>Hv*j*cmvQm0Ye}G zj{%PcKfvz49o!0@1D*)v6SxG(FYo~HkJ$RRf!_r>NAMPK99#nK3qFgT|61@oa0KWK z!GFQF{~Pev;8Q?*1LOnfT>szBwQb;EvF~pK9|Eri*MP&|=YZk>be3Q|JK!yRe-?Nq zm;-x&;sJgZoCW>{TmOCFW#A>?*T5vW5Ii2-fzAKN;00hOxDeb6d&YQuV`}9%_vaa1_fn_R#`nLre`NTL*_i}Y2-IA14nX}^l zB=cZLZEjI-74b!l!irns87}age`|Svk?Lv23)%;-FCLqk=|?}-GK#tb7lXFEq8YQP z+`eHLO%iK_axGL~wmhemBkZ2?ocG-ZRNGY^ft4MEg*mD? zLAent()O7HrgH-8B05&nhBzCpbDF9yvNrFvjN>5cZ;UQCD&VOcw7 zlNQd_p&FUjUVgW5WI~GUM z;6kixChx4^UT1$)ZMe`2$+a&x(JK1ZUHrKy_uMq6j7K_6EK6{yEbwM6?M+cFBf_z#Sc&5Z73GY*1Ck+PUugQ&XOvYF}&d zF<6|leMJpKd6KA}slB`z-1gLO38GSINV3&D^ABW-lHcptxm4^-K{`gIMCAv*jwQrk z@49Qu35&}_EJua^ajsd-BV&^+6%r7m3BiD;1c! zJBxaBn$U+vCMVyC#)eikPP!3aGm$On5ujA99V;zzD7cqOY5$2*6k1jti+Lvc38yOl znl3eQICDFAa9o^XYwCoYcv4IVY&P08UeO0l(9$Cs=v=k3thtAo)Si2 z_rzJf>FFFha${`j(4pKr8;3DX`&^+QlC7l|ufQ|gtt94oL*|b|PZhM#Rn2^}9y#OT zg$$RWD`@3e^_GJqwYaT2hYi~Et8LJxQw8Tq_|s$b>#=<)HT8Or8$E~R=sTqbejHOa zF8|jIF~LAiv<4r;3K&3AUZ*2Qe39Imh8AQZ7~5)g9F_TVhSh}OZW?8tuHVH|u0n!l z*E`S(H#CezF*Vgd7DIX#M3`z*4>~Db$Cx}gy*G8UOKx!ZgnV-fN5S1L5!DfnzR5Hl z1h4NdwL{U!<}ND%PGlmV3$p8B6dh}-&``zPWEy7&JMlmDerM(6oCt34_^Scv&}Qze z;OGsK=_+KGt5@s)nDBjH4(I|z+YpZzZbk2+yp9M9y}I&1Ka%TKx_GL1g`;?f?@DO zZ1it{_X64N$3O$<9RTNn`-Aslm%jtN6`TZha2nXHW&c|;1k&Ee+b?UPJ*X^dxEcEtG^4p3A`4Z0EfUu;L+f(X*adqmpab^q#YAJ zQ3H*GDE-b}rI~vnYWf<_M7vmu6OA5Ar%KnF`9F7rTOonZrAauvt#Hc&i93#Mvq{@J z+KT6x#d$T;B*W~Nd$!lR<)AY7w6hsM4p}{pe~WC&De9|ns;Iqps^kQbtL`>e=Xx)f z7$l5+He#d^^Ls7A3?;)_Cfnvq(%kwH3n(btnmfglj@_+k1Xz!XP;C$JH095B1@>>> zvVGu`mB>i7s-RS+>Lc2TJ85Iof2>1`RXIQGVWs!iIQ>DfDHv$p);($v<9I_T6i)XL zjTSC!hSF2kCT)_iAheQPgdjeJvAF~n#U{V!ytwGo(In&Xz*dMe{zY}F_>{=#+A$vr zc8hXIGLtUQbg0W|tyZc|&^}!4I~{2>vv6&VnW+xuF^CoR_6#jO63#y`A(p0)uFGkd zTBEx|l{|$5gCWl>Wb=2(#@F#`!5!m5s`$E7*QyUx$WJC#|C;9t;uzYx|RbOtFDrxNGf@N2SS?JgAnieC$wnw}PSSqdUL%;z-KexHvF9v47;rdZCEC-Fff1Q_+#Bu{akE zsoyatefP%S)prhFuj+y|C*!F;YC4?0)0M@AN%%rzkI#oOpjB%$k>>?lQdJHqOVTWd z5?-6Mv8CU`riTs_l0Cd}>*JriId^JYYo@Z)*i$9wU}n$kp7E*au}S2ff~QLhl__+- z{sHY8Q^=C7EvT%+lOr>GXW@Zl?Z&OVFYp|_M~P%>BkLMIPq^k9QFhHWIno+=!ipp$ z@L|-q?@anZRw*lrr0qEWTF9v`@XpHQ(D;Ep6Njdut5=h#2jv5dx)*_nMKuGTd0!pY z*X2zb^wY*RmkowfIH9pRn^YTIsW#Hzs;uOiy!4fx*`sTA$!glF*<&taUT)eEm2=t` zPA})|v0Tobi+tp7E=()dhh6rf7Ov*I7&r1iQznRQA~iC!Fgz7~#c2{zCHwyuu@$GW zCAI$l++hFzAME}A0d(e{_WbVvj|Gne4+sB>4e;yWTCfjX0Cs@C#b@w`K<5LT z0Clh(+zosme}T>lkT2m4;Ps#Zo&+|6yMzCW-{4!|GvJTFd%!EeN$_-VIncR)8^GD% zAMq#L3Z4(H1WyE)fV034Xn(!0@IBz|;3Uww0SAF(V+ySCqd7b1!hXIdgQM;IVjaSF zy2$+X!%Cvp`>2_~#W1+kT*b8WRh004m+d)i+e@&KE*NgBti_l=*9Mm}P13d;N!FcD zn<9I_qK&7StzwK)8orDplhX@5ZZtipNyeXe*2}`*<`#a=`)d`K^SWF(e97M0WS0gu zVY$g@;Z#@1(KQ>SDcBD*O0uJUbd;hmqVh+hP-gWr40UNnIlaQY17ZB{c$tm)DxFU0 zaty-r>^{~XGPSDM5BCjIaPc%*YhI&;O4<=M@ulyBMr1ZyxntjgY?a=r7~MNI`lQ*t zBM0{E8`F-qa1yMMgF>LQdJ;ZyQGhH1ud;HZ>_wk?mc|8qSD!RxYihOf6mL!>G_Vo^ z1k)pb%ykkyJ%z4x%Kn0J7@Q(u3FRlydG0z9|GC}8M80s2dfH(EuZqs2;4#nE*s)rQ z{Ro-KIz^nAp@MHsE>6G!5@TkeLbb9y{jCra_eAz^l6!O6f*<%z?{s z)KKlA*Uv%SGd9{?prv%eGAFl+_T2IT6XwRU(W4_czGr1)H@%a@jvqKv#7^bk8q}Lq zOp^U}GLJbB(S_kkpPOmL>nv)c>nnMOtxF#dq(g#rr`7c?P4q$YiVic23iRa=^T`ByWxwU94LkgU@}%~& zfiJf{!qAXOOV6d^{qp?>4|WeJ$ZSm4lcLs;=R3@a=EbiWxh!M=nH&bZMiO zbY^Pgz%=gzoSvDQxN6oXhRKPEeY3ouy6Xu)(V5C{{*g!dY3axE=@7s$`=%n;A$;L> zm-zJ>K`m|902<0qJJL!l%w#_|1Vy5wGn*!^EV1V=3!hor>$}$aK8mhbrr<#}&S~E4 X8I7VLU*SJlFp83@@A;kVNZtPjn%Mi$ From 0f63837e69b3f620196be6ebdfe583d02a37bc7e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 8 Aug 2022 13:25:02 +0800 Subject: [PATCH 09/17] refactor(sync): make leader life longer --- source/dnode/vnode/src/vnd/vnodeSync.c | 2 ++ source/libs/sync/src/syncMain.c | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index a269f81ddd..50ca81e58b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -722,10 +722,12 @@ void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); } bool vnodeIsLeader(SVnode *pVnode) { if (!syncIsReady(pVnode->sync)) { + vDebug("vgId:%d, vnode not ready", pVnode->config.vgId); return false; } if (!pVnode->restored) { + vDebug("vgId:%d, vnode not restored", pVnode->config.vgId); terrno = TSDB_CODE_APP_NOT_READY; return false; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 389eda0f8b..3d9b34c34c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2668,6 +2668,12 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p syncNodeEventLog(ths, "I am not follower, can not do leader transfer"); return 0; } + + if (!ths->restoreFinish) { + syncNodeEventLog(ths, "restore not finish, can not do leader transfer"); + return 0; + } + syncNodeEventLog(ths, "do leader transfer"); bool sameId = syncUtilSameId(&(pSyncLeaderTransfer->newLeaderId), &(ths->myRaftId)); From f15517b1d7f4decd06be411466346c685da39258 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 8 Aug 2022 13:28:49 +0800 Subject: [PATCH 10/17] fix: add checks for stream query --- source/libs/parser/src/parTranslater.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index be16289595..c12c4aa9c6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4729,15 +4729,8 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt return TSDB_CODE_SUCCESS; } - if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); - } - - SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; - - if (QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable) || - TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || - !pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect)) { + if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery) || + QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pStmt->pQuery)->pFromTable)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); } @@ -4770,12 +4763,23 @@ static int32_t addWstartTsToCreateStreamQuery(SNode* pStmt) { return code; } +static int32_t checkStreamQuery(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || + !pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); + } + return TSDB_CODE_SUCCESS; +} + static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SNode* pStmt, SCMCreateStreamReq* pReq) { pCxt->createStream = true; int32_t code = addWstartTsToCreateStreamQuery(pStmt); if (TSDB_CODE_SUCCESS == code) { code = translateQuery(pCxt, pStmt); } + if (TSDB_CODE_SUCCESS == code) { + code = checkStreamQuery(pCxt, (SSelectStmt*)pStmt); + } if (TSDB_CODE_SUCCESS == code) { getSourceDatabase(pStmt, pCxt->pParseCxt->acctId, pReq->sourceDB); code = nodesNodeToString(pStmt, false, &pReq->ast, NULL); From 020a656063452290abf5caeeb0587715d418e694 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 8 Aug 2022 13:32:20 +0800 Subject: [PATCH 11/17] fix: fix memory double free issue --- source/client/src/clientHb.c | 1 + source/client/src/tmq.c | 2 +- source/libs/catalog/src/ctgRemote.c | 4 +++- source/libs/catalog/src/ctgUtil.c | 10 +++++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 06bd3f3887..7031a1ebca 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -286,6 +286,7 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { if (pInst == NULL || NULL == *pInst) { taosThreadMutexUnlock(&appInfo.mutex); tscError("cluster not exist, key:%s", key); + taosMemoryFree(pMsg->pData); tFreeClientHbBatchRsp(&pRsp); return -1; } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index f7d45dc6ff..f7d04e67ee 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1007,7 +1007,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pParam); if (code != 0) { tscWarn("msg discard from vgId:%d, epoch %d, code:%x", vgId, epoch, code); - if (pMsg->pData) taosMemoryFree(pMsg->pData); + if (pMsg->pData) taosMemoryFreeClear(pMsg->pData); if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index a9f2d426bc..45f97865ce 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -467,6 +467,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (NULL == taosArrayPush(newBatch.pMsgs, &req)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + msg = NULL; if (NULL == taosArrayPush(newBatch.pTaskIds, &pTask->taskId)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -517,6 +518,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (NULL == taosArrayPush(pBatch->pMsgs, &req)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + msg = NULL; if (NULL == taosArrayPush(pBatch->pTaskIds, &pTask->taskId)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -545,7 +547,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } - tNameGetFullDbName(pName, newBatch.dbFName); + tNameGetFullDbName(pName, pBatch->dbFName); } ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), pBatch->batchId, diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 8e5fb90f1a..1ca60c89cd 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -438,6 +438,14 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) { } } +void ctgFreeTbMetasMsgCtx(SCtgMsgCtx* pCtx) { + ctgFreeMsgCtx(pCtx); + if (pCtx->lastOut) { + ctgFreeSTableMetaOutput((STableMetaOutput*)pCtx->lastOut); + pCtx->lastOut = NULL; + } +} + void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput) { if (NULL == pOutput) { return; @@ -641,7 +649,7 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { taosArrayDestroy(taskCtx->pFetchs); // NO NEED TO FREE pNames - taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeMsgCtx); + taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeTbMetasMsgCtx); if (pTask->msgCtx.lastOut) { ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.lastOut); From b23fe53327ae95879d99cc888b88abede1aa594b Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Mon, 8 Aug 2022 13:56:30 +0800 Subject: [PATCH 12/17] Update 03-package.md --- docs/zh/05-get-started/03-package.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 6dbf74f8bc..1cdfe20c8d 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -11,7 +11,7 @@ import TabItem from "@theme/TabItem"; ::: -TDengine 开源版本提供 deb 和 rpm 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 deb 支持 Debian/Ubuntu 及衍生系统，rpm 支持 CentOS/RHEL/SUSE 及衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包。 +TDengine 开源版本提供 deb 和 rpm 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 deb 支持 Debian/Ubuntu 及衍生系统，rpm 支持 CentOS/RHEL/SUSE 及衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包。也支持通过 `apt-get` 工具从线上进行安装。 ## 安装 @@ -293,4 +293,4 @@ taos> select avg(current), max(voltage), min(phase) from test.meters where group ```sql taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s); -``` \ No newline at end of file +``` From d9b8417d7b0791080096f684bfe381b9cc3c97d3 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 8 Aug 2022 14:01:31 +0800 Subject: [PATCH 13/17] refactor(sync): make leader life longer --- source/libs/sync/src/syncMain.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 3d9b34c34c..cdd729999f 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2674,7 +2674,16 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p return 0; } - syncNodeEventLog(ths, "do leader transfer"); + if (ths->vgId > 1) { + syncNodeEventLog(ths, "I am vnode, can not do leader transfer"); + return 0; + } + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "do leader transfer, index:%ld", pEntry->index); + syncNodeEventLog(ths, logBuf); + } while (0); bool sameId = syncUtilSameId(&(pSyncLeaderTransfer->newLeaderId), &(ths->myRaftId)); bool sameNodeInfo = strcmp(pSyncLeaderTransfer->newNodeInfo.nodeFqdn, ths->myNodeInfo.nodeFqdn) == 0 && From 7a559084ae1cdb33bbbf60531fc066f908b37e58 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 8 Aug 2022 14:38:04 +0800 Subject: [PATCH 14/17] fix: add checks for stream query --- source/libs/parser/src/parTranslater.c | 4 +- tests/script/tsim/stream/drop_stream.sim | 88 ++++++++++++------------ 2 files changed, 45 insertions(+), 47 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c12c4aa9c6..5c49a6e0ab 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2515,7 +2515,7 @@ static bool isPartitionByTbname(SNodeList* pPartitionByList) { return false; } SNode* pPartKey = nodesListGetNode(pPartitionByList, 0); - return QUERY_NODE_FUNCTION != nodeType(pPartKey) || FUNCTION_TYPE_TBNAME != ((SFunctionNode*)pPartKey)->funcType; + return QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType; } static int32_t checkStateWindowForStream(STranslateContext* pCxt, SSelectStmt* pSelect) { @@ -2566,7 +2566,6 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } - pSelect->isTimeLineResult = true; pCxt->currClause = SQL_CLAUSE_WINDOW; int32_t code = translateExpr(pCxt, &pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { @@ -2637,7 +2636,6 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec if (NULL == pSelect->pPartitionByList) { return TSDB_CODE_SUCCESS; } - pSelect->isTimeLineResult = false; pCxt->currClause = SQL_CLAUSE_PARTITION_BY; return translateExprList(pCxt, pSelect->pPartitionByList); } diff --git a/tests/script/tsim/stream/drop_stream.sim b/tests/script/tsim/stream/drop_stream.sim index bdd88bf780..747f59fe85 100644 --- a/tests/script/tsim/stream/drop_stream.sim +++ b/tests/script/tsim/stream/drop_stream.sim @@ -45,70 +45,70 @@ sql create table test.scalar_function_tb1 (ts timestamp, c1 tinyint, c2 smallint sql create table if not exists scalar_stb (ts timestamp, c1 int, c2 double, c3 binary(20), c4 binary(20), c5 nchar(20)) tags (t1 int); sql create table scalar_ct1 using scalar_stb tags(10); sql create table if not exists scalar_tb (ts timestamp, c1 int, c2 double, c3 binary(20), c4 binary(20), c5 nchar(20)); -sql create stream stb_abs_stream trigger at_once into output_abs_stb as select ts, abs(c1), abs(c2), c3 from scalar_stb; +sql create stream stb_abs_stream trigger at_once into output_abs_stb as select ts, abs(c1), abs(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_abs_stream trigger at_once into output_abs_ctb as select ts, abs(c1), abs(c2), c3 from scalar_ct1; sql create stream tb_abs_stream trigger at_once into output_abs_tb as select ts, abs(c1), abs(c2), c3 from scalar_tb; -sql create stream stb_acos_stream trigger at_once into output_acos_stb as select ts, acos(c1), acos(c2), c3 from scalar_stb; +sql create stream stb_acos_stream trigger at_once into output_acos_stb as select ts, acos(c1), acos(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_acos_stream trigger at_once into output_acos_ctb as select ts, acos(c1), acos(c2), c3 from scalar_ct1; sql create stream tb_acos_stream trigger at_once into output_acos_tb as select ts, acos(c1), acos(c2), c3 from scalar_tb; -sql create stream stb_asin_stream trigger at_once into output_asin_stb as select ts, asin(c1), asin(c2), c3 from scalar_stb; +sql create stream stb_asin_stream trigger at_once into output_asin_stb as select ts, asin(c1), asin(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_asin_stream trigger at_once into output_asin_ctb as select ts, asin(c1), asin(c2), c3 from scalar_ct1; sql create stream tb_asin_stream trigger at_once into output_asin_tb as select ts, asin(c1), asin(c2), c3 from scalar_tb; -sql create stream stb_atan_stream trigger at_once into output_atan_stb as select ts, atan(c1), atan(c2), c3 from scalar_stb; +sql create stream stb_atan_stream trigger at_once into output_atan_stb as select ts, atan(c1), atan(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_atan_stream trigger at_once into output_atan_ctb as select ts, atan(c1), atan(c2), c3 from scalar_ct1; sql create stream tb_atan_stream trigger at_once into output_atan_tb as select ts, atan(c1), atan(c2), c3 from scalar_tb; -sql create stream stb_ceil_stream trigger at_once into output_ceil_stb as select ts, ceil(c1), ceil(c2), c3 from scalar_stb; +sql create stream stb_ceil_stream trigger at_once into output_ceil_stb as select ts, ceil(c1), ceil(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_ceil_stream trigger at_once into output_ceil_ctb as select ts, ceil(c1), ceil(c2), c3 from scalar_ct1; sql create stream tb_ceil_stream trigger at_once into output_ceil_tb as select ts, ceil(c1), ceil(c2), c3 from scalar_tb; -sql create stream stb_cos_stream trigger at_once into output_cos_stb as select ts, cos(c1), cos(c2), c3 from scalar_stb; +sql create stream stb_cos_stream trigger at_once into output_cos_stb as select ts, cos(c1), cos(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_cos_stream trigger at_once into output_cos_ctb as select ts, cos(c1), cos(c2), c3 from scalar_ct1; sql create stream tb_cos_stream trigger at_once into output_cos_tb as select ts, cos(c1), cos(c2), c3 from scalar_tb; -sql create stream stb_floor_stream trigger at_once into output_floor_stb as select ts, floor(c1), floor(c2), c3 from scalar_stb; +sql create stream stb_floor_stream trigger at_once into output_floor_stb as select ts, floor(c1), floor(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_floor_stream trigger at_once into output_floor_ctb as select ts, floor(c1), floor(c2), c3 from scalar_ct1; sql create stream tb_floor_stream trigger at_once into output_floor_tb as select ts, floor(c1), floor(c2), c3 from scalar_tb; -sql create stream stb_log_stream trigger at_once into output_log_stb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_stb; +sql create stream stb_log_stream trigger at_once into output_log_stb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_stb partition by tbname; sql create stream ctb_log_stream trigger at_once into output_log_ctb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_ct1; sql create stream tb_log_stream trigger at_once into output_log_tb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_tb; -sql create stream stb_pow_stream trigger at_once into output_pow_stb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_stb; +sql create stream stb_pow_stream trigger at_once into output_pow_stb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_stb partition by tbname; sql create stream ctb_pow_stream trigger at_once into output_pow_ctb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_ct1; sql create stream tb_pow_stream trigger at_once into output_pow_tb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_tb; -sql create stream stb_round_stream trigger at_once into output_round_stb as select ts, round(c1), round(c2), c3 from scalar_stb; +sql create stream stb_round_stream trigger at_once into output_round_stb as select ts, round(c1), round(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_round_stream trigger at_once into output_round_ctb as select ts, round(c1), round(c2), c3 from scalar_ct1; sql create stream tb_round_stream trigger at_once into output_round_tb as select ts, round(c1), round(c2), c3 from scalar_tb; -sql create stream stb_sin_stream trigger at_once into output_sin_stb as select ts, sin(c1), sin(c2), c3 from scalar_stb; +sql create stream stb_sin_stream trigger at_once into output_sin_stb as select ts, sin(c1), sin(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_sin_stream trigger at_once into output_sin_ctb as select ts, sin(c1), sin(c2), c3 from scalar_ct1; sql create stream tb_sin_stream trigger at_once into output_sin_tb as select ts, sin(c1), sin(c2), c3 from scalar_tb; -sql create stream stb_sqrt_stream trigger at_once into output_sqrt_stb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_stb; +sql create stream stb_sqrt_stream trigger at_once into output_sqrt_stb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_sqrt_stream trigger at_once into output_sqrt_ctb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_ct1; sql create stream tb_sqrt_stream trigger at_once into output_sqrt_tb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_tb; -sql create stream stb_tan_stream trigger at_once into output_tan_stb as select ts, tan(c1), tan(c2), c3 from scalar_stb; +sql create stream stb_tan_stream trigger at_once into output_tan_stb as select ts, tan(c1), tan(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_tan_stream trigger at_once into output_tan_ctb as select ts, tan(c1), tan(c2), c3 from scalar_ct1; sql create stream tb_tan_stream trigger at_once into output_tan_tb as select ts, tan(c1), tan(c2), c3 from scalar_tb; -sql create stream stb_char_length_stream into output_char_length_stb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_stb; +sql create stream stb_char_length_stream into output_char_length_stb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_stb partition by tbname; sql create stream ctb_char_length_stream into output_char_length_ctb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_ct1; sql create stream tb_char_length_stream into output_char_length_tb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_tb; -sql create stream stb_concat_stream into output_concat_stb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_stb; +sql create stream stb_concat_stream into output_concat_stb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_stb partition by tbname; sql create stream ctb_concat_stream into output_concat_ctb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_ct1; sql create stream tb_concat_stream into output_concat_tb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_tb; -sql create stream stb_concat_ws_stream into output_concat_ws_stb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_stb; +sql create stream stb_concat_ws_stream into output_concat_ws_stb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_stb partition by tbname; sql create stream ctb_concat_ws_stream into output_concat_ws_ctb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_ct1; sql create stream tb_concat_ws_stream into output_concat_ws_tb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_tb; -sql create stream stb_length_stream into output_length_stb as select ts, length(c3), length(c4), length(c5) from scalar_stb; +sql create stream stb_length_stream into output_length_stb as select ts, length(c3), length(c4), length(c5) from scalar_stb partition by tbname; sql create stream ctb_length_stream into output_length_ctb as select ts, length(c3), length(c4), length(c5) from scalar_ct1; sql create stream tb_length_stream into output_length_tb as select ts, length(c3), length(c4), length(c5) from scalar_tb; -sql create stream stb_lower_stream into output_lower_stb as select ts, lower(c3), lower(c4), lower(c5) from scalar_stb; +sql create stream stb_lower_stream into output_lower_stb as select ts, lower(c3), lower(c4), lower(c5) from scalar_stb partition by tbname; sql create stream ctb_lower_stream into output_lower_ctb as select ts, lower(c3), lower(c4), lower(c5) from scalar_ct1; sql create stream tb_lower_stream into output_lower_tb as select ts, lower(c3), lower(c4), lower(c5) from scalar_tb; -sql create stream stb_ltrim_stream into output_ltrim_stb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_stb; +sql create stream stb_ltrim_stream into output_ltrim_stb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_stb partition by tbname; sql create stream ctb_ltrim_stream into output_ltrim_ctb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_ct1; sql create stream tb_ltrim_stream into output_ltrim_tb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_tb; -sql create stream stb_rtrim_stream into output_rtrim_stb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_stb; +sql create stream stb_rtrim_stream into output_rtrim_stb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_stb partition by tbname; sql create stream ctb_rtrim_stream into output_rtrim_ctb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_ct1; sql create stream tb_rtrim_stream into output_rtrim_tb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_tb; -sql create stream stb_substr_stream into output_substr_stb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_stb; +sql create stream stb_substr_stream into output_substr_stb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_stb partition by tbname; sql create stream ctb_substr_stream into output_substr_ctb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_ct1; sql create stream tb_substr_stream into output_substr_tb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_tb; -sql create stream stb_upper_stream into output_upper_stb as select ts, upper(c3), upper(c4), upper(c5) from scalar_stb; +sql create stream stb_upper_stream into output_upper_stb as select ts, upper(c3), upper(c4), upper(c5) from scalar_stb partition by tbname; sql create stream ctb_upper_stream into output_upper_ctb as select ts, upper(c3), upper(c4), upper(c5) from scalar_ct1; sql create stream tb_upper_stream into output_upper_tb as select ts, upper(c3), upper(c4), upper(c5) from scalar_tb; sql insert into scalar_ct1 values (1656668180503, 100, 100.1, "beijing", "taos", "Taos"); @@ -136,70 +136,70 @@ sql create table test.scalar_function_tb1 (ts timestamp, c1 tinyint, c2 smallint sql create table if not exists scalar_stb (ts timestamp, c1 int, c2 double, c3 binary(20), c4 binary(20), c5 nchar(20)) tags (t1 int); sql create table scalar_ct1 using scalar_stb tags(10); sql create table if not exists scalar_tb (ts timestamp, c1 int, c2 double, c3 binary(20), c4 binary(20), c5 nchar(20)); -sql create stream stb_abs_stream trigger at_once into output_abs_stb as select ts, abs(c1), abs(c2), c3 from scalar_stb; +sql create stream stb_abs_stream trigger at_once into output_abs_stb as select ts, abs(c1), abs(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_abs_stream trigger at_once into output_abs_ctb as select ts, abs(c1), abs(c2), c3 from scalar_ct1; sql create stream tb_abs_stream trigger at_once into output_abs_tb as select ts, abs(c1), abs(c2), c3 from scalar_tb; -sql create stream stb_acos_stream trigger at_once into output_acos_stb as select ts, acos(c1), acos(c2), c3 from scalar_stb; +sql create stream stb_acos_stream trigger at_once into output_acos_stb as select ts, acos(c1), acos(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_acos_stream trigger at_once into output_acos_ctb as select ts, acos(c1), acos(c2), c3 from scalar_ct1; sql create stream tb_acos_stream trigger at_once into output_acos_tb as select ts, acos(c1), acos(c2), c3 from scalar_tb; -sql create stream stb_asin_stream trigger at_once into output_asin_stb as select ts, asin(c1), asin(c2), c3 from scalar_stb; +sql create stream stb_asin_stream trigger at_once into output_asin_stb as select ts, asin(c1), asin(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_asin_stream trigger at_once into output_asin_ctb as select ts, asin(c1), asin(c2), c3 from scalar_ct1; sql create stream tb_asin_stream trigger at_once into output_asin_tb as select ts, asin(c1), asin(c2), c3 from scalar_tb; -sql create stream stb_atan_stream trigger at_once into output_atan_stb as select ts, atan(c1), atan(c2), c3 from scalar_stb; +sql create stream stb_atan_stream trigger at_once into output_atan_stb as select ts, atan(c1), atan(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_atan_stream trigger at_once into output_atan_ctb as select ts, atan(c1), atan(c2), c3 from scalar_ct1; sql create stream tb_atan_stream trigger at_once into output_atan_tb as select ts, atan(c1), atan(c2), c3 from scalar_tb; -sql create stream stb_ceil_stream trigger at_once into output_ceil_stb as select ts, ceil(c1), ceil(c2), c3 from scalar_stb; +sql create stream stb_ceil_stream trigger at_once into output_ceil_stb as select ts, ceil(c1), ceil(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_ceil_stream trigger at_once into output_ceil_ctb as select ts, ceil(c1), ceil(c2), c3 from scalar_ct1; sql create stream tb_ceil_stream trigger at_once into output_ceil_tb as select ts, ceil(c1), ceil(c2), c3 from scalar_tb; -sql create stream stb_cos_stream trigger at_once into output_cos_stb as select ts, cos(c1), cos(c2), c3 from scalar_stb; +sql create stream stb_cos_stream trigger at_once into output_cos_stb as select ts, cos(c1), cos(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_cos_stream trigger at_once into output_cos_ctb as select ts, cos(c1), cos(c2), c3 from scalar_ct1; sql create stream tb_cos_stream trigger at_once into output_cos_tb as select ts, cos(c1), cos(c2), c3 from scalar_tb; -sql create stream stb_floor_stream trigger at_once into output_floor_stb as select ts, floor(c1), floor(c2), c3 from scalar_stb; +sql create stream stb_floor_stream trigger at_once into output_floor_stb as select ts, floor(c1), floor(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_floor_stream trigger at_once into output_floor_ctb as select ts, floor(c1), floor(c2), c3 from scalar_ct1; sql create stream tb_floor_stream trigger at_once into output_floor_tb as select ts, floor(c1), floor(c2), c3 from scalar_tb; -sql create stream stb_log_stream trigger at_once into output_log_stb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_stb; +sql create stream stb_log_stream trigger at_once into output_log_stb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_stb partition by tbname; sql create stream ctb_log_stream trigger at_once into output_log_ctb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_ct1; sql create stream tb_log_stream trigger at_once into output_log_tb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_tb; -sql create stream stb_pow_stream trigger at_once into output_pow_stb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_stb; +sql create stream stb_pow_stream trigger at_once into output_pow_stb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_stb partition by tbname; sql create stream ctb_pow_stream trigger at_once into output_pow_ctb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_ct1; sql create stream tb_pow_stream trigger at_once into output_pow_tb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_tb; -sql create stream stb_round_stream trigger at_once into output_round_stb as select ts, round(c1), round(c2), c3 from scalar_stb; +sql create stream stb_round_stream trigger at_once into output_round_stb as select ts, round(c1), round(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_round_stream trigger at_once into output_round_ctb as select ts, round(c1), round(c2), c3 from scalar_ct1; sql create stream tb_round_stream trigger at_once into output_round_tb as select ts, round(c1), round(c2), c3 from scalar_tb; -sql create stream stb_sin_stream trigger at_once into output_sin_stb as select ts, sin(c1), sin(c2), c3 from scalar_stb; +sql create stream stb_sin_stream trigger at_once into output_sin_stb as select ts, sin(c1), sin(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_sin_stream trigger at_once into output_sin_ctb as select ts, sin(c1), sin(c2), c3 from scalar_ct1; sql create stream tb_sin_stream trigger at_once into output_sin_tb as select ts, sin(c1), sin(c2), c3 from scalar_tb; -sql create stream stb_sqrt_stream trigger at_once into output_sqrt_stb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_stb; +sql create stream stb_sqrt_stream trigger at_once into output_sqrt_stb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_sqrt_stream trigger at_once into output_sqrt_ctb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_ct1; sql create stream tb_sqrt_stream trigger at_once into output_sqrt_tb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_tb; -sql create stream stb_tan_stream trigger at_once into output_tan_stb as select ts, tan(c1), tan(c2), c3 from scalar_stb; +sql create stream stb_tan_stream trigger at_once into output_tan_stb as select ts, tan(c1), tan(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_tan_stream trigger at_once into output_tan_ctb as select ts, tan(c1), tan(c2), c3 from scalar_ct1; sql create stream tb_tan_stream trigger at_once into output_tan_tb as select ts, tan(c1), tan(c2), c3 from scalar_tb; -sql create stream stb_char_length_stream into output_char_length_stb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_stb; +sql create stream stb_char_length_stream into output_char_length_stb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_stb partition by tbname; sql create stream ctb_char_length_stream into output_char_length_ctb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_ct1; sql create stream tb_char_length_stream into output_char_length_tb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_tb; -sql create stream stb_concat_stream into output_concat_stb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_stb; +sql create stream stb_concat_stream into output_concat_stb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_stb partition by tbname; sql create stream ctb_concat_stream into output_concat_ctb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_ct1; sql create stream tb_concat_stream into output_concat_tb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_tb; -sql create stream stb_concat_ws_stream into output_concat_ws_stb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_stb; +sql create stream stb_concat_ws_stream into output_concat_ws_stb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_stb partition by tbname; sql create stream ctb_concat_ws_stream into output_concat_ws_ctb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_ct1; sql create stream tb_concat_ws_stream into output_concat_ws_tb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_tb; -sql create stream stb_length_stream into output_length_stb as select ts, length(c3), length(c4), length(c5) from scalar_stb; +sql create stream stb_length_stream into output_length_stb as select ts, length(c3), length(c4), length(c5) from scalar_stb partition by tbname; sql create stream ctb_length_stream into output_length_ctb as select ts, length(c3), length(c4), length(c5) from scalar_ct1; sql create stream tb_length_stream into output_length_tb as select ts, length(c3), length(c4), length(c5) from scalar_tb; -sql create stream stb_lower_stream into output_lower_stb as select ts, lower(c3), lower(c4), lower(c5) from scalar_stb; +sql create stream stb_lower_stream into output_lower_stb as select ts, lower(c3), lower(c4), lower(c5) from scalar_stb partition by tbname; sql create stream ctb_lower_stream into output_lower_ctb as select ts, lower(c3), lower(c4), lower(c5) from scalar_ct1; sql create stream tb_lower_stream into output_lower_tb as select ts, lower(c3), lower(c4), lower(c5) from scalar_tb; -sql create stream stb_ltrim_stream into output_ltrim_stb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_stb; +sql create stream stb_ltrim_stream into output_ltrim_stb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_stb partition by tbname; sql create stream ctb_ltrim_stream into output_ltrim_ctb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_ct1; sql create stream tb_ltrim_stream into output_ltrim_tb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_tb; -sql create stream stb_rtrim_stream into output_rtrim_stb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_stb; +sql create stream stb_rtrim_stream into output_rtrim_stb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_stb partition by tbname; sql create stream ctb_rtrim_stream into output_rtrim_ctb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_ct1; sql create stream tb_rtrim_stream into output_rtrim_tb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_tb; -sql create stream stb_substr_stream into output_substr_stb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_stb; +sql create stream stb_substr_stream into output_substr_stb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_stb partition by tbname; sql create stream ctb_substr_stream into output_substr_ctb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_ct1; sql create stream tb_substr_stream into output_substr_tb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_tb; -sql create stream stb_upper_stream into output_upper_stb as select ts, upper(c3), upper(c4), upper(c5) from scalar_stb; +sql create stream stb_upper_stream into output_upper_stb as select ts, upper(c3), upper(c4), upper(c5) from scalar_stb partition by tbname; sql create stream ctb_upper_stream into output_upper_ctb as select ts, upper(c3), upper(c4), upper(c5) from scalar_ct1; sql create stream tb_upper_stream into output_upper_tb as select ts, upper(c3), upper(c4), upper(c5) from scalar_tb; sql insert into scalar_ct1 values (1656668180503, 100, 100.1, "beijing", "taos", "Taos"); From 40d1ff775aa8aee8545be45b84a163ffdfd7469f Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 8 Aug 2022 15:04:02 +0800 Subject: [PATCH 15/17] build: add win package --- packaging/release.sh | 2 +- source/common/src/systable.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/release.sh b/packaging/release.sh index 09781dbe8e..2452ee1813 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -216,7 +216,7 @@ else fi # check support cpu type -if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "mips64" ]]; then +if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "arm64" ]] || [[ "$cpuType" == "arm32" ]] || [[ "$cpuType" == "mips64" ]]; then if [ "$verMode" != "cluster" ]; then # community-version compile cmake ../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} diff --git a/source/common/src/systable.c b/source/common/src/systable.c index a79082ab23..871a6e5210 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -24,7 +24,7 @@ #define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) static const SSysDbTableSchema dnodesSchema[] = { - {.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_SMALLINT}, {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, {.name = "support_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, @@ -66,7 +66,7 @@ static const SSysDbTableSchema bnodesSchema[] = { }; static const SSysDbTableSchema clusterSchema[] = { - {.name = "id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; From bb0ce39511c8a4eb2ce1b4a03fb16ac3c3ffe3b2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 8 Aug 2022 13:48:09 +0800 Subject: [PATCH 16/17] refactor(stream): add stream meta --- include/libs/stream/tstream.h | 6 +- source/dnode/vnode/src/inc/tq.h | 14 ++--- source/dnode/vnode/src/tq/tq.c | 89 +++++++++++++---------------- source/dnode/vnode/src/tq/tqPush.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/libs/stream/src/streamMeta.c | 62 ++++++++++++++++++++ 6 files changed, 116 insertions(+), 59 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 239fcdad8d..842e656b9d 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -476,8 +476,10 @@ typedef struct SStreamMeta { SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc); void streamMetaClose(SStreamMeta* streamMeta); -int32_t streamMetaAddTask(SStreamMeta* pMeta, SStreamTask* pTask); -int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); +int32_t streamMetaAddTask(SStreamMeta* pMeta, SStreamTask* pTask); +int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, char* msg, int32_t msgLen); +int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); +SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId); int32_t streamMetaBegin(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index c093b2cd5d..a1dba41c94 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -117,10 +117,9 @@ typedef struct { struct STQ { SVnode* pVnode; char* path; - SHashObj* pushMgr; // consumerId -> STqHandle* - SHashObj* handles; // subKey -> STqHandle - SHashObj* pStreamTasks; // taksId -> SStreamTask - SHashObj* pAlterInfo; // topic -> SAlterCheckInfo + SHashObj* pushMgr; // consumerId -> STqHandle* + SHashObj* handles; // subKey -> STqHandle + SHashObj* pAlterInfo; // topic -> SAlterCheckInfo STqOffsetStore* pOffsetStore; @@ -129,9 +128,7 @@ struct STQ { TTB* pAlterInfoStore; - TDB* pStreamStore; - TTB* pTaskDb; - TTB* pTaskState; + SStreamMeta* pStreamMeta; }; typedef struct { @@ -188,6 +185,9 @@ static FORCE_INLINE void tqOffsetResetToLog(STqOffsetVal* pOffsetVal, int64_t ve pOffsetVal->version = ver; } +// tqStream +int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 62e37f048e..c1c680fc56 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -62,8 +62,6 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { pTq->handles = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); - pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - pTq->pushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); pTq->pAlterInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); @@ -76,6 +74,11 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { ASSERT(0); } + pTq->pStreamMeta = streamMetaOpen(path, pTq, (FTaskExpand*)tqExpandTask); + if (pTq->pStreamMeta == NULL) { + ASSERT(0); + } + return pTq; } @@ -83,18 +86,11 @@ void tqClose(STQ* pTq) { if (pTq) { tqOffsetClose(pTq->pOffsetStore); taosHashCleanup(pTq->handles); - void* pIter = NULL; - while (1) { - pIter = taosHashIterate(pTq->pStreamTasks, pIter); - if (pIter == NULL) break; - SStreamTask* pTask = *(SStreamTask**)pIter; - tFreeSStreamTask(pTask); - } - taosHashCleanup(pTq->pStreamTasks); taosHashCleanup(pTq->pushMgr); taosHashCleanup(pTq->pAlterInfo); taosMemoryFree(pTq->path); tqMetaClose(pTq); + streamMetaClose(pTq->pStreamMeta); taosMemoryFree(pTq); } } @@ -672,6 +668,9 @@ FAIL: } int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { + // + return streamMetaAddSerializedTask(pTq->pStreamMeta, msg, msgLen); +#if 0 SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { return -1; @@ -695,6 +694,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { FAIL: if (pTask) taosMemoryFree(pTask); return -1; +#endif } int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq, int64_t ver) { @@ -710,7 +710,7 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq, int64_t ver) { } while (1) { - pIter = taosHashIterate(pTq->pStreamTasks, pIter); + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); if (pIter == NULL) break; SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; @@ -744,9 +744,9 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { // SStreamTaskRunReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - streamProcessRunReq(*ppTask); + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { + streamProcessRunReq(pTask); return 0; } else { return -1; @@ -762,14 +762,15 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecodeStreamDispatchReq(&decoder, &req); - int32_t taskId = req.taskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { + int32_t taskId = req.taskId; + + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { SRpcMsg rsp = { .info = pMsg->info, .code = 0, }; - streamProcessDispatchReq(*ppTask, &req, &rsp, exec); + streamProcessDispatchReq(pTask, &req, &rsp, exec); return 0; } else { return -1; @@ -779,9 +780,9 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - streamProcessRecoverReq(*ppTask, pReq, pMsg); + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { + streamProcessRecoverReq(pTask, pReq, pMsg); return 0; } else { return -1; @@ -791,9 +792,9 @@ int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t taskId = pRsp->taskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - streamProcessDispatchRsp(*ppTask, pRsp); + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { + streamProcessDispatchRsp(pTask, pRsp); return 0; } else { return -1; @@ -803,9 +804,10 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverRsp* pRsp = pMsg->pCont; int32_t taskId = pRsp->rspTaskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - streamProcessRecoverRsp(*ppTask, pRsp); + + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { + streamProcessRecoverRsp(pTask, pRsp); return 0; } else { return -1; @@ -815,18 +817,7 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &pReq->taskId, sizeof(int32_t)); - if (ppTask) { - SStreamTask* pTask = *ppTask; - taosHashRemove(pTq->pStreamTasks, &pReq->taskId, sizeof(int32_t)); - atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING); - } - // todo - // clear queue - // push drop req into queue - // launch exec to free memory - // remove from hash - return 0; + return streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId); } int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { @@ -837,18 +828,18 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { SDecoder decoder; tDecoderInit(&decoder, msgBody, msgLen); tDecodeStreamRetrieveReq(&decoder, &req); - int32_t taskId = req.dstTaskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { + int32_t taskId = req.dstTaskId; + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { SRpcMsg rsp = { .info = pMsg->info, .code = 0, }; - streamProcessRetrieveReq(*ppTask, &req, &rsp); + streamProcessRetrieveReq(pTask, &req, &rsp); + return 0; } else { return -1; } - return 0; } int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) { @@ -871,16 +862,18 @@ void vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { goto FAIL; } - int32_t taskId = req.taskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { + int32_t taskId = req.taskId; + + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { SRpcMsg rsp = { .info = pMsg->info, .code = 0, }; - streamProcessDispatchReq(*ppTask, &req, &rsp, false); + streamProcessDispatchReq(pTask, &req, &rsp, false); return; } + FAIL: if (pMsg->info.handle == NULL) return; SRpcMsg rsp = { diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 1c48ef7535..ae3fef9b4b 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -215,7 +215,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) walApplyVer(pTq->pVnode->pWal, ver); if (msgType == TDMT_VND_SUBMIT) { - if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0; + if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; void* data = taosMemoryMalloc(msgLen); if (data == NULL) { diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 5017893853..6ce8dbe5d9 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -413,7 +413,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } } while (1) { - pIter = taosHashIterate(pTq->pStreamTasks, pIter); + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); if (pIter == NULL) break; SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel == TASK_LEVEL__SOURCE) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 085a0e4ce7..8faa22d643 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -36,8 +36,18 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } + pMeta->pTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + if (pMeta->pTasks == NULL) { + goto _err; + } + + if (streamMetaBegin(pMeta) < 0) { + goto _err; + } + pMeta->ahandle = ahandle; pMeta->expandFunc = expandFunc; + return pMeta; _err: return NULL; @@ -48,6 +58,48 @@ void streamMetaClose(SStreamMeta* pMeta) { tdbTbClose(pMeta->pTaskDb); tdbTbClose(pMeta->pStateDb); tdbClose(pMeta->db); + + void* pIter = NULL; + while (1) { + pIter = taosHashIterate(pMeta->pTasks, pIter); + if (pIter == NULL) break; + SStreamTask* pTask = *(SStreamTask**)pIter; + tFreeSStreamTask(pTask); + } + taosHashCleanup(pMeta->pTasks); + taosMemoryFree(pMeta->path); + taosMemoryFree(pMeta); +} + +int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, char* msg, int32_t msgLen) { + SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); + if (pTask == NULL) { + return -1; + } + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + if (tDecodeSStreamTask(&decoder, pTask) < 0) { + ASSERT(0); + goto FAIL; + } + tDecoderClear(&decoder); + + if (pMeta->expandFunc(pMeta->ahandle, pTask) < 0) { + ASSERT(0); + goto FAIL; + } + + taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)); + + if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, &pMeta->txn) < 0) { + ASSERT(0); + return -1; + } + return 0; + +FAIL: + if (pTask) taosMemoryFree(pTask); + return -1; } int32_t streamMetaAddTask(SStreamMeta* pMeta, SStreamTask* pTask) { @@ -80,6 +132,16 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, SStreamTask* pTask) { return 0; } +SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId) { + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); + if (ppTask) { + ASSERT((*ppTask)->taskId == taskId); + return *ppTask; + } else { + return NULL; + } +} + int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); if (ppTask) { From 51f0cd3952ee2fea43215b8351d354f1ba93085c Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 8 Aug 2022 15:21:47 +0800 Subject: [PATCH 17/17] enh: adjust the content of the query data chapter --- docs/zh/07-develop/04-query-data/index.mdx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/zh/07-develop/04-query-data/index.mdx b/docs/zh/07-develop/04-query-data/index.mdx index 824f36ef2f..eecda92744 100644 --- a/docs/zh/07-develop/04-query-data/index.mdx +++ b/docs/zh/07-develop/04-query-data/index.mdx @@ -25,6 +25,7 @@ TDengine 采用 SQL 作为查询语言。应用程序可以通过 REST API 或 - 单列、多列数据查询 - 标签和数值的多种过滤条件：>, <, =, <\>, like 等 - 聚合结果的分组（Group by）、排序（Order by）、约束输出（Limit/Offset） +- 时间窗口（Interval）、会话窗口（Session）和状态窗口（State_window）等窗口切分聚合查询 - 数值列及聚合结果的四则运算 - 时间戳对齐的连接查询（Join Query: 隐式连接）操作 - 多种聚合/计算函数: count, max, min, avg, sum, twa, stddev, leastsquares, top, bottom, first, last, percentile, apercentile, last_row, spread, diff 等 @@ -40,7 +41,7 @@ taos> select * from d1001 where voltage > 215 order by ts desc limit 2; Query OK, 2 row(s) in set (0.001100s) ``` -为满足物联网场景的需求，TDengine 支持几个特殊的函数，比如 twa(时间加权平均)，spread (最大值与最小值的差)，last_row(最后一条记录)等，更多与物联网场景相关的函数将添加进来。TDengine 还支持连续查询。 +为满足物联网场景的需求，TDengine 支持几个特殊的函数，比如 twa(时间加权平均)，spread (最大值与最小值的差)，last_row(最后一条记录)等，更多与物联网场景相关的函数将添加进来。 具体的查询语法请看 [TAOS SQL 的数据查询](/taos-sql/select) 章节。 @@ -73,7 +74,7 @@ taos> SELECT count(*), max(current) FROM meters where groupId = 2 and ts > now - Query OK, 1 row(s) in set (0.002136s) ``` -TDengine 仅容许对属于同一个超级表的表之间进行聚合查询，不同超级表之间的聚合查询不支持。在 [TAOS SQL 的数据查询](/taos-sql/select) 一章，查询类操作都会注明是否支持超级表。 +在 [TAOS SQL 的数据查询](/taos-sql/select) 一章，查询类操作都会注明是否支持超级表。 ## 降采样查询、插值