diff --git a/README-CN.md b/README-CN.md index 189b7a059a..a6dfefc47a 100644 --- a/README-CN.md +++ b/README-CN.md @@ -52,7 +52,7 @@ TDengine 还提供一组辅助工具软件 taosTools，目前它包含 taosBench ### Ubuntu 18.04 及以上版本 & Debian： ```bash -sudo apt-get install -y gcc cmake build-essential git libssl-dev +sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev ``` #### 为 taos-tools 安装编译需要的软件 @@ -352,4 +352,4 @@ TDengine 提供了丰富的应用程序开发接口，其中包括 C/C++、Java # 加入技术交流群 -TDengine 官方社群「物联网大数据群」对外开放，欢迎您加入讨论。搜索微信号 "tdengine"，加小 T 为好友，即可入群。 +TDengine 官方社群「物联网大数据群」对外开放，欢迎您加入讨论。搜索微信号 "tdengine1"，加小 T 为好友，即可入群。 diff --git a/README.md b/README.md index 5d4bb309a4..a088404c85 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ To build TDengine, use [CMake](https://cmake.org/) 3.0.2 or higher versions in t ### Ubuntu 18.04 and above or Debian ```bash -sudo apt-get install -y gcc cmake build-essential git libssl-dev +sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev ``` #### Install build dependencies for taosTools diff --git a/cmake/cmake.define b/cmake/cmake.define index 5b65738c70..8484355a91 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -119,6 +119,9 @@ ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") MESSAGE(STATUS "Compile with Address Sanitizer!") + ELSEIF (${BUILD_RELEASE}) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") diff --git a/cmake/cmake.options b/cmake/cmake.options index 60ff00affc..14939c1d30 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -171,3 +171,8 @@ option( ON ) +option( + BUILD_RELEASE + "If build release version" + OFF +) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 4a8f4864b3..c67918351d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG ae8d51c + GIT_TAG 565ca21 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index eeb93a6558..5cf37f7946 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -64,6 +64,13 @@ taos = "*" taos = { version = "*", default-features = false, features = ["ws"] } ``` +当仅启用 `ws` 特性时，可同时指定 `r2d2` 使得在同步（blocking/sync）模式下使用 [r2d2] 作为连接池： + +```toml +[dependencies] +taos = { version = "*", default-features = false, features = ["r2d2", "ws"] } +``` + @@ -252,26 +259,24 @@ let conn: Taos = cfg.build(); ### 连接池 -在复杂应用中，建议启用连接池。[taos] 的连接池使用 [r2d2] 实现。 +在复杂应用中，建议启用连接池。[taos] 的连接池默认（异步模式）使用 [deadpool] 实现。 如下，可以生成一个默认参数的连接池。 ```rust -let pool = TaosBuilder::from_dsn(dsn)?.pool()?; +let pool: Pool = TaosBuilder::from_dsn("taos:///") + .unwrap() + .pool() + .unwrap(); ``` 同样可以使用连接池的构造器，对连接池参数进行设置： ```rust -let dsn = "taos://localhost:6030"; - -let opts = PoolBuilder::new() - .max_size(5000) // max connections - .max_lifetime(Some(Duration::from_secs(60 * 60))) // lifetime of each connection - .min_idle(Some(1000)) // minimal idle connections - .connection_timeout(Duration::from_secs(2)); - -let pool = TaosBuilder::from_dsn(dsn)?.with_pool_builder(opts)?; +let pool: Pool = Pool::builder(Manager::from_dsn(self.dsn.clone()).unwrap().0) + .max_size(88) // 最大连接数 + .build() + .unwrap(); ``` 在应用代码中，使用 `pool.get()?` 来获取一个连接对象 [Taos]。 @@ -511,6 +516,7 @@ consumer.unsubscribe().await; 其他相关结构体 API 使用说明请移步 Rust 文档托管网页：。 [taos]: https://github.com/taosdata/rust-connector-taos +[deadpool]: https://crates.io/crates/deadpool [r2d2]: https://crates.io/crates/r2d2 [TaosBuilder]: https://docs.rs/taos/latest/taos/struct.TaosBuilder.html [TaosCfg]: https://docs.rs/taos/latest/taos/struct.TaosCfg.html diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1ea9714bf9..9b01532839 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -942,6 +942,9 @@ int32_t tSerializeSVDropTtlTableReq(void* buf, int32_t bufLen, SVDropTtlTableReq int32_t tDeserializeSVDropTtlTableReq(void* buf, int32_t bufLen, SVDropTtlTableReq* pReq); typedef struct { + char db[TSDB_DB_FNAME_LEN]; + int64_t dbId; + int32_t cfgVersion; int32_t numOfVgroups; int32_t numOfStables; int32_t buffer; @@ -974,8 +977,13 @@ typedef struct { int16_t sstTrigger; } SDbCfgRsp; +typedef SDbCfgRsp SDbCfgInfo; + +int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp); int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp); int32_t tDeserializeSDbCfgRsp(void* buf, int32_t bufLen, SDbCfgRsp* pRsp); +int32_t tDeserializeSDbCfgRspImpl(SDecoder* decoder, SDbCfgRsp *pRsp); +void tFreeSDbCfgRsp(SDbCfgRsp *pRsp); typedef struct { int32_t rowNum; @@ -1032,12 +1040,17 @@ int32_t tDeserializeSDnodeListRsp(void* buf, int32_t bufLen, SDnodeListRsp* pRsp void tFreeSDnodeListRsp(SDnodeListRsp* pRsp); typedef struct { - SArray* pArray; // Array of SUseDbRsp -} SUseDbBatchRsp; + SUseDbRsp *useDbRsp; + SDbCfgRsp *cfgRsp; +} SDbHbRsp; -int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); -int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); -void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); +typedef struct { + SArray* pArray; // Array of SDbHbRsp +} SDbHbBatchRsp; + +int32_t tSerializeSDbHbBatchRsp(void* buf, int32_t bufLen, SDbHbBatchRsp* pRsp); +int32_t tDeserializeSDbHbBatchRsp(void* buf, int32_t bufLen, SDbHbBatchRsp* pRsp); +void tFreeSDbHbBatchRsp(SDbHbBatchRsp* pRsp); typedef struct { SArray* pArray; // Array of SGetUserAuthRsp diff --git a/include/common/ttime.h b/include/common/ttime.h index f189959f22..de74e48100 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -23,7 +23,7 @@ extern "C" { #endif -#define TIME_IS_VAR_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') +#define IS_CALENDAR_TIME_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') #define TIME_UNIT_NANOSECOND 'b' #define TIME_UNIT_MICROSECOND 'u' @@ -74,7 +74,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); +int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval); int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index d7084cfac4..9250bda592 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -129,13 +129,14 @@ typedef struct SSTableVersion { int32_t smaVer; } SSTableVersion; -typedef struct SDbVgVersion { +typedef struct SDbCacheInfo { char dbFName[TSDB_DB_FNAME_LEN]; int64_t dbId; int32_t vgVersion; + int32_t cfgVersion; int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT int64_t stateTs; -} SDbVgVersion; +} SDbCacheInfo; typedef struct STbSVersion { char* tbFName; @@ -148,7 +149,6 @@ typedef struct SUserAuthVersion { int32_t version; } SUserAuthVersion; -typedef SDbCfgRsp SDbCfgInfo; typedef SUserIndexRsp SIndexInfo; typedef void (*catalogCallback)(SMetaData* pResult, void* param, int32_t code); @@ -180,6 +180,8 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgInfo* dbInfo); +int32_t catalogUpdateDbCfg(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDbCfgInfo* cfgInfo); + int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId); int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName); @@ -304,7 +306,7 @@ int32_t catalogGetDnodeList(SCatalog* pCatalog, SRequestConnInfo* pConn, SArray* int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion** stables, uint32_t* num); -int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion** dbs, uint32_t* num); +int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* num); int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index b7e6c42e3b..5fa39b6bb1 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -186,7 +186,17 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len); int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len); -STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key); +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); +STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key); +/** + * return the scan info, in the form of tuple of two items, including table uid and current timestamp + * @param tinfo + * @param uid + * @param ts + * @return + */ +int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts); SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo); diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 42bc89f0b7..f097a2b2df 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -230,6 +230,7 @@ typedef enum EFuncDataRequired { FUNC_DATA_REQUIRED_SMA_LOAD, FUNC_DATA_REQUIRED_NOT_LOAD, FUNC_DATA_REQUIRED_FILTEROUT, + FUNC_DATA_REQUIRED_ALL_FILTEROUT, } EFuncDataRequired; EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 3556a8ac95..fd601bc3ba 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -307,6 +307,7 @@ void nodesDestroyAllocator(int64_t allocatorId); SNode* nodesMakeNode(ENodeType type); void nodesDestroyNode(SNode* pNode); +void nodesFree(void* p); SNodeList* nodesMakeList(); int32_t nodesListAppend(SNodeList* pList, SNode* pNode); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index ad4b59714c..a27237cab3 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -112,6 +112,7 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; bool isSingleTableJoin; EOrder inputTsOrder; + SNode* pColEqualOnConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -405,6 +406,7 @@ typedef struct SSortMergeJoinPhysiNode { SNode* pOnConditions; SNodeList* pTargets; EOrder inputTsOrder; + SNode* pColEqualOnConditions; } SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { @@ -446,7 +448,7 @@ typedef struct SMergePhysiNode { bool groupSort; } SMergePhysiNode; -typedef struct SWinodwPhysiNode { +typedef struct SWindowPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of parameter expression of function SNodeList* pFuncs; @@ -459,10 +461,10 @@ typedef struct SWinodwPhysiNode { EOrder inputTsOrder; EOrder outputTsOrder; bool mergeDataBlock; -} SWinodwPhysiNode; +} SWindowPhysiNode; typedef struct SIntervalPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; int64_t interval; int64_t offset; int64_t sliding; @@ -495,7 +497,7 @@ typedef struct SMultiTableIntervalPhysiNode { } SMultiTableIntervalPhysiNode; typedef struct SSessionWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; int64_t gap; } SSessionWinodwPhysiNode; @@ -504,14 +506,14 @@ typedef SSessionWinodwPhysiNode SStreamSemiSessionWinodwPhysiNode; typedef SSessionWinodwPhysiNode SStreamFinalSessionWinodwPhysiNode; typedef struct SStateWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; SNode* pStateKey; } SStateWinodwPhysiNode; typedef SStateWinodwPhysiNode SStreamStateWinodwPhysiNode; typedef struct SEventWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; SNode* pStartCond; SNode* pEndCond; } SEventWinodwPhysiNode; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 3841210076..69aa315ec2 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -260,6 +260,7 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst); void freeVgInfo(SDBVgInfo* vgInfo); +void freeDbCfgInfo(SDbCfgInfo *pInfo); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, void* (*mallocFp)(int64_t)); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 5fd9a8b12b..0d021e2fa2 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -135,7 +135,6 @@ typedef struct { typedef struct { int8_t type; int64_t ver; - int32_t* dataRef; SSDataBlock* pBlock; } SStreamRefDataBlock; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 46dc179295..7e106eefde 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -135,6 +135,7 @@ typedef struct { // int8_t scanUncommited; int8_t scanNotApplied; int8_t scanMeta; + int8_t deleteMsg; int8_t enableRef; } SWalFilterCond; @@ -193,9 +194,10 @@ SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond); void walCloseReader(SWalReader *pRead); void walReadReset(SWalReader *pReader); int32_t walReadVer(SWalReader *pRead, int64_t ver); -int32_t walReadSeekVer(SWalReader *pRead, int64_t ver); +int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver); int32_t walNextValidMsg(SWalReader *pRead); int64_t walReaderGetCurrentVer(const SWalReader* pReader); +int64_t walReaderGetValidFirstVer(const SWalReader* pReader); // only for tq usage void walSetReaderCapacity(SWalReader *pRead, int32_t capacity); @@ -208,8 +210,7 @@ SWalRef *walRefCommittedVer(SWal *); SWalRef *walOpenRef(SWal *); void walCloseRef(SWal *pWal, int64_t refId); -int32_t walRefVer(SWalRef *, int64_t ver); -void walUnrefVer(SWalRef *); +int32_t walSetRefVer(SWalRef *, int64_t ver); // helper function for raft bool walLogExist(SWal *, int64_t ver); diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index 908e470830..1a6d47d4f9 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -25,6 +25,7 @@ extern "C" { // clang-format off #define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", DEBUG_FATAL, cDebugFlag, __VA_ARGS__); }} while(0) #define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscErrorL(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLongString("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0) #define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0) #define tscWarnL(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLongString("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0) #define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) @@ -32,6 +33,8 @@ extern "C" { #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", 0, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscLog(...) do { taosPrintLog("TSC ", 0, DEBUG_FILE, __VA_ARGS__); } while(0) +#define tscLogL(...) do { taosPrintLongString("TSC ", 0, DEBUG_FILE, __VA_ARGS__); } while(0) // clang-format on #ifdef __cplusplus diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 418103f2a6..1d63955060 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -107,7 +107,7 @@ static void deregisterRequest(SRequestObj *pRequest) { if (duration >= SLOW_QUERY_INTERVAL) { atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); - tscWarnL("slow query: %s, duration:%" PRId64, pRequest->sqlstr, duration); + tscLogL("slow query: %s, duration:%" PRId64, pRequest->sqlstr, duration); } releaseTscObj(pTscObj->id); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 8d082ab60b..0cf9930911 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -94,47 +94,52 @@ _return: static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; - SUseDbBatchRsp batchUseRsp = {0}; - if (tDeserializeSUseDbBatchRsp(value, valueLen, &batchUseRsp) != 0) { + SDbHbBatchRsp batchRsp = {0}; + if (tDeserializeSDbHbBatchRsp(value, valueLen, &batchRsp) != 0) { terrno = TSDB_CODE_INVALID_MSG; - return -1; + code = terrno; + goto _return; } - int32_t numOfBatchs = taosArrayGetSize(batchUseRsp.pArray); + int32_t numOfBatchs = taosArrayGetSize(batchRsp.pArray); for (int32_t i = 0; i < numOfBatchs; ++i) { - SUseDbRsp *rsp = taosArrayGet(batchUseRsp.pArray, i); - tscDebug("hb db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->stateTs, - rsp->uid); - - if (rsp->vgVersion < 0) { - code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); - } else { - SDBVgInfo *vgInfo = NULL; - code = hbGenerateVgInfoFromRsp(&vgInfo, rsp); - if (TSDB_CODE_SUCCESS != code) { - goto _return; - } - - catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, vgInfo); - - if (IS_SYS_DBNAME(rsp->db)) { - code = hbGenerateVgInfoFromRsp(&vgInfo, rsp); + SDbHbRsp *rsp = taosArrayGet(batchRsp.pArray, i); + if (rsp->useDbRsp) { + tscDebug("hb use db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64, + rsp->useDbRsp->db, rsp->useDbRsp->vgVersion, rsp->useDbRsp->stateTs, rsp->useDbRsp->uid); + + if (rsp->useDbRsp->vgVersion < 0) { + code = catalogRemoveDB(pCatalog, rsp->useDbRsp->db, rsp->useDbRsp->uid); + } else { + SDBVgInfo *vgInfo = NULL; + code = hbGenerateVgInfoFromRsp(&vgInfo, rsp->useDbRsp); if (TSDB_CODE_SUCCESS != code) { goto _return; } - - catalogUpdateDBVgInfo(pCatalog, (rsp->db[0] == 'i') ? TSDB_PERFORMANCE_SCHEMA_DB : TSDB_INFORMATION_SCHEMA_DB, rsp->uid, vgInfo); + + catalogUpdateDBVgInfo(pCatalog, rsp->useDbRsp->db, rsp->useDbRsp->uid, vgInfo); + + if (IS_SYS_DBNAME(rsp->useDbRsp->db)) { + code = hbGenerateVgInfoFromRsp(&vgInfo, rsp->useDbRsp); + if (TSDB_CODE_SUCCESS != code) { + goto _return; + } + + catalogUpdateDBVgInfo(pCatalog, (rsp->useDbRsp->db[0] == 'i') ? TSDB_PERFORMANCE_SCHEMA_DB : TSDB_INFORMATION_SCHEMA_DB, rsp->useDbRsp->uid, vgInfo); + } } } - if (code) { - goto _return; + if (rsp->cfgRsp) { + tscDebug("hb db cfg rsp, db:%s, cfgVersion:%d", rsp->cfgRsp->db, rsp->cfgRsp->cfgVersion); + catalogUpdateDbCfg(pCatalog, rsp->cfgRsp->db, rsp->cfgRsp->dbId, rsp->cfgRsp); + rsp->cfgRsp = NULL; } } _return: - tFreeSUseDbBatchRsp(&batchUseRsp); + tFreeSDbHbBatchRsp(&batchRsp); return code; } @@ -510,7 +515,7 @@ int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S } int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { - SDbVgVersion *dbs = NULL; + SDbCacheInfo *dbs = NULL; uint32_t dbNum = 0; int32_t code = 0; @@ -525,19 +530,20 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl } for (int32_t i = 0; i < dbNum; ++i) { - SDbVgVersion *db = &dbs[i]; - tscDebug("the %dth expired dbFName:%s, dbId:%" PRId64 ", vgVersion:%d, numOfTable:%d, startTs:%" PRId64, - i, db->dbFName, db->dbId, db->vgVersion, db->numOfTable, db->stateTs); + SDbCacheInfo *db = &dbs[i]; + tscDebug("the %dth expired dbFName:%s, dbId:%" PRId64 ", vgVersion:%d, cfgVersion:%d, numOfTable:%d, startTs:%" PRId64, + i, db->dbFName, db->dbId, db->vgVersion, db->cfgVersion, db->numOfTable, db->stateTs); db->dbId = htobe64(db->dbId); db->vgVersion = htonl(db->vgVersion); + db->cfgVersion = htonl(db->cfgVersion); db->numOfTable = htonl(db->numOfTable); db->stateTs = htobe64(db->stateTs); } SKv kv = { .key = HEARTBEAT_KEY_DBINFO, - .valueLen = sizeof(SDbVgVersion) * dbNum, + .valueLen = sizeof(SDbCacheInfo) * dbNum, .value = dbs, }; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7102e556cc..fde3a89081 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2778,7 +2778,26 @@ int32_t tSerializeSUseDbRsp(void *buf, int32_t bufLen, const SUseDbRsp *pRsp) { return tlen; } -int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp) { +int32_t tSerializeSDbHbRspImp(SEncoder *pEncoder, const SDbHbRsp *pRsp) { + if (pRsp->useDbRsp) { + if (tEncodeI8(pEncoder, 1) < 0) return -1; + if (tSerializeSUseDbRspImp(pEncoder, pRsp->useDbRsp) < 0) return -1; + } else { + if (tEncodeI8(pEncoder, 0) < 0) return -1; + } + + if (pRsp->cfgRsp) { + if (tEncodeI8(pEncoder, 1) < 0) return -1; + if (tSerializeSDbCfgRspImpl(pEncoder, pRsp->cfgRsp) < 0) return -1; + } else { + if (tEncodeI8(pEncoder, 0) < 0) return -1; + } + + return 0; +} + + +int32_t tSerializeSDbHbBatchRsp(void *buf, int32_t bufLen, SDbHbBatchRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2787,8 +2806,8 @@ int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; for (int32_t i = 0; i < numOfBatch; ++i) { - SUseDbRsp *pUsedbRsp = taosArrayGet(pRsp->pArray, i); - if (tSerializeSUseDbRspImp(&encoder, pUsedbRsp) < 0) return -1; + SDbHbRsp *pDbRsp = taosArrayGet(pRsp->pArray, i); + if (tSerializeSDbHbRspImp(&encoder, pDbRsp) < 0) return -1; } tEndEncode(&encoder); @@ -2841,7 +2860,25 @@ int32_t tDeserializeSUseDbRsp(void *buf, int32_t bufLen, SUseDbRsp *pRsp) { return 0; } -int32_t tDeserializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp) { +int32_t tDeserializeSDbHbRspImp(SDecoder* decoder, SDbHbRsp* pRsp) { + int8_t flag = 0; + if (tDecodeI8(decoder, &flag) < 0) return -1; + if (flag) { + pRsp->useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp)); + if (NULL == pRsp->useDbRsp) return -1; + if (tDeserializeSUseDbRspImp(decoder, pRsp->useDbRsp) < 0) return -1; + } + if (tDecodeI8(decoder, &flag) < 0) return -1; + if (flag) { + pRsp->cfgRsp = taosMemoryCalloc(1, sizeof(SDbCfgRsp)); + if (NULL == pRsp->cfgRsp) return -1; + if (tDeserializeSDbCfgRspImpl(decoder, pRsp->cfgRsp) < 0) return -1; + } + + return 0; +} + +int32_t tDeserializeSDbHbBatchRsp(void *buf, int32_t bufLen, SDbHbBatchRsp *pRsp) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -2850,19 +2887,19 @@ int32_t tDeserializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pR int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; - pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SUseDbRsp)); + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SDbHbRsp)); if (pRsp->pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } for (int32_t i = 0; i < numOfBatch; ++i) { - SUseDbRsp usedbRsp = {0}; - if (tDeserializeSUseDbRspImp(&decoder, &usedbRsp) < 0) { + SDbHbRsp rsp = {0}; + if (tDeserializeSDbHbRspImp(&decoder, &rsp) < 0) { tDecoderClear(&decoder); return -1; } - taosArrayPush(pRsp->pArray, &usedbRsp); + taosArrayPush(pRsp->pArray, &rsp); } tEndDecode(&decoder); @@ -2872,11 +2909,27 @@ int32_t tDeserializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pR void tFreeSUsedbRsp(SUseDbRsp *pRsp) { taosArrayDestroy(pRsp->pVgroupInfos); } -void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { +void tFreeSDbHbRsp(SDbHbRsp *pDbRsp) { + if (NULL == pDbRsp) { + return; + } + + if (pDbRsp->useDbRsp) { + tFreeSUsedbRsp(pDbRsp->useDbRsp); + taosMemoryFree(pDbRsp->useDbRsp); + } + + if (pDbRsp->cfgRsp) { + tFreeSDbCfgRsp(pDbRsp->cfgRsp); + taosMemoryFree(pDbRsp->cfgRsp); + } +} + +void tFreeSDbHbBatchRsp(SDbHbBatchRsp *pRsp) { int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); for (int32_t i = 0; i < numOfBatch; ++i) { - SUseDbRsp *pUsedbRsp = taosArrayGet(pRsp->pArray, i); - tFreeSUsedbRsp(pUsedbRsp); + SDbHbRsp *pDbRsp = taosArrayGet(pRsp->pArray, i); + tFreeSDbHbRsp(pDbRsp); } taosArrayDestroy(pRsp->pArray); @@ -3039,85 +3092,93 @@ int32_t tDeserializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableR return 0; } +int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) { + if (tEncodeCStr(encoder, pRsp->db) < 0) return -1; + if (tEncodeI64(encoder, pRsp->dbId) < 0) return -1; + if (tEncodeI32(encoder, pRsp->cfgVersion) < 0) return -1; + if (tEncodeI32(encoder, pRsp->numOfVgroups) < 0) return -1; + if (tEncodeI32(encoder, pRsp->numOfStables) < 0) return -1; + if (tEncodeI32(encoder, pRsp->buffer) < 0) return -1; + if (tEncodeI32(encoder, pRsp->cacheSize) < 0) return -1; + if (tEncodeI32(encoder, pRsp->pageSize) < 0) return -1; + if (tEncodeI32(encoder, pRsp->pages) < 0) return -1; + if (tEncodeI32(encoder, pRsp->daysPerFile) < 0) return -1; + if (tEncodeI32(encoder, pRsp->daysToKeep0) < 0) return -1; + if (tEncodeI32(encoder, pRsp->daysToKeep1) < 0) return -1; + if (tEncodeI32(encoder, pRsp->daysToKeep2) < 0) return -1; + if (tEncodeI32(encoder, pRsp->minRows) < 0) return -1; + if (tEncodeI32(encoder, pRsp->maxRows) < 0) return -1; + if (tEncodeI32(encoder, pRsp->walFsyncPeriod) < 0) return -1; + if (tEncodeI16(encoder, pRsp->hashPrefix) < 0) return -1; + if (tEncodeI16(encoder, pRsp->hashSuffix) < 0) return -1; + if (tEncodeI8(encoder, pRsp->walLevel) < 0) return -1; + if (tEncodeI8(encoder, pRsp->precision) < 0) return -1; + if (tEncodeI8(encoder, pRsp->compression) < 0) return -1; + if (tEncodeI8(encoder, pRsp->replications) < 0) return -1; + if (tEncodeI8(encoder, pRsp->strict) < 0) return -1; + if (tEncodeI8(encoder, pRsp->cacheLast) < 0) return -1; + if (tEncodeI32(encoder, pRsp->tsdbPageSize) < 0) return -1; + if (tEncodeI32(encoder, pRsp->walRetentionPeriod) < 0) return -1; + if (tEncodeI32(encoder, pRsp->walRollPeriod) < 0) return -1; + if (tEncodeI64(encoder, pRsp->walRetentionSize) < 0) return -1; + if (tEncodeI64(encoder, pRsp->walSegmentSize) < 0) return -1; + if (tEncodeI32(encoder, pRsp->numOfRetensions) < 0) return -1; + for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { + SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i); + if (tEncodeI64(encoder, pRetension->freq) < 0) return -1; + if (tEncodeI64(encoder, pRetension->keep) < 0) return -1; + if (tEncodeI8(encoder, pRetension->freqUnit) < 0) return -1; + if (tEncodeI8(encoder, pRetension->keepUnit) < 0) return -1; + } + if (tEncodeI8(encoder, pRsp->schemaless) < 0) return -1; + if (tEncodeI16(encoder, pRsp->sstTrigger) < 0) return -1; + + return 0; +} + int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->numOfVgroups) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->numOfStables) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->buffer) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->cacheSize) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->pageSize) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->pages) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->daysPerFile) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->daysToKeep0) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->daysToKeep1) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->daysToKeep2) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->minRows) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->maxRows) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->walFsyncPeriod) < 0) return -1; - if (tEncodeI16(&encoder, pRsp->hashPrefix) < 0) return -1; - if (tEncodeI16(&encoder, pRsp->hashSuffix) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->walLevel) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->precision) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->strict) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->cacheLast) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->tsdbPageSize) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->walRetentionPeriod) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->walRollPeriod) < 0) return -1; - if (tEncodeI64(&encoder, pRsp->walRetentionSize) < 0) return -1; - if (tEncodeI64(&encoder, pRsp->walSegmentSize) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1; - for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { - SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i); - if (tEncodeI64(&encoder, pRetension->freq) < 0) return -1; - if (tEncodeI64(&encoder, pRetension->keep) < 0) return -1; - if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; - if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; - } - if (tEncodeI8(&encoder, pRsp->schemaless) < 0) return -1; - if (tEncodeI16(&encoder, pRsp->sstTrigger) < 0) return -1; + tSerializeSDbCfgRspImpl(&encoder, pRsp); tEndEncode(&encoder); int32_t tlen = encoder.pos; tEncoderClear(&encoder); return tlen; } -int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->numOfVgroups) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->numOfStables) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->buffer) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->cacheSize) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->pageSize) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->pages) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->daysPerFile) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->daysToKeep0) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->daysToKeep1) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->daysToKeep2) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->minRows) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->maxRows) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->walFsyncPeriod) < 0) return -1; - if (tDecodeI16(&decoder, &pRsp->hashPrefix) < 0) return -1; - if (tDecodeI16(&decoder, &pRsp->hashSuffix) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->walLevel) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->precision) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->strict) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->cacheLast) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->tsdbPageSize) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->walRetentionPeriod) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->walRollPeriod) < 0) return -1; - if (tDecodeI64(&decoder, &pRsp->walRetentionSize) < 0) return -1; - if (tDecodeI64(&decoder, &pRsp->walSegmentSize) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->numOfRetensions) < 0) return -1; +int32_t tDeserializeSDbCfgRspImpl(SDecoder* decoder, SDbCfgRsp *pRsp) { + if (tDecodeCStrTo(decoder, pRsp->db) < 0) return -1; + if (tDecodeI64(decoder, &pRsp->dbId) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->cfgVersion) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->numOfVgroups) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->numOfStables) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->buffer) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->cacheSize) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->pageSize) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->pages) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->daysPerFile) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->daysToKeep0) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->daysToKeep1) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->daysToKeep2) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->minRows) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->maxRows) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->walFsyncPeriod) < 0) return -1; + if (tDecodeI16(decoder, &pRsp->hashPrefix) < 0) return -1; + if (tDecodeI16(decoder, &pRsp->hashSuffix) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->walLevel) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->precision) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->compression) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->replications) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->strict) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->cacheLast) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->tsdbPageSize) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->walRetentionPeriod) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->walRollPeriod) < 0) return -1; + if (tDecodeI64(decoder, &pRsp->walRetentionSize) < 0) return -1; + if (tDecodeI64(decoder, &pRsp->walSegmentSize) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->numOfRetensions) < 0) return -1; if (pRsp->numOfRetensions > 0) { pRsp->pRetensions = taosArrayInit(pRsp->numOfRetensions, sizeof(SRetention)); if (pRsp->pRetensions == NULL) { @@ -3128,23 +3189,41 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { SRetention rentension = {0}; - if (tDecodeI64(&decoder, &rentension.freq) < 0) return -1; - if (tDecodeI64(&decoder, &rentension.keep) < 0) return -1; - if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; - if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; + if (tDecodeI64(decoder, &rentension.freq) < 0) return -1; + if (tDecodeI64(decoder, &rentension.keep) < 0) return -1; + if (tDecodeI8(decoder, &rentension.freqUnit) < 0) return -1; + if (tDecodeI8(decoder, &rentension.keepUnit) < 0) return -1; if (taosArrayPush(pRsp->pRetensions, &rentension) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } } - if (tDecodeI8(&decoder, &pRsp->schemaless) < 0) return -1; - if (tDecodeI16(&decoder, &pRsp->sstTrigger) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->schemaless) < 0) return -1; + if (tDecodeI16(decoder, &pRsp->sstTrigger) < 0) return -1; + + return 0; +} + +int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDeserializeSDbCfgRspImpl(&decoder, pRsp) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } +void tFreeSDbCfgRsp(SDbCfgRsp *pRsp) { + if (NULL == pRsp) { + return; + } + + taosArrayDestroy(pRsp->pRetensions); +} + int32_t tSerializeSUserIndexReq(void *buf, int32_t bufLen, SUserIndexReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index dcd539bd91..dbb31374de 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -82,6 +82,7 @@ static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_ static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static char* forwardToTimeStringEnd(char* str); static bool checkTzPresent(const char* str, int32_t len); +static int32_t parseTimezone(char* str, int64_t* tzOffset); static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) = { parseLocaltime, parseLocaltimeDst}; @@ -713,16 +714,12 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { return t; } - if (unit != 'n' && unit != 'y') { + if (!IS_CALENDAR_TIME_DURATION(unit)) { return t + duration; } // The following code handles the y/n time duration - int64_t numOfMonth = duration; - if (unit == 'y') { - numOfMonth *= 12; - } - + int64_t numOfMonth = (unit == 'y')? duration*12:duration; int64_t fraction = t % TSDB_TICK_PER_SECOND(precision); struct tm tm; @@ -764,13 +761,16 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char return (emon - smon) / (int32_t)interval; } -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision) { +int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { if (pInterval->sliding == 0 && pInterval->interval == 0) { - return t; + return ts; } - int64_t start = t; - if (pInterval->slidingUnit == 'n' || pInterval->slidingUnit == 'y') { + int64_t start = ts; + int32_t precision = pInterval->precision; + + if (IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { + start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; time_t tt = (time_t)start; @@ -792,44 +792,72 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio start = (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision)); } else { - int64_t delta = t - pInterval->interval; - int32_t factor = (delta >= 0) ? 1 : -1; + if (IS_CALENDAR_TIME_DURATION(pInterval->intervalUnit)) { + int64_t news = (ts / pInterval->sliding) * pInterval->sliding; + ASSERT(news <= ts); - start = (delta / pInterval->sliding + factor) * pInterval->sliding; + if (news <= ts) { + int64_t prev = news; + int64_t newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; - if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { - /* - * here we revised the start time of day according to the local time zone, - * but in case of DST, the start time of one day need to be dynamically decided. - */ - // todo refactor to extract function that is available for Linux/Windows/Mac platform -#if defined(WINDOWS) && _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char** tzname = _tzname; -#endif + if (newe < ts) { // move towards the greater endpoint + while(newe < ts && news < ts) { + news += pInterval->sliding; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } - start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); - } - - int64_t end = 0; - - // not enough time range - if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - while (end < t) { // move forward to the correct time window - start += pInterval->sliding; - - if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = start + pInterval->interval - 1; + prev = news; } else { - end = INT64_MAX; - break; + while (newe >= ts) { + prev = news; + news -= pInterval->sliding; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } } + + return prev; } } else { - end = INT64_MAX; + int64_t delta = ts - pInterval->interval; + int32_t factor = (delta >= 0) ? 1 : -1; + + start = (delta / pInterval->sliding + factor) * pInterval->sliding; + + if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { + /* + * here we revised the start time of day according to the local time zone, + * but in case of DST, the start time of one day need to be dynamically decided. + */ + // todo refactor to extract function that is available for Linux/Windows/Mac platform +#if defined(WINDOWS) && _MSC_VER >= 1900 + // see + // https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 + int64_t timezone = _timezone; + int32_t daylight = _daylight; + char** tzname = _tzname; +#endif + + start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); + } + + int64_t end = 0; + + // not enough time range + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; + while (end < ts) { // move forward to the correct time window + start += pInterval->sliding; + + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = start + pInterval->interval - 1; + } else { + end = INT64_MAX; + break; + } + } + } else { + end = INT64_MAX; + } } } @@ -841,10 +869,10 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio // try to move current window to the left-hande-side, due to the offset effect. int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - int64_t newEnd = end; - while (newEnd >= t) { - end = newEnd; - newEnd = taosTimeAdd(newEnd, -pInterval->sliding, pInterval->slidingUnit, precision); + int64_t newe = end; + while (newe >= ts) { + end = newe; + newe = taosTimeAdd(newe, -pInterval->sliding, pInterval->slidingUnit, precision); } start = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1; diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 989bff3984..da9a57387d 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -87,6 +87,18 @@ static void dmStopDnode(int signum, void *sigInfo, void *context) { } void dmLogCrash(int signum, void *sigInfo, void *context) { + // taosIgnSignal(SIGTERM); + // taosIgnSignal(SIGHUP); + // taosIgnSignal(SIGINT); + // taosIgnSignal(SIGBREAK); + +#ifndef WINDOWS + taosIgnSignal(SIGBUS); +#endif + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGFPE); + taosIgnSignal(SIGSEGV); + char *pMsg = NULL; const char *flags = "UTL FATAL "; ELogLevel level = DEBUG_FATAL; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 0152e5d0b1..bbd77c0828 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -120,6 +120,11 @@ int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + if (NULL == pMgmt->pMnode) { + const STraceId *trace = &pMsg->info.traceId; + dGError("msg:%p, stop to pre-process in mnode since mnode is NULL, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + return -1; + } pMsg->info.node = pMgmt->pMnode; if (mndPreProcessQueryMsg(pMsg) != 0) { const STraceId *trace = &pMsg->info.traceId; diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index 97d047d7a3..60d35ebe1d 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -26,7 +26,7 @@ int32_t mndInitDb(SMnode *pMnode); void mndCleanupDb(SMnode *pMnode); SDbObj *mndAcquireDb(SMnode *pMnode, const char *db); void mndReleaseDb(SMnode *pMnode, SDbObj *pDb); -int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen); +int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen); int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUseDbReq *pReq); bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 23b2b9d7c6..0f7e3de2f5 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -898,6 +898,42 @@ _OVER: return code; } +static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) { + strcpy(cfgRsp->db, pDb->name); + cfgRsp->dbId = pDb->uid; + cfgRsp->cfgVersion = pDb->cfgVersion; + cfgRsp->numOfVgroups = pDb->cfg.numOfVgroups; + cfgRsp->numOfStables = pDb->cfg.numOfStables; + cfgRsp->buffer = pDb->cfg.buffer; + cfgRsp->cacheSize = pDb->cfg.cacheLastSize; + cfgRsp->pageSize = pDb->cfg.pageSize; + cfgRsp->pages = pDb->cfg.pages; + cfgRsp->daysPerFile = pDb->cfg.daysPerFile; + cfgRsp->daysToKeep0 = pDb->cfg.daysToKeep0; + cfgRsp->daysToKeep1 = pDb->cfg.daysToKeep1; + cfgRsp->daysToKeep2 = pDb->cfg.daysToKeep2; + cfgRsp->minRows = pDb->cfg.minRows; + cfgRsp->maxRows = pDb->cfg.maxRows; + cfgRsp->walFsyncPeriod = pDb->cfg.walFsyncPeriod; + cfgRsp->hashPrefix = pDb->cfg.hashPrefix; + cfgRsp->hashSuffix = pDb->cfg.hashSuffix; + cfgRsp->walLevel = pDb->cfg.walLevel; + cfgRsp->precision = pDb->cfg.precision; + cfgRsp->compression = pDb->cfg.compression; + cfgRsp->replications = pDb->cfg.replications; + cfgRsp->strict = pDb->cfg.strict; + cfgRsp->cacheLast = pDb->cfg.cacheLast; + cfgRsp->tsdbPageSize = pDb->cfg.tsdbPageSize; + cfgRsp->walRetentionPeriod = pDb->cfg.walRetentionPeriod; + cfgRsp->walRollPeriod = pDb->cfg.walRollPeriod; + cfgRsp->walRetentionSize = pDb->cfg.walRetentionSize; + cfgRsp->walSegmentSize = pDb->cfg.walSegmentSize; + cfgRsp->numOfRetensions = pDb->cfg.numOfRetensions; + cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); + cfgRsp->schemaless = pDb->cfg.schemaless; + cfgRsp->sstTrigger = pDb->cfg.sstTrigger; +} + static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -910,41 +946,15 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { goto _OVER; } - pDb = mndAcquireDb(pMnode, cfgReq.db); - if (pDb == NULL) { - goto _OVER; - } + if (strcasecmp(cfgReq.db, TSDB_INFORMATION_SCHEMA_DB) && strcasecmp(cfgReq.db, TSDB_PERFORMANCE_SCHEMA_DB)) { + pDb = mndAcquireDb(pMnode, cfgReq.db); + if (pDb == NULL) { + goto _OVER; + } - cfgRsp.numOfVgroups = pDb->cfg.numOfVgroups; - cfgRsp.numOfStables = pDb->cfg.numOfStables; - cfgRsp.buffer = pDb->cfg.buffer; - cfgRsp.cacheSize = pDb->cfg.cacheLastSize; - cfgRsp.pageSize = pDb->cfg.pageSize; - cfgRsp.pages = pDb->cfg.pages; - cfgRsp.daysPerFile = pDb->cfg.daysPerFile; - cfgRsp.daysToKeep0 = pDb->cfg.daysToKeep0; - cfgRsp.daysToKeep1 = pDb->cfg.daysToKeep1; - cfgRsp.daysToKeep2 = pDb->cfg.daysToKeep2; - cfgRsp.minRows = pDb->cfg.minRows; - cfgRsp.maxRows = pDb->cfg.maxRows; - cfgRsp.walFsyncPeriod = pDb->cfg.walFsyncPeriod; - cfgRsp.hashPrefix = pDb->cfg.hashPrefix; - cfgRsp.hashSuffix = pDb->cfg.hashSuffix; - cfgRsp.walLevel = pDb->cfg.walLevel; - cfgRsp.precision = pDb->cfg.precision; - cfgRsp.compression = pDb->cfg.compression; - cfgRsp.replications = pDb->cfg.replications; - cfgRsp.strict = pDb->cfg.strict; - cfgRsp.cacheLast = pDb->cfg.cacheLast; - cfgRsp.tsdbPageSize = pDb->cfg.tsdbPageSize; - cfgRsp.walRetentionPeriod = pDb->cfg.walRetentionPeriod; - cfgRsp.walRollPeriod = pDb->cfg.walRollPeriod; - cfgRsp.walRetentionSize = pDb->cfg.walRetentionSize; - cfgRsp.walSegmentSize = pDb->cfg.walSegmentSize; - cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions; - cfgRsp.pRetensions = pDb->cfg.pRetensions; - cfgRsp.schemaless = pDb->cfg.schemaless; - cfgRsp.sstTrigger = pDb->cfg.sstTrigger; + mndDumpDbCfgInfo(&cfgRsp, pDb); + } + int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &cfgRsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { @@ -962,6 +972,8 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { _OVER: + tFreeSDbCfgRsp(&cfgRsp); + if (code != 0) { mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr()); } @@ -1341,103 +1353,119 @@ _OVER: return code; } -int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen) { - SUseDbBatchRsp batchUseRsp = {0}; - batchUseRsp.pArray = taosArrayInit(numOfDbs, sizeof(SUseDbRsp)); - if (batchUseRsp.pArray == NULL) { +int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen) { + SDbHbBatchRsp batchRsp = {0}; + batchRsp.pArray = taosArrayInit(numOfDbs, sizeof(SDbHbRsp)); + if (batchRsp.pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } for (int32_t i = 0; i < numOfDbs; ++i) { - SDbVgVersion *pDbVgVersion = &pDbs[i]; - pDbVgVersion->dbId = be64toh(pDbVgVersion->dbId); - pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion); - pDbVgVersion->numOfTable = htonl(pDbVgVersion->numOfTable); - pDbVgVersion->stateTs = be64toh(pDbVgVersion->stateTs); + SDbCacheInfo *pDbCacheInfo = &pDbs[i]; + pDbCacheInfo->dbId = be64toh(pDbCacheInfo->dbId); + pDbCacheInfo->vgVersion = htonl(pDbCacheInfo->vgVersion); + pDbCacheInfo->cfgVersion = htonl(pDbCacheInfo->cfgVersion); + pDbCacheInfo->numOfTable = htonl(pDbCacheInfo->numOfTable); + pDbCacheInfo->stateTs = be64toh(pDbCacheInfo->stateTs); - SUseDbRsp usedbRsp = {0}; + SDbHbRsp rsp = {0}; - if ((0 == strcasecmp(pDbVgVersion->dbFName, TSDB_INFORMATION_SCHEMA_DB) || - (0 == strcasecmp(pDbVgVersion->dbFName, TSDB_PERFORMANCE_SCHEMA_DB)))) { - memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); + if ((0 == strcasecmp(pDbCacheInfo->dbFName, TSDB_INFORMATION_SCHEMA_DB) || + (0 == strcasecmp(pDbCacheInfo->dbFName, TSDB_PERFORMANCE_SCHEMA_DB)))) { int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode); - if (pDbVgVersion->vgVersion < vgVersion) { - usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); - - mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); - usedbRsp.vgVersion = vgVersion++; - } else { - usedbRsp.vgVersion = pDbVgVersion->vgVersion; + if (pDbCacheInfo->vgVersion >= vgVersion) { + continue; } - usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + + rsp.useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp)); + memcpy(rsp.useDbRsp->db, pDbCacheInfo->dbFName, TSDB_DB_FNAME_LEN); + rsp.useDbRsp->pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); - taosArrayPush(batchUseRsp.pArray, &usedbRsp); + mndBuildDBVgroupInfo(NULL, pMnode, rsp.useDbRsp->pVgroupInfos); + rsp.useDbRsp->vgVersion = vgVersion++; + + rsp.useDbRsp->vgNum = taosArrayGetSize(rsp.useDbRsp->pVgroupInfos); + + taosArrayPush(batchRsp.pArray, &rsp); continue; } - SDbObj *pDb = mndAcquireDb(pMnode, pDbVgVersion->dbFName); + SDbObj *pDb = mndAcquireDb(pMnode, pDbCacheInfo->dbFName); if (pDb == NULL) { - mTrace("db:%s, no exist", pDbVgVersion->dbFName); - memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); - usedbRsp.uid = pDbVgVersion->dbId; - usedbRsp.vgVersion = -1; - taosArrayPush(batchUseRsp.pArray, &usedbRsp); + mTrace("db:%s, no exist", pDbCacheInfo->dbFName); + rsp.useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp)); + memcpy(rsp.useDbRsp->db, pDbCacheInfo->dbFName, TSDB_DB_FNAME_LEN); + rsp.useDbRsp->uid = pDbCacheInfo->dbId; + rsp.useDbRsp->vgVersion = -1; + taosArrayPush(batchRsp.pArray, &rsp); continue; } int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable && - pDbVgVersion->stateTs == pDb->stateTs) { - mTrace("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 - " numOfTables:%d, not changed vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", - pDbVgVersion->dbFName, pDbVgVersion->vgVersion, pDbVgVersion->stateTs, pDbVgVersion->numOfTable, - pDb->vgVersion, pDb->stateTs, numOfTable); + if (pDbCacheInfo->vgVersion >= pDb->vgVersion && + pDbCacheInfo->cfgVersion >= pDb->cfgVersion && + numOfTable == pDbCacheInfo->numOfTable && + pDbCacheInfo->stateTs == pDb->stateTs) { + mTrace("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64 + " numOfTables:%d, not changed vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d", + pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, pDbCacheInfo->numOfTable, + pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); mndReleaseDb(pMnode, pDb); continue; } else { - mInfo("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 - " numOfTables:%d, changed to vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", - pDbVgVersion->dbFName, pDbVgVersion->vgVersion, pDbVgVersion->stateTs, pDbVgVersion->numOfTable, - pDb->vgVersion, pDb->stateTs, numOfTable); + mInfo("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64 + " numOfTables:%d, changed to vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d", + pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, pDbCacheInfo->numOfTable, + pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); } - usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); - if (usedbRsp.pVgroupInfos == NULL) { - mndReleaseDb(pMnode, pDb); - mError("db:%s, failed to malloc usedb response", pDb->name); - continue; + if (pDbCacheInfo->cfgVersion < pDb->cfgVersion) { + rsp.cfgRsp = taosMemoryCalloc(1, sizeof(SDbCfgRsp)); + mndDumpDbCfgInfo(rsp.cfgRsp, pDb); } - mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); - memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); - usedbRsp.uid = pDb->uid; - usedbRsp.vgVersion = pDb->vgVersion; - usedbRsp.stateTs = pDb->stateTs; - usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos); - usedbRsp.hashMethod = pDb->cfg.hashMethod; - usedbRsp.hashPrefix = pDb->cfg.hashPrefix; - usedbRsp.hashSuffix = pDb->cfg.hashSuffix; + if (pDbCacheInfo->vgVersion < pDb->vgVersion || + numOfTable != pDbCacheInfo->numOfTable || + pDbCacheInfo->stateTs != pDb->stateTs) { + rsp.useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp)); + rsp.useDbRsp->pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (rsp.useDbRsp->pVgroupInfos == NULL) { + mndReleaseDb(pMnode, pDb); + mError("db:%s, failed to malloc usedb response", pDb->name); + continue; + } - taosArrayPush(batchUseRsp.pArray, &usedbRsp); + mndBuildDBVgroupInfo(pDb, pMnode, rsp.useDbRsp->pVgroupInfos); + memcpy(rsp.useDbRsp->db, pDb->name, TSDB_DB_FNAME_LEN); + rsp.useDbRsp->uid = pDb->uid; + rsp.useDbRsp->vgVersion = pDb->vgVersion; + rsp.useDbRsp->stateTs = pDb->stateTs; + rsp.useDbRsp->vgNum = (int32_t)taosArrayGetSize(rsp.useDbRsp->pVgroupInfos); + rsp.useDbRsp->hashMethod = pDb->cfg.hashMethod; + rsp.useDbRsp->hashPrefix = pDb->cfg.hashPrefix; + rsp.useDbRsp->hashSuffix = pDb->cfg.hashSuffix; + } + + taosArrayPush(batchRsp.pArray, &rsp); mndReleaseDb(pMnode, pDb); } - int32_t rspLen = tSerializeSUseDbBatchRsp(NULL, 0, &batchUseRsp); + int32_t rspLen = tSerializeSDbHbBatchRsp(NULL, 0, &batchRsp); void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tFreeSUseDbBatchRsp(&batchUseRsp); + tFreeSDbHbBatchRsp(&batchRsp); return -1; } - tSerializeSUseDbBatchRsp(pRsp, rspLen, &batchUseRsp); + tSerializeSDbHbBatchRsp(pRsp, rspLen, &batchRsp); *ppRsp = pRsp; *pRspLen = rspLen; - tFreeSUseDbBatchRsp(&batchUseRsp); + tFreeSDbHbBatchRsp(&batchRsp); return 0; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index ff6a2f460a..fbd845d448 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -533,7 +533,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb case HEARTBEAT_KEY_DBINFO: { void *rspMsg = NULL; int32_t rspLen = 0; - mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen); + mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbCacheInfo), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { SKv kv1 = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv1); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8b708c3e0f..dff0ef261f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2742,7 +2742,7 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); // // SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)stbName, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false); // // char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; // tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB); @@ -2750,29 +2750,29 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // varDataSetLen(db, strlen(varDataVal(db))); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)db, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)db, false); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->createdTime, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->createdTime, false); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables +// colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); // if (pStb->commentLen > 0) { // char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; // STR_TO_VARSTR(comment, pStb->comment); -// colDataAppend(pColInfo, numOfRows, comment, false); +// colDataSetVal(pColInfo, numOfRows, comment, false); // } else if (pStb->commentLen == 0) { // char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; // STR_TO_VARSTR(comment, ""); -// colDataAppend(pColInfo, numOfRows, comment, false); +// colDataSetVal(pColInfo, numOfRows, comment, false); // } else { // colDataSetNULL(pColInfo, numOfRows); // } @@ -2782,14 +2782,14 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // varDataSetLen(watermark, strlen(varDataVal(watermark))); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)watermark, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)watermark, false); // // char maxDelay[64 + VARSTR_HEADER_SIZE] = {0}; // sprintf(varDataVal(maxDelay), "%" PRId64 "a,%" PRId64 "a", pStb->maxdelay[0], pStb->maxdelay[1]); // varDataSetLen(maxDelay, strlen(varDataVal(maxDelay))); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)maxDelay, false); // // char rollup[160 + VARSTR_HEADER_SIZE] = {0}; // int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); @@ -2808,7 +2808,7 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // varDataSetLen(rollup, strlen(varDataVal(rollup))); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)rollup, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)rollup, false); // // numOfRows++; // sdbRelease(pSdb, pStb); @@ -3067,20 +3067,20 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p for (int32_t j = 0; j < pm->colNum; j++) { // table name SColumnInfoData *pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, tName, false); + colDataSetVal(pColInfoData, numOfRows, tName, false); // database name pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dName, false); + colDataSetVal(pColInfoData, numOfRows, dName, false); pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, typeName, false); + colDataSetVal(pColInfoData, numOfRows, typeName, false); // col name char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(colName, pm->schema[j].name); pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, colName, false); + colDataSetVal(pColInfoData, numOfRows, colName, false); // col type int8_t colType = pm->schema[j].type; @@ -3095,10 +3095,10 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p (int32_t)((pm->schema[j].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(colTypeStr, colTypeLen); - colDataAppend(pColInfoData, numOfRows, (char *)colTypeStr, false); + colDataSetVal(pColInfoData, numOfRows, (char *)colTypeStr, false); pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (const char *)&pm->schema[j].bytes, false); + colDataSetVal(pColInfoData, numOfRows, (const char *)&pm->schema[j].bytes, false); for (int32_t k = 6; k <= 8; ++k) { pColInfoData = taosArrayGet(p->pDataBlock, k); colDataSetNULL(pColInfoData, numOfRows); @@ -3192,19 +3192,19 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB for (int i = 0; i < pStb->numOfColumns; i++) { int32_t cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)stbName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)db, false); + colDataSetVal(pColInfo, numOfRows, (const char *)db, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, typeName, false); + colDataSetVal(pColInfo, numOfRows, typeName, false); // col name char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(colName, pStb->pColumns[i].name); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, colName, false); + colDataSetVal(pColInfo, numOfRows, colName, false); // col type int8_t colType = pStb->pColumns[i].type; @@ -3219,10 +3219,10 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB (int32_t)((pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(colTypeStr, colTypeLen); - colDataAppend(pColInfo, numOfRows, (char *)colTypeStr, false); + colDataSetVal(pColInfo, numOfRows, (char *)colTypeStr, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStb->pColumns[i].bytes, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->pColumns[i].bytes, false); while (cols < pShow->numOfColumns) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetNULL(pColInfo, numOfRows); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 88460cd3ca..cb83a561d2 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -123,6 +123,11 @@ int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, in int32_t payloadLen, double selectivityRatio); int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); +int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid); +int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); +int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen); + int64_t metaGetTbNum(SMeta *pMeta); int64_t metaGetNtbNum(SMeta *pMeta); typedef struct { @@ -176,7 +181,7 @@ typedef struct STsdbReader STsdbReader; #define CACHESCAN_RETRIEVE_LAST 0x8 int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly); + SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables); void tsdbReaderClose(STsdbReader *pReader); int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); @@ -192,6 +197,7 @@ int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int3 void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); void tsdbReaderSetCloseFlag(STsdbReader *pReader); +int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, uint64_t suid, void **pReader, const char *idstr); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids); @@ -258,7 +264,7 @@ int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); int32_t tqNextBlockInWal(STqReader* pReader); bool tqNextBlockImpl(STqReader *pReader); -int32_t extractSubmitMsgFromWal(SWalReader *pReader, SPackedData *pPackedData); +int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock(STqReader *pReader, SSubmitTbData **pSubmitTbDataRet); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index e431ca4a01..ef36b8429a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -46,23 +46,23 @@ typedef struct STqOffsetStore STqOffsetStore; // tqPush -typedef struct { - // msg info - int64_t consumerId; - int64_t reqOffset; - int64_t processedVer; - int32_t epoch; - // rpc info - int64_t reqId; - SRpcHandleInfo rpcInfo; - tmr_h timerId; - int8_t tmrStopped; - // exec - int8_t inputStatus; - int8_t execStatus; - SStreamQueue inputQ; - SRWLatch lock; -} STqPushHandle; +//typedef struct { +// // msg info +// int64_t consumerId; +// int64_t reqOffset; +// int64_t processedVer; +// int32_t epoch; +// // rpc info +// int64_t reqId; +// SRpcHandleInfo rpcInfo; +// tmr_h timerId; +// int8_t tmrStopped; +// // exec +// int8_t inputStatus; +// int8_t execStatus; +// SStreamQueue inputQ; +// SRWLatch lock; +//} STqPushHandle; // tqExec @@ -90,6 +90,11 @@ typedef struct { int32_t numOfCols; // number of out pout column, temporarily used } STqExecHandle; +typedef enum tq_handle_status{ + TMQ_HANDLE_STATUS_IDLE = 0, + TMQ_HANDLE_STATUS_EXEC = 1, +}tq_handle_status; + typedef struct { char subKey[TSDB_SUBSCRIBE_KEY_LEN]; int64_t consumerId; @@ -98,17 +103,18 @@ typedef struct { int64_t snapshotVer; SWalReader* pWalReader; SWalRef* pRef; - STqPushHandle pushHandle; // push +// STqPushHandle pushHandle; // push STqExecHandle execHandle; // exec SRpcMsg* msg; int32_t noDataPollCnt; + tq_handle_status status; } STqHandle; -typedef struct { - SMqDataRsp* pDataRsp; - char subKey[TSDB_SUBSCRIBE_KEY_LEN]; - SRpcHandleInfo info; -} STqPushEntry; +//typedef struct { +// SMqDataRsp* pDataRsp; +// char subKey[TSDB_SUBSCRIBE_KEY_LEN]; +// SRpcHandleInfo info; +//} STqPushEntry; struct STQ { SVnode* pVnode; @@ -181,8 +187,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); int32_t tqStreamTasksScanWal(STQ* pTq); // tq util +int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, SStreamRefDataBlock** pRefBlock); char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); -int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver); +int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 1aea479511..0173db976d 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -213,7 +213,7 @@ int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessSubmitReqForSubscribe(STQ* pTq); -int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); +int32_t tqProcessDeleteDataReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 795f281ab2..436ca1abd3 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -59,6 +59,13 @@ struct SMetaCache { SHashObj* pTableEntry; SLRUCache* pUidResCache; } sTagFilterResCache; + + struct STbGroupResCache { + TdThreadMutex lock; + uint32_t accTimes; + SHashObj* pTableEntry; + SLRUCache* pResCache; + } STbGroupResCache; }; static void entryCacheClose(SMeta* pMeta) { @@ -144,6 +151,25 @@ int32_t metaCacheOpen(SMeta* pMeta) { taosHashSetFreeFp(pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp); taosThreadMutexInit(&pCache->sTagFilterResCache.lock, NULL); + + pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); + if (pCache->STbGroupResCache.pResCache == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err2; + } + + pCache->STbGroupResCache.accTimes = 0; + pCache->STbGroupResCache.pTableEntry = + taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); + if (pCache->STbGroupResCache.pTableEntry == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err2; + } + + taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp); + taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL); + + pMeta->pCache = pCache; return code; @@ -165,6 +191,10 @@ void metaCacheClose(SMeta* pMeta) { taosThreadMutexDestroy(&pMeta->pCache->sTagFilterResCache.lock); taosHashCleanup(pMeta->pCache->sTagFilterResCache.pTableEntry); + taosLRUCacheCleanup(pMeta->pCache->STbGroupResCache.pResCache); + taosThreadMutexDestroy(&pMeta->pCache->STbGroupResCache.lock); + taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry); + taosMemoryFree(pMeta->pCache); pMeta->pCache = NULL; } @@ -520,7 +550,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK return TSDB_CODE_SUCCESS; } -static void freePayload(const void* key, size_t keyLen, void* value) { +static void freeUidCachePayload(const void* key, size_t keyLen, void* value) { if (value == NULL) { return; } @@ -626,7 +656,7 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int } // add to cache. - taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freePayload, NULL, + taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeUidCachePayload, NULL, TAOS_LRU_PRIORITY_LOW); _end: taosThreadMutexUnlock(pLock); @@ -671,3 +701,180 @@ int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) { metaDebug("vgId:%d suid:%"PRId64" cached related tag filter uid list cleared", vgId, suid); return TSDB_CODE_SUCCESS; } + +int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList) { + int32_t vgId = TD_VID(pMeta->pVnode); + + // generate the composed key for LRU cache + SLRUCache* pCache = pMeta->pCache->STbGroupResCache.pResCache; + SHashObj* pTableMap = pMeta->pCache->STbGroupResCache.pTableEntry; + TdThreadMutex* pLock = &pMeta->pCache->STbGroupResCache.lock; + + *pList = NULL; + uint64_t key[4]; + initCacheKey(key, pTableMap, suid, (const char*)pKey, keyLen); + + taosThreadMutexLock(pLock); + pMeta->pCache->STbGroupResCache.accTimes += 1; + + LRUHandle* pHandle = taosLRUCacheLookup(pCache, key, TAG_FILTER_RES_KEY_LEN); + if (pHandle == NULL) { + taosThreadMutexUnlock(pLock); + return TSDB_CODE_SUCCESS; + } + + STagFilterResEntry** pEntry = taosHashGet(pTableMap, &suid, sizeof(uint64_t)); + if (NULL == pEntry) { + metaDebug("suid %" PRIu64 " not in tb group cache", suid); + return TSDB_CODE_FAILED; + } + + *pList = taosArrayDup(taosLRUCacheValue(pCache, pHandle), NULL); + + (*pEntry)->hitTimes += 1; + + uint32_t acc = pMeta->pCache->STbGroupResCache.accTimes; + if ((*pEntry)->hitTimes % 5000 == 0 && (*pEntry)->hitTimes > 0) { + metaInfo("vgId:%d tb group cache hit:%d, total acc:%d, rate:%.2f", vgId, (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc); + } + + taosLRUCacheRelease(pCache, pHandle, false); + + // unlock meta + taosThreadMutexUnlock(pLock); + return TSDB_CODE_SUCCESS; +} + + +static void freeTbGroupCachePayload(const void* key, size_t keyLen, void* value) { + if (value == NULL) { + return; + } + + const uint64_t* p = key; + if (keyLen != sizeof(int64_t) * 4) { + metaError("tb group key length is invalid, length:%d, expect:%d", (int32_t)keyLen, (int32_t)sizeof(uint64_t) * 2); + return; + } + + SHashObj* pHashObj = (SHashObj*)p[0]; + + STagFilterResEntry** pEntry = taosHashGet(pHashObj, &p[1], sizeof(uint64_t)); + + if (pEntry != NULL && (*pEntry) != NULL) { + int64_t st = taosGetTimestampUs(); + + SListIter iter = {0}; + tdListInitIter((SList*)&((*pEntry)->list), &iter, TD_LIST_FORWARD); + + SListNode* pNode = NULL; + while ((pNode = tdListNext(&iter)) != NULL) { + uint64_t* digest = (uint64_t*)pNode->data; + if (digest[0] == p[2] && digest[1] == p[3]) { + void* tmp = tdListPopNode(&((*pEntry)->list), pNode); + taosMemoryFree(tmp); + + double el = (taosGetTimestampUs() - st) / 1000.0; + metaDebug("clear one item in tb group cache, remain cached item:%d, elapsed time:%.2fms", listNEles(&((*pEntry)->list)), + el); + break; + } + } + } + + taosArrayDestroy((SArray*)value); +} + + +int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen) { + int32_t code = 0; + int32_t vgId = TD_VID(pMeta->pVnode); + + if (payloadLen > tsTagFilterResCacheSize) { + metaDebug("vgId:%d, suid:%" PRIu64 + " ignore to add to tb group cache, due to payload length %d greater than threshold %d", + vgId, suid, payloadLen, tsTagFilterResCacheSize); + taosArrayDestroy((SArray*)pPayload); + return TSDB_CODE_SUCCESS; + } + + SLRUCache* pCache = pMeta->pCache->STbGroupResCache.pResCache; + SHashObj* pTableEntry = pMeta->pCache->STbGroupResCache.pTableEntry; + TdThreadMutex* pLock = &pMeta->pCache->STbGroupResCache.lock; + + uint64_t key[4] = {0}; + initCacheKey(key, pTableEntry, suid, pKey, keyLen); + + taosThreadMutexLock(pLock); + STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t)); + if (pEntry == NULL) { + code = addNewEntry(pTableEntry, pKey, keyLen, suid); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } else { // check if it exists or not + size_t size = listNEles(&(*pEntry)->list); + if (size == 0) { + tdListAppend(&(*pEntry)->list, pKey); + } else { + SListNode* pNode = listHead(&(*pEntry)->list); + uint64_t* p = (uint64_t*)pNode->data; + if (p[1] == ((uint64_t*)pKey)[1] && p[0] == ((uint64_t*)pKey)[0]) { + // we have already found the existed items, no need to added to cache anymore. + taosThreadMutexUnlock(pLock); + return TSDB_CODE_SUCCESS; + } else { // not equal, append it + tdListAppend(&(*pEntry)->list, pKey); + } + } + } + + // add to cache. + taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeTbGroupCachePayload, NULL, + TAOS_LRU_PRIORITY_LOW); +_end: + taosThreadMutexUnlock(pLock); + metaDebug("vgId:%d, suid:%" PRIu64 " tb group added into cache, total:%d, tables:%d", vgId, suid, + (int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry)); + + return code; +} + +// remove the lru cache that are expired due to the tags value update, or creating, or dropping, of child tables +int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { + uint64_t p[4] = {0}; + int32_t vgId = TD_VID(pMeta->pVnode); + SHashObj* pEntryHashMap = pMeta->pCache->STbGroupResCache.pTableEntry; + + uint64_t dummy[2] = {0}; + initCacheKey(p, pEntryHashMap, suid, (char*) &dummy[0], 16); + + TdThreadMutex* pLock = &pMeta->pCache->STbGroupResCache.lock; + taosThreadMutexLock(pLock); + + STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t)); + if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) { + taosThreadMutexUnlock(pLock); + return TSDB_CODE_SUCCESS; + } + + (*pEntry)->hitTimes = 0; + + SListIter iter = {0}; + tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD); + + SListNode* pNode = NULL; + while ((pNode = tdListNext(&iter)) != NULL) { + setMD5DigestInKey(p, pNode->data, 2 * sizeof(uint64_t)); + taosLRUCacheErase(pMeta->pCache->STbGroupResCache.pResCache, p, TAG_FILTER_RES_KEY_LEN); + } + + tdListEmpty(&(*pEntry)->list); + taosThreadMutexUnlock(pLock); + + metaDebug("vgId:%d suid:%"PRId64" cached related tb group cleared", vgId, suid); + return TSDB_CODE_SUCCESS; +} + + diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 83f2ece571..0164a82c69 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -767,6 +767,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe metaWLock(pMeta); metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1); metaUidCacheClear(pMeta, me.ctbEntry.suid); + metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); metaULock(pMeta); } else { me.ntbEntry.ctime = pReq->ctime; @@ -998,6 +999,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1); metaUidCacheClear(pMeta, e.ctbEntry.suid); + metaTbGroupCacheClear(pMeta, e.ctbEntry.suid); } else if (e.type == TSDB_NORMAL_TABLE) { // drop schema.db (todo) @@ -1009,6 +1011,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { metaStatsCacheDrop(pMeta, uid); metaUidCacheClear(pMeta, uid); + metaTbGroupCacheClear(pMeta, uid); --pMeta->pVnode->config.vndStats.numOfSTables; } @@ -1429,6 +1432,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn); metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid); + metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid); metaULock(pMeta); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 311d637be8..9730ad2877 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -22,6 +22,10 @@ static int32_t tqInitialize(STQ* pTq); +static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_STATUS_EXEC == pHandle->status; } +static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_EXEC;} +static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_IDLE;} + int32_t tqInit() { int8_t old; while (1) { @@ -61,6 +65,7 @@ void tqCleanUp() { static void destroyTqHandle(void* data) { STqHandle* pData = (STqHandle*)data; qDestroyTask(pData->execHandle.task); + if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { taosMemoryFreeClear(pData->execHandle.execCol.qmsg); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__DB) { @@ -292,10 +297,13 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t } if (offset.val.type == TMQ_OFFSET__LOG) { + taosWLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, offset.subKey, strlen(offset.subKey)); - if (pHandle && (walRefVer(pHandle->pRef, offset.val.version) < 0)) { + if (pHandle && (walSetRefVer(pHandle->pRef, offset.val.version) < 0)) { + taosWUnLockLatch(&pTq->lock); return -1; } + taosWUnLockLatch(&pTq->lock); } return 0; @@ -340,58 +348,64 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { STqOffsetVal reqOffset = req.reqOffset; int32_t vgId = TD_VID(pTq->pVnode); + taosWLockLatch(&pTq->lock); // 1. find handle STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); terrno = TSDB_CODE_INVALID_MSG; + taosWUnLockLatch(&pTq->lock); return -1; } + while (tqIsHandleExec(pHandle)) { + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", consumerId, vgId, req.subKey); + taosMsleep(5); + } + // 2. check re-balance status - taosRLockLatch(&pTq->lock); if (pHandle->consumerId != consumerId) { tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - taosRUnLockLatch(&pTq->lock); + taosWUnLockLatch(&pTq->lock); return -1; } - taosRUnLockLatch(&pTq->lock); + tqSetHandleExec(pHandle); + taosWUnLockLatch(&pTq->lock); // 3. update the epoch value - taosWLockLatch(&pTq->lock); int32_t savedEpoch = pHandle->epoch; if (savedEpoch < reqEpoch) { tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, reqEpoch); pHandle->epoch = reqEpoch; } - taosWUnLockLatch(&pTq->lock); char buf[80]; tFormatOffset(buf, 80, &reqOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); - return tqExtractDataForMq(pTq, pHandle, &req, pMsg); + int code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); + tqSetHandleIdle(pHandle); + return code; } int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; + int32_t vgId = TD_VID(pTq->pVnode); tqDebug("vgId:%d, tq process delete sub req %s", pTq->pVnode->config.vgId, pReq->subKey); int32_t code = 0; -// taosWLockLatch(&pTq->lock); -// int32_t code = taosHashRemove(pTq->pPushMgr, pReq->subKey, strlen(pReq->subKey)); -// if (code != 0) { -// tqDebug("vgId:%d, tq remove push handle %s", pTq->pVnode->config.vgId, pReq->subKey); -// } -// taosWUnLockLatch(&pTq->lock); + taosWLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { - // walCloseRef(pHandle->pWalReader->pWal, pHandle->pRef->refId); + while (tqIsHandleExec(pHandle)) { + tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey); + taosMsleep(5); + } if (pHandle->pRef) { walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId); } @@ -409,6 +423,8 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg if (tqMetaDeleteHandle(pTq, pReq->subKey) < 0) { tqError("cannot process tq delete req %s, since no such offset in tdb", pReq->subKey); } + taosWUnLockLatch(&pTq->lock); + return 0; } @@ -455,6 +471,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey, req.oldConsumerId, req.newConsumerId); + taosWLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { if (req.oldConsumerId != -1) { @@ -506,7 +523,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); pHandle->execHandle.execDb.pFilterOutTbUid = - taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta, (SSnapContext**)(&handle.sContext)); @@ -537,6 +554,11 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } else { + while (tqIsHandleExec(pHandle)) { + tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey); + taosMsleep(5); + } + if (pHandle->consumerId == req.newConsumerId) { // do nothing tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId); atomic_add_fetch_32(&pHandle->epoch, 1); @@ -552,22 +574,17 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg if (pTaskInfo != NULL) { qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); } - - taosWLockLatch(&pTq->lock); - // remove if it has been register in the push manager, and return one empty block to consumer - tqUnregisterPushHandle(pTq, pHandle); - - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { qStreamCloseTsdbReader(pTaskInfo); } - - taosWUnLockLatch(&pTq->lock); + // remove if it has been register in the push manager, and return one empty block to consumer + tqUnregisterPushHandle(pTq, pHandle); ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } end: + taosWUnLockLatch(&pTq->lock); taosMemoryFree(req.qmsg); return ret; } @@ -647,7 +664,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { } if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); + SWalFilterCond cond = {.deleteMsg = 1}; + pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond); } streamSetupTrigger(pTask); @@ -851,7 +869,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { memcpy(serializedReq, &req, len); // dispatch msg - tqDebug("s-task:%s start recover block stage", pTask->id.idStr); + tqDebug("s-task:%s start to recover blocking stage", pTask->id.idStr); SRpcMsg rpcMsg = { .code = 0, .contLen = len, .msgType = TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, .pCont = serializedReq}; @@ -875,6 +893,9 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } + qDebug("s-task:%s set the start wal offset to be:%"PRId64, pTask->id.idStr, sversion); + walReaderSeekVer(pTask->exec.pWalReader, sversion); + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; @@ -941,7 +962,60 @@ int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg) { return 0; } -int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { +int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, SStreamRefDataBlock** pRefBlock) { + SDecoder* pCoder = &(SDecoder){0}; + SDeleteRes* pRes = &(SDeleteRes){0}; + + *pRefBlock = NULL; + + pRes->uidList = taosArrayInit(0, sizeof(tb_uid_t)); + if (pRes->uidList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + tDecoderInit(pCoder, (uint8_t*)pData, len); + tDecodeDeleteRes(pCoder, pRes); + tDecoderClear(pCoder); + + int32_t numOfTables = taosArrayGetSize(pRes->uidList); + if (numOfTables == 0 || pRes->affectedRows == 0) { + taosArrayDestroy(pRes->uidList); + return TSDB_CODE_SUCCESS; + } + + SSDataBlock* pDelBlock = createSpecialDataBlock(STREAM_DELETE_DATA); + blockDataEnsureCapacity(pDelBlock, numOfTables); + pDelBlock->info.rows = numOfTables; + pDelBlock->info.version = ver; + + for (int32_t i = 0; i < numOfTables; i++) { + // start key column + SColumnInfoData* pStartCol = taosArrayGet(pDelBlock->pDataBlock, START_TS_COLUMN_INDEX); + colDataSetVal(pStartCol, i, (const char*)&pRes->skey, false); // end key column + SColumnInfoData* pEndCol = taosArrayGet(pDelBlock->pDataBlock, END_TS_COLUMN_INDEX); + colDataSetVal(pEndCol, i, (const char*)&pRes->ekey, false); + // uid column + SColumnInfoData* pUidCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX); + int64_t* pUid = taosArrayGet(pRes->uidList, i); + colDataSetVal(pUidCol, i, (const char*)pUid, false); + + colDataSetNULL(taosArrayGet(pDelBlock->pDataBlock, GROUPID_COLUMN_INDEX), i); + colDataSetNULL(taosArrayGet(pDelBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX), i); + colDataSetNULL(taosArrayGet(pDelBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX), i); + } + + taosArrayDestroy(pRes->uidList); + *pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); + if (pRefBlock == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pRefBlock)->type = STREAM_INPUT__REF_DATA_BLOCK; + (*pRefBlock)->pBlock = pDelBlock; + return TSDB_CODE_SUCCESS; +} + +int32_t tqProcessDeleteDataReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { bool failed = false; SDecoder* pCoder = &(SDecoder){0}; SDeleteRes* pRes = &(SDeleteRes){0}; @@ -961,6 +1035,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { taosArrayDestroy(pRes->uidList); return 0; } + SSDataBlock* pDelBlock = createSpecialDataBlock(STREAM_DELETE_DATA); blockDataEnsureCapacity(pDelBlock, sz); pDelBlock->info.rows = sz; @@ -1007,8 +1082,6 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); pRefBlock->type = STREAM_INPUT__REF_DATA_BLOCK; pRefBlock->pBlock = pDelBlock; - pRefBlock->dataRef = pRef; - atomic_add_fetch_32(pRefBlock->dataRef, 1); if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) { atomic_sub_fetch_32(pRef, 1); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index f3ecaa08f6..5654147b6d 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -54,7 +54,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execCol.qmsg) < 0) return -1; } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { pHandle->execHandle.execDb.pFilterOutTbUid = - taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); int32_t size = 0; if (tDecodeI32(pDecoder, &size) < 0) return -1; for (int32_t i = 0; i < size; i++) { @@ -295,7 +295,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { code = -1; goto end; } - walRefVer(handle.pRef, handle.snapshotVer); + walSetRefVer(handle.pRef, handle.snapshotVer); SReadHandle reader = { .meta = pTq->pVnode->pMeta, @@ -352,7 +352,9 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId); + taosWLockLatch(&pTq->lock); taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle)); + taosWUnLockLatch(&pTq->lock); } end: diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 34e93cec2d..377a5d1887 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -78,13 +78,15 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { // todo remove this if (offset.val.type == TMQ_OFFSET__LOG) { + taosWLockLatch(&pStore->pTq->lock); STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey)); if (pHandle) { - if (walRefVer(pHandle->pRef, offset.val.version) < 0) { + if (walSetRefVer(pHandle->pRef, offset.val.version) < 0) { // tqError("vgId: %d, tq handle %s ref ver %" PRId64 "error", pStore->pTq->pVnode->config.vgId, pHandle->subKey, // offset.val.version); } } + taosWUnLockLatch(&pStore->pTq->lock); } taosMemoryFree(pMemBuf); diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 950c5ea96b..50d94c5ed5 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -34,12 +34,12 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v return 0; } - if (msgType == TDMT_VND_SUBMIT) { + if (msgType == TDMT_VND_SUBMIT || msgType == TDMT_VND_DELETE) { tqStartStreamTasks(pTq); } if (msgType == TDMT_VND_DELETE) { - tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver); +// tqProcessDeleteDataReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver); } } @@ -55,6 +55,7 @@ int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) { memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); pHandle->msg->pCont = rpcMallocCont(pMsg->contLen); } else { + tqPushDataRsp(pTq, pHandle); void* tmp = pHandle->msg->pCont; memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); pHandle->msg->pCont = tmp; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 1fbdb25528..09574a6b62 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -294,32 +294,51 @@ void tqCloseReader(STqReader* pReader) { } int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { - if (walReadSeekVer(pReader->pWalReader, ver) < 0) { + if (walReaderSeekVer(pReader->pWalReader, ver) < 0) { return -1; } tqDebug("wal reader seek to ver:%"PRId64" %s", ver, id); return 0; } -int32_t extractSubmitMsgFromWal(SWalReader* pReader, SPackedData* pPackedData) { - if (walNextValidMsg(pReader) < 0) { - return -1; +int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id) { + int32_t code = walNextValidMsg(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; } - void* pBody = POINTER_SHIFT(pReader->pHead->head.body, sizeof(SSubmitReq2Msg)); - int32_t len = pReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); int64_t ver = pReader->pHead->head.version; - void* data = taosMemoryMalloc(len); - if (data == NULL) { - // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry - terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", 0); - return -1; + if (pReader->pHead->head.msgType == TDMT_VND_SUBMIT) { + void* pBody = POINTER_SHIFT(pReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + int32_t len = pReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); + + void* data = taosMemoryMalloc(len); + if (data == NULL) { + // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", 0); + return -1; + } + + memcpy(data, pBody, len); + SPackedData data1 = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data}; + + *pItem = (SStreamQueueItem*)streamDataSubmitNew(data1, STREAM_INPUT__DATA_SUBMIT); + if (*pItem == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("%s failed to create data submit for stream since out of memory", id); + return terrno; + } + } else if (pReader->pHead->head.msgType == TDMT_VND_DELETE) { + void* pBody = POINTER_SHIFT(pReader->pHead->head.body, sizeof(SMsgHead)); + int32_t len = pReader->pHead->head.bodyLen - sizeof(SMsgHead); + + extractDelDataBlock(pBody, len, ver, (SStreamRefDataBlock**)pItem); + } else { + ASSERT(0); } - memcpy(data, pBody, len); - *pPackedData = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data}; return 0; } @@ -503,6 +522,74 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap return 0; } +static int32_t buildResSDataBlock(SSDataBlock* pBlock, SSchemaWrapper* pSchema, const SArray* pColIdList) { + if (blockDataGetNumOfCols(pBlock) > 0) { + return TSDB_CODE_SUCCESS; + } + + int32_t numOfCols = taosArrayGetSize(pColIdList); + + if (numOfCols == 0) { // all columns are required + for (int32_t i = 0; i < pSchema->nCols; ++i) { + SSchema* pColSchema = &pSchema->pSchema[i]; + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + blockDataFreeRes(pBlock); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + } else { + if (numOfCols > pSchema->nCols) { + numOfCols = pSchema->nCols; + } + + int32_t i = 0; + int32_t j = 0; + while (i < pSchema->nCols && j < numOfCols) { + SSchema* pColSchema = &pSchema->pSchema[i]; + col_id_t colIdSchema = pColSchema->colId; + + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pColIdList, j); + if (colIdSchema < colIdNeed) { + i++; + } else if (colIdSchema > colIdNeed) { + j++; + } else { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + return -1; + } + i++; + j++; + } + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SColVal* pColVal) { + int32_t code = TSDB_CODE_SUCCESS; + + if (IS_STR_DATA_TYPE(pColVal->type)) { + char val[65535 + 2] = {0}; + if (pColVal->value.pData != NULL) { + memcpy(varDataVal(val), pColVal->value.pData, pColVal->value.nData); + varDataSetLen(val, pColVal->value.nData); + code = colDataSetVal(pColumnInfoData, rowIndex, val, !COL_VAL_IS_VALUE(pColVal)); + } else { + colDataSetNULL(pColumnInfoData, rowIndex); + } + } else { + code = colDataSetVal(pColumnInfoData, rowIndex, (void*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal)); + } + + return code; +} + int32_t tqRetrieveDataBlock(STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) { tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); @@ -538,53 +625,11 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSubmitTbData** pSubmitTbDataRet pReader->cachedSchemaSuid = suid; pReader->cachedSchemaVer = sversion; - SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - if (blockDataGetNumOfCols(pBlock) > 0) { - blockDataDestroy(pReader->pResBlock); - pReader->pResBlock = createDataBlock(); - pBlock = pReader->pResBlock; - - pBlock->info.id.uid = uid; - pBlock->info.version = pReader->msg.ver; - } - - int32_t numOfCols = taosArrayGetSize(pReader->pColIdList); - if (numOfCols == 0) { // all columns are required - for (int32_t i = 0; i < pSchemaWrapper->nCols; ++i) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[i]; - SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); - if (code != TSDB_CODE_SUCCESS) { - blockDataFreeRes(pBlock); - return -1; - } - } - } else { - if (numOfCols > pSchemaWrapper->nCols) { - numOfCols = pSchemaWrapper->nCols; - } - - int32_t i = 0; - int32_t j = 0; - while (i < pSchemaWrapper->nCols && j < numOfCols) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[i]; - col_id_t colIdSchema = pColSchema->colId; - - col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, j); - if (colIdSchema < colIdNeed) { - i++; - } else if (colIdSchema > colIdNeed) { - j++; - } else { - SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); - if (code != TSDB_CODE_SUCCESS) { - return -1; - } - i++; - j++; - } + ASSERT(pReader->cachedSchemaVer == pReader->pSchemaWrapper->version); + if (blockDataGetNumOfCols(pBlock) == 0) { + int32_t code = buildResSDataBlock(pReader->pResBlock, pReader->pSchemaWrapper, pReader->pColIdList); + if (code != TSDB_CODE_SUCCESS) { + return code; } } } @@ -632,30 +677,15 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSubmitTbData** pSubmitTbDataRet } else if (pCol->cid == pColData->info.colId) { for (int32_t i = 0; i < pCol->nVal; i++) { tColDataGetValue(pCol, i, &colVal); - if (IS_STR_DATA_TYPE(colVal.type)) { - if (colVal.value.pData != NULL) { - char val[65535 + 2] = {0}; - memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); - varDataSetLen(val, colVal.value.nData); - if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - return -1; - } - } else { - colDataSetNULL(pColData, i); - } - } else { - if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - return -1; - } + int32_t code = doSetVal(pColData, i, &colVal); + if (code != TSDB_CODE_SUCCESS) { + return code; } } sourceIdx++; targetIdx++; } else { - for (int32_t i = 0; i < pCol->nVal; i++) { - colDataSetNULL(pColData, i); - } - + colDataSetNNULL(pColData, 0, pCol->nVal); targetIdx++; } } @@ -681,21 +711,9 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSubmitTbData** pSubmitTbDataRet sourceIdx++; continue; } else if (colVal.cid == pColData->info.colId) { - if (IS_STR_DATA_TYPE(colVal.type)) { - if (colVal.value.pData != NULL) { - char val[65535 + 2] = {0}; - memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); - varDataSetLen(val, colVal.value.nData); - if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - return -1; - } - } else { - colDataSetNULL(pColData, i); - } - } else { - if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - return -1; - } + int32_t code = doSetVal(pColData, i, &colVal); + if (code != TSDB_CODE_SUCCESS) { + return code; } sourceIdx++; @@ -833,14 +851,14 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas char val[65535 + 2]; memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); varDataSetLen(val, colVal.value.nData); - if (colDataAppend(pColData, curRow - lastRow, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + if (colDataSetVal(pColData, curRow - lastRow, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { goto FAIL; } } else { colDataSetNULL(pColData, curRow - lastRow); } } else { - if (colDataAppend(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + if (colDataSetVal(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { goto FAIL; } } @@ -930,14 +948,14 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas char val[65535 + 2]; memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); varDataSetLen(val, colVal.value.nData); - if (colDataAppend(pColData, curRow - lastRow, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + if (colDataSetVal(pColData, curRow - lastRow, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { goto FAIL; } } else { colDataSetNULL(pColData, curRow - lastRow); } } else { - if (colDataAppend(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + if (colDataSetVal(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { goto FAIL; } } @@ -1017,6 +1035,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { int32_t vgId = TD_VID(pTq->pVnode); // update the table list for each consumer handle + taosWLockLatch(&pTq->lock); while (1) { pIter = taosHashIterate(pTq->pHandle, pIter); if (pIter == NULL) { @@ -1073,6 +1092,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } } } + taosWUnLockLatch(&pTq->lock); // update the table list handle for each stream scanner/wal reader taosWLockLatch(&pTq->pStreamMeta->lock); diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 0bb33b1215..7db078140d 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -107,42 +107,54 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { *pScanIdle = false; // seek the stored version and extract data from WAL - int32_t code = walReadSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); - if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit - SWal *pWal = pTask->exec.pWalReader->pWal; - if (pTask->chkInfo.currentVer < pWal->vers.firstVer ) { - pTask->chkInfo.currentVer = pWal->vers.firstVer; - code = walReadSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); - if (code != TSDB_CODE_SUCCESS) { + int64_t firstVer = walReaderGetValidFirstVer(pTask->exec.pWalReader); + if (pTask->chkInfo.currentVer < firstVer) { + pTask->chkInfo.currentVer = firstVer; + tqWarn("vgId:%d s-task:%s ver earlier than the first ver of wal range %" PRId64 ", forward to %" PRId64, vgId, + pTask->id.idStr, firstVer, pTask->chkInfo.currentVer); + + // todo need retry if failed + int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); + if (code != TSDB_CODE_SUCCESS) { + streamMetaReleaseTask(pStreamMeta, pTask); + continue; + } + + // append the data for the stream + tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer); + } else { + int64_t currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader); + if (currentVer == -1) { + int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); + if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit streamMetaReleaseTask(pStreamMeta, pTask); continue; } + + // append the data for the stream + tqDebug("vgId:%d s-task:%s wal reader initial seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer); } - streamMetaReleaseTask(pStreamMeta, pTask); - continue; } // append the data for the stream - tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer); +// tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer); - SPackedData packData = {0}; - code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData); + SStreamQueueItem* pItem = NULL; + int32_t code = extractMsgFromWal(pTask->exec.pWalReader, (void**) &pItem, pTask->id.idStr); if (code != TSDB_CODE_SUCCESS) { // failed, continue streamMetaReleaseTask(pStreamMeta, pTask); continue; } - SStreamDataSubmit2* p = streamDataSubmitNew(packData, STREAM_INPUT__DATA_SUBMIT); - if (p == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr); + // delete ignore + if (pItem == NULL) { streamMetaReleaseTask(pStreamMeta, pTask); continue; } noNewDataInWal = false; - code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver); + code = tqAddInputBlockNLaunchTask(pTask, pItem); if (code == TSDB_CODE_SUCCESS) { pTask->chkInfo.currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader); tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, @@ -151,8 +163,7 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { tqError("s-task:%s append input queue failed, ver:%"PRId64, pTask->id.idStr, pTask->chkInfo.currentVer); } - streamDataSubmitDestroy(p); - taosFreeQitem(p); + streamMetaReleaseTask(pStreamMeta, pTask); } diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 800bcc8b71..e4b2fa8821 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -82,6 +82,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; qStreamSetOpen(task); + tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); if (qExecTask(task, &pDataBlock, &ts) != TSDB_CODE_SUCCESS) { tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, terrstr()); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 94803ef438..622cc5b049 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -26,10 +26,10 @@ char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { return taosStrdup(buf); } -int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) { +int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem) { int32_t code = tAppendDataToInputQueue(pTask, pQueueItem); if (code < 0) { - tqError("s-task:%s failed to put into queue, too many, next start ver:%" PRId64, pTask->id.idStr, ver); + tqError("s-task:%s failed to put into queue, too many", pTask->id.idStr); return -1; } @@ -170,12 +170,6 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); - qTaskInfo_t task = pHandle->execHandle.task; - if(qTaskIsExecuting(task)){ - code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); - tDeleteSMqDataRsp(&dataRsp); - return code; - } qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); code = tqScanData(pTq, pHandle, &dataRsp, pOffset); @@ -200,10 +194,8 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, } } - - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP); - // NOTE: this pHandle->consumerId may have been changed already. + code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP); end: { @@ -211,31 +203,25 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, tFormatOffset(buf, 80, &dataRsp.rspOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 " code:%d", consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); -// taosWUnLockLatch(&pTq->lock); tDeleteSMqDataRsp(&dataRsp); } + return code; } static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal *offset) { - int code = 0; + int code = 0; int32_t vgId = TD_VID(pTq->pVnode); SWalCkHead* pCkHead = NULL; SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; tqInitTaosxRsp(&taosxRsp, pRequest); - qTaskInfo_t task = pHandle->execHandle.task; - if(qTaskIsExecuting(task)){ - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - return code; - } if (offset->type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { - tDeleteSTaosxRsp(&taosxRsp); - return -1; + code = -1; + goto end; } if (metaRsp.metaRspLen > 0) { @@ -243,30 +229,27 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 ",ts:%" PRId64, pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.ts); taosMemoryFree(metaRsp.metaRsp); - tDeleteSTaosxRsp(&taosxRsp); - return code; + goto end; } tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 ",ts:%" PRId64,pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid,taosxRsp.rspOffset.ts); if (taosxRsp.blockNum > 0) { code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - return code; + goto end; }else { *offset = taosxRsp.rspOffset; } } - if (offset->type == TMQ_OFFSET__LOG) { verifyOffset(pHandle->pWalReader, offset); int64_t fetchVer = offset->version + 1; pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); if (pCkHead == NULL) { - tDeleteSTaosxRsp(&taosxRsp); terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = -1; + goto end; } walSetReaderCapacity(pHandle->pWalReader, 2048); int totalRows = 0; @@ -281,9 +264,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); - return code; + goto end; } SWalCont* pHead = &pCkHead->head; @@ -295,9 +276,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if(totalRows > 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1); code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); - return code; + goto end; } tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); @@ -305,17 +284,8 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, metaRsp.resMsgType = pHead->msgType; metaRsp.metaRspLen = pHead->bodyLen; metaRsp.metaRsp = pHead->body; - if (tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp) < 0) { - code = -1; - taosMemoryFreeClear(pCkHead); - tDeleteSTaosxRsp(&taosxRsp); - return code; - } - - code = 0; - taosMemoryFreeClear(pCkHead); - tDeleteSTaosxRsp(&taosxRsp); - return code; + code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp); + goto end; } // process data @@ -325,29 +295,27 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, .ver = pHead->version, }; - if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows) < 0) { - tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId, - pRequest->subKey); - taosMemoryFreeClear(pCkHead); - tDeleteSTaosxRsp(&taosxRsp); - return -1; + code = tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows); + if (code < 0) { + tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId, pRequest->subKey); + goto end; } if (totalRows >= 4096 || taosxRsp.createTableNum > 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); - return code; + goto end; } else { fetchVer++; } } } +end: + tDeleteSTaosxRsp(&taosxRsp); taosMemoryFreeClear(pCkHead); - return 0; + return code; } int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 64d30c77a3..8f168e67a8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -142,6 +142,18 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id return TSDB_CODE_SUCCESS; } +int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOfTables) { + SCacheRowsReader* pReader = (SCacheRowsReader*)reader; + + pReader->pTableList = pTableIdList; + pReader->numOfTables = numOfTables; + pReader->lastTs = INT64_MIN; + + resetLastBlockLoadInfo(pReader->pLoadInfo); + + return TSDB_CODE_SUCCESS; +} + int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols, uint64_t suid, void** pReader, const char* idstr) { *pReader = NULL; @@ -213,6 +225,9 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, void* tsdbCacherowsReaderClose(void* pReader) { SCacheRowsReader* p = pReader; + if (p == NULL) { + return NULL; + } if (p->pSchema != NULL) { for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2fc2b8cdd6..4acc784aee 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -210,6 +210,7 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; + SHashObj** pIgnoreTables; STSchema* pSchema; // the newest version schema SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema SDataFReader* pFileReader; // the file reader @@ -2785,15 +2786,21 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { int64_t st = taosGetTimestampUs(); int32_t step = asc ? 1 : -1; double el = 0; + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo != NULL) { + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); + return code; + } + pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); if (pBlockScanInfo == NULL) { goto _end; } - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly @@ -2812,9 +2819,12 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } } else { // file blocks not exist pBlockScanInfo = *pReader->status.pTableIter; + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockScanInfo->uid, sizeof(pBlockScanInfo->uid))) { + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); + return code; + } } - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; while (1) { @@ -3101,6 +3111,18 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // load the last data block of current table STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) { + // reset the index in last block when handing a new file + doCleanupTableScanInfo(pScanInfo); + pStatus->mapDataCleaned = true; + + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + + continue; + } // reset the index in last block when handing a new file doCleanupTableScanInfo(pScanInfo); @@ -3156,20 +3178,24 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { static int32_t doBuildDataBlock(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; - SDataBlk* pBlock = NULL; SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; STableBlockScanInfo* pScanInfo = NULL; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + SDataBlk* pBlock = getCurrentBlock(pBlockIter); + + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order); + return code; + } pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); if (pScanInfo == NULL) { return terrno; } - pBlock = getCurrentBlock(pBlockIter); initLastBlockReader(pLastBlockReader, pScanInfo, pReader); TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); @@ -3402,6 +3428,13 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { } STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) { + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + } + initMemDataIterator(*pBlockScanInfo, pReader); int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? INT64_MAX : INT64_MIN; @@ -4356,7 +4389,7 @@ static void freeSchemaFunc(void* param) { // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, - SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr, bool countOnly) { + SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr, bool countOnly, SHashObj** pIgnoreTables) { STimeWindow window = pCond->twindows; int32_t capacity = pVnode->config.tsdbCfg.maxRows; @@ -4466,6 +4499,8 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL pReader->readMode = READ_MODE_COUNT_ONLY; } + pReader->pIgnoreTables = pIgnoreTables; + tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; diff --git a/test1 b/source/libs/cache/test/cacheTests.cpp similarity index 100% rename from test1 rename to source/libs/cache/test/cacheTests.cpp diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index c548a6c696..0293584487 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -82,6 +82,7 @@ enum { enum { CTG_OP_UPDATE_VGROUP = 0, + CTG_OP_UPDATE_DB_CFG, CTG_OP_UPDATE_TB_META, CTG_OP_DROP_DB_CACHE, CTG_OP_DROP_DB_VGROUP, @@ -248,20 +249,26 @@ typedef struct SCtgVgCache { SDBVgInfo* vgInfo; } SCtgVgCache; +typedef struct SCtgCfgCache { + SRWLatch cfgLock; + SDbCfgInfo* cfgInfo; +} SCtgCfgCache; + typedef struct SCtgDBCache { - SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads - uint64_t dbId; - int8_t deleted; - SCtgVgCache vgCache; - SHashObj* tbCache; // key:tbname, value:SCtgTbCache - SHashObj* stbCache; // key:suid, value:char* - uint64_t dbCacheNum[CTG_CI_MAX_VALUE]; + SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads + uint64_t dbId; + int8_t deleted; + SCtgVgCache vgCache; + SCtgCfgCache cfgCache; + SHashObj* tbCache; // key:tbname, value:SCtgTbCache + SHashObj* stbCache; // key:suid, value:char* + uint64_t dbCacheNum[CTG_CI_MAX_VALUE]; } SCtgDBCache; typedef struct SCtgRentSlot { SRWLatch lock; bool needSort; - SArray* meta; // element is SDbVgVersion or SSTableVersion + SArray* meta; // element is SDbCacheInfo or SSTableVersion } SCtgRentSlot; typedef struct SCtgRentMgmt { @@ -425,6 +432,13 @@ typedef struct SCtgUpdateVgMsg { SDBVgInfo* dbInfo; } SCtgUpdateVgMsg; +typedef struct SCtgUpdateDbCfgMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + uint64_t dbId; + SDbCfgInfo* cfgInfo; +} SCtgUpdateDbCfgMsg; + typedef struct SCtgUpdateTbMetaMsg { SCatalog* pCtg; STableMetaOutput* pMeta; @@ -815,8 +829,10 @@ int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t* fetchIdx, int32_t baseResIdx, SArray* pList); +void* ctgCloneDbCfgInfo(void* pSrc); int32_t ctgOpUpdateVgroup(SCtgCacheOperation* action); +int32_t ctgOpUpdateDbCfg(SCtgCacheOperation *operation); int32_t ctgOpUpdateTbMeta(SCtgCacheOperation* action); int32_t ctgOpDropDbCache(SCtgCacheOperation* action); int32_t ctgOpDropDbVgroup(SCtgCacheOperation* action); @@ -838,6 +854,7 @@ int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, bool syncReq); int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, const char* tbName, bool syncReq); int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq); +int32_t ctgUpdateDbCfgEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDbCfgInfo *cfgInfo, bool syncOp); int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput* output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp* pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char* dbFName, int32_t vgId, SEpSet* pEpSet); @@ -909,9 +926,9 @@ int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFNam void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); -int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2); +int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); -int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2); +int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); @@ -932,6 +949,9 @@ SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); int32_t ctgdGetOneHandle(SCatalog** pHandle); int ctgVgInfoComp(const void* lp, const void* rp); int32_t ctgMakeVgArray(SDBVgInfo* dbInfo); +int32_t ctgChkSetAuthRes(SCatalog *pCtg, SCtgAuthReq *req, SCtgAuthRsp* res); +int32_t ctgReadDBCfgFromCache(SCatalog *pCtg, const char* dbFName, SDbCfgInfo* pDbCfg); + int32_t ctgAcquireVgMetaFromCache(SCatalog* pCtg, const char* dbFName, const char* tbName, SCtgDBCache** pDb, SCtgTbCache** pTb); int32_t ctgCopyTbMeta(SCatalog* pCtg, SCtgTbMetaCtx* ctx, SCtgDBCache** pDb, SCtgTbCache** pTb, STableMeta** pTableMeta, diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 976a38c03d..0deeafda9f 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -668,6 +668,23 @@ _return: CTG_RET(code); } +int32_t ctgGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SDbCfgInfo* pDbCfg) { + CTG_ERR_RET(ctgReadDBCfgFromCache(pCtg, dbFName, pDbCfg)); + + if (pDbCfg->cfgVersion < 0) { + CTG_ERR_RET(ctgGetDBCfgFromMnode(pCtg, pConn, dbFName, pDbCfg, NULL)); + SDbCfgInfo *pCfg = ctgCloneDbCfgInfo(pDbCfg); + if (NULL == pCfg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + CTG_ERR_RET(ctgUpdateDbCfgEnqueue(pCtg, dbFName, pDbCfg->dbId, pCfg, false)); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t catalogInit(SCatalogCfg* cfg) { if (gCtgMgmt.pCluster) { qError("catalog already initialized"); @@ -958,6 +975,23 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogUpdateDbCfg(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDbCfgInfo* cfgInfo) { + CTG_API_ENTER(); + + int32_t code = 0; + + if (NULL == pCtg || NULL == dbFName || NULL == cfgInfo) { + freeDbCfgInfo(cfgInfo); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + + code = ctgUpdateDbCfgEnqueue(pCtg, dbFName, dbId, cfgInfo, false); + +_return: + + CTG_API_LEAVE(code); +} + int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { CTG_API_ENTER(); @@ -1371,14 +1405,14 @@ int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableVersion** stables, uint3 CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void**)stables, num, sizeof(SSTableVersion))); } -int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion** dbs, uint32_t* num) { +int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) { CTG_API_ENTER(); if (NULL == pCtg || NULL == dbs || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void**)dbs, num, sizeof(SDbVgVersion))); + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void**)dbs, num, sizeof(SDbCacheInfo))); } int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num) { @@ -1426,9 +1460,7 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbF CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_CACHE_NHIT_INC(CTG_CI_DB_CFG, 1); - - CTG_API_LEAVE(ctgGetDBCfgFromMnode(pCtg, pConn, dbFName, pDbCfg, NULL)); + CTG_API_LEAVE(ctgGetDBCfg(pCtg, pConn, dbFName, pDbCfg)); } int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* indexName, SIndexInfo* pInfo) { diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 56c79eac1f..2b78b8dd13 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1437,12 +1437,12 @@ _return: SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); pRes->code = code; pRes->pRes = NULL; + ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { TSWAP(pTask->res, ctx->pResList); taskDone = true; } - ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); } if (pTask->res && taskDone) { @@ -1587,7 +1587,6 @@ int32_t ctgHandleGetTbIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu CTG_ERR_JRET(ctgCloneTableIndex(pOut->pIndex, &pInfo)); pTask->res = pInfo; - SCtgTbIndexCtx* ctx = pTask->taskCtx; CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, (STableIndex**)&pTask->msgCtx.out, false)); _return: @@ -1660,8 +1659,14 @@ _return: int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + SCtgDbCfgCtx* ctx = pTask->taskCtx; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + SDbCfgInfo* pCfg = ctgCloneDbCfgInfo(pTask->msgCtx.out); + + CTG_ERR_RET(ctgUpdateDbCfgEnqueue(pTask->pJob->pCtg, ctx->dbFName, pCfg->dbId, pCfg, false)); + TSWAP(pTask->res, pTask->msgCtx.out); _return: @@ -2195,13 +2200,24 @@ int32_t ctgLaunchGetDbCfgTask(SCtgTask* pTask) { SCtgDbCfgCtx* pCtx = (SCtgDbCfgCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + SDbCfgInfo cfgInfo; if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } - CTG_CACHE_NHIT_INC(CTG_CI_DB_CFG, 1); + CTG_ERR_RET(ctgReadDBCfgFromCache(pCtg, pCtx->dbFName, &cfgInfo)); - CTG_ERR_RET(ctgGetDBCfgFromMnode(pCtg, pConn, pCtx->dbFName, NULL, pTask)); + if (cfgInfo.cfgVersion < 0) { + CTG_ERR_RET(ctgGetDBCfgFromMnode(pCtg, pConn, pCtx->dbFName, NULL, pTask)); + } else { + pTask->res = taosMemoryCalloc(1, sizeof(SDbCfgInfo)); + if (NULL == pTask->res) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pTask->res, &cfgInfo, sizeof(cfgInfo)); + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 433eacd30e..debd5f567c 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -20,6 +20,7 @@ #include "trpc.h" SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update vgInfo", ctgOpUpdateVgroup}, + {CTG_OP_UPDATE_DB_CFG, "update dbCfg", ctgOpUpdateDbCfg}, {CTG_OP_UPDATE_TB_META, "update tbMeta", ctgOpUpdateTbMeta}, {CTG_OP_DROP_DB_CACHE, "drop DB", ctgOpDropDbCache}, {CTG_OP_DROP_DB_VGROUP, "drop DBVgroup", ctgOpDropDbVgroup}, @@ -89,10 +90,15 @@ int32_t ctgWLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { return TSDB_CODE_SUCCESS; } -void ctgRUnlockVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); } +void ctgRLockDbCfgInfo(SCtgDBCache *dbCache) { CTG_LOCK(CTG_READ, &dbCache->cfgCache.cfgLock); } +void ctgWLockDbCfgInfo(SCtgDBCache *dbCache) { CTG_LOCK(CTG_WRITE, &dbCache->cfgCache.cfgLock); } +void ctgRUnlockVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); } void ctgWUnlockVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); } +void ctgRUnlockDbCfgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->cfgCache.cfgLock); } +void ctgWUnlockDbCfgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_WRITE, &dbCache->cfgCache.cfgLock); } + void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->dbLock); taosHashRelease(pCtg->dbCache, dbCache); @@ -703,6 +709,43 @@ _return: CTG_RET(code); } +int32_t ctgReadDBCfgFromCache(SCatalog *pCtg, const char* dbFName, SDbCfgInfo* pDbCfg) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + pDbCfg->cfgVersion = -1; + CTG_CACHE_NHIT_INC(CTG_CI_DB_CFG, 1); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->cfgCache.cfgLock); + + if (dbCache->cfgCache.cfgInfo) { + SDbCfgInfo *pInfo = ctgCloneDbCfgInfo(dbCache->cfgCache.cfgInfo); + if (NULL == pInfo) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pDbCfg, pInfo, sizeof(*pInfo)); + taosMemoryFree(pInfo); + CTG_CACHE_HIT_INC(CTG_CI_DB_CFG, 1); + } else { + pDbCfg->cfgVersion = -1; + CTG_CACHE_NHIT_INC(CTG_CI_DB_CFG, 1); + } + +_return: + + if (dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->cfgCache.cfgLock); + ctgReleaseDBCache(pCtg, dbCache); + } + + return code; +} + int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName) { *stbName = NULL; @@ -726,6 +769,7 @@ int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid return TSDB_CODE_SUCCESS; } + int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) { int32_t code = 0; if (IS_SYS_DBNAME(pReq->tbName.dbname)) { @@ -1006,6 +1050,44 @@ _return: CTG_RET(code); } + +int32_t ctgUpdateDbCfgEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDbCfgInfo *cfgInfo, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_UPDATE_DB_CFG; + op->syncOp = syncOp; + + SCtgUpdateDbCfgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateDbCfgMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateDbCfgMsg)); + taosMemoryFree(op); + freeDbCfgInfo(cfgInfo); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + char *p = strchr(dbFName, '.'); + if (p && IS_SYS_DBNAME(p + 1)) { + dbFName = p + 1; + } + + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->pCtg = pCtg; + msg->dbId = dbId; + msg->cfgInfo = cfgInfo; + + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; + +_return: + + freeDbCfgInfo(cfgInfo); + CTG_RET(code); +} + + int32_t ctgUpdateTbMetaEnqueue(SCatalog *pCtg, STableMetaOutput *output, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); @@ -1419,15 +1501,15 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { CTG_CACHE_NUM_INC(CTG_CI_DB, 1); - SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1, .stateTs = 0}; - tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + SDbCacheInfo dbCacheInfo = {.dbId = newDBCache.dbId, .vgVersion = -1, .stateTs = 0, .cfgVersion = -1}; + tstrncpy(dbCacheInfo.dbFName, dbFName, sizeof(dbCacheInfo.dbFName)); ctgDebug("db added to cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); if (!IS_SYS_DBNAME(dbFName)) { - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &dbCacheInfo, dbId, sizeof(SDbCacheInfo))); - ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, dbId); + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, dbCacheInfo.vgVersion, dbId); } return TSDB_CODE_SUCCESS; @@ -1471,7 +1553,7 @@ int32_t ctgRemoveDBFromCache(SCatalog *pCtg, SCtgDBCache *dbCache, const char *d CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock); - CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbId, ctgDbCacheInfoSortCompare, ctgDbCacheInfoSearchCompare)); ctgDebug("db removed from rent, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { @@ -1766,8 +1848,8 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { } bool newAdded = false; - SDbVgVersion vgVersion = { - .dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable, .stateTs = dbInfo->stateTs}; + SDbCacheInfo dbCacheInfo = { + .dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .cfgVersion = -1, .numOfTable = dbInfo->numOfTable, .stateTs = dbInfo->stateTs}; SCtgDBCache *dbCache = NULL; CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); @@ -1803,21 +1885,25 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { CTG_DB_NUM_RESET(CTG_CI_DB_VGROUP); } + if (dbCache->cfgCache.cfgInfo) { + dbCacheInfo.cfgVersion = dbCache->cfgCache.cfgInfo->cfgVersion; + } + vgCache->vgInfo = dbInfo; msg->dbInfo = NULL; CTG_DB_NUM_SET(CTG_CI_DB_VGROUP); ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, stateTs:%" PRId64 ", dbId:0x%" PRIx64, dbFName, - vgVersion.vgVersion, vgVersion.stateTs, vgVersion.dbId); + dbCacheInfo.vgVersion, dbCacheInfo.stateTs, dbCacheInfo.dbId); ctgWUnlockVgInfo(dbCache); dbCache = NULL; // if (!IS_SYS_DBNAME(dbFName)) { - tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), - ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + tstrncpy(dbCacheInfo.dbFName, dbFName, sizeof(dbCacheInfo.dbFName)); + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &dbCacheInfo, dbCacheInfo.dbId, sizeof(SDbCacheInfo), + ctgDbCacheInfoSortCompare, ctgDbCacheInfoSearchCompare)); //} _return: @@ -1828,6 +1914,67 @@ _return: CTG_RET(code); } +int32_t ctgOpUpdateDbCfg(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgUpdateDbCfgMsg *msg = operation->data; + SDbCfgInfo *cfgInfo = msg->cfgInfo; + char *dbFName = msg->dbFName; + SCatalog *pCtg = msg->pCtg; + + if (pCtg->stopUpdate || NULL == cfgInfo) { + goto _return; + } + + if (cfgInfo->cfgVersion < 0) { + ctgDebug("invalid db cfgInfo, dbFName:%s, cfgVersion:%d", dbFName, cfgInfo->cfgVersion); + CTG_ERR_JRET(TSDB_CODE_APP_ERROR); + } + + SCtgDBCache *dbCache = NULL; + CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%" PRIx64, dbFName, msg->dbId); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SDbCacheInfo cacheInfo = {0}; + cacheInfo.dbId = dbCache->dbId; + tstrncpy(cacheInfo.dbFName, dbFName, sizeof(cacheInfo.dbFName)); + cacheInfo.cfgVersion = cfgInfo->cfgVersion; + + SCtgVgCache *vgCache = &dbCache->vgCache; + if (vgCache->vgInfo) { + cacheInfo.vgVersion = vgCache->vgInfo->vgVersion; + cacheInfo.numOfTable = vgCache->vgInfo->numOfTable; + cacheInfo.stateTs = vgCache->vgInfo->stateTs; + } else { + cacheInfo.vgVersion = -1; + } + + ctgWLockDbCfgInfo(dbCache); + + freeDbCfgInfo(dbCache->cfgCache.cfgInfo); + dbCache->cfgCache.cfgInfo = cfgInfo; + cfgInfo = NULL; + + ctgWUnlockDbCfgInfo(dbCache); + + ctgDebug("db cfgInfo updated, dbFName:%s, cfgVer:%d", dbFName, dbCache->cfgCache.cfgInfo->cfgVersion); + + // if (!IS_SYS_DBNAME(dbFName)) { + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &cacheInfo, cacheInfo.dbId, sizeof(SDbCacheInfo), + ctgDbCacheInfoSortCompare, ctgDbCacheInfoSearchCompare)); + //} + +_return: + + freeDbCfgInfo(cfgInfo); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) { int32_t code = 0; SCtgDropDBMsg *msg = operation->data; @@ -2080,27 +2227,19 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { CTG_LOCK(CTG_WRITE, &pUser->lock); taosHashCleanup(pUser->userAuth.createdDbs); - pUser->userAuth.createdDbs = msg->userAuth.createdDbs; - msg->userAuth.createdDbs = NULL; - taosHashCleanup(pUser->userAuth.readDbs); - pUser->userAuth.readDbs = msg->userAuth.readDbs; - msg->userAuth.readDbs = NULL; - taosHashCleanup(pUser->userAuth.writeDbs); - pUser->userAuth.writeDbs = msg->userAuth.writeDbs; - msg->userAuth.writeDbs = NULL; - taosHashCleanup(pUser->userAuth.readTbs); - pUser->userAuth.readTbs = msg->userAuth.readTbs; - msg->userAuth.readTbs = NULL; - taosHashCleanup(pUser->userAuth.writeTbs); - pUser->userAuth.writeTbs = msg->userAuth.writeTbs; - msg->userAuth.writeTbs = NULL; - taosHashCleanup(pUser->userAuth.useDbs); - pUser->userAuth.useDbs = msg->userAuth.useDbs; + + memcpy(&pUser->userAuth, &msg->userAuth, sizeof(msg->userAuth)); + + msg->userAuth.createdDbs = NULL; + msg->userAuth.readDbs = NULL; + msg->userAuth.writeDbs = NULL; + msg->userAuth.readTbs = NULL; + msg->userAuth.writeTbs = NULL; msg->userAuth.useDbs = NULL; CTG_UNLOCK(CTG_WRITE, &pUser->lock); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 12ff8a7b38..8924e8c27c 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -501,6 +501,25 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { } } + if (dbCache->cfgCache.cfgInfo) { + SDbCfgInfo *pCfg = dbCache->cfgCache.cfgInfo; + ctgDebug("[%d] db [%.*s][0x%" PRIx64 + "] %s: cfgVersion:%d, numOfVgroups:%d, numOfStables:%d, buffer:%d, cacheSize:%d, pageSize:%d, pages:%d" + ", daysPerFile:%d, daysToKeep0:%d, daysToKeep1:%d, daysToKeep2:%d, minRows:%d, maxRows:%d, walFsyncPeriod:%d" + ", hashPrefix:%d, hashSuffix:%d, walLevel:%d, precision:%d, compression:%d, replications:%d, strict:%d" + ", cacheLast:%d, tsdbPageSize:%d, walRetentionPeriod:%d, walRollPeriod:%d, walRetentionSize:%" PRId64 "" + ", walSegmentSize:%" PRId64 ", numOfRetensions:%d, schemaless:%d, sstTrigger:%d", + i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", + pCfg->cfgVersion, pCfg->numOfVgroups, pCfg->numOfStables, pCfg->buffer, + pCfg->cacheSize, pCfg->pageSize, pCfg->pages, pCfg->daysPerFile, pCfg->daysToKeep0, + pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->minRows, pCfg->maxRows, pCfg->walFsyncPeriod, + pCfg->hashPrefix, pCfg->hashSuffix, pCfg->walLevel, pCfg->precision, pCfg->compression, + pCfg->replications, pCfg->strict, pCfg->cacheLast, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, + pCfg->walRollPeriod, pCfg->walRetentionSize, pCfg->walSegmentSize, pCfg->numOfRetensions, + pCfg->schemaless, pCfg->sstTrigger); + } + + ++i; pIter = taosHashIterate(dbHash, pIter); } } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 9274c1ef92..ae8fe0cca4 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -236,6 +236,7 @@ void ctgFreeTbCache(SCtgDBCache* dbCache) { } void ctgFreeVgInfoCache(SCtgDBCache* dbCache) { freeVgInfo(dbCache->vgCache.vgInfo); } +void ctgFreeCfgInfoCache(SCtgDBCache* dbCache) { freeDbCfgInfo(dbCache->cfgCache.cfgInfo); } void ctgFreeDbCache(SCtgDBCache* dbCache) { if (NULL == dbCache) { @@ -243,6 +244,7 @@ void ctgFreeDbCache(SCtgDBCache* dbCache) { } ctgFreeVgInfoCache(dbCache); + ctgFreeCfgInfoCache(dbCache); ctgFreeStbMetaCache(dbCache); ctgFreeTbCache(dbCache); } @@ -1073,10 +1075,10 @@ int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { } } -int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) { - if (*(int64_t*)key1 < ((SDbVgVersion*)key2)->dbId) { +int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2) { + if (*(int64_t*)key1 < ((SDbCacheInfo*)key2)->dbId) { return -1; - } else if (*(int64_t*)key1 > ((SDbVgVersion*)key2)->dbId) { + } else if (*(int64_t*)key1 > ((SDbCacheInfo*)key2)->dbId) { return 1; } else { return 0; @@ -1093,10 +1095,10 @@ int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { } } -int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2) { - if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) { +int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2) { + if (((SDbCacheInfo*)key1)->dbId < ((SDbCacheInfo*)key2)->dbId) { return -1; - } else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) { + } else if (((SDbCacheInfo*)key1)->dbId > ((SDbCacheInfo*)key2)->dbId) { return 1; } else { return 0; @@ -1267,16 +1269,20 @@ static void* ctgCloneDbVgroup(void* pSrc) { return taosArrayDup((const SArray*)p static void ctgFreeDbVgroup(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } -static void* ctgCloneDbCfgInfo(void* pSrc) { +void* ctgCloneDbCfgInfo(void* pSrc) { SDbCfgInfo* pDst = taosMemoryMalloc(sizeof(SDbCfgInfo)); if (NULL == pDst) { return NULL; } memcpy(pDst, pSrc, sizeof(SDbCfgInfo)); + pDst->pRetensions = taosArrayDup(((SDbCfgInfo *)pSrc)->pRetensions, NULL); return pDst; } -static void ctgFreeDbCfgInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } +static void ctgFreeDbCfgInfo(void* p) { + SDbCfgInfo* pDst = (SDbCfgInfo *)((SMetaRes*)p)->pRes; + freeDbCfgInfo(pDst); +} static void* ctgCloneDbInfo(void* pSrc) { SDbInfo* pDst = taosMemoryMalloc(sizeof(SDbInfo)); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 2cba433e84..66d2de083b 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -1331,7 +1331,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - SDbVgVersion *dbs = NULL; + SDbCacheInfo *dbs = NULL; SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; @@ -1443,7 +1443,7 @@ TEST(tableMeta, childTableCase) { taosMemoryFree(tableMeta); - SDbVgVersion *dbs = NULL; + SDbCacheInfo *dbs = NULL; SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; @@ -1584,7 +1584,7 @@ TEST(tableMeta, superTableCase) { taosMemoryFree(tableMeta); - SDbVgVersion *dbs = NULL; + SDbCacheInfo *dbs = NULL; SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; @@ -2680,7 +2680,7 @@ TEST(rentTest, allRent) { SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; - SDbVgVersion *dbs = NULL; + SDbCacheInfo *dbs = NULL; SSTableVersion *stable = NULL; uint32_t num = 0; diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 5a9f079c15..5b248f9a90 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -87,7 +87,25 @@ typedef struct SColMatchInfo { } SColMatchInfo; typedef struct SExecTaskInfo SExecTaskInfo; -typedef struct STableListInfo STableListInfo; + + +typedef struct STableListIdInfo { + uint64_t suid; + uint64_t uid; + int32_t tableType; +} STableListIdInfo; + +// If the numOfOutputGroups is 1, the data blocks that belongs to different groups will be provided randomly +// The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups +typedef struct STableListInfo { + bool oneTableForEachGroup; + int32_t numOfOuputGroups; // the data block will be generated one by one + int32_t* groupOffset; // keep the offset value for each group in the tableList + SArray* pTableList; + SHashObj* map; // speedup acquire the tableQueryInfo by table uid + STableListIdInfo idInfo; // this maybe the super table or ordinary table +} STableListInfo; + struct SqlFunctionCtx; int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, @@ -164,4 +182,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, void printDataBlock(SSDataBlock* pBlock, const char* flag); +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); + #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index a4f1e2ef94..5db79bba8b 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -213,6 +213,7 @@ typedef struct STableScanInfo { SScanInfo scanInfo; int32_t scanTimes; SSDataBlock* pResBlock; + SHashObj* pIgnoreTables; SSampleExecInfo sample; // sample execution info int32_t currentGroupId; int32_t currentTable; diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index eec34a6406..fad1c55b3d 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -226,13 +226,17 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } - code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, - taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, - pTaskInfo->id.str); - if (code != TSDB_CODE_SUCCESS) { - pInfo->currentGroupIndex += 1; - taosArrayClear(pInfo->pUidList); - continue; + if (NULL == pInfo->pLastrowReader) { + code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, + taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, + pTaskInfo->id.str); + if (code != TSDB_CODE_SUCCESS) { + pInfo->currentGroupIndex += 1; + taosArrayClear(pInfo->pUidList); + continue; + } + } else { + tsdbReuseCacherowsReader(pInfo->pLastrowReader, pList, num); } taosArrayClear(pInfo->pUidList); @@ -265,13 +269,14 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); + //pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); return pInfo->pRes; } else { - pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); + //pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); } } + pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); setOperatorCompleted(pOperator); return NULL; } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index ce8dc898a5..49e5bfdd19 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -194,9 +194,6 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; - // ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); - // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); - atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c51dc39b5b..34ead810f2 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -28,23 +28,6 @@ #include "querytask.h" #include "tcompression.h" -typedef struct STableListIdInfo { - uint64_t suid; - uint64_t uid; - int32_t tableType; -} STableListIdInfo; - -// If the numOfOutputGroups is 1, the data blocks that belongs to different groups will be provided randomly -// The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups -struct STableListInfo { - bool oneTableForEachGroup; - int32_t numOfOuputGroups; // the data block will be generated one by one - int32_t* groupOffset; // keep the offset value for each group in the tableList - SArray* pTableList; - SHashObj* map; // speedup acquire the tableQueryInfo by table uid - STableListIdInfo idInfo; // this maybe the super table or ordinary table -}; - typedef struct tagFilterAssist { SHashObj* colHash; int32_t index; @@ -62,7 +45,7 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArra static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond); static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, - SNode* pTagIndexCond, STableListInfo* pListInfo, const char* idstr); + SNode* pTagIndexCond, STableListInfo* pListInfo, uint8_t* digest, const char* idstr); static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* metaHandle); @@ -152,7 +135,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in size_t keyLen = 0; int32_t iter = 0; - int32_t bufLen = 0, offset = 0; + int64_t bufLen = 0, offset = 0; // todo move away and record this during create window while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { @@ -436,12 +419,49 @@ void freeItem(void* p) { } } -int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo) { +static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) { + if (pTagCond == NULL) { + return; + } + + char* payload = NULL; + int32_t len = 0; + nodesNodeToMsg(pTagCond, &payload, &len); + + tMD5Init(pContext); + tMD5Update(pContext, (uint8_t*)payload, (uint32_t)len); + tMD5Final(pContext); + + taosMemoryFree(payload); +} + + +static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_CTX* pContext) { + char* payload = NULL; + int32_t len = 0; + nodesNodeToMsg(pGroup, &payload, &len); + if (filterDigest[0]) { + payload = taosMemoryRealloc(payload, len + tListLen(pContext->digest)); + memcpy(payload + len, filterDigest + 1, tListLen(pContext->digest)); + len += tListLen(pContext->digest); + } + + tMD5Init(pContext); + tMD5Update(pContext, (uint8_t*)payload, (uint32_t)len); + tMD5Final(pContext); + + taosMemoryFree(payload); +} + + +int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo, uint8_t *digest) { int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; void* keyBuf = NULL; SArray* groupData = NULL; + SArray* pUidTagList = NULL; + SArray* tableList = NULL; int32_t rows = taosArrayGetSize(pTableListInfo->pTableList); if (rows == 0) { @@ -468,7 +488,23 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis REPLACE_NODE(pNode); } - SArray* pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo)); + T_MD5_CTX context = {0}; + if (tsTagFilterCache) { + SNodeListNode* listNode = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); + listNode->pNodeList = group; + genTbGroupDigest((SNode *)listNode, digest, &context); + nodesFree(listNode); + + metaGetCachedTbGroup(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList); + if (tableList) { + taosArrayDestroy(pTableListInfo->pTableList); + pTableListInfo->pTableList = tableList; + qDebug("retrieve tb group list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(pTableListInfo->pTableList)); + goto end; + } + } + + pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo)); for (int32_t i = 0; i < rows; ++i) { STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i); STUidTagInfo info = {.uid = pkeyInfo->uid}; @@ -594,6 +630,11 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis info->groupId = calcGroupId(keyBuf, len); } + if (tsTagFilterCache) { + tableList = taosArrayDup(pTableListInfo->pTableList, NULL); + metaPutTbGroupToCache(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); + } + // int64_t st2 = taosGetTimestampUs(); // qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); @@ -798,21 +839,6 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis return -1; } -static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) { - if (pTagCond == NULL) { - return; - } - - char* payload = NULL; - int32_t len = 0; - nodesNodeToMsg(pTagCond, &payload, &len); - - tMD5Init(pContext); - tMD5Update(pContext, (uint8_t*)payload, (uint32_t)len); - tMD5Final(pContext); - - taosMemoryFree(payload); -} static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* metaHandle) { @@ -1027,7 +1053,7 @@ end: } int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, - STableListInfo* pListInfo, const char* idstr) { + STableListInfo* pListInfo, uint8_t* digest, const char* idstr) { int32_t code = TSDB_CODE_SUCCESS; size_t numOfTables = 0; @@ -1057,6 +1083,8 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pUidList, &acquired); if (acquired) { + digest[0] = 1; + memcpy(digest + 1, context.digest, tListLen(context.digest)); qDebug("retrieve table uid list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(pUidList)); goto _end; } @@ -1100,6 +1128,8 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1); + digest[0] = 1; + memcpy(digest + 1, context.digest, tListLen(context.digest)); } } @@ -1484,14 +1514,23 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu return TSDB_CODE_OUT_OF_MEMORY; } + SHashObj *pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); for (int32_t i = 0; i < numOfOutput; ++i) { const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0)) { pValCtx[num++] = &pCtx[i]; } else if (fmIsSelectFunc(pCtx[i].functionId)) { - p = &pCtx[i]; + void* data = taosHashGet(pSelectFuncs, pName, strlen(pName)); + if (taosHashGetSize(pSelectFuncs) != 0 && data == NULL) { + p = NULL; + break; + } else { + taosHashPut(pSelectFuncs, pName, strlen(pName), &num, sizeof(num)); + p = &pCtx[i]; + } } } + taosHashCleanup(pSelectFuncs); if (p != NULL) { p->subsidiaries.pCtx = pValCtx; @@ -1703,12 +1742,12 @@ int32_t convertFillType(int32_t mode) { return type; } -static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { if (ascQuery) { - *w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts); + *w = getAlignQueryTimeWindow(pInterval, ts); } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - *w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts); + *w = getAlignQueryTimeWindow(pInterval, ts); int64_t key = w->skey; while (key < ts) { // moving towards end @@ -1725,7 +1764,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindo static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) { STimeWindow w = {0}; - w.skey = taosTimeTruncate(ts, pInterval, pInterval->precision); + w.skey = taosTimeTruncate(ts, pInterval); w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return w; } @@ -1759,6 +1798,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI if (pRow) { w = pRow->win; } + // in case of typical time window, we can calculate time window directly. if (w.skey > ts || w.ekey < ts) { w = doCalculateTimeWindow(ts, pInterval); @@ -1773,6 +1813,34 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI return w; } +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + if (!IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { + tw->skey += pInterval->sliding * factor; + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return; + } + + // convert key to second + int64_t key = convertTimePrecision(tw->skey, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; + + int64_t duration = pInterval->sliding; + if (pInterval->slidingUnit == 'y') { + duration *= 12; + } + + struct tm tm; + time_t t = (time_t) key; + taosLocalTime(&t, &tm, NULL); + + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + duration * factor); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); + + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; +} + bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo) { return (pLimitInfo->limit.limit != -1 || pLimitInfo->limit.offset != -1 || pLimitInfo->slimit.limit != -1 || pLimitInfo->slimit.offset != -1); @@ -1994,8 +2062,8 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { return TDB_CODE_SUCCESS; } -int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, - bool groupSort) { +int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group, + bool groupSort, uint8_t *digest) { int32_t code = TSDB_CODE_SUCCESS; bool groupByTbname = groupbyTbname(group); @@ -2015,7 +2083,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pTableListInfo->numOfOuputGroups = 1; } } else { - code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo); + code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo, digest); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2046,7 +2114,8 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return TSDB_CODE_INVALID_PARA; } - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, idStr); + uint8_t digest[17] = {0}; + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr); if (code != TSDB_CODE_SUCCESS) { qError("failed to getTableList, code: %s", tstrerror(code)); return code; @@ -2064,7 +2133,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return TSDB_CODE_SUCCESS; } - code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pGroupTags, groupSort); + code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pScanNode, pGroupTags, groupSort, digest); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 8bbbd3524d..c4a56d78ae 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1060,7 +1060,10 @@ void qStreamSetOpen(qTaskInfo_t tinfo) { void verifyOffset(void *pWalReader, STqOffsetVal* pOffset){ // if offset version is small than first version , let's seek to first version + taosThreadMutexLock(&((SWalReader*)pWalReader)->pWal->mutex); int64_t firstVer = walGetFirstVer(((SWalReader*)pWalReader)->pWal); + taosThreadMutexUnlock(&((SWalReader*)pWalReader)->pWal->mutex); + if (pOffset->version + 1 < firstVer){ pOffset->version = firstVer - 1; } @@ -1150,7 +1153,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (pScanBaseInfo->dataReader == NULL) { int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, - pScanInfo->pResBlock, &pScanBaseInfo->dataReader, id, false); + pScanInfo->pResBlock, &pScanBaseInfo->dataReader, id, false, NULL); if (code != TSDB_CODE_SUCCESS) { qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); terrno = code; @@ -1209,7 +1212,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT int32_t size = tableListGetSize(pTableListInfo); tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL, - false); + false, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 62ab2d9df2..89fa10b704 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -82,7 +82,7 @@ static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SC static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, - SGroupResInfo* pGroupResInfo, int32_t threshold); + SGroupResInfo* pGroupResInfo, int32_t threshold, bool ignoreGroup); SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize) { SFilePage* pData = NULL; @@ -442,15 +442,15 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB } ///////////////////////////////////////////////////////////////////////////////////////////// -STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key) { +STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key) { STimeWindow win = {0}; - win.skey = taosTimeTruncate(key, pInterval, precision); + win.skey = taosTimeTruncate(key, pInterval); /* * if the realSkey > INT64_MAX - pInterval->interval, the query duration between * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. */ - win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; if (win.ekey < win.skey) { win.ekey = INT64_MAX; } @@ -776,7 +776,7 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos } int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, - SGroupResInfo* pGroupResInfo, int32_t threshold) { + SGroupResInfo* pGroupResInfo, int32_t threshold, bool ignoreGroup) { SExprInfo* pExprInfo = pSup->pExprInfo; int32_t numOfExprs = pSup->numOfExprs; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; @@ -803,20 +803,23 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS continue; } - if (pBlock->info.id.groupId == 0) { - pBlock->info.id.groupId = pPos->groupId; - } else { - // current value belongs to different group, it can't be packed into one datablock - if (pBlock->info.id.groupId != pPos->groupId) { - releaseBufPage(pBuf, page); - break; + if (!ignoreGroup) { + if (pBlock->info.id.groupId == 0) { + pBlock->info.id.groupId = pPos->groupId; + } else { + // current value belongs to different group, it can't be packed into one datablock + if (pBlock->info.id.groupId != pPos->groupId) { + releaseBufPage(pBuf, page); + break; + } } } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - blockDataEnsureCapacity(pBlock, pBlock->info.rows + pRow->numOfRows); - qDebug("datablock capacity not sufficient, expand to required:%" PRId64 ", current capacity:%d, %s", - (pRow->numOfRows + pBlock->info.rows), pBlock->info.capacity, GET_TASKID(pTaskInfo)); + uint32_t newSize = pBlock->info.rows + pRow->numOfRows + (numOfRows - i) > 1 ? 1 : 0; + blockDataEnsureCapacity(pBlock, newSize); + qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", + newSize, pBlock->info.capacity, GET_TASKID(pTaskInfo)); // todo set the pOperator->resultInfo size } @@ -853,7 +856,7 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr // clear the existed group id pBlock->info.id.groupId = 0; ASSERT(!pbInfo->mergeResultBlock); - doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold); + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold, false); void* tbname = NULL; if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { @@ -880,10 +883,10 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG // clear the existed group id pBlock->info.id.groupId = 0; if (!pbInfo->mergeResultBlock) { - doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold); + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold, false); } else { while (hasRemainResults(pGroupResInfo)) { - doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold); + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold, true); if (pBlock->info.rows >= pOperator->resultInfo.threshold) { break; } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 0ac9e6097f..73222ee375 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -255,10 +255,10 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); + STimeWindow w = {0}; int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, startKey); - w = getFirstQualifiedTimeWindow(startKey, &w, pInterval, order); + getInitialStartTimeWindow(pInterval, startKey, &w, order); pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); @@ -400,13 +400,13 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_ASC); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); return win.skey; } TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_DESC); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC); return win.skey; } diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 754b5f4737..442f8162ed 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -30,11 +30,13 @@ typedef struct SJoinRowCtx { bool rowRemains; int64_t ts; SArray* leftRowLocations; - SArray* rightRowLocations; SArray* leftCreatedBlocks; SArray* rightCreatedBlocks; int32_t leftRowIdx; int32_t rightRowIdx; + + bool rightUseBuildTable; + SArray* rightRowLocations; } SJoinRowCtx; typedef struct SJoinOperatorInfo { @@ -50,7 +52,17 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; + SNode* pColEqualOnConditions; + SArray* leftEqOnCondCols; + char* leftEqOnCondKeyBuf; + int32_t leftEqOnCondKeyLen; + + SArray* rightEqOnCondCols; + char* rightEqOnCondKeyBuf; + int32_t rightEqOnCondKeyLen; + + SSHashObj* rightBuildTable; SJoinRowCtx rowCtx; } SJoinOperatorInfo; @@ -92,6 +104,100 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown setJoinColumnInfo(&pInfo->rightCol, rightTsCol); } +static void extractEqualOnCondColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode, + SColumn* pLeft, SColumn* pRight) { + SColumnNode* pLeftNode = (SColumnNode*)pOperNode->pLeft; + SColumnNode* pRightNode = (SColumnNode*)pOperNode->pRight; + if (pLeftNode->dataBlockId == pRightNode->dataBlockId || pLeftNode->dataBlockId == pDownstreams[0]->resultDataBlockId) { + *pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft); + *pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight); + } else { + *pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight); + *pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft); + } +} + +static void extractEqualOnCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pEqualOnCondNode, + SArray* leftTagEqCols, SArray* rightTagEqCols) { + SColumn left = {0}; + SColumn right = {0}; + if (nodeType(pEqualOnCondNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pEqualOnCondNode)->condType == LOGIC_COND_TYPE_AND) { + SNode* pNode = NULL; + FOREACH(pNode, ((SLogicConditionNode*)pEqualOnCondNode)->pParameterList) { + SOperatorNode* pOperNode = (SOperatorNode*)pNode; + extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + taosArrayPush(leftTagEqCols, &left); + taosArrayPush(rightTagEqCols, &right); + } + return; + } + + if (nodeType(pEqualOnCondNode) == QUERY_NODE_OPERATOR) { + SOperatorNode* pOperNode = (SOperatorNode*)pEqualOnCondNode; + extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + taosArrayPush(leftTagEqCols, &left); + taosArrayPush(rightTagEqCols, &right); + } +} + +static int32_t initTagColskeyBuf(int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { + int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = (SColumn*)taosArrayGet(pGroupColList, i); + (*keyLen) += pCol->bytes; // actual data + null_flag + } + + int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + (*keyLen) += nullFlagSize; + + (*keyBuf) = taosMemoryCalloc(1, (*keyLen)); + if ((*keyBuf) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t fillKeyBufFromTagCols(SArray* pCols, SSDataBlock* pBlock, int32_t rowIndex, void* pKey) { + SColumnDataAgg* pColAgg = NULL; + size_t numOfGroupCols = taosArrayGetSize(pCols); + char* isNull = (char*)pKey; + char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; + + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = (SColumn*) taosArrayGet(pCols, i); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + + // valid range check. todo: return error code. + if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) { + continue; + } + + if (pBlock->pBlockAgg != NULL) { + pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + } + + if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { + isNull[i] = 1; + } else { + isNull[i] = 0; + char* val = colDataGetData(pColInfoData, rowIndex); + if (pCol->type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(val); + memcpy(pStart, val, dataLen); + pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pCol->type)) { + varDataCopy(pStart, val); + pStart += varDataTLen(val); + } else { + memcpy(pStart, val, pCol->bytes); + pStart += pCol->bytes; + } + } + } + return (int32_t)(pStart - (char*)pKey); +} + SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); @@ -153,6 +259,16 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } + pInfo->pColEqualOnConditions = pJoinNode->pColEqualOnConditions; + if (pInfo->pColEqualOnConditions != NULL) { + pInfo->leftEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->rightEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); + extractEqualOnCondCols(pInfo, pDownstream, pInfo->pColEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols); + initTagColskeyBuf(&pInfo->leftEqOnCondKeyLen, &pInfo->leftEqOnCondKeyBuf, pInfo->leftEqOnCondCols); + initTagColskeyBuf(&pInfo->rightEqOnCondKeyLen, &pInfo->rightEqOnCondKeyBuf, pInfo->rightEqOnCondCols); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->rightBuildTable = tSimpleHashInit(256, hashFn); + } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { @@ -179,8 +295,28 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { pColumn->scale = pColumnNode->node.resType.scale; } +static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } + + tSimpleHashCleanup(pBuildTable); +} + void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; + if (pJoinOperator->pColEqualOnConditions != NULL) { + mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable); + taosMemoryFreeClear(pJoinOperator->rightEqOnCondKeyBuf); + taosArrayDestroy(pJoinOperator->rightEqOnCondCols); + + taosMemoryFreeClear(pJoinOperator->leftEqOnCondKeyBuf); + taosArrayDestroy(pJoinOperator->leftEqOnCondCols); + } nodesDestroyNode(pJoinOperator->pCondAfterMerge); pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); @@ -300,21 +436,122 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator return 0; } +static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) { + for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { + SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); + int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightEqOnCondCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightEqOnCondKeyBuf); + SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen); + if (!ppRows) { + SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); + taosArrayPush(rows, rightRow); + tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen, &rows, POINTER_BYTES); + } else { + taosArrayPush(*ppRows, rightRow); + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows, + const SArray* leftRowLocations, int32_t leftRowIdx, + int32_t rightRowIdx, bool useBuildTableTSRange, SArray* rightRowLocations, bool* pReachThreshold) { + *pReachThreshold = false; + uint32_t limitRowNum = pOperator->resultInfo.threshold; + SJoinOperatorInfo* pJoinInfo = pOperator->info; + size_t leftNumJoin = taosArrayGetSize(leftRowLocations); + + int32_t i,j; + + for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { + SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); + SArray* pRightRows = NULL; + if (useBuildTableTSRange) { + int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftEqOnCondCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftEqOnCondKeyBuf); + SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftEqOnCondKeyBuf, keyLen); + if (!ppRightRows) { + continue; + } + pRightRows = *ppRightRows; + } else { + pRightRows = rightRowLocations; + } + size_t rightRowsSize = taosArrayGetSize(pRightRows); + for (j = rightRowIdx; j < rightRowsSize; ++j) { + if (*nRows >= limitRowNum) { + *pReachThreshold = true; + break; + } + + SRowLocation* rightRow = taosArrayGet(pRightRows, j); + mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, + rightRow->pos); + ++*nRows; + } + if (*pReachThreshold) { + break; + } + } + + if (*pReachThreshold) { + pJoinInfo->rowCtx.rowRemains = true; + pJoinInfo->rowCtx.leftRowIdx = i; + pJoinInfo->rowCtx.rightRowIdx = j; + } + return TSDB_CODE_SUCCESS; +} + +static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks, + SArray* rightCreatedBlocks, bool rightUseBuildTable, SArray* rightRowLocations) { + for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); + blockDataDestroy(pBlock); + } + taosArrayDestroy(rightCreatedBlocks); + for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); + blockDataDestroy(pBlock); + } + if (rightRowLocations != NULL) { + taosArrayDestroy(rightRowLocations); + } + if (rightUseBuildTable) { + void* p = NULL; + int32_t iter = 0; + while ((p = tSimpleHashIterate(pJoinInfo->rightBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } + tSimpleHashClear(pJoinInfo->rightBuildTable); + } + + taosArrayDestroy(leftCreatedBlocks); + taosArrayDestroy(leftRowLocations); + + pJoinInfo->rowCtx.rowRemains = false; + pJoinInfo->rowCtx.leftRowLocations = NULL; + pJoinInfo->rowCtx.leftCreatedBlocks = NULL; + pJoinInfo->rowCtx.rightCreatedBlocks = NULL; + pJoinInfo->rowCtx.rightUseBuildTable = false; + pJoinInfo->rowCtx.rightRowLocations = NULL; +} + static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t timestamp, SSDataBlock* pRes, int32_t* nRows) { int32_t code = TSDB_CODE_SUCCESS; SJoinOperatorInfo* pJoinInfo = pOperator->info; SArray* leftRowLocations = NULL; - SArray* leftCreatedBlocks = NULL; SArray* rightRowLocations = NULL; + SArray* leftCreatedBlocks = NULL; SArray* rightCreatedBlocks = NULL; int32_t leftRowIdx = 0; int32_t rightRowIdx = 0; - int32_t i, j; - + SSHashObj* rightTableHash = NULL; + bool rightUseBuildTable = false; + if (pJoinInfo->rowCtx.rowRemains) { leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; + rightUseBuildTable = pJoinInfo->rowCtx.rightUseBuildTable; rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; @@ -330,78 +567,40 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); + if (pJoinInfo->pColEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { + mergeJoinFillBuildTable(pJoinInfo, rightRowLocations); + rightUseBuildTable = true; + taosArrayDestroy(rightRowLocations); + rightRowLocations = NULL; + } } size_t leftNumJoin = taosArrayGetSize(leftRowLocations); - size_t rightNumJoin = taosArrayGetSize(rightRowLocations); - uint32_t maxRowNum = *nRows + (leftNumJoin - leftRowIdx - 1) * rightNumJoin + rightNumJoin - rightRowIdx; - uint32_t limitRowNum = maxRowNum; - if (maxRowNum > pOperator->resultInfo.threshold) { - limitRowNum = pOperator->resultInfo.threshold; - if (!pJoinInfo->rowCtx.rowRemains) { + code = blockDataEnsureCapacity(pRes, pOperator->resultInfo.threshold); + if (code != TSDB_CODE_SUCCESS) { + qError("%s can not ensure block capacity for join. left: %zu", GET_TASKID(pOperator->pTaskInfo), + leftNumJoin); + } + + bool reachThreshold = false; + + if (code == TSDB_CODE_SUCCESS) { + mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx, + rightRowIdx, rightUseBuildTable, rightRowLocations, &reachThreshold); + } + + if (!reachThreshold) { + mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks, + rightUseBuildTable, rightRowLocations); + + } else { pJoinInfo->rowCtx.rowRemains = true; pJoinInfo->rowCtx.ts = timestamp; pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; - pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; - } - } - - code = blockDataEnsureCapacity(pRes, limitRowNum); - if (code != TSDB_CODE_SUCCESS) { - qError("%s can not ensure block capacity for join. left: %zu, right: %zu", GET_TASKID(pOperator->pTaskInfo), - leftNumJoin, rightNumJoin); - } - - - if (code == TSDB_CODE_SUCCESS) { - bool done = false; - for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { - for (j = rightRowIdx; j < rightNumJoin; ++j) { - if (*nRows >= limitRowNum) { - done = true; - break; - } - - SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); - SRowLocation* rightRow = taosArrayGet(rightRowLocations, j); - mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, - rightRow->pos); - ++*nRows; - } - if (done) { - break; - } - } - - if (maxRowNum > pOperator->resultInfo.threshold) { - pJoinInfo->rowCtx.leftRowIdx = i; - pJoinInfo->rowCtx.rightRowIdx = j; - } - } - - if (maxRowNum <= pOperator->resultInfo.threshold) { - for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(rightCreatedBlocks); - taosArrayDestroy(rightRowLocations); - for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(leftCreatedBlocks); - taosArrayDestroy(leftRowLocations); - - if (pJoinInfo->rowCtx.rowRemains) { - pJoinInfo->rowCtx.rowRemains = false; - pJoinInfo->rowCtx.leftRowLocations = NULL; - pJoinInfo->rowCtx.rightRowLocations = NULL; - pJoinInfo->rowCtx.leftCreatedBlocks = NULL; - pJoinInfo->rowCtx.rightCreatedBlocks = NULL; - } + pJoinInfo->rowCtx.rightUseBuildTable = rightUseBuildTable; + pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7cb3c00c1a..f98419f768 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -84,39 +84,6 @@ static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } -static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm, NULL); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - tw->ekey -= 1; -} - static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order) { STimeWindow w = {0}; @@ -126,7 +93,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } if (order == TSDB_ORDER_ASC) { - w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey); + w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.skey); ASSERT(w.ekey >= pBlockInfo->window.skey); if (w.ekey < pBlockInfo->window.ekey) { @@ -145,7 +112,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } } } else { - w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey); + w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.ekey); ASSERT(w.skey <= pBlockInfo->window.ekey); if (w.skey > pBlockInfo->window.skey) { @@ -194,8 +161,23 @@ static SResultRow* getTableGroupOutputBuf(SOperatorInfo* pOperator, uint64_t gro return (SResultRow*)((char*)(*pPage) + p1->offset); } +static int32_t insertTableToScanIgnoreList(STableScanInfo* pTableScanInfo, uint64_t uid) { + if (NULL == pTableScanInfo->pIgnoreTables) { + int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); + pTableScanInfo->pIgnoreTables = taosHashInit(tableNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (NULL == pTableScanInfo->pIgnoreTables) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + taosHashPut(pTableScanInfo->pIgnoreTables, &uid, sizeof(uid), &pTableScanInfo->scanTimes, sizeof(pTableScanInfo->scanTimes)); + + return TSDB_CODE_SUCCESS; +} + static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* pBlockInfo, uint32_t* status) { STableScanInfo* pTableScanInfo = pOperator->info; + int32_t code = TSDB_CODE_SUCCESS; if (pTableScanInfo->base.pdInfo.pExprSup == NULL) { return TSDB_CODE_SUCCESS; @@ -228,9 +210,10 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* if (notLoadBlock) { *status = FUNC_DATA_REQUIRED_NOT_LOAD; + code = insertTableToScanIgnoreList(pTableScanInfo, pBlockInfo->id.uid); } - return TSDB_CODE_SUCCESS; + return code; } static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsAgg, int32_t numOfCols, @@ -382,7 +365,13 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; tsdbReleaseDataBlock(pTableScanInfo->dataReader); - *status = FUNC_DATA_REQUIRED_FILTEROUT; + + STableScanInfo* pTableScanInfo = pOperator->info; + if (taosHashGetSize(pTableScanInfo->pIgnoreTables) == taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList)) { + *status = FUNC_DATA_REQUIRED_ALL_FILTEROUT; + } else { + *status = FUNC_DATA_REQUIRED_FILTEROUT; + } return TSDB_CODE_SUCCESS; } @@ -697,6 +686,10 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } + if (status == FUNC_DATA_REQUIRED_ALL_FILTEROUT) { + break; + } + // current block is filter out according to filter condition, continue load the next block if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) { continue; @@ -736,6 +729,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { } pTableScanInfo->scanTimes += 1; + taosHashClear(pTableScanInfo->pIgnoreTables); if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); @@ -763,6 +757,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { } pTableScanInfo->scanTimes += 1; + taosHashClear(pTableScanInfo->pIgnoreTables); if (pTableScanInfo->scanTimes < total) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); @@ -827,7 +822,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { ASSERT(pInfo->base.dataReader == NULL); int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, - (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly); + (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -896,6 +891,7 @@ static void destroyTableScanBase(STableScanBase* pBase) { static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); + taosHashCleanup(pTableScanInfo->pIgnoreTables); destroyTableScanBase(&pTableScanInfo->base); taosMemoryFreeClear(param); } @@ -1061,7 +1057,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, - (STsdbReader**)&pReader, GET_TASKID(pTaskInfo), false); + (STsdbReader**)&pReader, GET_TASKID(pTaskInfo), false, NULL); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -1792,7 +1788,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamScanInfo* pInfo = pOperator->info; - qDebug("stream scan called"); + qDebug("stream scan started, %s", GET_TASKID(pTaskInfo)); if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1 || pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE2) { @@ -1801,13 +1797,13 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1) { pTSInfo->base.cond.startVersion = 0; pTSInfo->base.cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer1; - qDebug("stream recover step 1, from %" PRId64 " to %" PRId64, pTSInfo->base.cond.startVersion, + qDebug("stream recover step1, verRange:%" PRId64 " - %" PRId64, pTSInfo->base.cond.startVersion, pTSInfo->base.cond.endVersion); pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN1; } else { pTSInfo->base.cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1; pTSInfo->base.cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2; - qDebug("stream recover step 2, from %" PRId64 " to %" PRId64, pTSInfo->base.cond.startVersion, + qDebug("stream recover step2, verRange:%" PRId64 " - %" PRId64, pTSInfo->base.cond.startVersion, pTSInfo->base.cond.endVersion); pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2; } @@ -1912,7 +1908,6 @@ FETCH_NEXT_BLOCK: if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { if (pInfo->validBlockIndex >= total) { doClearBufferedBlocks(pInfo); - /*pOperator->status = OP_EXEC_DONE;*/ return NULL; } @@ -2660,7 +2655,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SReadHandle* pHandle = &pInfo->base.readHandle; if (NULL == source->dataReader || !source->multiReader) { - code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo), false); + code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo), false, NULL); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } @@ -2708,6 +2703,10 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { T_LONG_JMP(pTaskInfo->env, code); } + if (status == FUNC_DATA_REQUIRED_ALL_FILTEROUT) { + break; + } + // current block is filter out according to filter condition, continue load the next block if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) { continue; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index c75c49fe77..04c6c619aa 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2268,7 +2268,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false); + code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false, NULL); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index fc4e82b57f..c98746c241 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -519,7 +519,7 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) pFillInfo->end = endKey; if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->interval.precision); + pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval); } pFillInfo->index = 0; diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 29e3668ec4..168bdf3f7c 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -244,6 +244,11 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } + if (end.key != INT64_MIN && end.key < pSliceInfo->current) { + hasInterp = false; + break; + } + if (start.key == INT64_MIN || end.key == INT64_MIN) { colDataSetNULL(pDst, rows); break; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index bea01fa0d8..8cf5813ad5 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -270,43 +270,6 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary return num; } -static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm, NULL); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); - - tw->ekey -= 1; -} - -void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { - getNextTimeWindow(pInterval, pInterval->precision, order, tw); -} - void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) { SqlFunctionCtx* pCtx = pSup->pCtx; @@ -462,7 +425,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, bool ascQuery = (order == TSDB_ORDER_ASC); int32_t precision = pInterval->precision; - getNextTimeWindow(pInterval, precision, order, pNext); + getNextTimeWindow(pInterval, pNext, order); // next time window is not in current block if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || @@ -507,7 +470,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, if (ascQuery && primaryKeys[startPos] > pNext->ekey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->skey = taosTimeTruncate(next, pInterval); pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; @@ -516,7 +479,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->skey = taosTimeTruncate(next, pInterval); pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; @@ -1377,7 +1340,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa do { if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i])) { - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); continue; } uint64_t winGpId = pGpDatas[i]; @@ -1389,7 +1352,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa if (pUpdatedMap) { tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); } - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); } while (win.ekey <= endTsCols[i]); } } diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp new file mode 100644 index 0000000000..2894c66587 --- /dev/null +++ b/source/libs/executor/test/timewindowTest.cpp @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "taos.h" +#include "thash.h" +#include "tsimplehash.h" +#include "executor.h" +#include "ttime.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +namespace { +SInterval createInterval(int64_t interval, int64_t sliding, int64_t offset, char intervalUnit, char slidingUnit, + char offsetUnit, int8_t precision) { + SInterval v = {0}; + v.interval = interval; + v.intervalUnit = intervalUnit; + v.sliding = sliding; + v.slidingUnit = slidingUnit; + v.offset = offset; + v.offsetUnit = offsetUnit; + v.precision = precision; + return v; +} + +void printTimeWindow(STimeWindow* pWindow, int8_t precision, int64_t ts) { + char buf[64] = {0}; + char bufs[64] = {0}; + char bufe[64] = {0}; + + taosFormatUtcTime(buf, tListLen(buf), ts, precision); + + taosFormatUtcTime(bufs, tListLen(bufs), pWindow->skey, precision); + taosFormatUtcTime(bufe, tListLen(bufe), pWindow->ekey, precision); + + printf("%s [%s - %s]\n", buf, bufs, bufe); +} +} // namespace + +TEST(testCase, timewindow_gen) { + // set correct time zone + osSetTimezone("UTC"); + int32_t precision = TSDB_TIME_PRECISION_MILLI; + + SInterval interval = + createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', precision); + + int64_t key = 1659312000L * 1000; // 2022-8-1 00:00:00 // UTC+8 (ms) + + STimeWindow w = {0}; + getInitialStartTimeWindow(&interval, key, &w, true); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&interval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, precision, key); + + printf("---------------------------------------------------\n"); + SInterval monthInterval = + createInterval(1, 1, 0, 'n', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&monthInterval, key, &w, true); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&monthInterval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, precision, key); + + printf("----------------------------------------------------------\n"); + SInterval slidingInterval = createInterval(1, 10*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&slidingInterval, key, &w, true); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + printf("-----------------calendar_interval_1n_sliding_1d-------\n"); + SInterval calendar_interval_1n = createInterval(1, 1*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + int64_t k1 = 1664409600 * 1000L; + getInitialStartTimeWindow(&calendar_interval_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + + printf("next\n"); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + printf("----------------interval_1d_clendar_sliding_1n---------\n"); + SInterval interval_1d_calendar_sliding_1n = createInterval(1*86400*1000L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + + k1 = 1664409600 * 1000L; + getInitialStartTimeWindow(&interval_1d_calendar_sliding_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + + printf("next time window:\n"); + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + printf("----------------interval_1d_sliding_1d_calendar_offset_1n---------\n"); + SInterval offset_1n = createInterval(10*86400*1000L, 10*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&offset_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + + +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index aa72309c62..b198865696 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -54,16 +54,15 @@ int32_t udfdCPluginOpen(SScriptUdfEnvItem *items, int numItems) { return 0; } int32_t udfdCPluginClose() { return 0; } const char *udfdCPluginUdfInitLoadInitDestoryFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) { - char initFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; + char initFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; char *initSuffix = "_init"; - strcpy(initFuncName, udfName); - strncat(initFuncName, initSuffix, strlen(initSuffix)); + snprintf(initFuncName, sizeof(initFuncName), "%s%s", udfName, initSuffix); uv_dlsym(&udfCtx->lib, initFuncName, (void **)(&udfCtx->initFunc)); - char destroyFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; + char destroyFuncName[TSDB_FUNC_NAME_LEN + 9] = {0}; char *destroySuffix = "_destroy"; strcpy(destroyFuncName, udfName); - strncat(destroyFuncName, destroySuffix, strlen(destroySuffix)); + snprintf(destroyFuncName, sizeof(destroyFuncName), "%s%s", udfName, destroySuffix); uv_dlsym(&udfCtx->lib, destroyFuncName, (void **)(&udfCtx->destroyFunc)); return udfName; } @@ -73,22 +72,19 @@ void udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) strcpy(processFuncName, udfName); uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->aggProcFunc)); - char startFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; + char startFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; char *startSuffix = "_start"; - strncpy(startFuncName, processFuncName, sizeof(startFuncName)); - strncat(startFuncName, startSuffix, strlen(startSuffix)); + snprintf(startFuncName, sizeof(startFuncName), "%s%s", processFuncName, startSuffix); uv_dlsym(&udfCtx->lib, startFuncName, (void **)(&udfCtx->aggStartFunc)); - char finishFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; + char finishFuncName[TSDB_FUNC_NAME_LEN + 8] = {0}; char *finishSuffix = "_finish"; - strncpy(finishFuncName, processFuncName, sizeof(finishFuncName)); - strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); + snprintf(finishFuncName, sizeof(finishFuncName), "%s%s", processFuncName, finishSuffix); uv_dlsym(&udfCtx->lib, finishFuncName, (void **)(&udfCtx->aggFinishFunc)); - char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; + char mergeFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; char *mergeSuffix = "_merge"; - strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName)); - strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix)); + snprintf(mergeFuncName, sizeof(mergeFuncName), "%s%s", processFuncName, mergeSuffix); uv_dlsym(&udfCtx->lib, mergeFuncName, (void **)(&udfCtx->aggMergeFunc)); } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index d9a4c5178f..f18e666741 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -401,6 +401,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); + CLONE_NODE_FIELD(pColEqualOnConditions); COPY_SCALAR_FIELD(isSingleTableJoin); COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; @@ -587,7 +588,7 @@ static int32_t physiSysTableScanCopy(const SSystemTableScanPhysiNode* pSrc, SSys return TSDB_CODE_SUCCESS; } -static int32_t physiWindowCopy(const SWinodwPhysiNode* pSrc, SWinodwPhysiNode* pDst) { +static int32_t physiWindowCopy(const SWindowPhysiNode* pSrc, SWindowPhysiNode* pDst) { COPY_BASE_OBJECT_FIELD(node, physiNodeCopy); CLONE_NODE_LIST_FIELD(pExprs); CLONE_NODE_LIST_FIELD(pFuncs); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 0aeb83ce08..f061fe60d2 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1416,6 +1416,7 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) { static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; static const char* jkJoinLogicPlanMergeCondition = "MergeConditions"; +static const char* jkJoinLogicPlanColEqualOnConditions = "ColumnEqualOnConditions"; static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj; @@ -1430,7 +1431,9 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinLogicPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions); + } return code; } @@ -1447,7 +1450,9 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); } - + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinLogicPlanColEqualOnConditions, &pNode->pColEqualOnConditions); + } return code; } @@ -1878,6 +1883,7 @@ static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition"; static const char* jkJoinPhysiPlanOnConditions = "OnConditions"; static const char* jkJoinPhysiPlanTargets = "Targets"; +static const char* jkJoinPhysiPlanColEqualOnConditions = "ColumnEqualOnConditions"; static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj; @@ -1898,7 +1904,9 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinPhysiPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions); + } return code; } @@ -1921,7 +1929,9 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); } - + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinPhysiPlanColEqualOnConditions, &pNode->pColEqualOnConditions); + } return code; } @@ -2125,7 +2135,7 @@ static const char* jkWindowPhysiPlanOutputTsOrder = "outputTsOrder"; static const char* jkWindowPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { - const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj; int32_t code = physicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { @@ -2166,7 +2176,7 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { } static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { - SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj; int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index 156744ef1d..477d27d271 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -82,9 +82,7 @@ static bool columnNodeEqual(const SColumnNode* a, const SColumnNode* b) { COMPARE_STRING_FIELD(dbName); COMPARE_STRING_FIELD(tableName); COMPARE_STRING_FIELD(colName); - if (0 == a->tableId) { - COMPARE_STRING_FIELD(tableAlias); - } + COMPARE_STRING_FIELD(tableAlias); return true; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 6c6b6c0e81..e200b5dac5 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2317,7 +2317,8 @@ enum { PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION, PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, PHY_SORT_MERGE_JOIN_CODE_TARGETS, - PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER + PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, + PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS }; static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -2339,7 +2340,9 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pColEqualOnConditions); + } return code; } @@ -2368,6 +2371,9 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { case PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER: code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); break; + case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColEqualOnConditions); + break; default: break; } @@ -2633,7 +2639,7 @@ enum { }; static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { - const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj; int32_t code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); if (TSDB_CODE_SUCCESS == code) { @@ -2674,7 +2680,7 @@ static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { } static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) { - SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj; int32_t code = TSDB_CODE_SUCCESS; STlv* pTlv = NULL; diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index 0181da92a9..b57bba0cc9 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -120,9 +120,9 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } if (colNode->tableAlias[0]) { - *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->node.userAlias); } else { - *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->node.userAlias); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index f71eef7969..830d5886f4 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -87,7 +87,7 @@ static void* nodesCalloc(int32_t num, int32_t size) { return (char*)p + 1; } -static void nodesFree(void* p) { +void nodesFree(void* p) { char* ptr = (char*)p - 1; if (0 == *ptr) { taosMemoryFree(ptr); @@ -599,7 +599,7 @@ static void destroyPhysiNode(SPhysiNode* pNode) { nodesDestroyNode(pNode->pSlimit); } -static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) { +static void destroyWinodwPhysiNode(SWindowPhysiNode* pNode) { destroyPhysiNode((SPhysiNode*)pNode); nodesDestroyList(pNode->pExprs); nodesDestroyList(pNode->pFuncs); @@ -1072,6 +1072,7 @@ void nodesDestroyNode(SNode* pNode) { destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyNode(pLogicNode->pMergeCondition); nodesDestroyNode(pLogicNode->pOnConditions); + nodesDestroyNode(pLogicNode->pColEqualOnConditions); break; } case QUERY_NODE_LOGIC_PLAN_AGG: { @@ -1204,6 +1205,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); nodesDestroyList(pPhyNode->pTargets); + nodesDestroyNode(pPhyNode->pColEqualOnConditions); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { @@ -1241,7 +1243,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: - destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: { @@ -1257,19 +1259,19 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: - destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: { SStateWinodwPhysiNode* pPhyNode = (SStateWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pStateKey); break; } case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: { SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pStartCond); nodesDestroyNode(pPhyNode->pEndCond); break; @@ -2054,7 +2056,7 @@ char* nodesGetNameFromColumnNode(SNode* pNode) { return "NULL"; } - return ((SColumnNode*)pNode)->colName; + return ((SColumnNode*)pNode)->node.userAlias; } int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots) { diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 5a47ed731d..6f866c027e 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -259,8 +259,19 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName); } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); - strncpy(pExpr->aliasName, pRawExpr->p, len); - pExpr->aliasName[len] = '\0'; + + // See TS-3398. + // Len of pRawExpr->p could be larger than len of aliasName[TSDB_COL_NAME_LEN]. + // If aliasName is truncated, hash value of aliasName could be the same. + T_MD5_CTX ctx; + tMD5Init(&ctx); + tMD5Update(&ctx, (uint8_t*)pRawExpr->p, pRawExpr->n); + tMD5Final(&ctx); + char* p = pExpr->aliasName; + for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) { + sprintf(p, "%02x", ctx.digest[i]); + p += 2; + } strncpy(pExpr->userAlias, pRawExpr->p, len); pExpr->userAlias[len] = '\0'; } diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index c25d0e7036..01b62a9051 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -388,6 +388,9 @@ static bool isSetUselessCol(SSetOperator* pSetOp, int32_t index, SExprNode* pPro } static int32_t calcConstSetOpProjections(SCalcConstContext* pCxt, SSetOperator* pSetOp, bool subquery) { + if (subquery && pSetOp->opType == SET_OP_TYPE_UNION) { + return TSDB_CODE_SUCCESS; + } int32_t index = 0; SNode* pProj = NULL; WHERE_EACH(pProj, pSetOp->pProjectionList) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f4c86d4849..e92cdca6b6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -827,7 +827,7 @@ static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColum strcpy(pCol->node.aliasName, pCol->colName); } if ('\0' == pCol->node.userAlias[0]) { - strcpy(pCol->node.userAlias, pCol->colName); + strcpy(pCol->node.userAlias, pExpr->userAlias); } pCol->node.resType = pExpr->resType; } @@ -1760,6 +1760,7 @@ static int32_t rewriteFuncToValue(STranslateContext* pCxt, char* pLiteral, SNode return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName); + strcpy(pVal->node.userAlias, ((SExprNode*)*pNode)->userAlias); pVal->node.resType = ((SExprNode*)*pNode)->resType; if (NULL == pLiteral) { pVal->isNull = true; @@ -2739,6 +2740,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { } else { len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); strncpy(pFunc->node.aliasName, buf, TMIN(len, sizeof(pFunc->node.aliasName) - 1)); + len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); strncpy(pFunc->node.userAlias, buf, TMIN(len, sizeof(pFunc->node.userAlias) - 1)); } @@ -3181,7 +3183,7 @@ static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode* int64_t timeRange = TABS(pFill->timeRange.skey - pFill->timeRange.ekey); int64_t intervalRange = 0; - if (TIME_IS_VAR_DURATION(pInterval->unit)) { + if (IS_CALENDAR_TIME_DURATION(pInterval->unit)) { int64_t f = 1; if (pInterval->unit == 'n') { f = 30LL * MILLISECOND_PER_DAY; @@ -3262,7 +3264,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; - bool valInter = TIME_IS_VAR_DURATION(pInter->unit); + bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit); if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime, getPrecisionStr(precision)); @@ -3276,7 +3278,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (pInter->unit == 'n' && pOffset->unit == 'y') { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); } - bool fixed = !TIME_IS_VAR_DURATION(pOffset->unit) && !valInter; + bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter; if ((fixed && pOffset->datum.i >= pInter->datum.i) || (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { @@ -3292,7 +3294,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* const static int32_t INTERVAL_SLIDING_FACTOR = 100; SValueNode* pSliding = (SValueNode*)pInterval->pSliding; - if (TIME_IS_VAR_DURATION(pSliding->unit)) { + if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); } if ((pSliding->datum.i < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, precision)) || @@ -5326,7 +5328,8 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable } if (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType) { - if (calcTypeBytes(pStmt->dataType) > TSDB_MAX_FIELD_LEN) { + if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } @@ -5351,6 +5354,11 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TOO_MANY_COLUMNS); } + if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + } + if (pTableMeta->tableInfo.rowSize + calcTypeBytes(pStmt->dataType) > TSDB_MAX_BYTES_PER_ROW) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW); } @@ -8322,6 +8330,11 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL); } + if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + } + if (pTableMeta->tableInfo.rowSize + pReq->colModBytes - pSchema->bytes > TSDB_MAX_BYTES_PER_ROW) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW); } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index ec6c69ea8d..c60257cc22 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -459,6 +459,8 @@ TEST_F(ParserSelectTest, joinSemanticCheck) { run("SELECT * FROM (SELECT tag1, SUM(c1) s FROM st1 GROUP BY tag1) t1, st1 t2 where t1.tag1 = t2.tag1", TSDB_CODE_PAR_NOT_SUPPORT_JOIN); + + run("SELECT count(*) FROM t1 a join t1 b on a.ts=b.ts where a.ts=b.ts"); } } // namespace ParserTest diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 39783868b3..d748f1fb7f 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -37,19 +37,24 @@ typedef struct SRewriteExprCxt { int32_t errCode; SNodeList* pExprs; bool* pOutputs; + bool isPartitionBy; } SRewriteExprCxt; -static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol) { +static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol, bool isPartitionBy) { switch (pFunc->funcType) { case FUNCTION_TYPE_TBNAME: pCol->colType = COLUMN_TYPE_TBNAME; break; case FUNCTION_TYPE_WSTART: - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (!isPartitionBy) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } pCol->colType = COLUMN_TYPE_WINDOW_START; break; case FUNCTION_TYPE_WEND: - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (!isPartitionBy) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } pCol->colType = COLUMN_TYPE_WINDOW_END; break; case FUNCTION_TYPE_WDURATION: @@ -100,9 +105,10 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); pCol->node.resType = pToBeRewrittenExpr->resType; strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); + strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { - setColumnInfo((SFunctionNode*)pExpr, pCol); + setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); } nodesDestroyNode(*pNode); *pNode = (SNode*)pCol; @@ -141,7 +147,8 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExpr(pExpr, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL}; + bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL, .isPartitionBy = isPartitionBy}; cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr); if (TSDB_CODE_SUCCESS == cxt.errCode) { nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); @@ -169,7 +176,8 @@ static int32_t cloneRewriteExprs(SNodeList* pExprs, bool* pOutputs, SNodeList** static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause, SNodeList** pRewriteExprs) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = isPartitionBy}; if (NULL != pRewriteExprs) { cxt.pOutputs = taosMemoryCalloc(LIST_LENGTH(pExprs), sizeof(bool)); if (NULL == cxt.pOutputs) { @@ -186,14 +194,14 @@ static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ES static int32_t rewriteExpr(SNodeList* pExprs, SNode** pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false}; nodesRewriteExpr(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false}; nodesRewriteExprs(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } @@ -543,11 +551,16 @@ static SNode* createGroupingSetNode(SNode* pExpr) { return (SNode*)pGroupingSet; } -static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { +static EGroupAction getDistinctGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; } +static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { + return ((pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) && !pSelect->isDistinct) ? GROUP_ACTION_KEEP + : GROUP_ACTION_NONE; +} + static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) { return needTimeline ? (NULL != pSelect->pPartitionByList ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) : DATA_ORDER_LEVEL_NONE; @@ -1151,7 +1164,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } - pAgg->node.groupAction = GROUP_ACTION_CLEAR; + pAgg->node.groupAction = GROUP_ACTION_CLEAR;//getDistinctGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index effbbc161e..d72ecb2af8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -740,6 +740,85 @@ static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoin return code; } +static bool pushDownCondOptIsTableColumn(SNode* pNode, SNodeList* pTableCols) { + if (QUERY_NODE_COLUMN != nodeType(pNode)) { + return false; + } + SColumnNode* pCol = (SColumnNode*)pNode; + return pushDownCondOptBelongThisTable(pNode, pTableCols); +} + +static bool pushDownCondOptIsColEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { + if (QUERY_NODE_OPERATOR != nodeType(pCond)) { + return false; + } + SOperatorNode* pOper = (SOperatorNode*)pCond; + if (OP_TYPE_EQUAL != pOper->opType) { + return false; + } + if (QUERY_NODE_COLUMN != nodeType(pOper->pLeft) || QUERY_NODE_COLUMN != nodeType(pOper->pRight)) { + return false; + } + SColumnNode* pLeft = (SColumnNode*)(pOper->pLeft); + SColumnNode* pRight = (SColumnNode*)(pOper->pRight); + //TODO: add cast to operator and remove this restriction of optimization + if (pLeft->node.resType.type != pRight->node.resType.type || pLeft->node.resType.bytes != pRight->node.resType.bytes) { + return false; + } + SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; + SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; + if (pushDownCondOptIsTableColumn(pOper->pLeft, pLeftCols)) { + return pushDownCondOptIsTableColumn(pOper->pRight, pRightCols); + } else if (pushDownCondOptIsTableColumn(pOper->pLeft, pRightCols)) { + return pushDownCondOptIsTableColumn(pOper->pRight, pLeftCols); + } + return false; +} + +static int32_t pushDownCondOptJoinExtractColEqualOnLogicCond(SJoinLogicNode* pJoin) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); + + int32_t code = TSDB_CODE_SUCCESS; + SNodeList* pEqualOnConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + if (pushDownCondOptIsColEqualOnCond(pJoin, pCond)) { + code = nodesListMakeAppend(&pEqualOnConds, nodesCloneNode(pCond)); + } + } + + SNode* pTempTagEqCond = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempTagEqCond, &pEqualOnConds); + } + + if (TSDB_CODE_SUCCESS == code) { + pJoin->pColEqualOnConditions = pTempTagEqCond; + return TSDB_CODE_SUCCESS; + } else { + nodesDestroyList(pEqualOnConds); + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t pushDownCondOptJoinExtractColEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (NULL == pJoin->pOnConditions) { + pJoin->pColEqualOnConditions = NULL; + return TSDB_CODE_SUCCESS; + } + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { + return pushDownCondOptJoinExtractColEqualOnLogicCond(pJoin); + } + + if (pushDownCondOptIsColEqualOnCond(pJoin, pJoin->pOnConditions)) { + pJoin->pColEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); + } + + return TSDB_CODE_SUCCESS; +} + static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) { return TSDB_CODE_SUCCESS; @@ -774,6 +853,10 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin); } + if (TSDB_CODE_SUCCESS == code) { + code = pushDownCondOptJoinExtractColEqualOnCond(pCxt, pJoin); + } + if (TSDB_CODE_SUCCESS == code) { OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; @@ -1259,8 +1342,8 @@ static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pW .sliding = pIndex->sliding, .slidingUnit = pIndex->slidingUnit, .precision = pScan->node.precision}; - return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval, pScan->node.precision)) && - (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval, pScan->node.precision)); + return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval)) && + (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval)); } return true; } @@ -2594,7 +2677,9 @@ static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pCondit if (NULL == pConditions) { return true; } - + if (LIST_LENGTH(pInfo->pAgg->pGroupKeys) != 0) { + return false; + } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pConditions)) { return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e2c2e4c655..f11f1244b4 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -705,6 +705,9 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); } + if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColEqualOnConditions) { + code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqualOnConditions, &pJoin->pColEqualOnConditions); + } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); } @@ -1147,7 +1150,7 @@ static int32_t createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNo } } -static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWinodwPhysiNode* pWindow, +static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowPhysiNode* pWindow, SWindowLogicNode* pWindowLogicNode) { pWindow->triggerType = pWindowLogicNode->triggerType; pWindow->watermark = pWindowLogicNode->watermark; @@ -1643,6 +1646,9 @@ static int32_t createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pChildren, (SNode*)pChild); } + if (TSDB_CODE_SUCCESS != code) { + break; + } } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index c6a4a97f6e..58b8e53478 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -19,6 +19,14 @@ #include "scalar.h" #include "tglobal.h" +static void debugPrintNode(SNode* pNode) { + char* pStr = NULL; + nodesNodeToString(pNode, false, &pStr, NULL); + printf("%s\n", pStr); + taosMemoryFree(pStr); + return; +} + static void dumpQueryPlan(SQueryPlan* pPlan) { if (!tsQueryPlannerTrace) { return; diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 9d8c170003..c70ea57401 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -548,3 +548,11 @@ int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { return TSDB_CODE_SUCCESS; } + +void freeDbCfgInfo(SDbCfgInfo *pInfo) { + if (pInfo) { + taosArrayDestroy(pInfo->pRetensions); + } + taosMemoryFree(pInfo); +} + diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index f198b73c7d..4bc357d7dd 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -120,7 +120,8 @@ typedef struct SQWTaskCtx { int8_t explain; int8_t needFetch; int8_t localExec; - int32_t msgType; + int32_t queryMsgType; + int32_t fetchMsgType; int32_t level; uint64_t sId; diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 59e63e9eae..f2e48918ab 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -126,10 +126,10 @@ void qwDbgDumpTasksInfo(SQWorker *mgmt) { void *key = taosHashGetKey(pIter, NULL); QW_GET_QTID(key, qId, tId, eId); - QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, msgType:%d, " + QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, queryMsgType:%d, " "sId:%" PRId64 ", level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, " "rspCode:%x, affectedRows:%" PRId64 ", taskHandle:%p, sinkHandle:%p, tbFName:%s, sver:%d, tver:%d, events:%d,%d,%d,%d,%d", - ctx, ctx->lock, ctx->phase, ctx->taskType, ctx->explain, ctx->needFetch, ctx->localExec, ctx->msgType, + ctx, ctx->lock, ctx->phase, ctx->taskType, ctx->explain, ctx->needFetch, ctx->localExec, ctx->queryMsgType, ctx->sId, ctx->level, ctx->queryGotData, ctx->queryRsped, ctx->queryEnd, ctx->queryContinue, ctx->queryInQueue, ctx->rspCode, ctx->affectedRows, ctx->taskHandle, ctx->sinkHandle, ctx->tbInfo.tbFName, ctx->tbInfo.sversion, ctx->tbInfo.tversion, ctx->events[QW_EVENT_CANCEL], ctx->events[QW_EVENT_READY], @@ -259,9 +259,9 @@ void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) { static int32_t ignoreTime = 0; if (++ignoreTime > 10 && 0 == taosRand() % 9) { - if (ctx->msgType == TDMT_SCH_FETCH) { + if (ctx->fetchMsgType == TDMT_SCH_FETCH) { qwBuildAndSendErrorRsp(TDMT_SCH_LINK_BROKEN, &ctx->ctrlConnInfo, TSDB_CODE_RPC_BROKEN_LINK); - qwBuildAndSendErrorRsp(ctx->msgType + 1, &ctx->dataConnInfo, TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); + qwBuildAndSendErrorRsp(ctx->fetchMsgType + 1, &ctx->dataConnInfo, TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); *rsped = true; taosSsleep(3); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 92c7852dbc..9db0495081 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -254,6 +254,7 @@ bool qwTaskNotInExec(SQWTaskCtx *ctx) { return false; } + int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { int32_t taskNum = 0; @@ -436,6 +437,40 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes return TSDB_CODE_SUCCESS; } + +int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx * ctx, SQWMsg *qwMsg, int32_t code) { + if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL) { + if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + void *rsp = NULL; + int32_t dataLen = 0; + SOutputData sOutput = {0}; + if (qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)) { + return TSDB_CODE_SUCCESS; + } + + if (rsp) { + bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + if (qComplete) { + atomic_store_8((int8_t *)&ctx->queryEnd, true); + } + + qwMsg->connInfo = ctx->dataConnInfo; + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); + + qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code); + rsp = NULL; + + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), + dataLen); + } + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t code = 0; SQWTaskCtx *ctx = NULL; @@ -584,7 +619,7 @@ _return: if (QW_PHASE_POST_QUERY == phase && ctx && !ctx->queryRsped) { bool rsped = false; - SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; + SQWMsg qwMsg = {.msgType = ctx->queryMsgType, .connInfo = ctx->ctrlConnInfo}; qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); if (!rsped) { @@ -634,6 +669,8 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); + qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); + _return: if (ctx) { @@ -660,7 +697,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { ctx->taskType = qwMsg->msgInfo.taskType; ctx->explain = qwMsg->msgInfo.explain; ctx->needFetch = qwMsg->msgInfo.needFetch; - ctx->msgType = qwMsg->msgType; + ctx->queryMsgType = qwMsg->msgType; ctx->localExec = false; // QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); @@ -684,7 +721,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(TSDB_CODE_APP_ERROR); } - qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); + //qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); ctx->level = plan->level; atomic_store_ptr(&ctx->taskHandle, pTaskInfo); @@ -701,32 +738,7 @@ _return: input.msgType = qwMsg->msgType; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); - if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { - void *rsp = NULL; - int32_t dataLen = 0; - SOutputData sOutput = {0}; - if (qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)) { - return TSDB_CODE_SUCCESS; - } - - if (rsp) { - bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); - - qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); - if (qComplete) { - atomic_store_8((int8_t *)&ctx->queryEnd, true); - } - - qwMsg->connInfo = ctx->dataConnInfo; - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - - qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); - rsp = NULL; - - QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), - dataLen); - } - } + qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code); QW_RET(TSDB_CODE_SUCCESS); } @@ -750,8 +762,10 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { atomic_store_8((int8_t *)&ctx->queryInQueue, 0); atomic_store_8((int8_t *)&ctx->queryContinue, 0); - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryStop)); - + if (!queryStop) { + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryStop)); + } + if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { SOutputData sOutput = {0}; QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); @@ -774,7 +788,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { qwMsg->connInfo = ctx->dataConnInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); + qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code); rsp = NULL; QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, @@ -796,7 +810,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { rsp = NULL; qwMsg->connInfo = ctx->dataConnInfo; - qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, NULL, 0, code); + qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, NULL, 0, code); QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), 0); } @@ -830,7 +844,7 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - ctx->msgType = qwMsg->msgType; + ctx->fetchMsgType = qwMsg->msgType; ctx->dataConnInfo = qwMsg->connInfo; SOutputData sOutput = {0}; @@ -896,6 +910,8 @@ _return: qwFreeFetchRsp(rsp); rsp = NULL; } + } else { + //qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code); } QW_RET(TSDB_CODE_SUCCESS); @@ -1307,7 +1323,7 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64 ctx->taskType = qwMsg->msgInfo.taskType; ctx->explain = qwMsg->msgInfo.explain; ctx->needFetch = qwMsg->msgInfo.needFetch; - ctx->msgType = qwMsg->msgType; + ctx->queryMsgType = qwMsg->msgType; ctx->localExec = true; ctx->explainRes = explainRes; @@ -1362,7 +1378,7 @@ int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64 QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - ctx->msgType = TDMT_SCH_MERGE_FETCH; + ctx->fetchMsgType = TDMT_SCH_MERGE_FETCH; ctx->explainRes = explainRes; SOutputData sOutput = {0}; diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 9ed297bd6b..f4ee0b98b7 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -288,14 +288,7 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { int8_t type = pItem->type; if (type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit2* pSubmitBlock = streamSubmitBlockClone((SStreamDataSubmit2*)pItem); - if (pSubmitBlock == NULL) { - qDebug("task %d %p submit enqueue failed since out of memory", pTask->id.taskId, pTask); - terrno = TSDB_CODE_OUT_OF_MEMORY; - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); - return -1; - } - + SStreamDataSubmit2* pSubmitBlock = (SStreamDataSubmit2*)pItem; int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1; qDebug("s-task:%s submit enqueue %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->id.idStr, pItem, pSubmitBlock->submit.msgStr, pSubmitBlock->submit.msgLen, @@ -304,6 +297,7 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) { qError("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY); streamDataSubmitDestroy(pSubmitBlock); + taosFreeQitem(pSubmitBlock); return -1; } diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index ae616260f3..96022850a3 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -194,13 +194,7 @@ void streamFreeQitem(SStreamQueueItem* data) { taosFreeQitem(pMerge); } else if (type == STREAM_INPUT__REF_DATA_BLOCK) { SStreamRefDataBlock* pRefBlock = (SStreamRefDataBlock*)data; - - int32_t ref = atomic_sub_fetch_32(pRefBlock->dataRef, 1); - ASSERT(ref >= 0); - if (ref == 0) { - blockDataDestroy(pRefBlock->pBlock); - taosMemoryFree(pRefBlock->dataRef); - } + blockDataDestroy(pRefBlock->pBlock); taosFreeQitem(pRefBlock); } } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index e10562f5cb..e093d45074 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -60,7 +60,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* SArray* pBlockList = pMerged->submits; int32_t numOfBlocks = taosArrayGetSize(pBlockList); - qDebug("st-task:%s %p set submit input (merged), batch num:%d", pTask->id.idStr, pTask, numOfBlocks); + qDebug("s-task:%s %p set submit input (merged), batch num:%d", pTask->id.idStr, pTask, numOfBlocks); qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)data; diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 0324580885..26429ea764 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -224,12 +224,19 @@ int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { void* exec = pTask->exec.pExecutor; + const char* id = pTask->id.idStr; - qDebug("s-task:%s recover step2(blocking stage) started", pTask->id.idStr); + int64_t st = taosGetTimestampMs(); + qDebug("s-task:%s recover step2(blocking stage) started", id); if (qStreamSourceRecoverStep2(exec, ver) < 0) { } - return streamScanExec(pTask, 100); + int32_t code = streamScanExec(pTask, 100); + + double el = (taosGetTimestampMs() - st) / 1000.0; + qDebug("s-task:%s recover step2(blocking stage) ended, elapsed time:%.2fs", id, el); + + return code; } int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index ea35f1cfe5..1709fc3cb1 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -966,7 +966,7 @@ static void cliSendCb(uv_write_t* req, int status) { } if (status == 0) { - tTrace("%s conn %p data already was written out", CONN_GET_INST_LABEL(pConn), pConn); + tDebug("%s conn %p data already was written out", CONN_GET_INST_LABEL(pConn), pConn); } else { if (!uv_is_closing((uv_handle_t*)&pConn->stream)) { tError("%s conn %p failed to write:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_err_name(status)); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 269c7ecf9b..da3b0ad626 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -236,7 +236,7 @@ static bool uvHandleReq(SSvrConn* pConn) { if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); if (cost >= EXCEPTION_LIMIT_US) { - tGWarn("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, recv exception", transLabel(pTransInst), + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, recv exception", transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost); } else { tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus", transLabel(pTransInst), pConn, @@ -244,7 +244,7 @@ static bool uvHandleReq(SSvrConn* pConn) { } } else { if (cost >= EXCEPTION_LIMIT_US) { - tGWarn("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus, recv exception", + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus, recv exception", transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, transMsg.code, (int)(cost)); } else { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 37d97b35a6..eed0c0dcfc 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -62,9 +62,6 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { void walCloseReader(SWalReader *pReader) { taosCloseFile(&pReader->pIdxFile); taosCloseFile(&pReader->pLogFile); - /*if (pReader->cond.enableRef) {*/ - /*taosHashRemove(pReader->pWal->pRefHash, &pReader->readerId, sizeof(int64_t));*/ - /*}*/ taosMemoryFreeClear(pReader->pHead); taosMemoryFree(pReader); } @@ -74,22 +71,25 @@ int32_t walNextValidMsg(SWalReader *pReader) { int64_t lastVer = walGetLastVer(pReader->pWal); int64_t committedVer = walGetCommittedVer(pReader->pWal); int64_t appliedVer = walGetAppliedVer(pReader->pWal); + if(appliedVer < committedVer){ // wait apply ver equal to commit ver, otherwise may lost data when consume data [TD-24010] wDebug("vgId:%d, wal apply ver:%"PRId64" smaller than commit ver:%"PRId64, pReader->pWal->cfg.vgId, appliedVer, committedVer); -// taosMsleep(10); } -// int64_t endVer = pReader->cond.scanUncommited ? lastVer : committedVer; + int64_t endVer = TMIN(appliedVer, committedVer); wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 ", applied index:%" PRId64", end index:%" PRId64, pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer, endVer); + while (fetchVer <= endVer) { if (walFetchHeadNew(pReader, fetchVer) < 0) { return -1; } - if (pReader->pHead->head.msgType == TDMT_VND_SUBMIT || - (IS_META_MSG(pReader->pHead->head.msgType) && pReader->cond.scanMeta)) { + + int32_t type = pReader->pHead->head.msgType; + if (type == TDMT_VND_SUBMIT || ((type == TDMT_VND_DELETE) && (pReader->cond.deleteMsg == 1)) || + (IS_META_MSG(type) && pReader->cond.scanMeta)) { if (walFetchBodyNew(pReader) < 0) { return -1; } @@ -98,13 +98,16 @@ int32_t walNextValidMsg(SWalReader *pReader) { if (walSkipFetchBodyNew(pReader) < 0) { return -1; } + fetchVer = pReader->curVersion; } } + return -1; } int64_t walReaderGetCurrentVer(const SWalReader *pReader) { return pReader->curVersion; } +int64_t walReaderGetValidFirstVer(const SWalReader *pReader) { return walGetFirstVer(pReader->pWal); } static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int64_t ver) { int64_t ret = 0; @@ -206,7 +209,7 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { return 0; } -int32_t walReadSeekVer(SWalReader *pReader, int64_t ver) { +int32_t walReaderSeekVer(SWalReader *pReader, int64_t ver) { SWal *pWal = pReader->pWal; if (ver == pReader->curVersion) { wDebug("vgId:%d, wal index:%" PRId64 " match, no need to reset", pReader->pWal->cfg.vgId, ver); @@ -236,7 +239,7 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { wDebug("vgId:%d, wal starts to fetch head, index:%" PRId64, pRead->pWal->cfg.vgId, fetchVer); if (pRead->curVersion != fetchVer) { - if (walReadSeekVer(pRead, fetchVer) < 0) { + if (walReaderSeekVer(pRead, fetchVer) < 0) { return -1; } seeked = true; @@ -247,7 +250,9 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { if (contLen == sizeof(SWalCkHead)) { break; } else if (contLen == 0 && !seeked) { - walReadSeekVerImpl(pRead, fetchVer); + if(walReadSeekVerImpl(pRead, fetchVer) < 0){ + return -1; + } seeked = true; continue; } else { @@ -276,6 +281,7 @@ static int32_t walFetchBodyNew(SWalReader *pReader) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + pReader->pHead = ptr; pReadHead = &pReader->pHead->head; pReader->capacity = pReadHead->bodyLen; @@ -291,14 +297,11 @@ static int32_t walFetchBodyNew(SWalReader *pReader) { pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } -// pRead->curInvalid = 1; return -1; } if (walValidBodyCksum(pReader->pHead) != 0) { wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, ver); -// pRead->curInvalid = 1; - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } @@ -340,7 +343,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { } if (pRead->curVersion != ver) { - code = walReadSeekVer(pRead, ver); + code = walReaderSeekVer(pRead, ver); if (code < 0) { // pRead->curVersion = ver; // pRead->curInvalid = 1; @@ -354,7 +357,9 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { if (contLen == sizeof(SWalCkHead)) { break; } else if (contLen == 0 && !seeked) { - walReadSeekVerImpl(pRead, ver); + if(walReadSeekVerImpl(pRead, ver) < 0){ + return -1; + } seeked = true; continue; } else { @@ -475,7 +480,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { taosThreadMutexLock(&pReader->mutex); if (pReader->curVersion != ver) { - if (walReadSeekVer(pReader, ver) < 0) { + if (walReaderSeekVer(pReader, ver) < 0) { wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; @@ -488,7 +493,10 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { if (contLen == sizeof(SWalCkHead)) { break; } else if (contLen == 0 && !seeked) { - walReadSeekVerImpl(pReader, ver); + if(walReadSeekVerImpl(pReader, ver) < 0){ + taosThreadMutexUnlock(&pReader->mutex); + return -1; + } seeked = true; continue; } else { diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index 4d451db0c0..6aba661926 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -45,7 +45,7 @@ void walCloseRef(SWal *pWal, int64_t refId) { taosMemoryFree(pRef); } -int32_t walRefVer(SWalRef *pRef, int64_t ver) { +int32_t walSetRefVer(SWalRef *pRef, int64_t ver) { SWal *pWal = pRef->pWal; wDebug("vgId:%d, wal ref version %" PRId64 ", refId %" PRId64, pWal->cfg.vgId, ver, pRef->refId); if (pRef->refVer != ver) { @@ -57,26 +57,12 @@ int32_t walRefVer(SWalRef *pRef, int64_t ver) { } pRef->refVer = ver; - // bsearch in fileSet - // SWalFileInfo tmpInfo; - // tmpInfo.firstVer = ver; - // SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - // ASSERT(pRet != NULL); - // pRef->refFile = pRet->firstVer; - taosThreadMutexUnlock(&pWal->mutex); } return 0; } -#if 1 -void walUnrefVer(SWalRef *pRef) { - pRef->refId = -1; - // pRef->refFile = -1; -} -#endif - SWalRef *walRefFirstVer(SWal *pWal, SWalRef *pRef) { if (pRef == NULL) { pRef = walOpenRef(pWal); @@ -87,12 +73,6 @@ SWalRef *walRefFirstVer(SWal *pWal, SWalRef *pRef) { taosThreadMutexLock(&pWal->mutex); int64_t ver = walGetFirstVer(pWal); pRef->refVer = ver; - // bsearch in fileSet - // SWalFileInfo tmpInfo; - // tmpInfo.firstVer = ver; - // SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - // ASSERT(pRet != NULL); - // pRef->refFile = pRet->firstVer; taosThreadMutexUnlock(&pWal->mutex); wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, ver); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index f8ce680531..6c7c5ddb0d 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -316,6 +316,10 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) { } SArray* taosArrayDup(const SArray* pSrc, __array_item_dup_fn_t fn) { + if (NULL == pSrc) { + return NULL; + } + if (pSrc->size == 0) { // empty array list return taosArrayInit(8, pSrc->elemSize); } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 7631da1b56..b05f1808e8 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -912,11 +912,11 @@ int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, ch return opos; } -uint64_t decodeDoubleValue(const char *const input, int32_t *const ipos, uint8_t flag) { +FORCE_INLINE uint64_t decodeDoubleValue(const char *const input, int32_t *const ipos, uint8_t flag) { uint64_t diff = 0ul; - int32_t nbytes = (flag & INT8MASK(3)) + 1; + int32_t nbytes = (flag & 0x7) + 1; for (int32_t i = 0; i < nbytes; i++) { - diff = diff | ((INT64MASK(8) & input[(*ipos)++]) << BITS_PER_BYTE * i); + diff |= (((uint64_t)0xff & input[(*ipos)++]) << BITS_PER_BYTE * i); } int32_t shift_width = (LONG_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); diff <<= shift_width; @@ -936,25 +936,22 @@ int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, uint8_t flags = 0; int32_t ipos = 1; int32_t opos = 0; - uint64_t prev_value = 0; + uint64_t diff = 0; + union { + uint64_t bits; + double real; + } curr; + + curr.bits = 0; for (int32_t i = 0; i < nelements; i++) { if ((i & 0x01) == 0) { flags = input[ipos++]; } - uint8_t flag = flags & INT8MASK(4); + diff = decodeDoubleValue(input, &ipos, flags & 0x0f); flags >>= 4; - - uint64_t diff = decodeDoubleValue(input, &ipos, flag); - union { - uint64_t bits; - double real; - } curr; - - uint64_t predicted = prev_value; - curr.bits = predicted ^ diff; - prev_value = curr.bits; + curr.bits ^= diff; ostream[opos++] = curr.real; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6f726c7eff..712c413e7c 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -372,7 +372,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, "Tag conditon too many TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of this query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many groups/time window in query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index d415379f92..e3b1c2fa2f 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -596,7 +596,7 @@ static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msg int32_t end = 0; int32_t remainSize = 0; static int64_t lostLine = 0; - char tmpBuf[128] = {0}; + char tmpBuf[128]; int32_t tmpBufLen = 0; if (pLogBuf == NULL || pLogBuf->stop) return -1; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4065ac5bee..bcfb392bb4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -78,7 +78,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +,,n,system-test,python3 ./test.py -f 7-tmq/tmqConsumerGroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAlterSchema.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -N 3 -n 3 @@ -118,6 +119,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3404.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py @@ -898,12 +900,14 @@ ,,y,script,./test.sh -f tsim/query/multi_order_by.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim +,,y,script,./test.sh -f tsim/query/groupby_distinct.sim ,,y,script,./test.sh -f tsim/query/event.sim ,,y,script,./test.sh -f tsim/query/forceFill.sim ,,y,script,./test.sh -f tsim/query/emptyTsRange.sim ,,y,script,./test.sh -f tsim/query/partitionby.sim ,,y,script,./test.sh -f tsim/query/tableCount.sim ,,y,script,./test.sh -f tsim/query/nullColSma.sim +,,y,script,./test.sh -f tsim/query/bug3398.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim ,,y,script,./test.sh -f tsim/mnode/basic1.sim diff --git a/tests/script/tsim/alter/table.sim b/tests/script/tsim/alter/table.sim index ded5d6f78a..5f45b446ca 100644 --- a/tests/script/tsim/alter/table.sim +++ b/tests/script/tsim/alter/table.sim @@ -657,17 +657,34 @@ if $data20 != null then return -1 endi -print =============== error +print =============== error for normal table sql create table tb2023(ts timestamp, f int); sql_error alter table tb2023 add column v varchar(16375); sql_error alter table tb2023 add column v varchar(16385); sql_error alter table tb2023 add column v varchar(33100); sql alter table tb2023 add column v varchar(16374); +sql_error alter table tb2023 modify column v varchar(16375); sql desc tb2023 sql alter table tb2023 drop column v sql_error alter table tb2023 add column v nchar(4094); sql alter table tb2023 add column v nchar(4093); +sql_error alter table tb2023 modify column v nchar(4094); sql desc tb2023 + +print =============== error for super table +sql create table stb2023(ts timestamp, f int) tags(t1 int); +sql_error alter table stb2023 add column v varchar(16375); +sql_error alter table stb2023 add column v varchar(16385); +sql_error alter table stb2023 add column v varchar(33100); +sql alter table stb2023 add column v varchar(16374); +sql_error alter table stb2023 modify column v varchar(16375); +sql desc stb2023 +sql alter table stb2023 drop column v +sql_error alter table stb2023 add column v nchar(4094); +sql alter table stb2023 add column v nchar(4093); +sql_error alter table stb2023 modify column v nchar(4094); +sql desc stb2023 + print ======= over sql drop database d1 sql select * from information_schema.ins_databases diff --git a/tests/script/tsim/parser/alter_column.sim b/tests/script/tsim/parser/alter_column.sim index c70a604c73..d569e47735 100644 --- a/tests/script/tsim/parser/alter_column.sim +++ b/tests/script/tsim/parser/alter_column.sim @@ -48,7 +48,7 @@ sql_error alter table tb modify column c2 binary(10); sql_error alter table tb modify column c2 binary(9); sql_error alter table tb modify column c2 binary(-9); sql_error alter table tb modify column c2 binary(0); -sql alter table tb modify column c2 binary(17000); +sql_error alter table tb modify column c2 binary(17000); sql_error alter table tb modify column c2 nchar(30); sql_error alter table tb modify column c3 double; sql_error alter table tb modify column c3 nchar(10); diff --git a/tests/script/tsim/parser/join_manyblocks.sim b/tests/script/tsim/parser/join_manyblocks.sim index a40a75f50c..7fd0df21b3 100644 --- a/tests/script/tsim/parser/join_manyblocks.sim +++ b/tests/script/tsim/parser/join_manyblocks.sim @@ -6,8 +6,8 @@ sql connect $dbPrefix = join_m_db $tbPrefix = join_tb $mtPrefix = join_mt -$tbNum = 3 -$rowNum = 2000 +$tbNum = 20 +$rowNum = 200 $totalNum = $tbNum * $rowNum print =============== join_manyBlocks.sim @@ -78,8 +78,8 @@ print ==============> td-3313 sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1; print $row -if $row != 6000 then - print expect 6000, actual: $row +if $row != 4000 then + print expect 4000, actual: $row return -1 endi diff --git a/tests/script/tsim/query/bug3398.sim b/tests/script/tsim/query/bug3398.sim new file mode 100644 index 0000000000..3ca88cf459 --- /dev/null +++ b/tests/script/tsim/query/bug3398.sim @@ -0,0 +1,30 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database +sql create database test + +print =============== create super table and child table +sql use test + +sql CREATE STABLE st (day timestamp, c2 int) TAGS (vin binary(32)) + +sql insert into test.g using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) +sql insert into test.t using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) +sql insert into test.tg using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) + +sql select sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end), sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end) from test.t t, test.g g, test.tg tg where t.day = g.day and t.day = tg.day and t.day between '2021-05-03' and '2023-05-04' and t.vin = 'TAG1' and t.vin = g.vin and t.vin = tg.vin group by t.day; + +print $rows $data00 $data01 +if $rows != 1 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi + +if $data01 != 3.000000000 then + return -1 +endi diff --git a/tests/script/tsim/query/groupby_distinct.sim b/tests/script/tsim/query/groupby_distinct.sim new file mode 100644 index 0000000000..8b16bb1910 --- /dev/null +++ b/tests/script/tsim/query/groupby_distinct.sim @@ -0,0 +1,30 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + + +sql drop database if exists db1; +sql create database db1; +sql use db1; + +sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create table tba1 using sta tags(1, 1, 1); +sql insert into tba1 values ('2022-04-26 15:15:08', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:07', 1, "b"); +sql insert into tba1 values ('2022-04-26 15:15:06', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:05', 1, "b"); +sql insert into tba1 values ('2022-04-26 15:15:04', 1, "c"); +sql insert into tba1 values ('2022-04-26 15:15:03', 1, "c"); +sql insert into tba1 values ('2022-04-26 15:15:02', 1, "d"); +sql insert into tba1 values ('2022-04-26 15:15:01', 1, "d"); +sql select distinct avg(f1) as avgv from sta group by f2; +if $rows != 1 then + return -1 +endi +sql select distinct avg(f1) as avgv from sta group by f2 limit 1,10; +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/partitionby.sim b/tests/script/tsim/query/partitionby.sim index 8babd1aa8d..4657713780 100644 --- a/tests/script/tsim/query/partitionby.sim +++ b/tests/script/tsim/query/partitionby.sim @@ -24,16 +24,83 @@ sql use $db sql create table $mt1 (ts timestamp, f1 int) TAGS(tag1 int, tag2 binary(500)) sql create table tb0 using $mt1 tags(0, 'a'); sql create table tb1 using $mt1 tags(1, 'b'); -sql create table tb2 using $mt1 tags(1, 'a'); -sql create table tb3 using $mt1 tags(1, 'a'); -sql create table tb4 using $mt1 tags(3, 'b'); -sql create table tb5 using $mt1 tags(3, 'a'); -sql create table tb6 using $mt1 tags(3, 'b'); -sql create table tb7 using $mt1 tags(3, 'b'); +sql create table tb2 using $mt1 tags(2, 'a'); +sql create table tb3 using $mt1 tags(3, 'a'); +sql create table tb4 using $mt1 tags(4, 'b'); +sql create table tb5 using $mt1 tags(5, 'a'); +sql create table tb6 using $mt1 tags(6, 'b'); +sql create table tb7 using $mt1 tags(7, 'b'); sql select * from $mt1 partition by tag1,tag2 limit 1; if $rows != 0 then return -1 endi +sql insert into tb0 values ('2022-04-26 15:15:08', 1); +sql insert into tb1 values ('2022-04-26 15:15:07', 2); +sql insert into tb2 values ('2022-04-26 15:15:06', 3); +sql insert into tb3 values ('2022-04-26 15:15:05', 4); +sql insert into tb4 values ('2022-04-26 15:15:04', 5); +sql insert into tb5 values ('2022-04-26 15:15:03', 6); +sql insert into tb6 values ('2022-04-26 15:15:02', 7); +sql insert into tb7 values ('2022-04-26 15:15:01', 8); + +sql select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart; +if $rows != 8 then + return -1 +endi +if $data00 != @22-04-26 15:15:01.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data10 != @22-04-26 15:15:02.000@ then + return -1 +endi +if $data11 != 1 then + return -1 +endi +if $data20 != @22-04-26 15:15:03.000@ then + return -1 +endi +if $data21 != 1 then + return -1 +endi +if $data30 != @22-04-26 15:15:04.000@ then + return -1 +endi +if $data31 != 1 then + return -1 +endi +if $data40 != @22-04-26 15:15:05.000@ then + return -1 +endi +if $data41 != 1 then + return -1 +endi +if $data50 != @22-04-26 15:15:06.000@ then + return -1 +endi +if $data51 != 1 then + return -1 +endi +if $data60 != @22-04-26 15:15:07.000@ then + return -1 +endi +if $data61 != 1 then + return -1 +endi +if $data70 != @22-04-26 15:15:08.000@ then + return -1 +endi +if $data71 != 1 then + return -1 +endi +sql select * from (select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart) order by ts; +sql select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart; +sql select * from (select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart) order by ts; +sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 interval(1s) order by _wstart); +sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 partition by tag1 interval(1s) order by _wstart); + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/tableCount.sim b/tests/script/tsim/query/tableCount.sim index ac5e23273a..6e65852dcc 100644 --- a/tests/script/tsim/query/tableCount.sim +++ b/tests/script/tsim/query/tableCount.sim @@ -103,7 +103,14 @@ endi if $data62 != 5 then return -1 endi - +sql select count(table_name) from information_schema.ins_tables where db_name='db1' and stable_name='sta' group by stable_name +print $rows , $data00 +if $rows != 1 then + return -1 +endi +if $data00 != 8 then + return -1 +endi sql select distinct db_name from information_schema.ins_tables; print $rows if $rows != 4 then diff --git a/tests/script/tsim/query/unionall_as_table.sim b/tests/script/tsim/query/unionall_as_table.sim index dc3d2cbec4..4d8f990718 100644 --- a/tests/script/tsim/query/unionall_as_table.sim +++ b/tests/script/tsim/query/unionall_as_table.sim @@ -25,4 +25,21 @@ if $data05 != @0021001@ then return -1 endi +sql create table st (ts timestamp, f int) tags (t int); +sql insert into ct1 using st tags(1) values(now, 1)(now+1s, 2) +sql insert into ct2 using st tags(2) values(now+2s, 3)(now+3s, 4) +sql select count(*) from (select * from ct1 union all select * from ct2) +if $rows != 1 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +sql select count(*) from (select * from ct1 union select * from ct2) +if $rows != 1 then + return -1 +endi +if $data00 != 4 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/user_privilege.py b/tests/system-test/0-others/user_privilege.py index 6d49ebfbfe..d1b93f6942 100644 --- a/tests/system-test/0-others/user_privilege.py +++ b/tests/system-test/0-others/user_privilege.py @@ -29,6 +29,7 @@ class TDTestCase: self.stbname = 'stb' self.binary_length = 20 # the length of binary for column_dict self.nchar_length = 20 # the length of nchar for column_dict + self.dbnames = ['db1', 'db2'] self.column_dict = { 'ts': 'timestamp', 'col1': 'float', @@ -57,21 +58,25 @@ class TDTestCase: def create_user(self): user_name = 'test' tdSql.execute(f'create user {user_name} pass "test"') - tdSql.execute(f'grant read on db.stb with t2 = "Beijing" to {user_name}') + tdSql.execute(f'grant read on {self.dbnames[0]}.{self.stbname} with t2 = "Beijing" to {user_name}') + tdSql.execute(f'grant write on {self.dbnames[1]}.{self.stbname} with t1 = 2 to {user_name}') def prepare_data(self): - tdSql.execute(self.setsql.set_create_stable_sql(self.stbname, self.column_dict, self.tag_dict)) - for i in range(self.tbnum): - tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})') - for j in self.values_list: - tdSql.execute(f'insert into {self.stbname}_{i} values({j})') + for db in self.dbnames: + tdSql.execute(f"create database {db}") + tdSql.execute(f"use {db}") + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname, self.column_dict, self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})') + for j in self.values_list: + tdSql.execute(f'insert into {self.stbname}_{i} values({j})') - def user_privilege_check(self): + def user_read_privilege_check(self, dbname): testconn = taos.connect(user='test', password='test') expectErrNotOccured = False try: - sql = "select count(*) from db.stb where t2 = 'Beijing'" + sql = f"select count(*) from {dbname}.stb where t2 = 'Beijing'" res = testconn.query(sql) data = res.fetch_all() count = data[0][0] @@ -85,11 +90,30 @@ class TDTestCase: tdLog.exit(f"{sql}, expect result doesn't match") pass + def user_write_privilege_check(self, dbname): + testconn = taos.connect(user='test', password='test') + expectErrNotOccured = False + + try: + sql = f"insert into {dbname}.stb_1 values(now, 1.1, 200, 0.3)" + testconn.execute(sql) + except BaseException: + expectErrNotOccured = True + + if expectErrNotOccured: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{sql}, expect error not occured") + else: + pass + def user_privilege_error_check(self): testconn = taos.connect(user='test', password='test') expectErrNotOccured = False - sql_list = ["alter talbe db.stb_1 set t2 = 'Wuhan'", "drop table db.stb_1"] + sql_list = [f"alter talbe {self.dbnames[0]}.stb_1 set t2 = 'Wuhan'", + f"insert into {self.dbnames[0]}.stb_1 values(now, 1.1, 200, 0.3)", + f"drop table {self.dbnames[0]}.stb_1", + f"select count(*) from {self.dbnames[1]}.stb"] for sql in sql_list: try: @@ -104,11 +128,11 @@ class TDTestCase: tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{sql}, expect error not occured") pass - def run(self): - tdSql.prepare() + def run(self): self.prepare_data() self.create_user() - self.user_privilege_check() + self.user_read_privilege_check(self.dbnames[0]) + self.user_write_privilege_check(self.dbnames[1]) self.user_privilege_error_check() def stop(self): diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 44eef8bf24..6b633fa193 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -145,29 +145,25 @@ class TMQCom: processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") tdLog.debug("%s is stopped by kill -INT" % (processorName)) - def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): + def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb',rows=1): loopFlag = 1 while loopFlag: - tdSql.query("select * from %s.notifyinfo"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + tdSql.query("select * from %s.notifyinfo where cmdid = 0"%cdbName) actRows = tdSql.getRows() - for i in range(actRows): - if tdSql.getData(i, 1) == 0: + tdLog.info("row: %d"%(actRows)) + if (actRows >= rows): loopFlag = 0 - break time.sleep(0.02) return - def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): + def getStartCommitNotifyFromTmqsim(self,cdbName='cdb',rows=1): loopFlag = 1 while loopFlag: - tdSql.query("select * from %s.notifyinfo"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + tdSql.query("select * from %s.notifyinfo where cmdid = 1"%cdbName) actRows = tdSql.getRows() - for i in range(actRows): - if tdSql.getData(i, 1) == 1: - loopFlag = 0 - break + tdLog.info("row: %d"%(actRows)) + if (actRows >= rows): + loopFlag = 0 time.sleep(0.02) return diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py index ae9671bcf4..f05f600f27 100644 --- a/tests/system-test/7-tmq/tmqConsumerGroup.py +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -100,7 +100,7 @@ class TDTestCase: tdLog.info("wait consumer commit notify") # tmqCom.getStartCommitNotifyFromTmqsim(rows=4) - tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartConsumeNotifyFromTmqsim(rows=2) tdLog.info("pkill one consume processor") tmqCom.stopTmqSimProcess('tmq_sim_new') diff --git a/tests/system-test/99-TDcase/TS-3404.py b/tests/system-test/99-TDcase/TS-3404.py new file mode 100644 index 0000000000..3fdfb9a193 --- /dev/null +++ b/tests/system-test/99-TDcase/TS-3404.py @@ -0,0 +1,101 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def create_tables(self): + tdSql.execute(f"CREATE STABLE `stb5` (`ts` TIMESTAMP, `ip_value` FLOAT, `ip_quality` INT) TAGS (`t1` INT)") + tdSql.execute(f"CREATE TABLE `t_11` USING `stb5` (`t1`) TAGS (1)") + + def insert_data(self): + tdLog.debug("start to insert data ............") + + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-10 09:30:47.722', 10.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-10 09:30:56.383', 12.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-10 09:48:55.778', 13.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-10 09:51:50.821', 9.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-10 09:58:07.162', 9.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-10 13:41:16.075', 9.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-13 14:12:58.318', 21.00000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-13 14:13:21.328', 1.10000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-13 14:35:24.258', 1.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-13 16:56:49.033', 1.80000, 100)") + + tdLog.debug("insert data ............ [OK]") + + def run(self): + tdSql.prepare() + self.create_tables() + self.insert_data() + tdLog.printNoPrefix("======== test TS-3404") + + tdSql.query(f"select _irowts, interp(ip_value) from t_11 range('2023-05-13 14:00:00', '2023-05-13 15:00:00') every(300s) fill(linear);") + tdSql.checkRows(13) + + tdSql.checkData(0, 0, '2023-05-13 14:00:00.000') + tdSql.checkData(1, 0, '2023-05-13 14:05:00.000') + tdSql.checkData(2, 0, '2023-05-13 14:10:00.000') + tdSql.checkData(3, 0, '2023-05-13 14:15:00.000') + tdSql.checkData(4, 0, '2023-05-13 14:20:00.000') + tdSql.checkData(5, 0, '2023-05-13 14:25:00.000') + tdSql.checkData(6, 0, '2023-05-13 14:30:00.000') + tdSql.checkData(7, 0, '2023-05-13 14:35:00.000') + tdSql.checkData(8, 0, '2023-05-13 14:40:00.000') + tdSql.checkData(9, 0, '2023-05-13 14:45:00.000') + tdSql.checkData(10, 0, '2023-05-13 14:50:00.000') + tdSql.checkData(11, 0, '2023-05-13 14:55:00.000') + tdSql.checkData(12, 0, '2023-05-13 15:00:00.000') + + tdSql.checkData(0, 1, 20.96512) + tdSql.checkData(1, 1, 20.97857) + tdSql.checkData(2, 1, 20.99201) + tdSql.checkData(3, 1, 1.114917) + tdSql.checkData(4, 1, 1.160271) + tdSql.checkData(5, 1, 1.205625) + tdSql.checkData(6, 1, 1.250978) + tdSql.checkData(7, 1, 1.296333) + tdSql.checkData(8, 1, 1.316249) + tdSql.checkData(9, 1, 1.333927) + tdSql.checkData(10, 1, 1.351607) + tdSql.checkData(11, 1, 1.369285) + tdSql.checkData(12, 1, 1.386964) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase())