diff --git a/cmake/cmake.define b/cmake/cmake.define index 1d34896f9a..0de3fba0c1 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -46,7 +46,7 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(COMMON_FLAGS "/w /D_WIN32") + SET(COMMON_FLAGS "/w /D_WIN32 /Zi") SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index aba955ff3b..31b9936f3e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -100,8 +100,10 @@ endif(${BUILD_WITH_NURAFT}) # addr2line if(${BUILD_ADDR2LINE}) - cat("${TD_SUPPORT_DIR}/libdwarf_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - cat("${TD_SUPPORT_DIR}/addr2line_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + if(NOT ${TD_WINDOWS}) + cat("${TD_SUPPORT_DIR}/libdwarf_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + cat("${TD_SUPPORT_DIR}/addr2line_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + endif(NOT ${TD_WINDOWS}) endif(${BUILD_ADDR2LINE}) # download dependencies @@ -335,45 +337,47 @@ endif(${BUILD_WITH_SQLITE}) # addr2line if(${BUILD_ADDR2LINE}) - check_include_file( "sys/types.h" HAVE_SYS_TYPES_H) - check_include_file( "sys/stat.h" HAVE_SYS_STAT_H ) - check_include_file( "inttypes.h" HAVE_INTTYPES_H ) - check_include_file( "stddef.h" HAVE_STDDEF_H ) - check_include_file( "stdlib.h" HAVE_STDLIB_H ) - check_include_file( "string.h" HAVE_STRING_H ) - check_include_file( "memory.h" HAVE_MEMORY_H ) - check_include_file( "strings.h" HAVE_STRINGS_H ) - check_include_file( "stdint.h" HAVE_STDINT_H ) - check_include_file( "unistd.h" HAVE_UNISTD_H ) - check_include_file( "sgidefs.h" HAVE_SGIDEFS_H ) - check_include_file( "stdafx.h" HAVE_STDAFX_H ) - check_include_file( "elf.h" HAVE_ELF_H ) - check_include_file( "libelf.h" HAVE_LIBELF_H ) - check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H) - check_include_file( "alloca.h" HAVE_ALLOCA_H ) - check_include_file( "elfaccess.h" HAVE_ELFACCESS_H) - check_include_file( "sys/elf_386.h" HAVE_SYS_ELF_386_H ) - check_include_file( "sys/elf_amd64.h" HAVE_SYS_ELF_AMD64_H) - check_include_file( "sys/elf_sparc.h" HAVE_SYS_ELF_SPARC_H) - check_include_file( "sys/ia64/elf.h" HAVE_SYS_IA64_ELF_H ) - set(VERSION 0.3.1) - set(PACKAGE_VERSION "\"${VERSION}\"") - configure_file(libdwarf/cmake/config.h.cmake config.h) - file(GLOB_RECURSE LIBDWARF_SOURCES "libdwarf/src/lib/libdwarf/*.c") - add_library(libdwarf STATIC ${LIBDWARF_SOURCES}) - set_target_properties(libdwarf PROPERTIES OUTPUT_NAME "libdwarf") - if(HAVE_LIBELF_H OR HAVE_LIBELF_LIBELF_H) - target_link_libraries(libdwarf PUBLIC libelf) - endif() - target_include_directories(libdwarf SYSTEM PUBLIC "libdwarf/src/lib/libdwarf" ${CMAKE_CURRENT_BINARY_DIR}) - file(READ "addr2line/addr2line.c" ADDR2LINE_CONTENT) - string(REPLACE "static int" "int" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") - string(REPLACE "static void" "void" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") - string(REPLACE "main(" "main_addr2line(" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") - file(WRITE "addr2line/addr2line.c" "${ADDR2LINE_CONTENT}") - add_library(addr2line STATIC "addr2line/addr2line.c") - target_link_libraries(addr2line PUBLIC libdwarf dl z) - target_include_directories(addr2line PUBLIC "libdwarf/src/lib/libdwarf" ) + if(NOT ${TD_WINDOWS}) + check_include_file( "sys/types.h" HAVE_SYS_TYPES_H) + check_include_file( "sys/stat.h" HAVE_SYS_STAT_H ) + check_include_file( "inttypes.h" HAVE_INTTYPES_H ) + check_include_file( "stddef.h" HAVE_STDDEF_H ) + check_include_file( "stdlib.h" HAVE_STDLIB_H ) + check_include_file( "string.h" HAVE_STRING_H ) + check_include_file( "memory.h" HAVE_MEMORY_H ) + check_include_file( "strings.h" HAVE_STRINGS_H ) + check_include_file( "stdint.h" HAVE_STDINT_H ) + check_include_file( "unistd.h" HAVE_UNISTD_H ) + check_include_file( "sgidefs.h" HAVE_SGIDEFS_H ) + check_include_file( "stdafx.h" HAVE_STDAFX_H ) + check_include_file( "elf.h" HAVE_ELF_H ) + check_include_file( "libelf.h" HAVE_LIBELF_H ) + check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H) + check_include_file( "alloca.h" HAVE_ALLOCA_H ) + check_include_file( "elfaccess.h" HAVE_ELFACCESS_H) + check_include_file( "sys/elf_386.h" HAVE_SYS_ELF_386_H ) + check_include_file( "sys/elf_amd64.h" HAVE_SYS_ELF_AMD64_H) + check_include_file( "sys/elf_sparc.h" HAVE_SYS_ELF_SPARC_H) + check_include_file( "sys/ia64/elf.h" HAVE_SYS_IA64_ELF_H ) + set(VERSION 0.3.1) + set(PACKAGE_VERSION "\"${VERSION}\"") + configure_file(libdwarf/cmake/config.h.cmake config.h) + file(GLOB_RECURSE LIBDWARF_SOURCES "libdwarf/src/lib/libdwarf/*.c") + add_library(libdwarf STATIC ${LIBDWARF_SOURCES}) + set_target_properties(libdwarf PROPERTIES OUTPUT_NAME "libdwarf") + if(HAVE_LIBELF_H OR HAVE_LIBELF_LIBELF_H) + target_link_libraries(libdwarf PUBLIC libelf) + endif() + target_include_directories(libdwarf SYSTEM PUBLIC "libdwarf/src/lib/libdwarf" ${CMAKE_CURRENT_BINARY_DIR}) + file(READ "addr2line/addr2line.c" ADDR2LINE_CONTENT) + string(REPLACE "static int" "int" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") + string(REPLACE "static void" "void" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") + string(REPLACE "main(" "main_addr2line(" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") + file(WRITE "addr2line/addr2line.c" "${ADDR2LINE_CONTENT}") + add_library(addr2line STATIC "addr2line/addr2line.c") + target_link_libraries(addr2line PUBLIC libdwarf dl z) + target_include_directories(addr2line PUBLIC "libdwarf/src/lib/libdwarf" ) + endif(NOT ${TD_WINDOWS}) endif(${BUILD_ADDR2LINE}) diff --git a/docs-cn/07-develop/01-connect/index.md b/docs-cn/07-develop/01-connect/index.md index ebdefc77b9..3a15d03f93 100644 --- a/docs-cn/07-develop/01-connect/index.md +++ b/docs-cn/07-develop/01-connect/index.md @@ -33,7 +33,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 关键不同点在于： 1. 使用 REST 连接，用户无需安装客户端驱动程序 taosc，具有跨平台易用的优势，但性能要下降 30%左右。 -2. 使用原生连接可以体验 TDengine 的全部功能，如[参数绑定接口](/reference/connector/cpp#参数绑定-api)、[订阅](reference/connector/cpp#数据订阅接口)等等。 +2. 使用原生连接可以体验 TDengine 的全部功能，如[参数绑定接口](/reference/connector/cpp#参数绑定-api)、[订阅](/reference/connector/cpp#订阅和消费-api)等等。 ## 安装客户端驱动 taosc diff --git a/docs-cn/12-taos-sql/02-database.md b/docs-cn/12-taos-sql/02-database.md index 1454d1d344..566fec3241 100644 --- a/docs-cn/12-taos-sql/02-database.md +++ b/docs-cn/12-taos-sql/02-database.md @@ -20,21 +20,21 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1]; 3. 数据库名最大长度为 33； 4. 一条 SQL 语句的最大长度为 65480 个字符； 5. 创建数据库时可用的参数有： - - cache: [Description](/reference/config/#cache) - - blocks: [Description](/reference/config/#blocks) - - days: [Description](/reference/config/#days) - - keep: [Description](/reference/config/#keep) - - minRows: [Description](/reference/config/#minrows) - - maxRows: [Description](/reference/config/#maxrows) - - wal: [Description](/reference/config/#wallevel) - - fsync: [Description](/reference/config/#fsync) - - update: [Description](/reference/config/#update) - - cacheLast: [Description](/reference/config/#cachelast) - - replica: [Description](/reference/config/#replica) - - quorum: [Description](/reference/config/#quorum) - - maxVgroupsPerDb: [Description](/reference/config/#maxvgroupsperdb) - - comp: [Description](/reference/config/#comp) - - precision: [Description](/reference/config/#precision) + - cache: [详细说明](/reference/config/#cache) + - blocks: [详细说明](/reference/config/#blocks) + - days: [详细说明](/reference/config/#days) + - keep: [详细说明](/reference/config/#keep) + - minRows: [详细说明](/reference/config/#minrows) + - maxRows: [详细说明](/reference/config/#maxrows) + - wal: [详细说明](/reference/config/#wallevel) + - fsync: [详细说明](/reference/config/#fsync) + - update: [详细说明](/reference/config/#update) + - cacheLast: [详细说明](/reference/config/#cachelast) + - replica: [详细说明](/reference/config/#replica) + - quorum: [详细说明](/reference/config/#quorum) + - maxVgroupsPerDb: [详细说明](/reference/config/#maxvgroupsperdb) + - comp: [详细说明](/reference/config/#comp) + - precision: [详细说明](/reference/config/#precision) 6. 请注意上面列出的所有参数都可以配置在配置文件 `taosd.cfg` 中作为创建数据库时使用的默认配置， `create database` 的参数中明确指定的会覆盖配置文件中的设置。 ::: diff --git a/docs-cn/14-reference/03-connector/csharp.mdx b/docs-cn/14-reference/03-connector/csharp.mdx index c2fbb3b67f..1e23df9286 100644 --- a/docs-cn/14-reference/03-connector/csharp.mdx +++ b/docs-cn/14-reference/03-connector/csharp.mdx @@ -18,7 +18,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx" `TDengine.Connector` 是 TDengine 提供的 C# 语言连接器。C# 开发人员可以通过它开发存取 TDengine 集群数据的 C# 应用软件。 -`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动（taosc）建立与 TDengine 运行实例的连接，提供数据写入、查询、订阅、schemaless 数据写入、参数绑定接口数据写入等功能 `TDengine.Connector` 目前暂未提供 REST 连接方式，用户可以参考 [RESTful APIs](https://docs.taosdata.com//reference/restful-api/) 文档自行编写。 +`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动（taosc）建立与 TDengine 运行实例的连接，提供数据写入、查询、订阅、schemaless 数据写入、参数绑定接口数据写入等功能 `TDengine.Connector` 目前暂未提供 REST 连接方式，用户可以参考 [REST API](/reference/rest-api/) 文档自行编写。 本文介绍如何在 Linux 或 Windows 环境中安装 `TDengine.Connector`，并通过 `TDengine.Connector` 连接 TDengine 集群，进行数据写入、查询等基本操作。 diff --git a/docs-en/07-develop/01-connect/index.md b/docs-en/07-develop/01-connect/index.md index ecb8caa308..ee11c8f544 100644 --- a/docs-en/07-develop/01-connect/index.md +++ b/docs-en/07-develop/01-connect/index.md @@ -33,7 +33,7 @@ Either way, same or similar APIs are provided by connectors to access database o Key differences： 1. With REST connection, it's not necessary to install TDengine client driver (taosc), it's more friendly for cross-platform with the cost of 30% performance downgrade. When taosc has an upgrade, application does not need to make changes. -2. With native connection, full compatibility of TDengine can be utilized, like [Parameter Binding](/reference/connector/cpp#Parameter Binding-api), [Subscription](reference/connector/cpp#Subscription), etc. But taosc has to be installed, some platforms may not be supported. +2. With native connection, full compatibility of TDengine can be utilized, like [Parameter Binding](/reference/connector/cpp#parameter-binding-api), [Subscription](/reference/connector/cpp#subscription-and-consumption-api), etc. But taosc has to be installed, some platforms may not be supported. ## Install Client Driver taosc diff --git a/docs-en/12-taos-sql/02-database.md b/docs-en/12-taos-sql/02-database.md index 12e2edf8ba..85b71bbde7 100644 --- a/docs-en/12-taos-sql/02-database.md +++ b/docs-en/12-taos-sql/02-database.md @@ -34,7 +34,7 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1]; - quorum: [Description](/reference/config/#quorum) - maxVgroupsPerDb: [Description](/reference/config/#maxvgroupsperdb) - comp: [Description](/reference/config/#comp) - - precision: [Description](reference/config/#precision) + - precision: [Description](/reference/config/#precision) 6. Please be noted that all of the parameters mentioned in this section can be configured in configuration file `taosd.cfg` at server side and used by default, can be override if they are specified in `create database` statement. ::: diff --git a/docs-en/14-reference/03-connector/csharp.mdx b/docs-en/14-reference/03-connector/csharp.mdx index ca4b1b9ece..2969392a05 100644 --- a/docs-en/14-reference/03-connector/csharp.mdx +++ b/docs-en/14-reference/03-connector/csharp.mdx @@ -19,7 +19,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx" `TDengine.Connector` is a C# language connector provided by TDengine that allows C# developers to develop C# applications that access TDengine cluster data. -The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc. The `TDengine.Connector` currently does not provide a REST connection interface. Developers can write their RESTful application by referring to the [RESTful APIs](https://docs.taosdata.com//reference/restful-api/) documentation. +The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc. The `TDengine.Connector` currently does not provide a REST connection interface. Developers can write their RESTful application by referring to the [REST API](/reference/rest-api/) documentation. This article describes how to install `TDengine.Connector` in a Linux or Windows environment and connect to TDengine clusters via `TDengine.Connector` to perform basic operations such as data writing and querying. diff --git a/docs-en/14-reference/03-connector/node.mdx b/docs-en/14-reference/03-connector/node.mdx index 48f724426a..3d30148e8e 100644 --- a/docs-en/14-reference/03-connector/node.mdx +++ b/docs-en/14-reference/03-connector/node.mdx @@ -78,7 +78,7 @@ Manually install the following tools. - Install [Python](https://www.python.org/downloads/) 2.7 (`v3.x.x` is not supported) and execute `npm config set python python2.7`. - Go to the `cmd` command-line interface, `npm config set msvs_version 2017` -Refer to Microsoft's Node.js User Manual [Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows- environment. md#compiling-native-addon-modules). +Refer to Microsoft's Node.js User Manual [Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules). If using ARM64 Node.js on Windows 10 ARM, you must add "Visual C++ compilers and libraries for ARM64" and "Visual C++ ATL for ARM64". diff --git a/docs-en/25-application/03-immigrate.md b/docs-en/25-application/03-immigrate.md index 81d5f512bf..4cfeb892d8 100644 --- a/docs-en/25-application/03-immigrate.md +++ b/docs-en/25-application/03-immigrate.md @@ -32,7 +32,7 @@ We will explain how to migrate OpenTSDB applications to TDengine quickly, secure The following figure (Figure 1) shows the system's overall architecture for a typical DevOps application scenario. **Figure 1. Typical architecture in a DevOps scenario** -Figure 1. [IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch](/img/IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch.jpg "Figure 1. Typical architecture in a DevOps scenario") +![IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch](/img/IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch.jpg "Figure 1. Typical architecture in a DevOps scenario") In this application scenario, there are Agent tools deployed in the application environment to collect machine metrics, network metrics, and application metrics. Data collectors to aggregate information collected by agents, systems for persistent data storage and management, and tools for monitoring data visualization (e.g., Grafana, etc.). @@ -75,7 +75,7 @@ After writing the data to TDengine properly, you can adapt Grafana to visualize TDengine provides two sets of Dashboard templates by default, and users only need to import the templates from the Grafana directory into Grafana to activate their use. **Importing Grafana Templates** Figure 2. -! [](/img/IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard.jpg "Figure 2. Importing a Grafana Template") +![](/img/IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard.jpg "Figure 2. Importing a Grafana Template") After the above steps, you completed the migration to replace OpenTSDB with TDengine. You can see that the whole process is straightforward, there is no need to write any code, and only some configuration files need to be adjusted to meet the migration work. @@ -88,7 +88,7 @@ In most DevOps scenarios, if you have a small OpenTSDB cluster (3 or fewer nodes Suppose your application is particularly complex, or the application domain is not a DevOps scenario. You can continue reading subsequent chapters for a more comprehensive and in-depth look at the advanced topics of migrating an OpenTSDB application to TDengine. **Figure 3. System architecture after migration** -! [IT-DevOps-Solutions-Immigrate-TDengine-Arch](/img/IT-DevOps-Solutions-Immigrate-TDengine-Arch.jpg "Figure 3. System architecture after migration completion") +![IT-DevOps-Solutions-Immigrate-TDengine-Arch](/img/IT-DevOps-Solutions-Immigrate-TDengine-Arch.jpg "Figure 3. System architecture after migration completion") ## Migration evaluation and strategy for other scenarios @@ -96,7 +96,7 @@ Suppose your application is particularly complex, or the application domain is n This chapter describes the differences between OpenTSDB and TDengine at the system functionality level. After reading this chapter, you can fully evaluate whether you can migrate some complex OpenTSDB-based applications to TDengine, and what you should pay attention to after migration. -TDengine currently only supports Grafana for visual kanban rendering, so if your application uses front-end kanban boards other than Grafana (e.g., [TSDash](https://github.com/facebook/tsdash), [Status Wolf](https://github) .com/box/StatusWolf), etc.). You cannot directly migrate those front-end kanbans to TDengine, and the front-end kanban will need to be ported to Grafana to work correctly. +TDengine currently only supports Grafana for visual kanban rendering, so if your application uses front-end kanban boards other than Grafana (e.g., [TSDash](https://github.com/facebook/tsdash), [Status Wolf](https://github.com/box/StatusWolf), etc.). You cannot directly migrate those front-end kanbans to TDengine, and the front-end kanban will need to be ported to Grafana to work correctly. TDengine version 2.3.0.x only supports collectd and StatsD as data collection aggregation software but will provide more data collection aggregation software in the future. If you use other data aggregators on the collection side, your application needs to be ported to these two data aggregation systems to write data correctly. In addition to the two data aggregator software protocols mentioned above, TDengine also supports writing data directly via InfluxDB's line protocol and OpenTSDB's data writing protocol, JSON format. You can rewrite the logic on the data push side to write data using the line protocols supported by TDengine. diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b893099064..f27dea27ad 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1697,7 +1697,7 @@ int32_t tDecodeSRSmaParam(SDecoder* pCoder, SRSmaParam* pRSmaParam); // TDMT_VND_CREATE_STB ============== typedef struct SVCreateStbReq { - const char* name; + char* name; tb_uid_t suid; int8_t rollup; SSchemaWrapper schema; @@ -1710,8 +1710,8 @@ int tDecodeSVCreateStbReq(SDecoder* pCoder, SVCreateStbReq* pReq); // TDMT_VND_DROP_STB ============== typedef struct SVDropStbReq { - const char* name; - tb_uid_t suid; + char* name; + tb_uid_t suid; } SVDropStbReq; int32_t tEncodeSVDropStbReq(SEncoder* pCoder, const SVDropStbReq* pReq); @@ -1720,16 +1720,16 @@ int32_t tDecodeSVDropStbReq(SDecoder* pCoder, SVDropStbReq* pReq); // TDMT_VND_CREATE_TABLE ============== #define TD_CREATE_IF_NOT_EXISTS 0x1 typedef struct SVCreateTbReq { - int32_t flags; - tb_uid_t uid; - int64_t ctime; - const char* name; - int32_t ttl; - int8_t type; + int32_t flags; + tb_uid_t uid; + int64_t ctime; + char* name; + int32_t ttl; + int8_t type; union { struct { - tb_uid_t suid; - const uint8_t* pTag; + tb_uid_t suid; + uint8_t* pTag; } ctb; struct { SSchemaWrapper schema; @@ -1777,8 +1777,8 @@ int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatc // TDMT_VND_DROP_TABLE ================= typedef struct { - const char* name; - int8_t igNotExists; + char* name; + int8_t igNotExists; } SVDropTbReq; typedef struct { @@ -1809,9 +1809,9 @@ int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp); // TDMT_VND_ALTER_TABLE ===================== typedef struct { - const char* tbName; - int8_t action; - const char* colName; + char* tbName; + int8_t action; + char* colName; // TSDB_ALTER_TABLE_ADD_COLUMN int8_t type; int8_t flags; @@ -1820,17 +1820,17 @@ typedef struct { // TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES int32_t colModBytes; // TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME - const char* colNewName; + char* colNewName; // TSDB_ALTER_TABLE_UPDATE_TAG_VAL - const char* tagName; - int8_t isNull; - uint32_t nTagVal; - const uint8_t* pTagVal; + char* tagName; + int8_t isNull; + uint32_t nTagVal; + uint8_t* pTagVal; // TSDB_ALTER_TABLE_UPDATE_OPTIONS - int8_t updateTTL; - int32_t newTTL; - int8_t updateComment; - const char* newComment; + int8_t updateTTL; + int32_t newTTL; + int8_t updateComment; + char* newComment; } SVAlterTbReq; int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); @@ -2020,7 +2020,7 @@ static FORCE_INLINE void tFreeClientHbBatchRsp(void* pRsp) { int32_t tSerializeSClientHbBatchRsp(void* buf, int32_t bufLen, const SClientHbBatchRsp* pBatchRsp); int32_t tDeserializeSClientHbBatchRsp(void* buf, int32_t bufLen, SClientHbBatchRsp* pBatchRsp); -void tFreeSClientHbBatchRsp(SClientHbBatchRsp *pBatchRsp); +void tFreeSClientHbBatchRsp(SClientHbBatchRsp* pBatchRsp); static FORCE_INLINE int32_t tEncodeSKv(SEncoder* pEncoder, const SKv* pKv) { if (tEncodeI32(pEncoder, pKv->key) < 0) return -1; @@ -2255,20 +2255,20 @@ int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); typedef struct { - int8_t version; // for compatibility(default 0) - int8_t intervalUnit; // MACRO: TIME_UNIT_XXX - int8_t slidingUnit; // MACRO: TIME_UNIT_XXX - int8_t timezoneInt; // sma data expired if timezone changes. - char indexName[TSDB_INDEX_NAME_LEN]; - int32_t exprLen; - int32_t tagsFilterLen; - int64_t indexUid; - tb_uid_t tableUid; // super/child/common table uid - int64_t interval; - int64_t offset; // use unit by precision of DB - int64_t sliding; - const char* expr; // sma expression - const char* tagsFilter; + int8_t version; // for compatibility(default 0) + int8_t intervalUnit; // MACRO: TIME_UNIT_XXX + int8_t slidingUnit; // MACRO: TIME_UNIT_XXX + int8_t timezoneInt; // sma data expired if timezone changes. + char indexName[TSDB_INDEX_NAME_LEN]; + int32_t exprLen; + int32_t tagsFilterLen; + int64_t indexUid; + tb_uid_t tableUid; // super/child/common table uid + int64_t interval; + int64_t offset; // use unit by precision of DB + int64_t sliding; + char* expr; // sma expression + char* tagsFilter; } STSma; // Time-range-wise SMA typedef STSma SVCreateTSmaReq; @@ -2596,12 +2596,12 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { #define TD_AUTO_CREATE_TABLE 0x1 typedef struct { - int64_t suid; - int64_t uid; - int32_t sver; - uint32_t nData; - const uint8_t* pData; - SVCreateTbReq cTbReq; + int64_t suid; + int64_t uid; + int32_t sver; + uint32_t nData; + uint8_t* pData; + SVCreateTbReq cTbReq; } SVSubmitBlk; typedef struct { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index f9d8fc0de1..e64fb4235c 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -46,24 +46,34 @@ typedef enum { AUTH_TYPE_OTHER, } AUTH_TYPE; +typedef struct SUserAuthInfo { + char user[TSDB_USER_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + AUTH_TYPE type; +} SUserAuthInfo; + typedef struct SCatalogReq { - SArray *pTableName; // element is SNAME - SArray *pUdf; // udf name + SArray *pTableMeta; // element is SNAME + SArray *pDbVgroup; // element is db full name + SArray *pTableHash; // element is SNAME + SArray *pUdf; // element is udf name + SArray *pDbCfg; // element is db full name + SArray *pIndex; // element is index name + SArray *pUser; // element is SUserAuthInfo bool qNodeRequired; // valid qnode } SCatalogReq; typedef struct SMetaData { - SArray *pTableMeta; // STableMeta array - SArray *pVgroupInfo; // SVgroupInfo list - SArray *pUdfList; // udf info list - SArray *pQnodeList; // qnode list, SArray + SArray *pTableMeta; // SArray + SArray *pDbVgroup; // SArray*> + SArray *pTableHash; // SArray + SArray *pUdfList; // SArray + SArray *pDbCfg; // SArray + SArray *pIndex; // SArray + SArray *pUser; // SArray + SArray *pQnodeList; // SArray } SMetaData; -typedef struct STbSVersion { - char* tbFName; - int32_t sver; -} STbSVersion; - typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; @@ -88,6 +98,11 @@ typedef struct SDbVgVersion { int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT } SDbVgVersion; +typedef struct STbSVersion { + char* tbFName; + int32_t sver; +} STbSVersion; + typedef struct SUserAuthVersion { char user[TSDB_USER_LEN]; int32_t version; @@ -96,6 +111,8 @@ typedef struct SUserAuthVersion { typedef SDbCfgRsp SDbCfgInfo; typedef SUserIndexRsp SIndexInfo; +typedef void (*catalogCallback)(SMetaData* pResult, void* param, int32_t code); + int32_t catalogInit(SCatalogCfg *cfg); /** @@ -131,7 +148,7 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t d int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId); -int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName); +int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName); int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid); @@ -241,9 +258,9 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg); -int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); +int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); -int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo); +int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo); int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); diff --git a/include/util/tencode.h b/include/util/tencode.h index 938e3018a8..90d7247ea9 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -39,11 +39,11 @@ typedef struct { } SEncoder; typedef struct { - const uint8_t* data; - uint32_t size; - uint32_t pos; - SCoderMem* mList; - SDecoderNode* dStack; + uint8_t* data; + uint32_t size; + uint32_t pos; + SCoderMem* mList; + SDecoderNode* dStack; } SDecoder; #define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL) @@ -120,7 +120,7 @@ static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t le static int32_t tEncodeCStr(SEncoder* pCoder, const char* val); /* ------------------------ DECODE ------------------------ */ -void tDecoderInit(SDecoder* pCoder, const uint8_t* data, uint32_t size); +void tDecoderInit(SDecoder* pCoder, uint8_t* data, uint32_t size); void tDecoderClear(SDecoder* SDecoder); int32_t tStartDecode(SDecoder* pCoder); void tEndDecode(SDecoder* pCoder); @@ -141,9 +141,9 @@ static int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val); static int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val); static int32_t tDecodeFloat(SDecoder* pCoder, float* val); static int32_t tDecodeDouble(SDecoder* pCoder, double* val); -static int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, uint32_t* len); -static int32_t tDecodeCStrAndLen(SDecoder* pCoder, const char** val, uint32_t* len); -static int32_t tDecodeCStr(SDecoder* pCoder, const char** val); +static int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len); +static int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len); +static int32_t tDecodeCStr(SDecoder* pCoder, char** val); static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); /* ------------------------ IMPL ------------------------ */ @@ -377,7 +377,7 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) { return 0; } -static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, uint32_t* len) { +static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) { if (tDecodeU32v(pCoder, len) < 0) return -1; if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, *len)) return -1; @@ -389,20 +389,20 @@ static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, return 0; } -static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, const char** val, uint32_t* len) { - if (tDecodeBinary(pCoder, (const uint8_t**)val, len) < 0) return -1; +static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) { + if (tDecodeBinary(pCoder, (uint8_t**)val, len) < 0) return -1; (*len) -= 1; return 0; } -static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, const char** val) { +static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) { uint32_t len; return tDecodeCStrAndLen(pCoder, val, &len); } static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { - const char* pStr; - uint32_t len; + char* pStr; + uint32_t len; if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1; memcpy(val, pStr, len + 1); diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 2b26674324..f07705ff44 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -6,16 +6,36 @@ set -e #set -x +verMode=edge +pagMode=full + +iplist="" +serverFqdn="" + # -----------------------Variables definition--------------------- script_dir=$(dirname $(readlink -f "$0")) # Dynamic directory -data_dir="/var/lib/taos" -log_dir="/var/log/taos" -data_link_dir="/usr/local/taos/data" -log_link_dir="/usr/local/taos/log" +clientName="taos" +serverName="taosd" +configFile="taos.cfg" +productName="TDengine" +emailName="taosdata.com" +uninstallScript="rmtaos" +historyFile="taos_history" +tarName="taos.tar.gz" +dataDir="/var/lib/taos" +logDir="/var/log/taos" +configDir="/etc/taos" +installDir="/usr/local/taos" +adapterName="taosadapter" +benchmarkName="taosBenchmark" +dumpName="taosdump" +demoName="taosdemo" -cfg_install_dir="/etc/taos" +data_dir=${dataDir} +log_dir=${logDir} +cfg_install_dir=${configDir} bin_link_dir="/usr/bin" lib_link_dir="/usr/lib" @@ -23,21 +43,13 @@ lib64_link_dir="/usr/lib64" inc_link_dir="/usr/include" #install main path -install_main_dir="/usr/local/taos" - +install_main_dir=${installDir} # old bin dir -bin_dir="/usr/local/taos/bin" +bin_dir="${installDir}/bin" service_config_dir="/etc/systemd/system" - -#taos-tools para -demoName="taosdemo" -benchmarkName="taosBenchmark" -dumpName="taosdump" -emailName="taosdata.com" -taosName="taos" -toolsName="taostools" - +nginx_port=6060 +nginx_dir="/usr/local/nginxd" # Color setting RED='\033[0;31m' @@ -47,8 +59,8 @@ GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' csudo="" -if command -v sudo > /dev/null; then - csudo="sudo" +if command -v sudo >/dev/null; then + csudo="sudo " fi update_flag=0 @@ -56,52 +68,51 @@ prompt_force=0 initd_mod=0 service_mod=2 -if pidof systemd &> /dev/null; then - service_mod=0 -elif $(which service &> /dev/null); then - service_mod=1 - service_config_dir="/etc/init.d" - if $(which chkconfig &> /dev/null); then - initd_mod=1 - elif $(which insserv &> /dev/null); then - initd_mod=2 - elif $(which update-rc.d &> /dev/null); then - initd_mod=3 - else - service_mod=2 - fi -else +if pidof systemd &>/dev/null; then + service_mod=0 +elif $(which service &>/dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &>/dev/null); then + initd_mod=1 + elif $(which insserv &>/dev/null); then + initd_mod=2 + elif $(which update-rc.d &>/dev/null); then + initd_mod=3 + else service_mod=2 + fi +else + service_mod=2 fi - # get the operating system type for using the corresponding init file # ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification #osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) if [[ -e /etc/os-release ]]; then - osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) || : else osinfo="" fi #echo "osinfo: ${osinfo}" os_type=0 -if echo $osinfo | grep -qwi "ubuntu" ; then -# echo "This is ubuntu system" +if echo $osinfo | grep -qwi "ubuntu"; then + # echo "This is ubuntu system" os_type=1 -elif echo $osinfo | grep -qwi "debian" ; then -# echo "This is debian system" +elif echo $osinfo | grep -qwi "debian"; then + # echo "This is debian system" os_type=1 -elif echo $osinfo | grep -qwi "Kylin" ; then -# echo "This is Kylin system" +elif echo $osinfo | grep -qwi "Kylin"; then + # echo "This is Kylin system" os_type=1 -elif echo $osinfo | grep -qwi "centos" ; then -# echo "This is centos system" +elif echo $osinfo | grep -qwi "centos"; then + # echo "This is centos system" os_type=2 -elif echo $osinfo | grep -qwi "fedora" ; then -# echo "This is fedora system" +elif echo $osinfo | grep -qwi "fedora"; then + # echo "This is fedora system" os_type=2 -elif echo $osinfo | grep -qwi "Linx" ; then -# echo "This is Linx system" +elif echo $osinfo | grep -qwi "Linx"; then + # echo "This is Linx system" os_type=1 service_mod=0 initd_mod=0 @@ -110,43 +121,41 @@ else echo " osinfo: ${osinfo}" echo " This is an officially unverified linux system," echo " if there are any problems with the installation and operation, " - echo " please feel free to contact taosdata.com for support." + echo " please feel free to contact ${emailName} for support." os_type=1 fi - # ============================= get input parameters ================================================= # install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...] # set parameters by default value -interactiveFqdn=yes # [yes | no] -verType=server # [server | client] -initType=systemd # [systemd | service | ...] +interactiveFqdn=yes # [yes | no] +verType=server # [server | client] +initType=systemd # [systemd | service | ...] -while getopts "hv:e:i:" arg -do +while getopts "hv:e:i:" arg; do case $arg in - e) - #echo "interactiveFqdn=$OPTARG" - interactiveFqdn=$( echo $OPTARG ) - ;; - v) - #echo "verType=$OPTARG" - verType=$(echo $OPTARG) - ;; - i) - #echo "initType=$OPTARG" - initType=$(echo $OPTARG) - ;; - h) - echo "Usage: `basename $0` -v [server | client] -e [yes | no]" - exit 0 - ;; - ?) #unknow option - echo "unkonw argument" - exit 1 - ;; + e) + #echo "interactiveFqdn=$OPTARG" + interactiveFqdn=$(echo $OPTARG) + ;; + v) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; + i) + #echo "initType=$OPTARG" + initType=$(echo $OPTARG) + ;; + h) + echo "Usage: $(basename $0) -v [server | client] -e [yes | no]" + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; esac done @@ -155,98 +164,163 @@ done function kill_process() { pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then - ${csudo} kill -9 $pid || : + ${csudo}kill -9 $pid || : fi } function install_main_path() { - #create install main dir and all sub dir - ${csudo} rm -rf ${install_main_dir} || : - ${csudo} mkdir -p ${install_main_dir} - ${csudo} mkdir -p ${install_main_dir}/cfg - ${csudo} mkdir -p ${install_main_dir}/bin - ${csudo} mkdir -p ${install_main_dir}/connector - ${csudo} mkdir -p ${install_main_dir}/lib - ${csudo} mkdir -p ${install_main_dir}/examples - ${csudo} mkdir -p ${install_main_dir}/include - ${csudo} mkdir -p ${install_main_dir}/init.d - if [ "$verMode" == "cluster" ]; then - ${csudo} mkdir -p ${nginx_dir} - fi + #create install main dir and all sub dir + ${csudo}rm -rf ${install_main_dir} || : + ${csudo}mkdir -p ${install_main_dir} + ${csudo}mkdir -p ${install_main_dir}/cfg + ${csudo}mkdir -p ${install_main_dir}/bin + # ${csudo}mkdir -p ${install_main_dir}/connector + ${csudo}mkdir -p ${install_main_dir}/driver + ${csudo}mkdir -p ${install_main_dir}/examples + ${csudo}mkdir -p ${install_main_dir}/include + # ${csudo}mkdir -p ${install_main_dir}/init.d + if [ "$verMode" == "cluster" ]; then + ${csudo}mkdir -p ${nginx_dir} + fi - if [[ -e ${script_dir}/email ]]; then - ${csudo} cp ${script_dir}/email ${install_main_dir}/ ||: - fi + if [[ -e ${script_dir}/email ]]; then + ${csudo}cp ${script_dir}/email ${install_main_dir}/ || : + fi } function install_bin() { - # Remove links - ${csudo} rm -f ${bin_link_dir}/taos || : - ${csudo} rm -f ${bin_link_dir}/taosd || : - ${csudo} rm -f ${bin_link_dir}/taosadapter || : - ${csudo} rm -f ${bin_link_dir}/create_table || : - ${csudo} rm -f ${bin_link_dir}/tmq_sim || : - ${csudo} rm -f ${bin_link_dir}/taosdump || : - ${csudo} rm -f ${bin_link_dir}/rmtaos || : - #${csudo} rm -f ${bin_link_dir}/set_core || : + # Remove links + ${csudo}rm -f ${bin_link_dir}/${clientName} || : + ${csudo}rm -f ${bin_link_dir}/${serverName} || : + ${csudo}rm -f ${bin_link_dir}/${adapterName} || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : + ${csudo}rm -f ${bin_link_dir}/tarbitrator || : + ${csudo}rm -f ${bin_link_dir}/set_core || : + ${csudo}rm -f ${bin_link_dir}/run_${serverName}_and_${adapterName}.sh || : + ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : - ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* - #Make link - [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : - [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : - [ -x ${install_main_dir}/bin/create_table ] && ${csudo} ln -s ${install_main_dir}/bin/create_table ${bin_link_dir}/create_table || : - [ -x ${install_main_dir}/bin/tmq_sim ] && ${csudo} ln -s ${install_main_dir}/bin/tmq_sim ${bin_link_dir}/tmq_sim || : -# [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : -# [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : -# [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + #Make link + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : + [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : + [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -s ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || : + [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : + [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : + [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : + [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + [ -x ${install_main_dir}/bin/run_${serverName}_and_${adapterName}.sh ] && ${csudo}ln -s ${install_main_dir}/bin/run_${serverName}_and_${adapterName}.sh ${bin_link_dir}/run_${serverName}_and_${adapterName}.sh || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo}ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : + + if [ "$verMode" == "cluster" ]; then + ${csudo}cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo}chmod 0555 ${nginx_dir}/* + ${csudo}mkdir -p ${nginx_dir}/logs + ${csudo}chmod 777 ${nginx_dir}/sbin/nginx + fi } function install_lib() { - # Remove links - ${csudo} rm -f ${lib_link_dir}/libtaos.* || : - ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : - ${csudo} rm -f ${lib_link_dir}/libtdb.* || : - ${csudo} rm -f ${lib64_link_dir}/libtdb.* || : + # Remove links + ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo}rm -rf ${v15_java_app_dir} || : + ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* - ${csudo} cp -rf ${script_dir}/lib/* ${install_main_dir}/lib && ${csudo} chmod 777 ${install_main_dir}/lib/* + ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1 - ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then + ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi - if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then - ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : - ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + ${csudo}ldconfig +} + +function install_avro() { + if [ "$osType" != "Darwin" ]; then + avro_dir=${script_dir}/avro + if [ -f "${avro_dir}/lib/libavro.so.23.0.0" ] && [ -d /usr/local/$1 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/$1 + ${csudo}/usr/bin/install -c -m 755 ${avro_dir}/lib/libavro.so.23.0.0 /usr/local/$1 + ${csudo}ln -sf /usr/local/$1/libavro.so.23.0.0 /usr/local/$1/libavro.so.23 + ${csudo}ln -sf /usr/local/$1/libavro.so.23 /usr/local/$1/libavro.so + + ${csudo}/usr/bin/install -c -d /usr/local/$1 + [ -f ${avro_dir}/lib/libavro.a ] && + ${csudo}/usr/bin/install -c -m 755 ${avro_dir}/lib/libavro.a /usr/local/$1 + + if [ -d /etc/ld.so.conf.d ]; then + echo "/usr/local/$1" | ${csudo}tee /etc/ld.so.conf.d/libavro.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/libavro.conf" + ${csudo}ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi + fi +} + +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo}/usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo}/usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi - ${csudo} ldconfig + if [ -d /etc/ld.so.conf.d ]; then + echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf" + ${csudo}ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi } function install_header() { - ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : - ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* - ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h -# ${csudo} ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h - ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h -} - -# temp install taosBenchmark -function install_taosTools() { - ${csudo} rm -f ${bin_link_dir}/${benchmarkName} || : - ${csudo} rm -f ${bin_link_dir}/${dumpName} || : - ${csudo} rm -f ${bin_link_dir}/rm${toolsName} || : - - ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${dumpName} ${install_main_dir}/bin/${dumpName} - ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${benchmarkName} - ${csudo} ln -sf ${install_main_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${demoName} - #Make link - [[ -x ${install_main_dir}/bin/${benchmarkName} ]] && \ - ${csudo} ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : - [[ -x ${install_main_dir}/bin/${demoName} ]] && \ - ${csudo} ln -s ${install_main_dir}/bin/${demoName} ${bin_link_dir}/${demoName} || : - [[ -x ${install_main_dir}/bin/${dumpName} ]] && \ - ${csudo} ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : + ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* + ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h + ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } function add_newHostname_to_hosts() { @@ -256,18 +330,17 @@ function add_newHostname_to_hosts() { iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') arr=($iphost) IFS="$OLD_IFS" - for s in "${arr[@]}" - do + for s in "${arr[@]}"; do if [[ "$s" == "$localIp" ]]; then return fi done - ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: + ${csudo}echo "127.0.0.1 $1" >>/etc/hosts || : } function set_hostname() { echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" - read newHostname + read newHostname while true; do if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then break @@ -276,28 +349,25 @@ function set_hostname() { fi done - ${csudo} hostname $newHostname ||: - retval=`echo $?` + ${csudo}hostname $newHostname || : + retval=$(echo $?) if [[ $retval != 0 ]]; then - echo - echo "set hostname fail!" - return + echo + echo "set hostname fail!" + return fi - #echo -e -n "$(hostnamectl status --static)" - #echo -e -n "$(hostnamectl status --transient)" - #echo -e -n "$(hostnamectl status --pretty)" #ubuntu/centos /etc/hostname if [[ -e /etc/hostname ]]; then - ${csudo} echo $newHostname > /etc/hostname ||: + ${csudo}echo $newHostname >/etc/hostname || : fi #debian: #HOSTNAME=yourname if [[ -e /etc/sysconfig/network ]]; then - ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + ${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network || : fi - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile} serverFqdn=$newHostname if [[ -e /etc/hosts ]]; then @@ -311,20 +381,19 @@ function is_correct_ipaddr() { IFS=" " arr=($iplist) IFS="$OLD_IFS" - for s in "${arr[@]}" - do - if [[ "$s" == "$newIp" ]]; then - return 0 - fi + for s in "${arr[@]}"; do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi done return 1 } function set_ipAsFqdn() { - iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + iplist=$(ip address | grep inet | grep -v inet6 | grep -v 127.0.0.1 | awk '{print $2}' | awk -F "/" '{print $1}') || : if [ -z "$iplist" ]; then - iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + iplist=$(ifconfig | grep inet | grep -v inet6 | grep -v 127.0.0.1 | awk '{print $2}' | awk -F ":" '{print $2}') || : fi if [ -z "$iplist" ]; then @@ -332,7 +401,7 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile} serverFqdn=$localFqdn echo return @@ -345,23 +414,23 @@ function set_ipAsFqdn() { echo echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" read localFqdn - while true; do - if [ ! -z "$localFqdn" ]; then - # Check if correct ip address - is_correct_ipaddr $localFqdn - retval=`echo $?` - if [[ $retval != 0 ]]; then - read -p "Please choose an IP from local IP list:" localFqdn - else - # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg - serverFqdn=$localFqdn - break - fi - else + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=$(echo $?) + if [[ $retval != 0 ]]; then read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile} + serverFqdn=$localFqdn + break fi - done + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done } function local_fqdn_check() { @@ -369,205 +438,553 @@ function local_fqdn_check() { echo echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" echo - if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" echo - while true - do - read -r -p "Set hostname now? [Y/n] " input - if [ ! -n "$input" ]; then - set_hostname - break - else - case $input in - [yY][eE][sS]|[yY]) - set_hostname - break - ;; + while true; do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS] | [yY]) + set_hostname + break + ;; - [nN][oO]|[nN]) - set_ipAsFqdn - break - ;; + [nN][oO] | [nN]) + set_ipAsFqdn + break + ;; - *) - echo "Invalid input..." - ;; - esac - fi + *) + echo "Invalid input..." + ;; + esac + fi done fi } -function install_log() { - ${csudo} rm -rf ${log_dir} || : - ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} +function install_adapter_config() { + if [ ! -f "${cfg_install_dir}/${adapterName}.toml" ]; then + ${csudo}mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/${adapterName}.toml ] && ${csudo}cp ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir} + [ -f ${cfg_install_dir}/${adapterName}.toml ] && ${csudo}chmod 644 ${cfg_install_dir}/${adapterName}.toml + fi - ${csudo} ln -s ${log_dir} ${install_main_dir}/log -} + [ -f ${script_dir}/cfg/${adapterName}.toml ] && + ${csudo}cp -f ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir}/${adapterName}.toml.new -function install_data() { - ${csudo} mkdir -p ${data_dir} + [ -f ${cfg_install_dir}/${adapterName}.toml ] && + ${csudo}ln -s ${cfg_install_dir}/${adapterName}.toml ${install_main_dir}/cfg/${adapterName}.toml - ${csudo} ln -s ${data_dir} ${install_main_dir}/data -} + [ ! -z $1 ] && return 0 || : # only install client -function clean_service_on_systemd() { - taosd_service_config="${service_config_dir}/taosd.service" - if systemctl is-active --quiet taosd; then - echo "TDengine is running, stopping it..." - ${csudo} systemctl stop taosd &> /dev/null || echo &> /dev/null - fi - ${csudo} systemctl disable taosd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${taosd_service_config} - - tarbitratord_service_config="${service_config_dir}/tarbitratord.service" - if systemctl is-active --quiet tarbitratord; then - echo "tarbitrator is running, stopping it..." - ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null - fi - ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${tarbitratord_service_config} - - if [ "$verMode" == "cluster" ]; then - nginx_service_config="${service_config_dir}/nginxd.service" - if systemctl is-active --quiet nginxd; then - echo "Nginx for TDengine is running, stopping it..." - ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null - fi - ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${nginx_service_config} - fi -} - -# taos:2345:respawn:/etc/init.d/taosd start - -function install_service_on_systemd() { - clean_service_on_systemd - - taosd_service_config="${service_config_dir}/taosd.service" - ${csudo} bash -c "echo '[Unit]' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'After=network-online.target taosadapter.service' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'Wants=network-online.target taosadapter.service' >> ${taosd_service_config}" - ${csudo} bash -c "echo >> ${taosd_service_config}" - ${csudo} bash -c "echo '[Service]' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'StandardOutput=null' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'Restart=always' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}" - #${csudo} bash -c "echo 'StartLimitIntervalSec=60s' >> ${taosd_service_config}" - ${csudo} bash -c "echo >> ${taosd_service_config}" - ${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" - ${csudo} systemctl enable taosd - - ${csudo} systemctl daemon-reload -} - -function install_service() { - # if ((${service_mod}==0)); then - # install_service_on_systemd - # elif ((${service_mod}==1)); then - # install_service_on_sysvinit - # else - # # must manual stop taosd - kill_process taosd - # fi } function install_config() { - if [ ! -f ${cfg_install_dir}/${configFile} ]; then - ${csudo}mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir} - ${csudo}chmod 644 ${cfg_install_dir}/* - fi - ${csudo}cp -f ${script_dir}/cfg/${configFile} ${install_main_dir}/cfg/${configFile}.org - ${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg + if [ ! -f "${cfg_install_dir}/${configFile}" ]; then + ${csudo}mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir} + ${csudo}chmod 644 ${cfg_install_dir}/* + fi + + ${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new + ${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg + + [ ! -z $1 ] && return 0 || : # only install client + + if ((${update_flag} == 1)); then + return 0 + fi + + if [ "$interactiveFqdn" == "no" ]; then + return 0 + fi + + local_fqdn_check + + echo + echo -e -n "${GREEN}Enter FQDN:port (like h1.${emailName}:6030) of an existing ${productName} cluster node to join${NC}" + echo + echo -e -n "${GREEN}OR leave it blank to build one${NC}:" + read firstEp + while true; do + if [ ! -z "$firstEp" ]; then + ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile} + break + else + break + fi + done + + echo + echo -e -n "${GREEN}Enter your email address for priority support or enter empty to skip${NC}: " + read emailAddr + while true; do + if [ ! -z "$emailAddr" ]; then + email_file="${install_main_dir}/email" + ${csudo}bash -c "echo $emailAddr > ${email_file}" + break + else + break + fi + done } -function install_TDengine() { - # Start to install - echo -e "${GREEN}Start to install TDengine...${NC}" +function install_log() { + ${csudo}rm -rf ${log_dir} || : + ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} - install_main_path - install_data - install_log - install_header - install_lib - install_taosTools + ${csudo}ln -s ${log_dir} ${install_main_dir}/log +} - if [ -z $1 ]; then # install service and client - # For installing new - install_bin - install_service - install_config +function install_data() { + ${csudo}mkdir -p ${data_dir} - # Ask if to start the service - #echo - #echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" - echo - echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" - if ((${service_mod}==0)); then - echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} systemctl start taosd${NC}" - elif ((${service_mod}==1)); then - echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} service taosd start${NC}" + ${csudo}ln -s ${data_dir} ${install_main_dir}/data +} + +function install_connector() { + [ -d "${script_dir}/connector/" ] && ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ +} + +function install_examples() { + if [ -d ${script_dir}/examples ]; then + ${csudo}cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + fi +} + +function clean_service_on_sysvinit() { + if pidof ${serverName} &>/dev/null; then + ${csudo}service ${serverName} stop || : + fi + + if pidof tarbitrator &>/dev/null; then + ${csudo}service tarbitratord stop || : + fi + + if ((${initd_mod} == 1)); then + if [ -e ${service_config_dir}/${serverName} ]; then + ${csudo}chkconfig --del ${serverName} || : + fi + + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}chkconfig --del tarbitratord || : + fi + elif ((${initd_mod} == 2)); then + if [ -e ${service_config_dir}/${serverName} ]; then + ${csudo}insserv -r ${serverName} || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}insserv -r tarbitratord || : + fi + elif ((${initd_mod} == 3)); then + if [ -e ${service_config_dir}/${serverName} ]; then + ${csudo}update-rc.d -f ${serverName} remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo}rm -f ${service_config_dir}/${serverName} || : + ${csudo}rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &>/dev/null); then + ${csudo}init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + if ((${os_type} == 1)); then + # ${csudo}cp -f ${script_dir}/init.d/${serverName}.deb ${install_main_dir}/init.d/${serverName} + ${csudo}cp ${script_dir}/init.d/${serverName}.deb ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName} + # ${csudo}cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo}cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type} == 2)); then + # ${csudo}cp -f ${script_dir}/init.d/${serverName}.rpm ${install_main_dir}/init.d/${serverName} + ${csudo}cp ${script_dir}/init.d/${serverName}.rpm ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName} + # ${csudo}cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo}cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord + fi + + if ((${initd_mod} == 1)); then + ${csudo}chkconfig --add ${serverName} || : + ${csudo}chkconfig --level 2345 ${serverName} on || : + ${csudo}chkconfig --add tarbitratord || : + ${csudo}chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod} == 2)); then + ${csudo}insserv ${serverName} || : + ${csudo}insserv -d ${serverName} || : + ${csudo}insserv tarbitratord || : + ${csudo}insserv -d tarbitratord || : + elif ((${initd_mod} == 3)); then + ${csudo}update-rc.d ${serverName} defaults || : + ${csudo}update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + taosd_service_config="${service_config_dir}/${serverName}.service" + if systemctl is-active --quiet ${serverName}; then + echo "${productName} is running, stopping it..." + ${csudo}systemctl stop ${serverName} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${serverName} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${taosd_service_config} + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo}systemctl stop tarbitratord &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${tarbitratord_service_config} + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/nginxd.service" + if systemctl is-active --quiet nginxd; then + echo "Nginx for ${productName} is running, stopping it..." + ${csudo}systemctl stop nginxd &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable nginxd &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${nginx_service_config} + fi +} + +function install_service_on_systemd() { + clean_service_on_systemd + + [ -f ${script_dir}/cfg/${serverName}.service ] && + ${csudo}cp ${script_dir}/cfg/${serverName}.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + + ${csudo}systemctl enable ${serverName} + + [ -f ${script_dir}/cfg/tarbitratord.service ] && + ${csudo}cp ${script_dir}/cfg/tarbitratord.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + + if [ "$verMode" == "cluster" ]; then + [ -f ${script_dir}/cfg/nginxd.service ] && + ${csudo}cp ${script_dir}/cfg/nginxd.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + + if ! ${csudo}systemctl enable nginxd &>/dev/null; then + ${csudo}systemctl daemon-reexec + ${csudo}systemctl enable nginxd + fi + ${csudo}systemctl start nginxd + fi +} + +function install_adapter_service() { + if ((${service_mod} == 0)); then + [ -f ${script_dir}/cfg/${adapterName}.service ] && + ${csudo}cp ${script_dir}/cfg/${adapterName}.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + fi +} + +function install_service() { + if ((${service_mod} == 0)); then + install_service_on_systemd + elif ((${service_mod} == 1)); then + install_service_on_sysvinit + else + kill_process ${serverName} + fi +} + +vercomp() { + if [[ $1 == $2 ]]; then + return 0 + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i = ${#ver1[@]}; i < ${#ver2[@]}; i++)); do + ver1[i]=0 + done + + for ((i = 0; i < ${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})); then + return 1 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})); then + return 2 + fi + done + return 0 +} + +function is_version_compatible() { + + curr_version=$(ls ${script_dir}/driver/libtaos.so* | awk -F 'libtaos.so.' '{print $2}') + + if [ -f ${script_dir}/driver/vercomp.txt ]; then + min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt) + else + min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5) + fi + + exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3) + vercomp $exist_version "2.0.16.0" + case $? in + 2) + prompt_force=1 + ;; + esac + + vercomp $curr_version $min_compatible_version + echo "" # avoid $? value not update + + case $? in + 0) return 0 ;; + 1) return 0 ;; + 2) return 1 ;; + esac +} + +function updateProduct() { + # Check if version compatible + if ! is_version_compatible; then + echo -e "${RED}Version incompatible${NC}" + return 1 + fi + + # Start to update + if [ ! -e ${tarName} ]; then + echo "File ${tarName} does not exist" + exit 1 + fi + tar -zxf ${tarName} + install_jemalloc + + echo -e "${GREEN}Start to update ${productName}...${NC}" + # Stop the service if running + if pidof ${serverName} &>/dev/null; then + if ((${service_mod} == 0)); then + ${csudo}systemctl stop ${serverName} || : + elif ((${service_mod} == 1)); then + ${csudo}service ${serverName} stop || : + else + kill_process ${serverName} + fi + sleep 1 + fi + + if [ "$verMode" == "cluster" ]; then + if pidof nginx &>/dev/null; then + if ((${service_mod} == 0)); then + ${csudo}systemctl stop nginxd || : + elif ((${service_mod} == 1)); then + ${csudo}service nginxd stop || : + else + kill_process nginx + fi + sleep 1 + fi + fi + + install_main_path + + install_log + install_header + install_lib + + if [ "$verMode" == "cluster" ]; then + install_connector + fi + + install_examples + if [ -z $1 ]; then + install_bin + install_service + install_adapter_service + install_config + install_adapter_config + + openresty_work=false + if [ "$verMode" == "cluster" ]; then + # Check if openresty is installed + # Check if nginx is installed successfully + if type curl &>/dev/null; then + if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then + echo -e "\033[44;32;1mNginx for ${productName} is updated successfully!${NC}" + openresty_work=true else - echo -e "${GREEN_DARK}To start TDengine ${NC}: taosd${NC}" + echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m" fi - - if [ ! -z "$firstEp" ]; then - tmpFqdn=${firstEp%%:*} - substr=":" - if [[ $firstEp =~ $substr ]];then - tmpPort=${firstEp#*:} - else - tmpPort="" - fi - if [[ "$tmpPort" != "" ]];then - echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" - else - echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" - fi - echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" - echo - elif [ ! -z "$serverFqdn" ]; then - echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}" - echo - fi - - echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" - echo - else # Only install client - install_bin - install_config - echo - echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}" + fi fi - touch ~/.taos_history + echo + echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" + echo -e "${GREEN_DARK}To configure Adapter (if has) ${NC}: edit ${cfg_install_dir}/${adapterName}.toml" + if ((${service_mod} == 0)); then + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" + elif ((${service_mod} == 1)); then + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" + else + echo -e "${GREEN_DARK}To start Adapter (if has)${NC}: ${adapterName} &${NC}" + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ./${serverName}${NC}" + fi + + if [ ${openresty_work} = 'true' ]; then + echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + else + echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell${NC}" + fi + + if ((${prompt_force} == 1)); then + echo "" + echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName} $exist_version!${NC}" + fi + echo + echo -e "\033[44;32;1m${productName} is updated successfully!${NC}" + else + install_bin + install_config + + echo + echo -e "\033[44;32;1m${productName} client is updated successfully!${NC}" + fi + + rm -rf $(tar -tf ${tarName} | grep -v "^\./$") } +function installProduct() { + # Start to install + if [ ! -e ${tarName} ]; then + echo "File ${tarName} does not exist" + exit 1 + fi + tar -zxf ${tarName} + + echo -e "${GREEN}Start to install ${productName}...${NC}" + + install_main_path + + if [ -z $1 ]; then + install_data + fi + + install_log + install_header + install_lib + install_jemalloc + #install_avro lib + #install_avro lib64 + + if [ "$verMode" == "cluster" ]; then + install_connector + fi + install_examples + + if [ -z $1 ]; then # install service and client + # For installing new + install_bin + install_service + install_adapter_service + install_adapter_config + + openresty_work=false + if [ "$verMode" == "cluster" ]; then + # Check if nginx is installed successfully + if type curl &>/dev/null; then + if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then + echo -e "\033[44;32;1mNginx for ${productName} is installed successfully!${NC}" + openresty_work=true + else + echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m" + fi + fi + fi + + install_config + + # Ask if to start the service + echo + echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" + echo -e "${GREEN_DARK}To configure ${adapterName} (if has) ${NC}: edit ${cfg_install_dir}/${adapterName}.toml" + if ((${service_mod} == 0)); then + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" + elif ((${service_mod} == 1)); then + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" + else + echo -e "${GREEN_DARK}To start Adapter (if has)${NC}: ${adapterName} &${NC}" + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" + fi + + if [ ! -z "$firstEp" ]; then + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]]; then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]]; then + echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $serverFqdn${GREEN_DARK} to login into ${productName} server${NC}" + echo + fi + + echo -e "\033[44;32;1m${productName} is installed successfully!${NC}" + echo + else # Only install client + install_bin + install_config + echo + echo -e "\033[44;32;1m${productName} client is installed successfully!${NC}" + fi + + touch ~/.${historyFile} + rm -rf $(tar -tf ${tarName} | grep -v "^\./$") +} ## ==============================Main program starts from here============================ serverFqdn=$(hostname) if [ "$verType" == "server" ]; then - # Install server and client - install_TDengine + # Install server and client + if [ -x ${bin_dir}/${serverName} ]; then + update_flag=1 + updateProduct + else + installProduct + fi elif [ "$verType" == "client" ]; then - interactiveFqdn=no - # Only install client - install_TDengine client + interactiveFqdn=no + # Only install client + if [ -x ${bin_dir}/${clientName} ]; then + update_flag=1 + updateProduct client + else + installProduct client + fi else - echo "please input correct verType" + echo "please input correct verType" fi diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 516b289f08..d9f3351008 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -121,7 +121,7 @@ struct SAppInstInfo { SCorEpSet mgmtEp; SInstanceSummary summary; SList* pConnList; // STscObj linked list - int64_t clusterId; + uint64_t clusterId; void* pTransporter; SAppHbMgr* pAppHbMgr; }; @@ -286,6 +286,8 @@ void initMsgHandleFp(); TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, uint16_t port, int connType); +SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen); + int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb); int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index eec8cab7ad..53eb443b36 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -565,10 +565,32 @@ const char *taos_get_server_info(TAOS *taos) { void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { if (taos == NULL || sql == NULL) { - // todo directly call fp + fp(param, NULL, TSDB_CODE_INVALID_PARA); + return; } - taos_query_l(taos, sql, (int32_t)strlen(sql)); + SRequestObj* pRequest = NULL; + int32_t retryNum = 0; + int32_t code = 0; + + size_t sqlLen = strlen(sql); + + while (retryNum++ < REQUEST_MAX_TRY_TIMES) { + pRequest = launchQuery(taos, sql, sqlLen); + if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { + break; + } + + code = refreshMeta(taos, pRequest); + if (code) { + pRequest->code = code; + break; + } + + destroyRequest(pRequest); + } + + fp(param, pRequest, code); } void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 11c6971e3d..dfce01dd63 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -125,10 +125,10 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { struct SCatalog* pCatalog = NULL; if (usedbRsp.vgVersion >= 0) { - int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + uint64_t clusterId = pRequest->pTscObj->pAppInfo->clusterId; + int32_t code1 = catalogGetHandle(clusterId, &pCatalog); if (code1 != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId, - tstrerror(code1)); + tscWarn("0x%" PRIx64 "catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->requestId, clusterId, tstrerror(code1)); } else { catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid); } @@ -158,7 +158,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { if (output.dbVgroup) taosHashCleanup(output.dbVgroup->vgHash); taosMemoryFreeClear(output.dbVgroup); - tscError("failed to build use db output since %s", terrstr()); + tscError("0x%" PRIx64" failed to build use db output since %s", pRequest->requestId, terrstr()); } else if (output.dbVgroup) { struct SCatalog* pCatalog = NULL; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index db992f85d4..e34ce1150d 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -126,7 +126,7 @@ STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList); int32_t tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); int32_t tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); -int32_t tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList); +int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); @@ -174,20 +174,20 @@ typedef struct { } STableKeyInfo; struct SMetaEntry { - int64_t version; - int8_t type; - tb_uid_t uid; - const char *name; + int64_t version; + int8_t type; + tb_uid_t uid; + char *name; union { struct { SSchemaWrapper schema; SSchemaWrapper schemaTag; } stbEntry; struct { - int64_t ctime; - int32_t ttlDays; - tb_uid_t suid; - const uint8_t *pTags; + int64_t ctime; + int32_t ttlDays; + tb_uid_t suid; + uint8_t *pTags; } ctbEntry; struct { int64_t ctime; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 6cda56a589..4ec54d695f 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -30,9 +30,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int vLen = 0; const void *pKey = NULL; const void *pVal = NULL; - void * pBuf = NULL; + void *pBuf = NULL; int32_t szBuf = 0; - void * p = NULL; + void *p = NULL; SMetaReader mr = {0}; // validate req @@ -71,9 +71,9 @@ _err: } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { - TBC * pNameIdxc = NULL; - TBC * pUidIdxc = NULL; - TBC * pCtbIdxc = NULL; + TBC *pNameIdxc = NULL; + TBC *pUidIdxc = NULL; + TBC *pCtbIdxc = NULL; SCtbIdxKey *pCtbIdxKey; const void *pKey = NULL; int nKey; @@ -134,8 +134,8 @@ _err: int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry oStbEntry = {0}; SMetaEntry nStbEntry = {0}; - TBC * pUidIdxc = NULL; - TBC * pTbDbc = NULL; + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; const void *pData; int nData; int64_t oversion; @@ -165,7 +165,9 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ASSERT(ret == 0); - tDecoderInit(&dc, pData, nData); + oStbEntry.pBuf = taosMemoryMalloc(nData); + memcpy(oStbEntry.pBuf, pData, nData); + tDecoderInit(&dc, oStbEntry.pBuf, nData); metaDecodeEntry(&dc, &oStbEntry); nStbEntry.version = version; @@ -193,6 +195,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // update uid index tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); metaULock(pMeta); tDecoderClear(&dc); tdbTbcClose(pTbDbc); @@ -256,9 +259,9 @@ _err: } int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) { - TBC * pTbDbc = NULL; - TBC * pUidIdxc = NULL; - TBC * pNameIdxc = NULL; + TBC *pTbDbc = NULL; + TBC *pUidIdxc = NULL; + TBC *pNameIdxc = NULL; const void *pData; int nData; tb_uid_t uid; @@ -377,14 +380,14 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi } static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - void * pVal = NULL; + void *pVal = NULL; int nVal = 0; - const void * pData = NULL; + const void *pData = NULL; int nData = 0; int ret = 0; tb_uid_t uid; int64_t oversion; - SSchema * pColumn = NULL; + SSchema *pColumn = NULL; SMetaEntry entry = {0}; SSchemaWrapper *pSchema; int c; @@ -420,7 +423,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // get table entry SDecoder dc = {0}; - tDecoderInit(&dc, pData, nData); + entry.pBuf = taosMemoryMalloc(nData); + memcpy(entry.pBuf, pData, nData); + tDecoderInit(&dc, entry.pBuf, nData); ret = metaDecodeEntry(&dc, &entry); ASSERT(ret == 0); @@ -530,7 +535,7 @@ _err: static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { SMetaEntry ctbEntry = {0}; SMetaEntry stbEntry = {0}; - void * pVal = NULL; + void *pVal = NULL; int nVal = 0; int ret; int c; @@ -561,7 +566,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA oversion = *(int64_t *)pData; // search table.db - TBC * pTbDbc = NULL; + TBC *pTbDbc = NULL; SDecoder dc1 = {0}; SDecoder dc2 = {0}; @@ -585,7 +590,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaDecodeEntry(&dc2, &stbEntry); SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - SSchema * pColumn = NULL; + SSchema *pColumn = NULL; int32_t iCol = 0; for (;;) { pColumn = NULL; @@ -681,8 +686,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { STbDbKey tbDbKey; - void * pKey = NULL; - void * pVal = NULL; + void *pKey = NULL; + void *pVal = NULL; int kLen = 0; int vLen = 0; SEncoder coder = {0}; @@ -797,14 +802,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { } static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { - void * pData = NULL; + void *pData = NULL; int nData = 0; STbDbKey tbDbKey = {0}; SMetaEntry stbEntry = {0}; - STagIdxKey * pTagIdxKey = NULL; + STagIdxKey *pTagIdxKey = NULL; int32_t nTagIdxKey; const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0]; - const void * pTagData = NULL; // + const void *pTagData = NULL; // SDecoder dc = {0}; // get super table @@ -846,7 +851,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { SEncoder coder = {0}; - void * pVal = NULL; + void *pVal = NULL; int vLen = 0; int rcode = 0; SSkmDbKey skmDbKey = {0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 2acca738fb..025b2ab580 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -38,7 +38,7 @@ struct SMemTable { struct SMemSkipListNode { int8_t level; - SMemSkipListNode *forwards[1]; // Windows does not allow 0 + SMemSkipListNode *forwards[1]; // Windows does not allow 0 }; struct SMemSkipList { @@ -46,7 +46,7 @@ struct SMemSkipList { int8_t maxLevel; int8_t level; int32_t size; - SMemSkipListNode pHead[1]; // Windows does not allow 0 + SMemSkipListNode pHead[1]; // Windows does not allow 0 }; struct SMemData { @@ -217,7 +217,7 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p if (tDecodeIsEnd(&dc)) break; // decode row - if (tDecodeBinary(&dc, (const uint8_t **)&tRow.pRow, &tRow.szRow) < 0) { + if (tDecodeBinary(&dc, (uint8_t **)&tRow.pRow, &tRow.szRow) < 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -273,7 +273,7 @@ static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pR static FORCE_INLINE int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow) { if (tDecodeI64(pDecoder, &pRow->version) < 0) return -1; - if (tDecodeBinary(pDecoder, (const uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1; + if (tDecodeBinary(pDecoder, (uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1; return 0; } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 857c708852..9f66b6c598 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -58,6 +58,17 @@ enum { CTG_ACT_MAX }; +typedef enum { + CTG_TASK_GET_QNODE = 0, + CTG_TASK_GET_DB_VGROUP, + CTG_TASK_GET_DB_CFG, + CTG_TASK_GET_TB_META, + CTG_TASK_GET_TB_HASH, + CTG_TASK_GET_INDEX, + CTG_TASK_GET_UDF, + CTG_TASK_GET_USER, +} CTG_TASK_TYPE; + typedef struct SCtgDebug { bool lockEnable; bool cacheEnable; @@ -66,6 +77,43 @@ typedef struct SCtgDebug { uint32_t showCachePeriodSec; } SCtgDebug; +typedef struct SCtgTbCacheInfo { + bool inCache; + uint64_t dbId; + uint64_t suid; + int32_t tbType; +} SCtgTbCacheInfo; + +typedef struct SCtgTbMetaCtx { + SCtgTbCacheInfo tbInfo; + SName* pName; + int32_t flag; +} SCtgTbMetaCtx; + +typedef struct SCtgDbVgCtx { + char dbFName[TSDB_DB_FNAME_LEN]; +} SCtgDbVgCtx; + +typedef struct SCtgDbCfgCtx { + char dbFName[TSDB_DB_FNAME_LEN]; +} SCtgDbCfgCtx; + +typedef struct SCtgTbHashCtx { + char dbFName[TSDB_DB_FNAME_LEN]; + SName* pName; +} SCtgTbHashCtx; + +typedef struct SCtgIndexCtx { + char indexFName[TSDB_INDEX_FNAME_LEN]; +} SCtgIndexCtx; + +typedef struct SCtgUdfCtx { + char udfName[TSDB_FUNC_NAME_LEN]; +} SCtgUdfCtx; + +typedef struct SCtgUserCtx { + SUserAuthInfo user; +} SCtgUserCtx; typedef struct SCtgTbMetaCache { SRWLatch stbLock; @@ -113,6 +161,55 @@ typedef struct SCatalog { SCtgRentMgmt stbRent; } SCatalog; +typedef struct SCtgJob { + int64_t refId; + SArray* pTasks; + int32_t taskDone; + SMetaData jobRes; + int32_t rspCode; + + uint64_t queryId; + SCatalog* pCtg; + void* pTrans; + const SEpSet* pMgmtEps; + void* userParam; + catalogCallback userFp; + int32_t tbMetaNum; + int32_t tbHashNum; + int32_t dbVgNum; + int32_t udfNum; + int32_t qnodeNum; + int32_t dbCfgNum; + int32_t indexNum; + int32_t userNum; +} SCtgJob; + +typedef struct SCtgMsgCtx { + int32_t reqType; + void* lastOut; + void* out; + char* target; +} SCtgMsgCtx; + +typedef struct SCtgTask { + CTG_TASK_TYPE type; + int32_t taskId; + SCtgJob *pJob; + void* taskCtx; + SCtgMsgCtx msgCtx; + void* res; +} SCtgTask; + +typedef int32_t (*ctgLanchTaskFp)(SCtgTask*); +typedef int32_t (*ctgHandleTaskMsgRspFp)(SCtgTask*, int32_t, const SDataBuf *, int32_t); +typedef int32_t (*ctgDumpTaskResFp)(SCtgTask*); + +typedef struct SCtgAsyncFps { + ctgLanchTaskFp launchFp; + ctgHandleTaskMsgRspFp handleRspFp; + ctgDumpTaskResFp dumpResFp; +} SCtgAsyncFps; + typedef struct SCtgApiStat { #ifdef WINDOWS @@ -214,6 +311,7 @@ typedef struct SCtgQueue { typedef struct SCatalogMgmt { bool exit; + int32_t jobPool; SRWLatch lock; SCtgQueue queue; TdThread updateThread; @@ -327,10 +425,80 @@ typedef struct SCtgAction { #define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); CTG_RET(__code); } while (0) #define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) +#define CTG_PARAMS SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps +#define CTG_PARAMS_LIST() pCtg, pTrans, pMgmtEps -extern void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p); -extern void ctgdShowClusterCache(SCatalog* pCtg); -extern int32_t ctgdShowCacheInfo(void); +void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p); +void ctgdShowClusterCache(SCatalog* pCtg); +int32_t ctgdShowCacheInfo(void); + +int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq); +int32_t ctgGetTbMetaFromCache(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); + +int32_t ctgActUpdateVg(SCtgMetaAction *action); +int32_t ctgActUpdateTb(SCtgMetaAction *action); +int32_t ctgActRemoveDB(SCtgMetaAction *action); +int32_t ctgActRemoveStb(SCtgMetaAction *action); +int32_t ctgActRemoveTb(SCtgMetaAction *action); +int32_t ctgActUpdateUser(SCtgMetaAction *action); +int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); +void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache); +void ctgReleaseVgInfo(SCtgDBCache *dbCache); +int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); +int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist); +int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); +int32_t ctgReadTbSverFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tbType, uint64_t *suid, char *stbName); +int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass); +int32_t ctgPutRmDBToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId); +int32_t ctgPutRmStbToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq); +int32_t ctgPutRmTbToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq); +int32_t ctgPutUpdateVgToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq); +int32_t ctgPutUpdateTbToQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq); +int32_t ctgPutUpdateUserToQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); +int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); +int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); +int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); +int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); +int32_t ctgStartUpdateThread(); +int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask); + + + +int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target); +int32_t ctgGetDBVgInfoFromMnode(CTG_PARAMS, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTask* pTask); +int32_t ctgGetQnodeListFromMnode(CTG_PARAMS, SArray *out, SCtgTask* pTask); +int32_t ctgGetDBCfgFromMnode(CTG_PARAMS, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask); +int32_t ctgGetIndexInfoFromMnode(CTG_PARAMS, const char *indexName, SIndexInfo *out, SCtgTask* pTask); +int32_t ctgGetUdfInfoFromMnode(CTG_PARAMS, const char *funcName, SFuncInfo *out, SCtgTask* pTask); +int32_t ctgGetUserDbAuthFromMnode(CTG_PARAMS, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask); +int32_t ctgGetTbMetaFromMnodeImpl(CTG_PARAMS, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask); +int32_t ctgGetTbMetaFromMnode(CTG_PARAMS, const SName* pTableName, STableMetaOutput* out, SCtgTask* pTask); +int32_t ctgGetTbMetaFromVnode(CTG_PARAMS, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTask* pTask); + +int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param); +int32_t ctgLaunchJob(SCtgJob *pJob); +int32_t ctgMakeAsyncRes(SCtgJob *pJob); + +int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst); +int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput); +int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList); +void ctgFreeJob(void* job); +void ctgFreeHandle(SCatalog* pCtg); +void ctgFreeVgInfo(SDBVgInfo *vgInfo); +int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup); +void ctgResetTbMetaTask(SCtgTask* pTask); +void ctgFreeDbCache(SCtgDBCache *dbCache); +int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); +int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2); +int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); +int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2); +void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); +int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); + + +extern SCatalogMgmt gCtgMgmt; +extern SCtgDebug gCTGDebug; +extern SCtgAsyncFps gCtgAsyncFps[]; #ifdef __cplusplus } diff --git a/source/libs/catalog/inc/ctgRemote.h b/source/libs/catalog/inc/ctgRemote.h new file mode 100644 index 0000000000..cd88863c1b --- /dev/null +++ b/source/libs/catalog/inc/ctgRemote.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_CATALOG_REMOTE_H_ +#define _TD_CATALOG_REMOTE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SCtgTaskCallbackParam { + uint64_t queryId; + int64_t refId; + uint64_t taskId; + int32_t reqType; +} SCtgTaskCallbackParam; + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_CATALOG_REMOTE_H_*/ diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 2d4355191e..4afebf9951 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -13,1795 +13,66 @@ * along with this program. If not, see . */ -#include "catalogInt.h" -#include "query.h" -#include "systable.h" -#include "tname.h" #include "trpc.h" - -int32_t ctgActUpdateVg(SCtgMetaAction *action); -int32_t ctgActUpdateTbl(SCtgMetaAction *action); -int32_t ctgActRemoveDB(SCtgMetaAction *action); -int32_t ctgActRemoveStb(SCtgMetaAction *action); -int32_t ctgActRemoveTbl(SCtgMetaAction *action); -int32_t ctgActUpdateUser(SCtgMetaAction *action); - -extern SCtgDebug gCTGDebug; -SCatalogMgmt gCtgMgmt = {0}; -SCtgAction gCtgAction[CTG_ACT_MAX] = { - {CTG_ACT_UPDATE_VG, "update vgInfo", ctgActUpdateVg}, {CTG_ACT_UPDATE_TBL, "update tbMeta", ctgActUpdateTbl}, - {CTG_ACT_REMOVE_DB, "remove DB", ctgActRemoveDB}, {CTG_ACT_REMOVE_STB, "remove stbMeta", ctgActRemoveStb}, - {CTG_ACT_REMOVE_TBL, "remove tbMeta", ctgActRemoveTbl}, {CTG_ACT_UPDATE_USER, "update user", ctgActUpdateUser}}; - -void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { - if (NULL == mgmt->slots) { - return; - } - - for (int32_t i = 0; i < mgmt->slotNum; ++i) { - SCtgRentSlot *slot = &mgmt->slots[i]; - if (slot->meta) { - taosArrayDestroy(slot->meta); - slot->meta = NULL; - } - } - - taosMemoryFreeClear(mgmt->slots); -} - -void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - int32_t stblNum = taosHashGetSize(cache->stbCache); - taosHashCleanup(cache->stbCache); - cache->stbCache = NULL; - CTG_CACHE_STAT_SUB(stblNum, stblNum); - } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); - - CTG_LOCK(CTG_WRITE, &cache->metaLock); - if (cache->metaCache) { - int32_t tblNum = taosHashGetSize(cache->metaCache); - taosHashCleanup(cache->metaCache); - cache->metaCache = NULL; - CTG_CACHE_STAT_SUB(tblNum, tblNum); - } - CTG_UNLOCK(CTG_WRITE, &cache->metaLock); -} - -void ctgFreeVgInfo(SDBVgInfo *vgInfo) { - if (NULL == vgInfo) { - return; - } - - if (vgInfo->vgHash) { - taosHashCleanup(vgInfo->vgHash); - vgInfo->vgHash = NULL; - } - - taosMemoryFreeClear(vgInfo); -} - -void ctgFreeDbCache(SCtgDBCache *dbCache) { - if (NULL == dbCache) { - return; - } - - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - ctgFreeVgInfo(dbCache->vgInfo); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - - ctgFreeTableMetaCache(&dbCache->tbCache); -} - -void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) { - taosHashCleanup(userCache->createdDbs); - taosHashCleanup(userCache->readDbs); - taosHashCleanup(userCache->writeDbs); -} - -void ctgFreeHandle(SCatalog *pCtg) { - ctgFreeMetaRent(&pCtg->dbRent); - ctgFreeMetaRent(&pCtg->stbRent); - - if (pCtg->dbCache) { - int32_t dbNum = taosHashGetSize(pCtg->dbCache); - - void *pIter = taosHashIterate(pCtg->dbCache, NULL); - while (pIter) { - SCtgDBCache *dbCache = pIter; - - atomic_store_8(&dbCache->deleted, 1); - - ctgFreeDbCache(dbCache); - - pIter = taosHashIterate(pCtg->dbCache, pIter); - } - - taosHashCleanup(pCtg->dbCache); - - CTG_CACHE_STAT_SUB(dbNum, dbNum); - } - - if (pCtg->userCache) { - int32_t userNum = taosHashGetSize(pCtg->userCache); - - void *pIter = taosHashIterate(pCtg->userCache, NULL); - while (pIter) { - SCtgUserAuth *userCache = pIter; - - ctgFreeSCtgUserAuth(userCache); - - pIter = taosHashIterate(pCtg->userCache, pIter); - } - - taosHashCleanup(pCtg->userCache); - - CTG_CACHE_STAT_SUB(userNum, userNum); - } - - taosMemoryFree(pCtg); -} - -void ctgWaitAction(SCtgMetaAction *action) { - while (true) { - tsem_wait(&gCtgMgmt.queue.rspSem); - - if (atomic_load_8((int8_t *)&gCtgMgmt.exit)) { - tsem_post(&gCtgMgmt.queue.rspSem); - break; - } - - if (gCtgMgmt.queue.seqDone >= action->seqId) { - break; - } - - tsem_post(&gCtgMgmt.queue.rspSem); - sched_yield(); - } -} - -void ctgPopAction(SCtgMetaAction **action) { - SCtgQNode *orig = gCtgMgmt.queue.head; - - SCtgQNode *node = gCtgMgmt.queue.head->next; - gCtgMgmt.queue.head = gCtgMgmt.queue.head->next; - - CTG_QUEUE_SUB(); - - taosMemoryFreeClear(orig); - - *action = &node->action; -} - -int32_t ctgPushAction(SCatalog *pCtg, SCtgMetaAction *action) { - SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); - if (NULL == node) { - qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); - CTG_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - action->seqId = atomic_add_fetch_64(&gCtgMgmt.queue.seqId, 1); - - node->action = *action; - - CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); - gCtgMgmt.queue.tail->next = node; - gCtgMgmt.queue.tail = node; - CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); - - CTG_QUEUE_ADD(); - CTG_RUNTIME_STAT_ADD(qNum, 1); - - tsem_post(&gCtgMgmt.queue.reqSem); - - ctgDebug("action [%s] added into queue", gCtgAction[action->act].name); - - if (action->syncReq) { - ctgWaitAction(action); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgPushRmDBMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) { - int32_t code = 0; - SCtgMetaAction action = {.act = CTG_ACT_REMOVE_DB}; - SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - char *p = strchr(dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { - dbFName = p + 1; - } - - msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - msg->dbId = dbId; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - - return TSDB_CODE_SUCCESS; - -_return: - - taosMemoryFreeClear(action.data); - CTG_RET(code); -} - -int32_t ctgPushRmStbMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, - bool syncReq) { - int32_t code = 0; - SCtgMetaAction action = {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq}; - SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - strncpy(msg->stbName, stbName, sizeof(msg->stbName)); - msg->dbId = dbId; - msg->suid = suid; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - - return TSDB_CODE_SUCCESS; - -_return: - - taosMemoryFreeClear(action.data); - CTG_RET(code); -} - -int32_t ctgPushRmTblMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action = {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq}; - SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - strncpy(msg->tbName, tbName, sizeof(msg->tbName)); - msg->dbId = dbId; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - - return TSDB_CODE_SUCCESS; - -_return: - - taosMemoryFreeClear(action.data); - CTG_RET(code); -} - -int32_t ctgPushUpdateVgMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDBVgInfo *dbInfo, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action = {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq}; - SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); - ctgFreeVgInfo(dbInfo); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - char *p = strchr(dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { - dbFName = p + 1; - } - - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - msg->pCtg = pCtg; - msg->dbId = dbId; - msg->dbInfo = dbInfo; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - - return TSDB_CODE_SUCCESS; - -_return: - - ctgFreeVgInfo(dbInfo); - taosMemoryFreeClear(action.data); - CTG_RET(code); -} - -int32_t ctgPushUpdateTblMsgInQueue(SCatalog *pCtg, STableMetaOutput *output, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action = {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq}; - SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - char *p = strchr(output->dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { - memmove(output->dbFName, p + 1, strlen(p + 1)); - } - - msg->pCtg = pCtg; - msg->output = output; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - - return TSDB_CODE_SUCCESS; - -_return: - - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - -int32_t ctgPushUpdateUserMsgInQueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action = {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; - SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - msg->userAuth = *pAuth; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - - return TSDB_CODE_SUCCESS; - -_return: - - tFreeSGetUserAuthRsp(pAuth); - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - -int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { - CTG_LOCK(CTG_READ, &dbCache->vgLock); - - if (dbCache->deleted) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - - ctgDebug("db is dropping, dbId:%" PRIx64, dbCache->dbId); - - *inCache = false; +#include "query.h" +#include "tname.h" +#include "catalogInt.h" +#include "systable.h" +#include "tref.h" + +SCatalogMgmt gCtgMgmt = {0}; + + +int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq) { + int32_t code = 0; + STableMeta *tblMeta = NULL; + SCtgTbMetaCtx tbCtx = {0}; + tbCtx.flag = CTG_FLAG_UNKNOWN_STB; + tbCtx.pName = pTableName; + + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &tbCtx, &tblMeta)); + + if (NULL == tblMeta) { + ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname); return TSDB_CODE_SUCCESS; } - if (NULL == dbCache->vgInfo) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - - *inCache = false; - ctgDebug("db vgInfo is empty, dbId:%" PRIx64, dbCache->dbId); - return TSDB_CODE_SUCCESS; - } - - *inCache = true; - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - - if (dbCache->deleted) { - ctgDebug("db is dropping, dbId:%" PRIx64, dbCache->dbId); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } - - return TSDB_CODE_SUCCESS; -} - -void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { taosHashRelease(pCtg->dbCache, dbCache); } - -void ctgReleaseVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->vgLock); } - -void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); } - -int32_t ctgAcquireDBCacheImpl(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { - char *p = strchr(dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { - dbFName = p + 1; - } - - SCtgDBCache *dbCache = NULL; - if (acquire) { - dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); - } else { - dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); - } - - if (NULL == dbCache) { - *pCache = NULL; - ctgDebug("db not in cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; - } - - if (dbCache->deleted) { - if (acquire) { - ctgReleaseDBCache(pCtg, dbCache); - } - - *pCache = NULL; - ctgDebug("db is removing from cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; - } - - *pCache = dbCache; - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgAcquireDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) { - CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); -} - -int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) { - CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); -} - -int32_t ctgAcquireVgInfoFromCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { - SCtgDBCache *dbCache = NULL; - - if (NULL == pCtg->dbCache) { - ctgDebug("empty db cache, dbFName:%s", dbFName); - goto _return; - } - - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %s not in cache", dbFName); - goto _return; - } - - ctgAcquireVgInfo(pCtg, dbCache, inCache); - if (!(*inCache)) { - ctgDebug("vgInfo of db %s not in cache", dbFName); - goto _return; - } - - *pCache = dbCache; - *inCache = true; - - CTG_CACHE_STAT_ADD(vgHitNum, 1); - - ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); - - return TSDB_CODE_SUCCESS; - -_return: - - if (dbCache) { - ctgReleaseDBCache(pCtg, dbCache); - } - - *pCache = NULL; - *inCache = false; - - CTG_CACHE_STAT_ADD(vgMissNum, 1); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetQnodeListFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, SArray *out) { - char *msg = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get qnode list from mnode, mgmtEpInUse:%d", pMgmtEps->inUse); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_QNODE_LIST)](NULL, &msg, 0, &msgLen); - if (code) { - ctgError("Build qnode list msg failed, error:%s", tstrerror(code)); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_QNODE_LIST, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("error rsp for qnode list, error:%s", tstrerror(rpcRsp.code)); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_QNODE_LIST)](out, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process qnode list rsp failed, error:%s", tstrerror(rpcRsp.code)); - CTG_ERR_RET(code); - } - - ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(out)); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetDBVgInfoFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, SBuildUseDBInput *input, - SUseDbOutput *out) { - char *msg = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)](input, &msg, 0, &msgLen); - if (code) { - ctgError("Build use db msg failed, code:%x, db:%s", code, input->db); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_USE_DB, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("error rsp for use db, error:%s, db:%s", tstrerror(rpcRsp.code), input->db); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)](out, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process use db rsp failed, code:%x, db:%s", code, input->db); - CTG_ERR_RET(code); - } - - ctgDebug("Got db vgInfo from mnode, dbFName:%s", input->db); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetDBCfgFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, SDbCfgInfo *out) { - char *msg = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get db cfg from mnode, dbFName:%s", dbFName); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)]((void *)dbFName, &msg, 0, &msgLen); - if (code) { - ctgError("Build get db cfg msg failed, code:%x, db:%s", code, dbFName); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_GET_DB_CFG, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("error rsp for get db cfg, error:%s, db:%s", tstrerror(rpcRsp.code), dbFName); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)](out, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process get db cfg rsp failed, code:%x, db:%s", code, dbFName); - CTG_ERR_RET(code); - } - - ctgDebug("Got db cfg from mnode, dbFName:%s", dbFName); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetIndexInfoFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *indexName, - SIndexInfo *out) { - char *msg = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get index from mnode, indexName:%s", indexName); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)]((void *)indexName, &msg, 0, &msgLen); - if (code) { - ctgError("Build get index msg failed, code:%x, db:%s", code, indexName); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_GET_INDEX, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("error rsp for get index, error:%s, indexName:%s", tstrerror(rpcRsp.code), indexName); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)](out, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process get index rsp failed, code:%x, indexName:%s", code, indexName); - CTG_ERR_RET(code); - } - - ctgDebug("Got index from mnode, indexName:%s", indexName); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetUdfInfoFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *funcName, - SFuncInfo **out) { - char *msg = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get udf info from mnode, funcName:%s", funcName); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)]((void *)funcName, &msg, 0, &msgLen); - if (code) { - ctgError("Build get udf msg failed, code:%x, db:%s", code, funcName); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_RETRIEVE_FUNC, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - if (TSDB_CODE_MND_FUNC_NOT_EXIST == rpcRsp.code) { - ctgDebug("funcName %s not exist in mnode", funcName); - taosMemoryFreeClear(*out); - CTG_RET(TSDB_CODE_SUCCESS); - } - - ctgError("error rsp for get udf, error:%s, funcName:%s", tstrerror(rpcRsp.code), funcName); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)](*out, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process get udf rsp failed, code:%x, funcName:%s", code, funcName); - CTG_ERR_RET(code); - } - - ctgDebug("Got udf from mnode, funcName:%s", funcName); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetUserDbAuthFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *user, - SGetUserAuthRsp *authRsp) { - char *msg = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get user auth from mnode, user:%s", user); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)]((void *)user, &msg, 0, &msgLen); - if (code) { - ctgError("Build get user auth msg failed, code:%x, db:%s", code, user); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_GET_USER_AUTH, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("error rsp for get user auth, error:%s, user:%s", tstrerror(rpcRsp.code), user); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)](authRsp, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process get user auth rsp failed, code:%x, user:%s", code, user); - CTG_ERR_RET(code); - } - - ctgDebug("Got user auth from mnode, user:%s", user); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgIsTableMetaExistInCache(SCatalog *pCtg, char *dbFName, char *tbName, int32_t *exist) { - if (NULL == pCtg->dbCache) { - *exist = 0; - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; - } - - SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - *exist = 0; - return TSDB_CODE_SUCCESS; - } - - size_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == tbMeta) { - ctgReleaseDBCache(pCtg, dbCache); - - *exist = 0; - ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; - } - - *exist = 1; - - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetTableMetaFromCache(SCatalog *pCtg, const SName *pTableName, STableMeta **pTableMeta, bool *inCache, - int32_t flag, uint64_t *dbId) { - if (NULL == pCtg->dbCache) { - ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); - goto _return; - } - - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - if (CTG_FLAG_IS_SYS_DB(flag)) { - strcpy(dbFName, pTableName->dbname); - } else { - tNameGetFullDbName(pTableName, dbFName); - } - - *pTableMeta = NULL; - - SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %s not in cache", pTableName->tname); - goto _return; - } - - int32_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - int32_t code = taosHashGetDup_m(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), - (void **)pTableMeta, &sz); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == *pTableMeta) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); - goto _return; - } - - if (dbId) { - *dbId = dbCache->dbId; - } - - STableMeta *tbMeta = *pTableMeta; - - if (tbMeta->tableType != TSDB_CHILD_TABLE) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); - - *inCache = true; - CTG_CACHE_STAT_ADD(tblHitNum, 1); - - return TSDB_CODE_SUCCESS; - } - - ctgDebug("Got subtable meta from cache, type:%d, dbFName:%s, tbName:%s, suid:%" PRIx64, tbMeta->tableType, dbFName, - pTableName->tname, tbMeta->suid); - - CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - - STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); - if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgError("stb not in stbCache, suid:%" PRIx64, tbMeta->suid); - taosMemoryFreeClear(*pTableMeta); - goto _return; - } - - if ((*stbMeta)->suid != tbMeta->suid) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - taosMemoryFreeClear(*pTableMeta); - ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, tbMeta->suid, - (*stbMeta)->suid); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - int32_t metaSize = CTG_META_SIZE(*stbMeta); - *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); - if (NULL == *pTableMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgError("realloc size[%d] failed", metaSize); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); - - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - - ctgReleaseDBCache(pCtg, dbCache); - - *inCache = true; - CTG_CACHE_STAT_ADD(tblHitNum, 1); - - ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); - - return TSDB_CODE_SUCCESS; - -_return: - - *inCache = false; - CTG_CACHE_STAT_ADD(tblMissNum, 1); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetTableTypeFromCache(SCatalog *pCtg, const char *dbFName, const char *tableName, int32_t *tbType) { - if (NULL == pCtg->dbCache) { - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); - return TSDB_CODE_SUCCESS; - } - - SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, tableName, strlen(tableName)); - - if (NULL == pTableMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); - ctgReleaseDBCache(pCtg, dbCache); - - return TSDB_CODE_SUCCESS; - } - - *tbType = atomic_load_8(&pTableMeta->tableType); - - taosHashRelease(dbCache->tbCache.metaCache, pTableMeta); - - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgChkAuthFromCache(SCatalog *pCtg, const char *user, const char *dbFName, AUTH_TYPE type, bool *inCache, - bool *pass) { - if (NULL == pCtg->userCache) { - ctgDebug("empty user auth cache, user:%s", user); - goto _return; - } - - SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); - if (NULL == pUser) { - ctgDebug("user not in cache, user:%s", user); - goto _return; - } - - *inCache = true; - - ctgDebug("Got user from cache, user:%s", user); - CTG_CACHE_STAT_ADD(userHitNum, 1); - - if (pUser->superUser) { - *pass = true; - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_READ, &pUser->lock); - if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) { - *pass = true; - CTG_UNLOCK(CTG_READ, &pUser->lock); - return TSDB_CODE_SUCCESS; - } - - if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { - *pass = true; - } - - if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { - *pass = true; - } - - CTG_UNLOCK(CTG_READ, &pUser->lock); - - return TSDB_CODE_SUCCESS; - -_return: - - *inCache = false; - CTG_CACHE_STAT_ADD(userMissNum, 1); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetTableMetaFromMnodeImpl(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, char *dbFName, char *tbName, - STableMetaOutput *output) { - SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; - char *msg = NULL; - SEpSet *pVnodeEpSet = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)](&bInput, &msg, 0, &msgLen); - if (code) { - ctgError("Build mnode stablemeta msg failed, code:%x", code); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_TABLE_META, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pTrans, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); - - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { - SET_META_TYPE_NULL(output->metaType); - ctgDebug("stablemeta not exist in mnode, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; - } - - ctgError("error rsp for stablemeta from mnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, - tbName); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process mnode stablemeta rsp failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tbName); - CTG_ERR_RET(code); - } - - ctgDebug("Got table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetTableMetaFromMnode(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, - STableMetaOutput *output) { - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); - - return ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, dbFName, (char *)pTableName->tname, output); -} - -int32_t ctgGetTableMetaFromVnodeImpl(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, - SVgroupInfo *vgroupInfo, STableMetaOutput *output) { - if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || - NULL == output) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - - ctgDebug("try to get table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); - - SBuildTableMetaInput bInput = { - .vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; - char *msg = NULL; - int32_t msgLen = 0; - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen); - if (code) { - ctgError("Build vnode tablemeta msg failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, - tNameGetTableName(pTableName)); - CTG_ERR_RET(code); + + if (TSDB_SUPER_TABLE == tblMeta->tableType) { + CTG_ERR_JRET(ctgPutRmStbToQueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, tblMeta->suid, syncReq)); + } else { + CTG_ERR_JRET(ctgPutRmTbToQueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, syncReq)); } - - SRpcMsg rpcMsg = { - .msgType = TDMT_VND_TABLE_META, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - rpcSendRecv(pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); - - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { - SET_META_TYPE_NULL(output->metaType); - ctgDebug("tablemeta not exist in vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); - return TSDB_CODE_SUCCESS; - } - - ctgError("error rsp for table meta from vnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, - tNameGetTableName(pTableName)); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process vnode tablemeta rsp failed, code:%s, dbFName:%s, tbName:%s", tstrerror(code), dbFName, - tNameGetTableName(pTableName)); - CTG_ERR_RET(code); - } - - ctgDebug("Got table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetTableMetaFromVnode(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, - SVgroupInfo *vgroupInfo, STableMetaOutput *output) { - int32_t code = 0; - int32_t retryNum = 0; - - while (retryNum < CTG_DEFAULT_MAX_RETRY_TIMES) { - code = ctgGetTableMetaFromVnodeImpl(pCtg, pTrans, pMgmtEps, pTableName, vgroupInfo, output); - if (code) { - if (TSDB_CODE_VND_HASH_MISMATCH == code) { - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); - - code = catalogRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName); - if (code != TSDB_CODE_SUCCESS) { - break; - } - - ++retryNum; - continue; - } - } - - break; - } - - CTG_RET(code); -} - -int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { - switch (hashMethod) { - default: - *fp = MurmurHash3_32; - break; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray **pList) { - SHashObj *vgroupHash = NULL; - SVgroupInfo *vgInfo = NULL; - SArray *vgList = NULL; - int32_t code = 0; - int32_t vgNum = taosHashGetSize(vgHash); - - vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); - if (NULL == vgList) { - ctgError("taosArrayInit failed, num:%d", vgNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - void *pIter = taosHashIterate(vgHash, NULL); - while (pIter) { - vgInfo = pIter; - - if (NULL == taosArrayPush(vgList, vgInfo)) { - ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); - taosHashCancelIterate(vgHash, pIter); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - pIter = taosHashIterate(vgHash, pIter); - vgInfo = NULL; - } - - *pList = vgList; - - ctgDebug("Got vgList from cache, vgNum:%d", vgNum); - - return TSDB_CODE_SUCCESS; - + _return: - if (vgList) { - taosArrayDestroy(vgList); - } + taosMemoryFreeClear(tblMeta); CTG_RET(code); } -int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { +int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { int32_t code = 0; - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); - char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache)); - if (vgNum <= 0) { - ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", db, vgNum); - CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); - } - - tableNameHashFp fp = NULL; - SVgroupInfo *vgInfo = NULL; - - CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); - - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); - - uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); - - void *pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - vgInfo = pIter; - if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { - taosHashCancelIterate(dbInfo->vgHash, pIter); - break; - } - - pIter = taosHashIterate(dbInfo->vgHash, pIter); - vgInfo = NULL; - } - - if (NULL == vgInfo) { - ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, - taosHashGetSize(dbInfo->vgHash)); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - *pVgroup = *vgInfo; - - CTG_RET(code); -} - -int32_t ctgStbVersionSearchCompare(const void *key1, const void *key2) { - if (*(uint64_t *)key1 < ((SSTableMetaVersion *)key2)->suid) { - return -1; - } else if (*(uint64_t *)key1 > ((SSTableMetaVersion *)key2)->suid) { - return 1; - } else { - return 0; - } -} - -int32_t ctgDbVgVersionSearchCompare(const void *key1, const void *key2) { - if (*(int64_t *)key1 < ((SDbVgVersion *)key2)->dbId) { - return -1; - } else if (*(int64_t *)key1 > ((SDbVgVersion *)key2)->dbId) { - return 1; - } else { - return 0; - } -} - -int32_t ctgStbVersionSortCompare(const void *key1, const void *key2) { - if (((SSTableMetaVersion *)key1)->suid < ((SSTableMetaVersion *)key2)->suid) { - return -1; - } else if (((SSTableMetaVersion *)key1)->suid > ((SSTableMetaVersion *)key2)->suid) { - return 1; - } else { - return 0; - } -} - -int32_t ctgDbVgVersionSortCompare(const void *key1, const void *key2) { - if (((SDbVgVersion *)key1)->dbId < ((SDbVgVersion *)key2)->dbId) { - return -1; - } else if (((SDbVgVersion *)key1)->dbId > ((SDbVgVersion *)key2)->dbId) { - return 1; - } else { - return 0; - } -} - -int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { - mgmt->slotRIdx = 0; - mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; - mgmt->type = type; - - size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; - - mgmt->slots = taosMemoryCalloc(1, msgSize); - if (NULL == mgmt->slots) { - qError("calloc %d failed", (int32_t)msgSize); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { - int16_t widx = abs((int)(id % mgmt->slotNum)); - - SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; - - CTG_LOCK(CTG_WRITE, &slot->lock); - if (NULL == slot->meta) { - slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size); - if (NULL == slot->meta) { - qError("taosArrayInit %d failed, id:%" PRIx64 ", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, - mgmt->type); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - } - - if (NULL == taosArrayPush(slot->meta, meta)) { - qError("taosArrayPush meta to rent failed, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - slot->needSort = true; - - qDebug("add meta to rent, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); - -_return: - - CTG_UNLOCK(CTG_WRITE, &slot->lock); - CTG_RET(code); -} - -int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, - __compar_fn_t searchCompare) { - int16_t widx = abs((int)(id % mgmt->slotNum)); - - SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; - - CTG_LOCK(CTG_WRITE, &slot->lock); - if (NULL == slot->meta) { - qError("empty meta slot, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - if (slot->needSort) { - qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type, - (int32_t)taosArrayGetSize(slot->meta)); - taosArraySort(slot->meta, sortCompare); - slot->needSort = false; - qDebug("meta slot sorted, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); - } - - void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ); - if (NULL == orig) { - qError("meta not found in slot, id:%" PRIx64 ", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, - (int32_t)taosArrayGetSize(slot->meta)); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - memcpy(orig, meta, size); - - qDebug("meta in rent updated, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); - -_return: - - CTG_UNLOCK(CTG_WRITE, &slot->lock); - - if (code) { - qWarn("meta in rent update failed, will try to add it, code:%x, id:%" PRIx64 ", slot idx:%d, type:%d", code, id, - widx, mgmt->type); - CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size)); - } - - CTG_RET(code); -} - -int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare) { - int16_t widx = abs((int)(id % mgmt->slotNum)); - - SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; - - CTG_LOCK(CTG_WRITE, &slot->lock); - if (NULL == slot->meta) { - qError("empty meta slot, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - if (slot->needSort) { - taosArraySort(slot->meta, sortCompare); - slot->needSort = false; - qDebug("meta slot sorted, slot idx:%d, type:%d", widx, mgmt->type); - } - - int32_t idx = taosArraySearchIdx(slot->meta, &id, searchCompare, TD_EQ); - if (idx < 0) { - qError("meta not found in slot, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - taosArrayRemove(slot->meta, idx); - - qDebug("meta in rent removed, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); - -_return: - - CTG_UNLOCK(CTG_WRITE, &slot->lock); - - CTG_RET(code); -} - -int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { - int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1); - if (ridx >= mgmt->slotNum) { - ridx %= mgmt->slotNum; - atomic_store_16(&mgmt->slotRIdx, ridx); - } - - SCtgRentSlot *slot = &mgmt->slots[ridx]; - int32_t code = 0; - - CTG_LOCK(CTG_READ, &slot->lock); - if (NULL == slot->meta) { - qDebug("empty meta in slot:%d, type:%d", ridx, mgmt->type); - *num = 0; - goto _return; - } - - size_t metaNum = taosArrayGetSize(slot->meta); - if (metaNum <= 0) { - qDebug("no meta in slot:%d, type:%d", ridx, mgmt->type); - *num = 0; - goto _return; - } - - size_t msize = metaNum * size; - *res = taosMemoryMalloc(msize); - if (NULL == *res) { - qError("malloc %d failed", (int32_t)msize); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - void *meta = taosArrayGet(slot->meta, 0); - - memcpy(*res, meta, msize); - - *num = (uint32_t)metaNum; - - qDebug("Got %d meta from rent, type:%d", (int32_t)metaNum, mgmt->type); - -_return: - - CTG_UNLOCK(CTG_READ, &slot->lock); - - CTG_RET(code); -} - -int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { - while (true) { - int64_t msec = taosGetTimestampMs(); - int64_t lsec = atomic_load_64(&mgmt->lastReadMsec); - if ((msec - lsec) < CTG_RENT_SLOT_SECOND * 1000) { - *res = NULL; - *num = 0; - qDebug("too short time period to get expired meta, type:%d", mgmt->type); - return TSDB_CODE_SUCCESS; - } - - if (lsec != atomic_val_compare_exchange_64(&mgmt->lastReadMsec, lsec, msec)) { - continue; - } - - break; - } - - CTG_ERR_RET(ctgMetaRentGetImpl(mgmt, res, num, size)); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { - int32_t code = 0; - - SCtgDBCache newDBCache = {0}; - newDBCache.dbId = dbId; - - newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == newDBCache.tbCache.metaCache) { - ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); - if (NULL == newDBCache.tbCache.stbCache) { - ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache)); - if (code) { - if (HASH_NODE_EXIST(code)) { - ctgDebug("db already in cache, dbFName:%s", dbFName); - goto _return; - } - - ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - CTG_CACHE_STAT_ADD(dbNum, 1); - - SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - - ctgDebug("db added to cache, dbFName:%s, dbId:%" PRIx64, dbFName, dbId); - - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); - - ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%" PRIx64, dbFName, vgVersion.vgVersion, dbId); - - return TSDB_CODE_SUCCESS; - -_return: - - ctgFreeDbCache(&newDBCache); - - CTG_RET(code); -} - -void ctgRemoveStbRent(SCatalog *pCtg, SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - void *pIter = taosHashIterate(cache->stbCache, NULL); - while (pIter) { - uint64_t *suid = NULL; - suid = taosHashGetKey(pIter, NULL); - - if (TSDB_CODE_SUCCESS == - ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { - ctgDebug("stb removed from rent, suid:%" PRIx64, *suid); - } - - pIter = taosHashIterate(cache->stbCache, pIter); - } - } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); -} - -int32_t ctgRemoveDB(SCatalog *pCtg, SCtgDBCache *dbCache, const char *dbFName) { - uint64_t dbId = dbCache->dbId; - - ctgInfo("start to remove db from cache, dbFName:%s, dbId:%" PRIx64, dbFName, dbCache->dbId); - - atomic_store_8(&dbCache->deleted, 1); - - ctgRemoveStbRent(pCtg, &dbCache->tbCache); - - ctgFreeDbCache(dbCache); - - CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - ctgDebug("db removed from rent, dbFName:%s, dbId:%" PRIx64, dbFName, dbCache->dbId); - - if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { - ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } - - CTG_CACHE_STAT_SUB(dbNum, 1); - - ctgInfo("db removed from cache, dbFName:%s, dbId:%" PRIx64, dbFName, dbId); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { - int32_t code = 0; - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, dbFName, &dbCache); - - if (dbCache) { - // TODO OPEN IT -#if 0 - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#else - if (0 == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbId && (dbCache->dbId == 0)) { - dbCache->dbId = dbId; - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#endif - CTG_ERR_RET(ctgRemoveDB(pCtg, dbCache, dbFName)); - } - - CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); - - ctgGetDBCache(pCtg, dbFName, &dbCache); - - *pCache = dbCache; - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgUpdateDBVgInfo(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SDBVgInfo **pDbInfo) { - int32_t code = 0; - SDBVgInfo *dbInfo = *pDbInfo; - - if (NULL == dbInfo->vgHash) { + if (*dbCache) { return TSDB_CODE_SUCCESS; } - if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", dbFName, dbInfo->vgHash, - dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; - - SCtgDBCache *dbCache = NULL; - CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); - if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, dbFName, dbId); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - SDBVgInfo *vgInfo = NULL; - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); - - if (dbCache->vgInfo) { - if (dbInfo->vgVersion < dbCache->vgInfo->vgVersion) { - ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, - dbCache->vgInfo->vgVersion); - ctgWReleaseVgInfo(dbCache); - - return TSDB_CODE_SUCCESS; - } - - if (dbInfo->vgVersion == dbCache->vgInfo->vgVersion && dbInfo->numOfTable == dbCache->vgInfo->numOfTable) { - ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, - dbInfo->numOfTable); - ctgWReleaseVgInfo(dbCache); - - return TSDB_CODE_SUCCESS; - } - - ctgFreeVgInfo(dbCache->vgInfo); - } - - dbCache->vgInfo = dbInfo; - - *pDbInfo = NULL; - - ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); - - ctgWReleaseVgInfo(dbCache); - - dbCache = NULL; - - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), - ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - CTG_RET(code); -} - -int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, - STableMeta *meta, int32_t metaSize) { - SCtgTbMetaCache *tbCache = &dbCache->tbCache; - - CTG_LOCK(CTG_READ, &tbCache->metaLock); - if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("db is dropping, dbId:%" PRIx64, dbCache->dbId); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } - - int8_t origType = 0; - uint64_t origSuid = 0; - bool isStb = meta->tableType == TSDB_SUPER_TABLE; - STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); - if (orig) { - origType = orig->tableType; - - if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && - orig->tversion >= meta->tversion) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - return TSDB_CODE_SUCCESS; - } - - if (origType == TSDB_SUPER_TABLE) { - if ((!isStb) || orig->suid != meta->suid) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { - ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%" PRIx64, dbFName, tbName, orig->suid); - } else { - CTG_CACHE_STAT_SUB(stblNum, 1); - } - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%" PRIx64, dbFName, tbName, orig->suid); - - ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); - } - - origSuid = orig->suid; - } - } - - if (isStb) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - } - - if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { - if (isStb) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - } - - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - if (NULL == orig) { - CTG_CACHE_STAT_ADD(tblNum, 1); - } - - ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64, dbFName, tbName, meta->tableType, - meta->suid); - ctgdShowTableMeta(pCtg, tbName, meta); - - if (!isStb) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - return TSDB_CODE_SUCCESS; - } - - STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); - if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut stable to stable cache failed, suid:%" PRIx64, meta->suid); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - CTG_CACHE_STAT_ADD(stblNum, 1); - - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - - ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64 ",ma:%p", dbFName, tbName, - meta->tableType, meta->suid, tbMeta); - - SSTableMetaVersion metaRent = { - .dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; - strcpy(metaRent.dbFName, dbFName); - strcpy(metaRent.stbName, tbName); - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { - *dst = taosMemoryMalloc(sizeof(SDBVgInfo)); - if (NULL == *dst) { - qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - memcpy(*dst, src, sizeof(SDBVgInfo)); - - size_t hashSize = taosHashGetSize(src->vgHash); - (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == (*dst)->vgHash) { - qError("taosHashInit %d failed", (int32_t)hashSize); - taosMemoryFreeClear(*dst); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - int32_t *vgId = NULL; - void *pIter = taosHashIterate(src->vgHash, NULL); - while (pIter) { - vgId = taosHashGetKey(pIter, NULL); - - if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) { - qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize); - taosHashCancelIterate(src->vgHash, pIter); - taosHashCleanup((*dst)->vgHash); - taosMemoryFreeClear(*dst); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - pIter = taosHashIterate(src->vgHash, pIter); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, SCtgDBCache **dbCache, - SDBVgInfo **pInfo) { - bool inCache = false; - int32_t code = 0; - - CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); - - if (inCache) { - return TSDB_CODE_SUCCESS; - } - - SUseDbOutput DbOut = {0}; + SUseDbOutput DbOut = {0}; SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - code = ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut); - if (code) { - if (CTG_DB_NOT_EXIST(code) && input.vgVersion > CTG_DEFAULT_INVALID_VERSION) { - ctgDebug("db no longer exist, dbFName:%s, dbId:%" PRIx64, input.db, input.dbId); - ctgPushRmDBMsgInQueue(pCtg, input.db, input.dbId); - } - - CTG_ERR_RET(code); - } + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, &DbOut, NULL)); CTG_ERR_JRET(ctgCloneVgInfo(DbOut.dbVgroup, pInfo)); - CTG_ERR_RET(ctgPushUpdateVgMsgInQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, false)); + CTG_ERR_RET(ctgPutUpdateVgToQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, false)); return TSDB_CODE_SUCCESS; @@ -1809,81 +80,53 @@ _return: taosMemoryFreeClear(*pInfo); *pInfo = DbOut.dbVgroup; - + CTG_RET(code); } -int32_t ctgRefreshDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName) { - bool inCache = false; - int32_t code = 0; - SCtgDBCache *dbCache = NULL; +int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) { + int32_t code = 0; + SCtgDBCache* dbCache = NULL; - CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); - SUseDbOutput DbOut = {0}; + SUseDbOutput DbOut = {0}; SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); - if (inCache) { + if (NULL != dbCache) { input.dbId = dbCache->dbId; ctgReleaseVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } - + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; input.numOfTable = 0; - code = ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut); + code = ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, &DbOut, NULL); if (code) { - if (CTG_DB_NOT_EXIST(code) && inCache) { + if (CTG_DB_NOT_EXIST(code) && (NULL != dbCache)) { ctgDebug("db no longer exist, dbFName:%s, dbId:%" PRIx64, input.db, input.dbId); - ctgPushRmDBMsgInQueue(pCtg, input.db, input.dbId); + ctgPutRmDBToQueue(pCtg, input.db, input.dbId); } CTG_ERR_RET(code); } - CTG_ERR_RET(ctgPushUpdateVgMsgInQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, true)); + CTG_ERR_RET(ctgPutUpdateVgToQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, true)); return TSDB_CODE_SUCCESS; } -int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) { - *pOutput = taosMemoryMalloc(sizeof(STableMetaOutput)); - if (NULL == *pOutput) { - qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - memcpy(*pOutput, output, sizeof(STableMetaOutput)); - - if (output->tbMeta) { - int32_t metaSize = CTG_META_SIZE(output->tbMeta); - (*pOutput)->tbMeta = taosMemoryMalloc(metaSize); - if (NULL == (*pOutput)->tbMeta) { - qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); - taosMemoryFreeClear(*pOutput); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgRefreshTblMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, int32_t flag, - STableMetaOutput **pOutput, bool syncReq) { - if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } +int32_t ctgRefreshTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMetaOutput **pOutput, bool syncReq) { SVgroupInfo vgroupInfo = {0}; - int32_t code = 0; + int32_t code = 0; - if (!CTG_FLAG_IS_SYS_DB(flag)) { - CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo)); + if (!CTG_FLAG_IS_SYS_DB(ctx->flag)) { + CTG_ERR_RET(catalogGetTableHashVgroup(CTG_PARAMS_LIST(), ctx->pName, &vgroupInfo)); } STableMetaOutput moutput = {0}; @@ -1893,75 +136,72 @@ int32_t ctgRefreshTblMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - if (CTG_FLAG_IS_SYS_DB(flag)) { - ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(pTableName)); + if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { + ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(ctx->pName)); - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, (char *)pTableName->dbname, - (char *)pTableName->tname, output)); - } else if (CTG_FLAG_IS_STB(flag)) { - ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), (char *)ctx->pName->dbname, (char *)ctx->pName->tname, output, NULL)); + } else if (CTG_FLAG_IS_STB(ctx->flag)) { + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName)); // if get from mnode failed, will not try vnode - CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTrans, pMgmtEps, pTableName, output)); + CTG_ERR_JRET(ctgGetTbMetaFromMnode(CTG_PARAMS_LIST(), ctx->pName, output, NULL)); if (CTG_IS_META_NULL(output->metaType)) { - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgroupInfo, output, NULL)); } } else { - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pTableName), flag); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); // if get from vnode failed or no table meta, will not try mnode - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgroupInfo, output, NULL)); if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { - ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pTableName)); + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(ctx->pName)); taosMemoryFreeClear(output->tbMeta); - - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output)); + + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), output->dbFName, output->tbName, output, NULL)); } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; - if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { - CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); + if (!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) { + CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); } if (0 == exist) { - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, &moutput)); + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), output->dbFName, output->tbName, &moutput, NULL)); if (CTG_IS_META_NULL(moutput.metaType)) { SET_META_TYPE_NULL(output->metaType); } - + taosMemoryFreeClear(output->tbMeta); output->tbMeta = moutput.tbMeta; moutput.tbMeta = NULL; } else { taosMemoryFreeClear(output->tbMeta); - - SET_META_TYPE_CTABLE(output->metaType); + + SET_META_TYPE_CTABLE(output->metaType); } } } if (CTG_IS_META_NULL(output->metaType)) { - ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pTableName)); - catalogRemoveTableMeta(pCtg, pTableName); + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName)); + ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } if (CTG_IS_META_TABLE(output->metaType)) { - ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName, - output->tbMeta->tableType); + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName, output->tbMeta->tableType); } else { - ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, - output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType)); + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType)); } if (pOutput) { CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput)); } - CTG_ERR_JRET(ctgPushUpdateTblMsgInQueue(pCtg, output, syncReq)); + CTG_ERR_JRET(ctgPutUpdateTbToQueue(pCtg, output, syncReq)); return TSDB_CODE_SUCCESS; @@ -1969,48 +209,44 @@ _return: taosMemoryFreeClear(output->tbMeta); taosMemoryFreeClear(output); - + CTG_RET(code); } -int32_t ctgGetTableMeta(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const SName *pTableName, - STableMeta **pTableMeta, int32_t flag) { - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); +int32_t ctgGetTbMetaFromCache(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { + if (CTG_IS_SYS_DBNAME(ctx->pName->dbname)) { + CTG_FLAG_SET_SYS_DB(ctx->flag); } - bool inCache = false; - int32_t code = 0; - uint64_t dbId = 0; - uint64_t suid = 0; - STableMetaOutput *output = NULL; + CTG_ERR_RET(ctgReadTbMetaFromCache(pCtg, ctx, pTableMeta)); - if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { - CTG_FLAG_SET_SYS_DB(flag); - } - - CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &inCache, flag, &dbId)); - - int32_t tbType = 0; - - if (inCache) { - if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && - ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_SYS_DB(flag)))) { - goto _return; + if (*pTableMeta) { + if (CTG_FLAG_MATCH_STB(ctx->flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) || (CTG_FLAG_IS_SYS_DB(ctx->flag)))) { + return TSDB_CODE_SUCCESS; } - tbType = (*pTableMeta)->tableType; - suid = (*pTableMeta)->suid; - taosMemoryFreeClear(*pTableMeta); } - if (CTG_FLAG_IS_UNKNOWN_STB(flag)) { - CTG_FLAG_SET_STB(flag, tbType); + if (CTG_FLAG_IS_UNKNOWN_STB(ctx->flag)) { + CTG_FLAG_SET_STB(ctx->flag, ctx->tbInfo.tbType); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { + int32_t code = 0; + STableMetaOutput *output = NULL; + + CTG_ERR_RET(ctgGetTbMetaFromCache(CTG_PARAMS_LIST(), ctx, pTableMeta)); + if (*pTableMeta) { + goto _return; } while (true) { - CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, flag, &output, false)); + CTG_ERR_JRET(ctgRefreshTbMeta(CTG_PARAMS_LIST(), ctx, &output, false)); if (CTG_IS_META_TABLE(output->metaType)) { *pTableMeta = output->tbMeta; @@ -2019,7 +255,7 @@ int32_t ctgGetTableMeta(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, cons if (CTG_IS_META_BOTH(output->metaType)) { memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); - + *pTableMeta = output->tbMeta; goto _return; } @@ -2032,15 +268,17 @@ int32_t ctgGetTableMeta(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, cons // HANDLE ONLY CHILD TABLE META - SName stbName = *pTableName; - strcpy(stbName.tname, output->tbName); - taosMemoryFreeClear(output->tbMeta); - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL)); - if (!inCache) { - ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); - + SName stbName = *ctx->pName; + strcpy(stbName.tname, output->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = ctx->flag; + stbCtx.pName = &stbName; + + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, pTableMeta)); + if (NULL == *pTableMeta) { + ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, ctx->pName->tname); continue; } @@ -2051,38 +289,38 @@ int32_t ctgGetTableMeta(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, cons _return: - if (CTG_TABLE_NOT_EXIST(code) && inCache) { + if (CTG_TABLE_NOT_EXIST(code) && ctx->tbInfo.inCache) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; - if (CTG_FLAG_IS_SYS_DB(flag)) { - strcpy(dbFName, pTableName->dbname); + if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { + strcpy(dbFName, ctx->pName->dbname); } else { - tNameGetFullDbName(pTableName, dbFName); + tNameGetFullDbName(ctx->pName, dbFName); } - if (TSDB_SUPER_TABLE == tbType) { - ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, suid, false); + if (TSDB_SUPER_TABLE == ctx->tbInfo.tbType) { + ctgPutRmStbToQueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, ctx->tbInfo.suid, false); } else { - ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, false); + ctgPutRmTbToQueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, false); } } taosMemoryFreeClear(output); if (*pTableMeta) { - ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); - ctgdShowTableMeta(pCtg, pTableName->tname, *pTableMeta); + ctgDebug("tbmeta returned, tbName:%s, tbType:%d", ctx->pName->tname, (*pTableMeta)->tableType); + ctgdShowTableMeta(pCtg, ctx->pName->tname, *pTableMeta); } CTG_RET(code); } -int32_t ctgChkAuth(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *user, const char *dbFName, - AUTH_TYPE type, bool *pass) { - bool inCache = false; + +int32_t ctgChkAuth(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { + bool inCache = false; int32_t code = 0; - + *pass = false; - + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); if (inCache) { @@ -2090,8 +328,8 @@ int32_t ctgChkAuth(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const cha } SGetUserAuthRsp authRsp = {0}; - CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pRpc, pMgmtEps, user, &authRsp)); - + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(CTG_PARAMS_LIST(), user, &authRsp, NULL)); + if (authRsp.superAuth) { *pass = true; goto _return; @@ -2102,330 +340,39 @@ int32_t ctgChkAuth(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const cha goto _return; } - if (authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + if (type == AUTH_TYPE_READ && authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName))) { *pass = true; - } - - if (authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + } else if (type == AUTH_TYPE_WRITE && authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName))) { *pass = true; } _return: - ctgPushUpdateUserMsgInQueue(pCtg, &authRsp, false); + ctgPutUpdateUserToQueue(pCtg, &authRsp, false); return TSDB_CODE_SUCCESS; } -int32_t ctgActUpdateVg(SCtgMetaAction *action) { - int32_t code = 0; - SCtgUpdateVgMsg *msg = action->data; - - CTG_ERR_JRET(ctgUpdateDBVgInfo(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); - -_return: - - ctgFreeVgInfo(msg->dbInfo); - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - -int32_t ctgActRemoveDB(SCtgMetaAction *action) { - int32_t code = 0; - SCtgRemoveDBMsg *msg = action->data; - SCatalog *pCtg = msg->pCtg; - - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); - if (NULL == dbCache) { - goto _return; - } - - if (dbCache->dbId != msg->dbId) { - ctgInfo("dbId already updated, dbFName:%s, dbId:%" PRIx64 ", targetId:%" PRIx64, msg->dbFName, dbCache->dbId, - msg->dbId); - goto _return; - } - - CTG_ERR_JRET(ctgRemoveDB(pCtg, dbCache, msg->dbFName)); - -_return: - - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - -int32_t ctgActUpdateTbl(SCtgMetaAction *action) { - int32_t code = 0; - SCtgUpdateTblMsg *msg = action->data; - SCatalog *pCtg = msg->pCtg; - STableMetaOutput *output = msg->output; - SCtgDBCache *dbCache = NULL; - - if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { - ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { - ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); - if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, output->dbFName, output->dbId); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - int32_t metaSize = CTG_META_SIZE(output->tbMeta); - - CTG_ERR_JRET( - ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); - } - - if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, - (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); - } - -_return: - - if (output) { - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); - } - - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - -int32_t ctgActRemoveStb(SCtgMetaAction *action) { - int32_t code = 0; - SCtgRemoveStbMsg *msg = action->data; - SCatalog *pCtg = msg->pCtg; - - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); - if (NULL == dbCache) { - return TSDB_CODE_SUCCESS; - } - - if (msg->dbId && (dbCache->dbId != msg->dbId)) { - ctgDebug("dbId already modified, dbFName:%s, current:%" PRIx64 ", dbId:%" PRIx64 ", stb:%s, suid:%" PRIx64, - msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { - ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, - msg->suid); - } else { - CTG_CACHE_STAT_SUB(stblNum, 1); - } - - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { - ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, msg->suid); - } else { - CTG_CACHE_STAT_SUB(tblNum, 1); - } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - - ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, msg->suid); - - CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); - - ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, msg->suid); - -_return: - - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - -int32_t ctgActRemoveTbl(SCtgMetaAction *action) { - int32_t code = 0; - SCtgRemoveTblMsg *msg = action->data; - SCatalog *pCtg = msg->pCtg; - - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); - if (NULL == dbCache) { - return TSDB_CODE_SUCCESS; - } - - if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId already modified, dbFName:%s, current:%" PRIx64 ", dbId:%" PRIx64 ", tbName:%s", msg->dbFName, - dbCache->dbId, msg->dbId, msg->tbName); - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } else { - CTG_CACHE_STAT_SUB(tblNum, 1); - } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); - -_return: - - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - -int32_t ctgActUpdateUser(SCtgMetaAction *action) { - int32_t code = 0; - SCtgUpdateUserMsg *msg = action->data; - SCatalog *pCtg = msg->pCtg; - - if (NULL == pCtg->userCache) { - pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), - false, HASH_ENTRY_LOCK); - if (NULL == pCtg->userCache) { - ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - } - - SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); - if (NULL == pUser) { - SCtgUserAuth userAuth = {0}; - - userAuth.version = msg->userAuth.version; - userAuth.superUser = msg->userAuth.superAuth; - userAuth.createdDbs = msg->userAuth.createdDbs; - userAuth.readDbs = msg->userAuth.readDbs; - userAuth.writeDbs = msg->userAuth.writeDbs; - - if (taosHashPut(pCtg->userCache, msg->userAuth.user, sizeof(msg->userAuth.user), &userAuth, sizeof(userAuth))) { - ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - - taosMemoryFreeClear(msg); - - return TSDB_CODE_SUCCESS; - } - - pUser->version = msg->userAuth.version; - - CTG_LOCK(CTG_WRITE, &pUser->lock); - - taosHashCleanup(pUser->createdDbs); - pUser->createdDbs = msg->userAuth.createdDbs; - msg->userAuth.createdDbs = NULL; - - taosHashCleanup(pUser->readDbs); - pUser->readDbs = msg->userAuth.readDbs; - msg->userAuth.readDbs = NULL; - - taosHashCleanup(pUser->writeDbs); - pUser->writeDbs = msg->userAuth.writeDbs; - msg->userAuth.writeDbs = NULL; - - CTG_UNLOCK(CTG_WRITE, &pUser->lock); - -_return: - - taosHashCleanup(msg->userAuth.createdDbs); - taosHashCleanup(msg->userAuth.readDbs); - taosHashCleanup(msg->userAuth.writeDbs); - - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - -void *ctgUpdateThreadFunc(void *param) { - setThreadName("catalog"); - - qInfo("catalog update thread started"); - - CTG_LOCK(CTG_READ, &gCtgMgmt.lock); - - while (true) { - if (tsem_wait(&gCtgMgmt.queue.reqSem)) { - qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); - } - - if (atomic_load_8((int8_t *)&gCtgMgmt.exit)) { - tsem_post(&gCtgMgmt.queue.rspSem); - break; - } - - SCtgMetaAction *action = NULL; - ctgPopAction(&action); - SCatalog *pCtg = ((SCtgUpdateMsgHeader *)action->data)->pCtg; - - ctgDebug("process [%s] action", gCtgAction[action->act].name); - - (*gCtgAction[action->act].func)(action); - - gCtgMgmt.queue.seqDone = action->seqId; - - if (action->syncReq) { - tsem_post(&gCtgMgmt.queue.rspSem); - } - - CTG_RUNTIME_STAT_ADD(qDoneNum, 1); - - ctgdShowClusterCache(pCtg); - } - - CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); - - qInfo("catalog update thread stopped"); - - return NULL; -} - -int32_t ctgStartUpdateThread() { - TdThreadAttr thAttr; - taosThreadAttrInit(&thAttr); - taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - - if (taosThreadCreate(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - CTG_ERR_RET(terrno); - } - - taosThreadAttrDestroy(&thAttr); - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const SName *pTableName, - SArray **pVgList) { - STableMeta *tbMeta = NULL; - int32_t code = 0; - SVgroupInfo vgroupInfo = {0}; - SCtgDBCache *dbCache = NULL; - SArray *vgList = NULL; - SDBVgInfo *vgInfo = NULL; +int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SName* pTableName, SArray** pVgList) { + STableMeta *tbMeta = NULL; + int32_t code = 0; + SVgroupInfo vgroupInfo = {0}; + SCtgDBCache* dbCache = NULL; + SArray *vgList = NULL; + SDBVgInfo *vgInfo = NULL; + SCtgTbMetaCtx ctx = {0}; + ctx.pName = pTableName; + ctx.flag = CTG_FLAG_UNKNOWN_STB; *pVgList = NULL; - - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, &tbMeta, CTG_FLAG_UNKNOWN_STB)); + + CTG_ERR_JRET(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, &tbMeta)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - SHashObj *vgHash = NULL; - CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, db, &dbCache, &vgInfo)); + SHashObj *vgHash = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, db, &dbCache, &vgInfo)); if (dbCache) { vgHash = dbCache->vgInfo->vgHash; @@ -2439,7 +386,7 @@ int32_t ctgGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps // USE HASH METHOD INSTEAD OF VGID IN TBMETA ctgError("invalid method to get none stb vgInfo, tbType:%d", tbMeta->tableType); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); - + #if 0 int32_t vgId = tbMeta->vgId; if (taosHashGetDup(vgHash, &vgId, sizeof(vgId), &vgroupInfo) != 0) { @@ -2460,7 +407,7 @@ int32_t ctgGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps *pVgList = vgList; vgList = NULL; -#endif +#endif } _return: @@ -2491,7 +438,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - atomic_store_8((int8_t *)&gCtgMgmt.exit, false); + atomic_store_8((int8_t*)&gCtgMgmt.exit, false); if (cfg) { memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); @@ -2518,8 +465,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } - gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), - false, HASH_ENTRY_LOCK); + gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); if (NULL == gCtgMgmt.pCluster) { qError("taosHashInit %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -2529,7 +475,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); } - + if (tsem_init(&gCtgMgmt.queue.rspSem, 0, 0)) { qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); @@ -2542,33 +488,38 @@ int32_t catalogInit(SCatalogCfg *cfg) { } gCtgMgmt.queue.tail = gCtgMgmt.queue.head; + gCtgMgmt.jobPool = taosOpenRef(200, ctgFreeJob); + if (gCtgMgmt.jobPool < 0) { + qError("taosOpenRef failed, error:%s", tstrerror(terrno)); + CTG_ERR_RET(terrno); + } + CTG_ERR_RET(ctgStartUpdateThread()); - qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, - gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; } -int32_t catalogGetHandle(uint64_t clusterId, SCatalog **catalogHandle) { +int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { if (NULL == catalogHandle) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } if (NULL == gCtgMgmt.pCluster) { - qError("catalog cluster cache are not ready, clusterId:%" PRIx64, clusterId); + qError("catalog cluster cache are not ready, clusterId:%"PRIx64, clusterId); CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } - int32_t code = 0; + int32_t code = 0; SCatalog *clusterCtg = NULL; while (true) { - SCatalog **ctg = (SCatalog **)taosHashGet(gCtgMgmt.pCluster, (char *)&clusterId, sizeof(clusterId)); + SCatalog **ctg = (SCatalog **)taosHashGet(gCtgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); if (ctg && (*ctg)) { *catalogHandle = *ctg; - qDebug("got catalog handle from cache, clusterId:%" PRIx64 ", CTG:%p", clusterId, *ctg); + qDebug("got catalog handle from cache, clusterId:%"PRIx64", CTG:%p", clusterId, *ctg); return TSDB_CODE_SUCCESS; } @@ -2583,25 +534,30 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog **catalogHandle) { CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB)); CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); - clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), - false, HASH_ENTRY_LOCK); + clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == clusterCtg->dbCache) { qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } + SHashObj *metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == metaCache) { + qError("taosHashInit failed, num:%d", gCtgMgmt.cfg.maxTblCacheNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + code = taosHashPut(gCtgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); if (code) { if (HASH_NODE_EXIST(code)) { ctgFreeHandle(clusterCtg); continue; } - - qError("taosHashPut CTG to cache failed, clusterId:%" PRIx64, clusterId); + + qError("taosHashPut CTG to cache failed, clusterId:%"PRIx64, clusterId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("add CTG to cache, clusterId:%" PRIx64 ", CTG:%p", clusterId, clusterCtg); + qDebug("add CTG to cache, clusterId:%"PRIx64", CTG:%p", clusterId, clusterCtg); break; } @@ -2609,36 +565,36 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog **catalogHandle) { *catalogHandle = clusterCtg; CTG_CACHE_STAT_ADD(clusterNum, 1); - + return TSDB_CODE_SUCCESS; _return: ctgFreeHandle(clusterCtg); - + CTG_RET(code); } -void catalogFreeHandle(SCatalog *pCtg) { +void catalogFreeHandle(SCatalog* pCtg) { if (NULL == pCtg) { return; } if (taosHashRemove(gCtgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) { - ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%" PRIx64, pCtg->clusterId); + ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%"PRIx64, pCtg->clusterId); return; } CTG_CACHE_STAT_SUB(clusterNum, 1); uint64_t clusterId = pCtg->clusterId; - + ctgFreeHandle(pCtg); - - ctgInfo("handle freed, culsterId:%" PRIx64, clusterId); + + ctgInfo("handle freed, culsterId:%"PRIx64, clusterId); } -int32_t catalogGetDBVgVersion(SCatalog *pCtg, const char *dbFName, int32_t *version, int64_t *dbId, int32_t *tableNum) { +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t *tableNum) { CTG_API_ENTER(); if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId) { @@ -2646,11 +602,10 @@ int32_t catalogGetDBVgVersion(SCatalog *pCtg, const char *dbFName, int32_t *vers } SCtgDBCache *dbCache = NULL; - bool inCache = false; - int32_t code = 0; + int32_t code = 0; - CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); - if (!inCache) { + CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + if (NULL == dbCache) { *version = CTG_DEFAULT_INVALID_VERSION; CTG_API_LEAVE(TSDB_CODE_SUCCESS); } @@ -2671,20 +626,19 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, - SArray **vgroupList) { +int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { + if (NULL == pCtg || NULL == dbFName || NULL == pTrans || NULL == pMgmtEps || NULL == vgroupList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - SCtgDBCache *dbCache = NULL; - int32_t code = 0; - SArray *vgList = NULL; - SHashObj *vgHash = NULL; - SDBVgInfo *vgInfo = NULL; - CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName, &dbCache, &vgInfo)); + SCtgDBCache* dbCache = NULL; + int32_t code = 0; + SArray *vgList = NULL; + SHashObj *vgHash = NULL; + SDBVgInfo *vgInfo = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName, &dbCache, &vgInfo)); if (dbCache) { vgHash = dbCache->vgInfo->vgHash; } else { @@ -2708,31 +662,33 @@ _return: taosMemoryFreeClear(vgInfo); } - CTG_API_LEAVE(code); + CTG_API_LEAVE(code); } -int32_t catalogUpdateDBVgInfo(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SDBVgInfo *dbInfo) { + +int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo* dbInfo) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName || NULL == dbInfo) { ctgFreeVgInfo(dbInfo); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } - code = ctgPushUpdateVgMsgInQueue(pCtg, dbFName, dbId, dbInfo, false); + code = ctgPutUpdateVgToQueue(pCtg, dbFName, dbId, dbInfo, false); _return: CTG_API_LEAVE(code); } -int32_t catalogRemoveDB(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { + +int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2741,22 +697,24 @@ int32_t catalogRemoveDB(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - CTG_ERR_JRET(ctgPushRmDBMsgInQueue(pCtg, dbFName, dbId)); + CTG_ERR_JRET(ctgPutRmDBToQueue(pCtg, dbFName, dbId)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); - + _return: CTG_API_LEAVE(code); } -int32_t catalogUpdateVgEpSet(SCatalog *pCtg, const char *dbFName, int32_t vgId, SEpSet *epSet) { return 0; } +int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet *epSet) { + return 0; +} -int32_t catalogRemoveTableMeta(SCatalog *pCtg, const SName *pTableName) { +int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == pTableName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2765,39 +723,19 @@ int32_t catalogRemoveTableMeta(SCatalog *pCtg, const SName *pTableName) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - STableMeta *tblMeta = NULL; - bool inCache = false; - uint64_t dbId = 0; - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &inCache, 0, &dbId)); + CTG_ERR_JRET(ctgRemoveTbMetaFromCache(pCtg, pTableName, true)); - if (!inCache) { - ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname); - goto _return; - } - - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); - - if (TSDB_SUPER_TABLE == tblMeta->tableType) { - CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, tblMeta->suid, true)); - } else { - CTG_ERR_JRET(ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, true)); - } - - ctgDebug("table meta %s.%s removed", dbFName, pTableName->tname); - _return: - - taosMemoryFreeClear(tblMeta); - + CTG_API_LEAVE(code); } -int32_t catalogRemoveStbMeta(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *stbName, uint64_t suid) { + +int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName || NULL == stbName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2806,35 +744,36 @@ int32_t catalogRemoveStbMeta(SCatalog *pCtg, const char *dbFName, uint64_t dbId, CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid, true)); + CTG_ERR_JRET(ctgPutRmStbToQueue(pCtg, dbFName, dbId, stbName, suid, true)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); - + _return: CTG_API_LEAVE(code); } -int32_t catalogGetIndexMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, - const char *pIndexName, SIndexMeta **pIndexMeta) { - return 0; -} - -int32_t catalogGetTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, - STableMeta **pTableMeta) { +int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_UNKNOWN_STB)); + SCtgTbMetaCtx ctx = {0}; + ctx.pName = (SName*)pTableName; + ctx.flag = CTG_FLAG_UNKNOWN_STB; + + CTG_API_LEAVE(ctgGetTbMeta(pCtg, pTrans, pMgmtEps, &ctx, pTableMeta)); } -int32_t catalogGetSTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, - STableMeta **pTableMeta) { +int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_STB)); + SCtgTbMetaCtx ctx = {0}; + ctx.pName = (SName*)pTableName; + ctx.flag = CTG_FLAG_STB; + + CTG_API_LEAVE(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, pTableMeta)); } -int32_t catalogUpdateSTableMeta(SCatalog *pCtg, STableMetaRsp *rspMsg) { +int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { CTG_API_ENTER(); if (NULL == pCtg || NULL == rspMsg) { @@ -2846,117 +785,38 @@ int32_t catalogUpdateSTableMeta(SCatalog *pCtg, STableMetaRsp *rspMsg) { ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); } - + int32_t code = 0; strcpy(output->dbFName, rspMsg->dbFName); strcpy(output->tbName, rspMsg->tbName); output->dbId = rspMsg->dbId; - + SET_META_TYPE_TABLE(output->metaType); - + CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output->tbMeta)); - CTG_ERR_JRET(ctgPushUpdateTblMsgInQueue(pCtg, output, false)); + CTG_ERR_JRET(ctgPutUpdateTbToQueue(pCtg, output, false)); CTG_API_LEAVE(code); - + _return: taosMemoryFreeClear(output->tbMeta); taosMemoryFreeClear(output); - + CTG_API_LEAVE(code); } -int32_t ctgGetTbSverFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tbType, uint64_t *suid, - char *stbName) { - *sver = -1; - - if (NULL == pCtg->dbCache) { - ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); - return TSDB_CODE_SUCCESS; - } - - SCtgDBCache *dbCache = NULL; - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); - - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %s not in cache", pTableName->tname); - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); - if (tbMeta) { - *tbType = tbMeta->tableType; - *suid = tbMeta->suid; - if (*tbType != TSDB_CHILD_TABLE) { - *sver = tbMeta->sversion; - } - } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == tbMeta) { - ctgReleaseDBCache(pCtg, dbCache); - return TSDB_CODE_SUCCESS; - } - - if (*tbType != TSDB_CHILD_TABLE) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname); - - return TSDB_CODE_SUCCESS; - } - - ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, *suid); - - CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - - STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, suid, sizeof(*suid)); - if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid); - return TSDB_CODE_SUCCESS; - } - - if ((*stbMeta)->suid != *suid) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, *suid, - (*stbMeta)->suid); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - size_t nameLen = 0; - char *name = taosHashGetKey(*stbMeta, &nameLen); - - strncpy(stbName, name, nameLen); - stbName[nameLen] = 0; - - *sver = (*stbMeta)->sversion; - - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname); - - return TSDB_CODE_SUCCESS; -} - -int32_t catalogChkTbMetaVersion(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, SArray *pTables) { +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTables) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - SName name; + SName name; int32_t sver = 0; int32_t tbNum = taosArrayGetSize(pTables); for (int32_t i = 0; i < tbNum; ++i) { @@ -2974,7 +834,7 @@ int32_t catalogChkTbMetaVersion(SCatalog *pCtg, void *pTrans, const SEpSet *pMgm int32_t tbType = 0; uint64_t suid = 0; char stbName[TSDB_TABLE_FNAME_LEN]; - ctgGetTbSverFromCache(pCtg, &name, &sver, &tbType, &suid, stbName); + ctgReadTbSverFromCache(pCtg, &name, &sver, &tbType, &suid, stbName); if (sver >= 0 && sver < pTb->sver) { switch (tbType) { case TSDB_CHILD_TABLE: { @@ -2997,7 +857,8 @@ int32_t catalogChkTbMetaVersion(SCatalog *pCtg, void *pTrans, const SEpSet *pMgm CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogRefreshDBVgInfo(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const char *dbFName) { + +int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == dbFName) { @@ -3007,31 +868,34 @@ int32_t catalogRefreshDBVgInfo(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmt CTG_API_LEAVE(ctgRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName)); } -int32_t catalogRefreshTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, - int32_t isSTable) { +int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, - CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable), NULL, true)); + SCtgTbMetaCtx ctx = {0}; + ctx.pName = (SName*)pTableName; + ctx.flag = CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable); + + CTG_API_LEAVE(ctgRefreshTbMeta(CTG_PARAMS_LIST(), &ctx, NULL, true)); } -int32_t catalogRefreshGetTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, - STableMeta **pTableMeta, int32_t isSTable) { +int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, - CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable))); + SCtgTbMetaCtx ctx = {0}; + ctx.pName = (SName*)pTableName; + ctx.flag = CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable); + + CTG_API_LEAVE(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, pTableMeta)); } -int32_t catalogGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const SName *pTableName, - SArray **pVgList) { +int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3040,43 +904,21 @@ int32_t catalogGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgm CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + CTG_API_LEAVE(ctgGetTbDistVgInfo(pCtg, pTrans, pMgmtEps, (SName*)pTableName, pVgList)); +} + + +int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { + CTG_API_ENTER(); + + if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { + ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + SCtgDBCache* dbCache = NULL; int32_t code = 0; - - while (true) { - code = ctgGetTableDistVgInfo(pCtg, pRpc, pMgmtEps, pTableName, pVgList); - if (code) { - if (TSDB_CODE_CTG_VG_META_MISMATCH == code) { - CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, - CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(CTG_FLAG_UNKNOWN_STB), NULL, true)); - - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); - CTG_ERR_JRET(ctgRefreshDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName)); - - continue; - } - } - - break; - } - -_return: - - CTG_API_LEAVE(code); -} - -int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, - SVgroupInfo *pVgroup) { - CTG_API_ENTER(); - - if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { - ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); - CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); - } - - SCtgDBCache *dbCache = NULL; - int32_t code = 0; - char db[TSDB_DB_FNAME_LEN] = {0}; + char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); SDBVgInfo *vgInfo = NULL; @@ -3099,8 +941,8 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SCatalogReq *pReq, - SMetaData *pRsp) { + +int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { @@ -3110,8 +952,8 @@ int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, int32_t code = 0; pRsp->pTableMeta = NULL; - if (pReq->pTableName) { - int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableName); + if (pReq->pTableMeta) { + int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableMeta); if (tbNum <= 0) { ctgError("empty table name list, tbNum:%d", tbNum); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); @@ -3122,12 +964,15 @@ int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, ctgError("taosArrayInit %d failed", tbNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - + for (int32_t i = 0; i < tbNum; ++i) { - SName *name = taosArrayGet(pReq->pTableName, i); + SName *name = taosArrayGet(pReq->pTableMeta, i); STableMeta *pTableMeta = NULL; - - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, name, &pTableMeta, CTG_FLAG_UNKNOWN_STB)); + SCtgTbMetaCtx ctx = {0}; + ctx.pName = name; + ctx.flag = CTG_FLAG_UNKNOWN_STB; + + CTG_ERR_JRET(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, &pTableMeta)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); @@ -3139,12 +984,12 @@ int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, if (pReq->qNodeRequired) { pRsp->pQnodeList = taosArrayInit(10, sizeof(SQueryNodeAddr)); - CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, pRsp->pQnodeList)); + CTG_ERR_JRET(ctgGetQnodeListFromMnode(CTG_PARAMS_LIST(), pRsp->pQnodeList, NULL)); } CTG_API_LEAVE(TSDB_CODE_SUCCESS); -_return: +_return: if (pRsp->pTableMeta) { int32_t aSize = taosArrayGetSize(pRsp->pTableMeta); @@ -3152,30 +997,58 @@ _return: STableMeta *pMeta = taosArrayGetP(pRsp->pTableMeta, i); taosMemoryFreeClear(pMeta); } - + taosArrayDestroy(pRsp->pTableMeta); pRsp->pTableMeta = NULL; } - + CTG_API_LEAVE(code); } -int32_t catalogGetQnodeList(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, SArray *pQnodeList) { +int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int64_t* jobId) { CTG_API_ENTER(); - int32_t code = 0; - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == fp || NULL == param) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, pQnodeList)); + int32_t code = 0; + SCtgJob *pJob = NULL; + CTG_ERR_JRET(ctgInitJob(CTG_PARAMS_LIST(), &pJob, reqId, pReq, fp, param)); + + CTG_ERR_JRET(ctgLaunchJob(pJob)); + + *jobId = pJob->refId; + +_return: + + if (pJob) { + taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); + + if (code) { + taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + } + } + + CTG_API_LEAVE(code); +} + +int32_t catalogGetQnodeList(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pQnodeList) { + CTG_API_ENTER(); + + int32_t code = 0; + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pQnodeList) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_ERR_JRET(ctgGetQnodeListFromMnode(CTG_PARAMS_LIST(), pQnodeList, NULL)); _return: CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetExpiredSTables(SCatalog *pCtg, SSTableMetaVersion **stables, uint32_t *num) { +int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, uint32_t *num) { CTG_API_ENTER(); if (NULL == pCtg || NULL == stables || NULL == num) { @@ -3185,9 +1058,9 @@ int32_t catalogGetExpiredSTables(SCatalog *pCtg, SSTableMetaVersion **stables, u CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); } -int32_t catalogGetExpiredDBs(SCatalog *pCtg, SDbVgVersion **dbs, uint32_t *num) { +int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == dbs || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3195,9 +1068,9 @@ int32_t catalogGetExpiredDBs(SCatalog *pCtg, SDbVgVersion **dbs, uint32_t *num) CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); } -int32_t catalogGetExpiredUsers(SCatalog *pCtg, SUserAuthVersion **users, uint32_t *num) { +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == users || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3211,102 +1084,96 @@ int32_t catalogGetExpiredUsers(SCatalog *pCtg, SUserAuthVersion **users, uint32_ } } - uint32_t i = 0; + uint32_t i = 0; SCtgUserAuth *pAuth = taosHashIterate(pCtg->userCache, NULL); while (pAuth != NULL) { - void *key = taosHashGetKey(pAuth, NULL); - strncpy((*users)[i].user, key, sizeof((*users)[i].user)); + size_t len = 0; + void *key = taosHashGetKey(pAuth, &len); + strncpy((*users)[i].user, key, len); + (*users)[i].user[len] = 0; (*users)[i].version = pAuth->version; + ++i; pAuth = taosHashIterate(pCtg->userCache, pAuth); } CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetDBCfg(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, SDbCfgInfo *pDbCfg) { - CTG_API_ENTER(); - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == dbFName || NULL == pDbCfg) { +int32_t catalogGetDBCfg(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == dbFName || NULL == pDbCfg) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgGetDBCfgFromMnode(pCtg, pRpc, pMgmtEps, dbFName, pDbCfg)); + CTG_API_LEAVE(ctgGetDBCfgFromMnode(CTG_PARAMS_LIST(), dbFName, pDbCfg, NULL)); } -int32_t catalogGetIndexInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *indexName, - SIndexInfo *pInfo) { +int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo) { CTG_API_ENTER(); - - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == indexName || NULL == pInfo) { + + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == indexName || NULL == pInfo) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgGetIndexInfoFromMnode(pCtg, pRpc, pMgmtEps, indexName, pInfo)); + CTG_API_LEAVE(ctgGetIndexInfoFromMnode(CTG_PARAMS_LIST(), indexName, pInfo, NULL)); } -int32_t catalogGetUdfInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *funcName, SFuncInfo **pInfo) { +int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo) { CTG_API_ENTER(); - - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) { + + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; - *pInfo = taosMemoryMalloc(sizeof(SFuncInfo)); - if (NULL == *pInfo) { - CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); - } - - CTG_ERR_JRET(ctgGetUdfInfoFromMnode(pCtg, pRpc, pMgmtEps, funcName, pInfo)); - + CTG_ERR_JRET(ctgGetUdfInfoFromMnode(CTG_PARAMS_LIST(), funcName, pInfo, NULL)); + _return: - - if (code) { - taosMemoryFreeClear(*pInfo); - } - + CTG_API_LEAVE(code); } -int32_t catalogChkAuth(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *user, const char *dbFName, - AUTH_TYPE type, bool *pass) { +int32_t catalogChkAuth(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { CTG_API_ENTER(); - - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { + + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; - CTG_ERR_JRET(ctgChkAuth(pCtg, pRpc, pMgmtEps, user, dbFName, type, pass)); - + CTG_ERR_JRET(ctgChkAuth(CTG_PARAMS_LIST(), user, dbFName, type, pass)); + _return: CTG_API_LEAVE(code); } -int32_t catalogUpdateUserAuthInfo(SCatalog *pCtg, SGetUserAuthRsp *pAuth) { +int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pAuth) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgPushUpdateUserMsgInQueue(pCtg, pAuth, false)); + CTG_API_LEAVE(ctgPutUpdateUserToQueue(pCtg, pAuth, false)); } + void catalogDestroy(void) { qInfo("start to destroy catalog"); - - if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t *)&gCtgMgmt.exit)) { + + if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t*)&gCtgMgmt.exit)) { return; } - atomic_store_8((int8_t *)&gCtgMgmt.exit, true); + atomic_store_8((int8_t*)&gCtgMgmt.exit, true); if (tsem_post(&gCtgMgmt.queue.reqSem)) { qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); } - + if (tsem_post(&gCtgMgmt.queue.rspSem)) { qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); } @@ -3314,21 +1181,21 @@ void catalogDestroy(void) { while (CTG_IS_LOCKED(&gCtgMgmt.lock)) { taosUsleep(1); } - + CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock); SCatalog *pCtg = NULL; - void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); while (pIter) { pCtg = *(SCatalog **)pIter; if (pCtg) { catalogFreeHandle(pCtg); } - + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); } - + taosHashCleanup(gCtgMgmt.pCluster); gCtgMgmt.pCluster = NULL; @@ -3337,3 +1204,5 @@ void catalogDestroy(void) { qInfo("catalog destroyed"); } + + diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c new file mode 100644 index 0000000000..4908dc5101 --- /dev/null +++ b/source/libs/catalog/src/ctgAsync.c @@ -0,0 +1,1015 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trpc.h" +#include "query.h" +#include "tname.h" +#include "catalogInt.h" +#include "systable.h" +#include "tref.h" + +int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_TB_META; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbMetaCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbMetaCtx* ctx = pTask->taskCtx; + ctx->pName = taosMemoryMalloc(sizeof(*name)); + if (NULL == ctx->pName) { + taosMemoryFree(pTask->taskCtx); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(ctx->pName, name, sizeof(*name)); + ctx->flag = CTG_FLAG_UNKNOWN_STB; + + qDebug("QID:%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, pTask->type, name->tname); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_DB_VGROUP; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgDbVgCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgDbVgCtx* ctx = pTask->taskCtx; + + memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + + qDebug("QID:%" PRIx64 " task %d type %d initialized, dbFName:%s", pJob->queryId, taskIdx, pTask->type, dbFName); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetDbCfgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_DB_CFG; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgDbCfgCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgDbCfgCtx* ctx = pTask->taskCtx; + + memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + + qDebug("QID:%" PRIx64 " task %d type %d initialized, dbFName:%s", pJob->queryId, taskIdx, pTask->type, dbFName); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_TB_HASH; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbHashCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbHashCtx* ctx = pTask->taskCtx; + ctx->pName = taosMemoryMalloc(sizeof(*name)); + if (NULL == ctx->pName) { + taosMemoryFree(pTask->taskCtx); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(ctx->pName, name, sizeof(*name)); + tNameGetFullDbName(ctx->pName, ctx->dbFName); + + qDebug("QID:%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, pTask->type, name->tname); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_QNODE; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + pTask->taskCtx = NULL; + + qDebug("QID:%" PRIx64 " task %d type %d initialized", pJob->queryId, taskIdx, pTask->type); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetIndexTask(SCtgJob *pJob, int32_t taskIdx, char *name) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_INDEX; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgIndexCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgIndexCtx* ctx = pTask->taskCtx; + + strcpy(ctx->indexFName, name); + + qDebug("QID:%" PRIx64 " task %d type %d initialized, indexFName:%s", pJob->queryId, taskIdx, pTask->type, name); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetUdfTask(SCtgJob *pJob, int32_t taskIdx, char *name) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_UDF; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgUdfCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgUdfCtx* ctx = pTask->taskCtx; + + strcpy(ctx->udfName, name); + + qDebug("QID:%" PRIx64 " task %d type %d initialized, udfName:%s", pJob->queryId, taskIdx, pTask->type, name); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, SUserAuthInfo *user) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_USER; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgUserCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgUserCtx* ctx = pTask->taskCtx; + + memcpy(&ctx->user, user, sizeof(*user)); + + qDebug("QID:%" PRIx64 " task %d type %d initialized, user:%s", pJob->queryId, taskIdx, pTask->type, user->user); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param) { + int32_t code = 0; + int32_t tbMetaNum = (int32_t)taosArrayGetSize(pReq->pTableMeta); + int32_t dbVgNum = (int32_t)taosArrayGetSize(pReq->pDbVgroup); + int32_t tbHashNum = (int32_t)taosArrayGetSize(pReq->pTableHash); + int32_t udfNum = (int32_t)taosArrayGetSize(pReq->pUdf); + int32_t qnodeNum = pReq->qNodeRequired ? 1 : 0; + int32_t dbCfgNum = (int32_t)taosArrayGetSize(pReq->pDbCfg); + int32_t indexNum = (int32_t)taosArrayGetSize(pReq->pIndex); + int32_t userNum = (int32_t)taosArrayGetSize(pReq->pUser); + + int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dbCfgNum + indexNum + userNum; + if (taskNum <= 0) { + ctgError("empty input for job, taskNum:%d", taskNum); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + *job = taosMemoryCalloc(1, sizeof(SCtgJob)); + if (NULL == *job) { + ctgError("calloc %d failed", (int32_t)sizeof(SCtgJob)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgJob *pJob = *job; + + pJob->queryId = reqId; + pJob->userFp = fp; + pJob->pCtg = pCtg; + pJob->pTrans = pTrans; + pJob->pMgmtEps = pMgmtEps; + pJob->userParam = param; + + pJob->tbMetaNum = tbMetaNum; + pJob->tbHashNum = tbHashNum; + pJob->qnodeNum = qnodeNum; + pJob->dbVgNum = dbVgNum; + pJob->udfNum = udfNum; + pJob->dbCfgNum = dbCfgNum; + pJob->indexNum = indexNum; + pJob->userNum = userNum; + + pJob->pTasks = taosArrayInit(taskNum, sizeof(SCtgTask)); + + if (NULL == pJob->pTasks) { + ctgError("taosArrayInit %d tasks failed", taskNum); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + int32_t taskIdx = 0; + for (int32_t i = 0; i < dbVgNum; ++i) { + char *dbFName = taosArrayGet(pReq->pDbVgroup, i); + CTG_ERR_JRET(ctgInitGetDbVgTask(pJob, taskIdx++, dbFName)); + } + + for (int32_t i = 0; i < dbCfgNum; ++i) { + char *dbFName = taosArrayGet(pReq->pDbCfg, i); + CTG_ERR_JRET(ctgInitGetDbCfgTask(pJob, taskIdx++, dbFName)); + } + + for (int32_t i = 0; i < tbMetaNum; ++i) { + SName *name = taosArrayGet(pReq->pTableMeta, i); + CTG_ERR_JRET(ctgInitGetTbMetaTask(pJob, taskIdx++, name)); + } + + for (int32_t i = 0; i < tbHashNum; ++i) { + SName *name = taosArrayGet(pReq->pTableHash, i); + CTG_ERR_JRET(ctgInitGetTbHashTask(pJob, taskIdx++, name)); + } + + for (int32_t i = 0; i < indexNum; ++i) { + char *indexName = taosArrayGet(pReq->pIndex, i); + CTG_ERR_JRET(ctgInitGetIndexTask(pJob, taskIdx++, indexName)); + } + + for (int32_t i = 0; i < udfNum; ++i) { + char *udfName = taosArrayGet(pReq->pUdf, i); + CTG_ERR_JRET(ctgInitGetUdfTask(pJob, taskIdx++, udfName)); + } + + for (int32_t i = 0; i < userNum; ++i) { + SUserAuthInfo *user = taosArrayGet(pReq->pUser, i); + CTG_ERR_JRET(ctgInitGetUserTask(pJob, taskIdx++, user)); + } + + if (qnodeNum) { + CTG_ERR_JRET(ctgInitGetQnodeTask(pJob, taskIdx++)); + } + + pJob->refId = taosAddRef(gCtgMgmt.jobPool, pJob); + if (pJob->refId < 0) { + ctgError("add job to ref failed, error: %s", tstrerror(terrno)); + CTG_ERR_JRET(terrno); + } + + taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); + + qDebug("QID:%" PRIx64 ", job %" PRIx64 " initialized, task num %d", pJob->queryId, pJob->refId, taskNum); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(*job); + + CTG_RET(code); +} + +int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pTableMeta) { + pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, sizeof(STableMeta)); + if (NULL == pJob->jobRes.pTableMeta) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pTableMeta, pTask->res); + + taosMemoryFreeClear(pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpDbVgRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pDbVgroup) { + pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, POINTER_BYTES); + if (NULL == pJob->jobRes.pDbVgroup) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pDbVgroup, &pTask->res); + pTask->res = NULL; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpTbHashRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pTableHash) { + pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SVgroupInfo)); + if (NULL == pJob->jobRes.pTableHash) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pTableHash, &pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpIndexRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pIndex) { + pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SIndexInfo)); + if (NULL == pJob->jobRes.pIndex) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pIndex, pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpQnodeRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + + TSWAP(pJob->jobRes.pQnodeList, pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpDbCfgRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pDbCfg) { + pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SDbCfgInfo)); + if (NULL == pJob->jobRes.pDbCfg) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pDbCfg, &pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpUdfRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pUdfList) { + pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SFuncInfo)); + if (NULL == pJob->jobRes.pUdfList) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pUdfList, pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpUserRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pUser) { + pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(bool)); + if (NULL == pJob->jobRes.pUser) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pUser, pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { + SCtgJob* pJob = pTask->pJob; + int32_t code = 0; + + qDebug("QID:%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); + + if (rspCode) { + int32_t lastCode = atomic_val_compare_exchange_32(&pJob->rspCode, 0, rspCode); + if (0 == lastCode) { + CTG_ERR_JRET(rspCode); + } + + return TSDB_CODE_SUCCESS; + } + + int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1); + if (taskDone < taosArrayGetSize(pJob->pTasks)) { + qDebug("task done: %d, total: %d", taskDone, (int32_t)taosArrayGetSize(pJob->pTasks)); + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_JRET(ctgMakeAsyncRes(pJob)); + +_return: + + qDebug("QID:%" PRIx64 " user callback with rsp %s", pJob->queryId, tstrerror(code)); + + (*pJob->userFp)(&pJob->jobRes, pJob->userParam, code); + + taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + + CTG_RET(code); +} + +int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + + switch (reqType) { + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out; + + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, ctx->pName, &vgInfo)); + + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + + CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgInfo, NULL, pTask)); + + return TSDB_CODE_SUCCESS; + } + case TDMT_MND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + + if (CTG_IS_META_NULL(pOut->metaType)) { + if (CTG_FLAG_IS_STB(ctx->flag)) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(ctx->pName, dbFName); + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + if (NULL != dbCache) { + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo)); + + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + + CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgInfo, NULL, pTask)); + + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } else { + SBuildUseDBInput input = {0}; + + tstrncpy(input.db, dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, NULL, pTask)); + } + + return TSDB_CODE_SUCCESS; + } + + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName)); + ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (pTask->msgCtx.lastOut) { + TSWAP(pTask->msgCtx.out, pTask->msgCtx.lastOut); + STableMetaOutput* pLastOut = (STableMetaOutput*)pTask->msgCtx.out; + TSWAP(pLastOut->tbMeta, pOut->tbMeta); + } + + break; + } + case TDMT_VND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + + if (CTG_IS_META_NULL(pOut->metaType)) { + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName)); + ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (CTG_FLAG_IS_STB(ctx->flag)) { + break; + } + + if (CTG_IS_META_TABLE(pOut->metaType) && TSDB_SUPER_TABLE == pOut->tbMeta->tableType) { + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(ctx->pName)); + + taosMemoryFreeClear(pOut->tbMeta); + + CTG_ERR_JRET(ctgGetTbMetaFromMnode(CTG_PARAMS_LIST(), ctx->pName, NULL, pTask)); + } else if (CTG_IS_META_BOTH(pOut->metaType)) { + int32_t exist = 0; + if (!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) { + CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pOut->dbFName, pOut->tbName, &exist)); + } + + if (0 == exist) { + TSWAP(pTask->msgCtx.lastOut, pTask->msgCtx.out); + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), pOut->dbFName, pOut->tbName, NULL, pTask)); + } else { + taosMemoryFreeClear(pOut->tbMeta); + + SET_META_TYPE_CTABLE(pOut->metaType); + } + } + break; + } + default: + ctgError("invalid reqType %d", reqType); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + + STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + + ctgUpdateTbMetaToCache(pCtg, pOut, false); + + if (CTG_IS_META_BOTH(pOut->metaType)) { + memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + } else if (CTG_IS_META_CTABLE(pOut->metaType)) { + SName stbName = *ctx->pName; + strcpy(stbName.tname, pOut->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = ctx->flag; + stbCtx.pName = &stbName; + + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); + if (NULL == pOut->tbMeta) { + ctgDebug("stb no longer exist, stbName:%s", stbName.tname); + CTG_ERR_JRET(ctgRelaunchGetTbMetaTask(pTask)); + + return TSDB_CODE_SUCCESS; + } + + memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + } + + TSWAP(pTask->res, pOut->tbMeta); + +_return: + + if (dbCache) { + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetDbVgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + SCtgDbVgCtx* ctx = (SCtgDbVgCtx*)pTask->taskCtx; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + + switch (reqType) { + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out; + + CTG_ERR_JRET(ctgGenerateVgList(pCtg, pOut->dbVgroup->vgHash, (SArray**)&pTask->res)); + + CTG_ERR_JRET(ctgPutUpdateVgToQueue(pCtg, ctx->dbFName, pOut->dbId, pOut->dbVgroup, false)); + pOut->dbVgroup = NULL; + + break; + } + default: + ctgError("invalid reqType %d", reqType); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetTbHashRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + SCtgTbHashCtx* ctx = (SCtgTbHashCtx*)pTask->taskCtx; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + + switch (reqType) { + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out; + + pTask->res = taosMemoryMalloc(sizeof(SVgroupInfo)); + if (NULL == pTask->res) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, ctx->pName, (SVgroupInfo*)pTask->res)); + + CTG_ERR_JRET(ctgPutUpdateVgToQueue(pCtg, ctx->dbFName, pOut->dbId, pOut->dbVgroup, false)); + pOut->dbVgroup = NULL; + + break; + } + default: + ctgError("invalid reqType %d", reqType); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetDbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetQnodeRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetUdfRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetUserRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + SCtgUserCtx* ctx = (SCtgUserCtx*)pTask->taskCtx; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + bool pass = false; + SGetUserAuthRsp* pOut = (SGetUserAuthRsp*)pTask->msgCtx.out; + + if (pOut->superAuth) { + pass = true; + goto _return; + } + + if (pOut->createdDbs && taosHashGet(pOut->createdDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) { + pass = true; + goto _return; + } + + if (ctx->user.type == AUTH_TYPE_READ && pOut->readDbs && taosHashGet(pOut->readDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) { + pass = true; + } else if (ctx->user.type == AUTH_TYPE_WRITE && pOut->writeDbs && taosHashGet(pOut->writeDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) { + pass = true; + } + +_return: + + if (TSDB_CODE_SUCCESS == code) { + pTask->res = taosMemoryCalloc(1, sizeof(bool)); + if (NULL == pTask->res) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + *(bool*)pTask->res = pass; + } + } + + ctgPutUpdateUserToQueue(pCtg, pOut, false); + pTask->msgCtx.out = NULL; + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + int32_t code = 0; + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + + if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { + ctgDebug("will refresh sys db tbmeta, tbName:%s", tNameGetTableName(ctx->pName)); + + CTG_RET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), (char *)ctx->pName->dbname, (char *)ctx->pName->tname, NULL, pTask)); + } + + if (CTG_FLAG_IS_STB(ctx->flag)) { + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName)); + + // if get from mnode failed, will not try vnode + CTG_RET(ctgGetTbMetaFromMnode(CTG_PARAMS_LIST(), ctx->pName, NULL, pTask)); + } + + SCtgDBCache *dbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(ctx->pName, dbFName); + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + if (NULL == dbCache) { + SVgroupInfo vgInfo = {0}; + CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo)); + + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + + CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgInfo, NULL, pTask)); + } else { + SBuildUseDBInput input = {0}; + + tstrncpy(input.db, dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, NULL, pTask)); + } + +_return: + + if (dbCache) { + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + CTG_RET(code); +} + +int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + + CTG_ERR_RET(ctgGetTbMetaFromCache(CTG_PARAMS_LIST(), (SCtgTbMetaCtx*)pTask->taskCtx, (STableMeta**)&pTask->res)); + if (pTask->res) { + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { + int32_t code = 0; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + SCtgDBCache *dbCache = NULL; + SCtgDbVgCtx* pCtx = (SCtgDbVgCtx*)pTask->taskCtx; + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); + if (NULL != dbCache) { + CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgInfo->vgHash, (SArray**)&pTask->res)); + + CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); + } else { + SBuildUseDBInput input = {0}; + + tstrncpy(input.db, pCtx->dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, NULL, pTask)); + } + +_return: + + if (dbCache) { + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + CTG_RET(code); +} + +int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { + int32_t code = 0; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + SCtgDBCache *dbCache = NULL; + SCtgTbHashCtx* pCtx = (SCtgTbHashCtx*)pTask->taskCtx; + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); + if (NULL != dbCache) { + pTask->res = taosMemoryMalloc(sizeof(SVgroupInfo)); + if (NULL == pTask->res) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); + + CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); + } else { + SBuildUseDBInput input = {0}; + + tstrncpy(input.db, pCtx->dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, NULL, pTask)); + } + +_return: + + if (dbCache) { + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + CTG_RET(code); +} + +int32_t ctgLaunchGetQnodeTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + + CTG_ERR_RET(ctgGetQnodeListFromMnode(CTG_PARAMS_LIST(), NULL, pTask)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgLaunchGetDbCfgTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + SCtgDbCfgCtx* pCtx = (SCtgDbCfgCtx*)pTask->taskCtx; + + CTG_ERR_RET(ctgGetDBCfgFromMnode(CTG_PARAMS_LIST(), pCtx->dbFName, NULL, pTask)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgLaunchGetIndexTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + SCtgIndexCtx* pCtx = (SCtgIndexCtx*)pTask->taskCtx; + + CTG_ERR_RET(ctgGetIndexInfoFromMnode(CTG_PARAMS_LIST(), pCtx->indexFName, NULL, pTask)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgLaunchGetUdfTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + SCtgUdfCtx* pCtx = (SCtgUdfCtx*)pTask->taskCtx; + + CTG_ERR_RET(ctgGetUdfInfoFromMnode(CTG_PARAMS_LIST(), pCtx->udfName, NULL, pTask)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgLaunchGetUserTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + SCtgUserCtx* pCtx = (SCtgUserCtx*)pTask->taskCtx; + bool inCache = false; + bool pass = false; + + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, pCtx->user.user, pCtx->user.dbFName, pCtx->user.type, &inCache, &pass)); + if (inCache) { + pTask->res = taosMemoryCalloc(1, sizeof(bool)); + if (NULL == pTask->res) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + *(bool*)pTask->res = pass; + + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(CTG_PARAMS_LIST(), pCtx->user.user, NULL, pTask)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask) { + ctgResetTbMetaTask(pTask); + + CTG_ERR_RET(ctgLaunchGetTbMetaTask(pTask)); + + return TSDB_CODE_SUCCESS; +} + +SCtgAsyncFps gCtgAsyncFps[] = { + {ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes}, + {ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes}, + {ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes}, + {ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes}, + {ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes}, + {ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes}, + {ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes}, + {ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes}, +}; + +int32_t ctgMakeAsyncRes(SCtgJob *pJob) { + int32_t code = 0; + int32_t taskNum = taosArrayGetSize(pJob->pTasks); + + for (int32_t i = 0; i < taskNum; ++i) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, i); + CTG_ERR_RET((*gCtgAsyncFps[pTask->type].dumpResFp)(pTask)); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgLaunchJob(SCtgJob *pJob) { + int32_t taskNum = taosArrayGetSize(pJob->pTasks); + + for (int32_t i = 0; i < taskNum; ++i) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, i); + + qDebug("QID:%" PRIx64 " start to launch task %d", pJob->queryId, pTask->taskId); + CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); + } + + return TSDB_CODE_SUCCESS; +} + + + diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c new file mode 100644 index 0000000000..6335a056b9 --- /dev/null +++ b/source/libs/catalog/src/ctgCache.c @@ -0,0 +1,1520 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trpc.h" +#include "query.h" +#include "tname.h" +#include "catalogInt.h" +#include "systable.h" + +SCtgAction gCtgAction[CTG_ACT_MAX] = { + { + CTG_ACT_UPDATE_VG, + "update vgInfo", + ctgActUpdateVg + }, + { + CTG_ACT_UPDATE_TBL, + "update tbMeta", + ctgActUpdateTb + }, + { + CTG_ACT_REMOVE_DB, + "remove DB", + ctgActRemoveDB + }, + { + CTG_ACT_REMOVE_STB, + "remove stbMeta", + ctgActRemoveStb + }, + { + CTG_ACT_REMOVE_TBL, + "remove tbMeta", + ctgActRemoveTb + }, + { + CTG_ACT_UPDATE_USER, + "update user", + ctgActUpdateUser + } +}; + + + + +int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { + CTG_LOCK(CTG_READ, &dbCache->vgLock); + + if (dbCache->deleted) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + + ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + + *inCache = false; + return TSDB_CODE_SUCCESS; + } + + + if (NULL == dbCache->vgInfo) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + + *inCache = false; + ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); + return TSDB_CODE_SUCCESS; + } + + *inCache = true; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + + if (dbCache->deleted) { + ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + return TSDB_CODE_SUCCESS; +} + +void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { + taosHashRelease(pCtg->dbCache, dbCache); +} + +void ctgReleaseVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); +} + +void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); +} + + +int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { + char *p = strchr(dbFName, '.'); + if (p && CTG_IS_SYS_DBNAME(p + 1)) { + dbFName = p + 1; + } + + SCtgDBCache *dbCache = NULL; + if (acquire) { + dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); + } else { + dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); + } + + if (NULL == dbCache) { + *pCache = NULL; + ctgDebug("db not in cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + if (dbCache->deleted) { + if (acquire) { + ctgReleaseDBCache(pCtg, dbCache); + } + + *pCache = NULL; + ctgDebug("db is removing from cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgAcquireDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); +} + +int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); +} + + +int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + SCtgDBCache *dbCache = NULL; + + if (NULL == pCtg->dbCache) { + ctgDebug("empty db cache, dbFName:%s", dbFName); + goto _return; + } + + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + bool inCache = false; + ctgAcquireVgInfo(pCtg, dbCache, &inCache); + if (!inCache) { + ctgDebug("vgInfo of db %s not in cache", dbFName); + goto _return; + } + + *pCache = dbCache; + + CTG_CACHE_STAT_ADD(vgHitNum, 1); + + ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); + + return TSDB_CODE_SUCCESS; + +_return: + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } + + *pCache = NULL; + + CTG_CACHE_STAT_ADD(vgMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { + if (NULL == pCtg->dbCache) { + *exist = 0; + ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); + return TSDB_CODE_SUCCESS; + } + + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + size_t sz = 0; + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + if (NULL == tbMeta) { + ctgReleaseDBCache(pCtg, dbCache); + + *exist = 0; + ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); + return TSDB_CODE_SUCCESS; + } + + *exist = 1; + + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + + *pTableMeta = NULL; + + if (NULL == pCtg->dbCache) { + ctgDebug("empty tbmeta cache, tbName:%s", ctx->pName->tname); + return TSDB_CODE_SUCCESS; + } + + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { + strcpy(dbFName, ctx->pName->dbname); + } else { + tNameGetFullDbName(ctx->pName, dbFName); + } + + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", ctx->pName->tname); + return TSDB_CODE_SUCCESS; + } + + int32_t sz = 0; + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + taosHashGetDup_m(dbCache->tbCache.metaCache, ctx->pName->tname, strlen(ctx->pName->tname), (void **)pTableMeta, &sz); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + if (NULL == *pTableMeta) { + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); + return TSDB_CODE_SUCCESS; + } + + STableMeta* tbMeta = *pTableMeta; + ctx->tbInfo.inCache = true; + ctx->tbInfo.dbId = dbCache->dbId; + ctx->tbInfo.suid = tbMeta->suid; + ctx->tbInfo.tbType = tbMeta->tableType; + + if (tbMeta->tableType != TSDB_CHILD_TABLE) { + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, ctx->pName->tname); + + CTG_CACHE_STAT_ADD(tblHitNum, 1); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); + + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); + if (NULL == stbMeta || NULL == *stbMeta) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); + goto _return; + } + + if ((*stbMeta)->suid != tbMeta->suid) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + int32_t metaSize = CTG_META_SIZE(*stbMeta); + *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); + if (NULL == *pTableMeta) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgError("realloc size[%d] failed", metaSize); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); + + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + + ctgReleaseDBCache(pCtg, dbCache); + + CTG_CACHE_STAT_ADD(tblHitNum, 1); + + ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseDBCache(pCtg, dbCache); + taosMemoryFreeClear(*pTableMeta); + + CTG_CACHE_STAT_ADD(tblMissNum, 1); + + CTG_RET(code); +} + +int32_t ctgReadTbSverFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tbType, uint64_t *suid, + char *stbName) { + *sver = -1; + + if (NULL == pCtg->dbCache) { + ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + SCtgDBCache *dbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); + + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); + if (tbMeta) { + *tbType = tbMeta->tableType; + *suid = tbMeta->suid; + if (*tbType != TSDB_CHILD_TABLE) { + *sver = tbMeta->sversion; + } + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + if (NULL == tbMeta) { + ctgReleaseDBCache(pCtg, dbCache); + return TSDB_CODE_SUCCESS; + } + + if (*tbType != TSDB_CHILD_TABLE) { + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname); + + return TSDB_CODE_SUCCESS; + } + + ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, *suid); + + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); + + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, suid, sizeof(*suid)); + if (NULL == stbMeta || NULL == *stbMeta) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid); + return TSDB_CODE_SUCCESS; + } + + if ((*stbMeta)->suid != *suid) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseDBCache(pCtg, dbCache); + ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, *suid, + (*stbMeta)->suid); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + size_t nameLen = 0; + char *name = taosHashGetKey(*stbMeta, &nameLen); + + strncpy(stbName, name, nameLen); + stbName[nameLen] = 0; + + *sver = (*stbMeta)->sversion; + + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { + if (NULL == pCtg->dbCache) { + ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); + return TSDB_CODE_SUCCESS; + } + + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, tableName, strlen(tableName)); + + if (NULL == pTableMeta) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); + ctgReleaseDBCache(pCtg, dbCache); + + return TSDB_CODE_SUCCESS; + } + + *tbType = atomic_load_8(&pTableMeta->tableType); + + taosHashRelease(dbCache->tbCache.metaCache, pTableMeta); + + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { + if (NULL == pCtg->userCache) { + ctgDebug("empty user auth cache, user:%s", user); + goto _return; + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); + if (NULL == pUser) { + ctgDebug("user not in cache, user:%s", user); + goto _return; + } + + *inCache = true; + + ctgDebug("Got user from cache, user:%s", user); + CTG_CACHE_STAT_ADD(userHitNum, 1); + + if (pUser->superUser) { + *pass = true; + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &pUser->lock); + if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) { + *pass = true; + CTG_UNLOCK(CTG_READ, &pUser->lock); + return TSDB_CODE_SUCCESS; + } + + if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + *pass = true; + } + + if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + *pass = true; + } + + CTG_UNLOCK(CTG_READ, &pUser->lock); + + return TSDB_CODE_SUCCESS; + +_return: + + *inCache = false; + CTG_CACHE_STAT_ADD(userMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + + +void ctgWaitAction(SCtgMetaAction *action) { + while (true) { + tsem_wait(&gCtgMgmt.queue.rspSem); + + if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + tsem_post(&gCtgMgmt.queue.rspSem); + break; + } + + if (gCtgMgmt.queue.seqDone >= action->seqId) { + break; + } + + tsem_post(&gCtgMgmt.queue.rspSem); + sched_yield(); + } +} + +void ctgPopAction(SCtgMetaAction **action) { + SCtgQNode *orig = gCtgMgmt.queue.head; + + SCtgQNode *node = gCtgMgmt.queue.head->next; + gCtgMgmt.queue.head = gCtgMgmt.queue.head->next; + + CTG_QUEUE_SUB(); + + taosMemoryFreeClear(orig); + + *action = &node->action; +} + + +int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { + SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); + if (NULL == node) { + qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); + CTG_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + action->seqId = atomic_add_fetch_64(&gCtgMgmt.queue.seqId, 1); + + node->action = *action; + + CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); + gCtgMgmt.queue.tail->next = node; + gCtgMgmt.queue.tail = node; + CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); + + CTG_QUEUE_ADD(); + CTG_RUNTIME_STAT_ADD(qNum, 1); + + tsem_post(&gCtgMgmt.queue.reqSem); + + ctgDebug("action [%s] added into queue", gCtgAction[action->act].name); + + if (action->syncReq) { + ctgWaitAction(action); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgPutRmDBToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; + SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + char *p = strchr(dbFName, '.'); + if (p && CTG_IS_SYS_DBNAME(p + 1)) { + dbFName = p + 1; + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->dbId = dbId; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(action.data); + CTG_RET(code); +} + + +int32_t ctgPutRmStbToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq}; + SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + strncpy(msg->stbName, stbName, sizeof(msg->stbName)); + msg->dbId = dbId; + msg->suid = suid; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(action.data); + CTG_RET(code); +} + + + +int32_t ctgPutRmTbToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq}; + SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + strncpy(msg->tbName, tbName, sizeof(msg->tbName)); + msg->dbId = dbId; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(action.data); + CTG_RET(code); +} + +int32_t ctgPutUpdateVgToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq}; + SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); + ctgFreeVgInfo(dbInfo); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + char *p = strchr(dbFName, '.'); + if (p && CTG_IS_SYS_DBNAME(p + 1)) { + dbFName = p + 1; + } + + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->pCtg = pCtg; + msg->dbId = dbId; + msg->dbInfo = dbInfo; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgFreeVgInfo(dbInfo); + taosMemoryFreeClear(action.data); + CTG_RET(code); +} + +int32_t ctgPutUpdateTbToQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq}; + SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + char *p = strchr(output->dbFName, '.'); + if (p && CTG_IS_SYS_DBNAME(p + 1)) { + memmove(output->dbFName, p + 1, strlen(p + 1)); + } + + msg->pCtg = pCtg; + msg->output = output; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgPutUpdateUserToQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; + SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->userAuth = *pAuth; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + tFreeSGetUserAuthRsp(pAuth); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { + mgmt->slotRIdx = 0; + mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; + mgmt->type = type; + + size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; + + mgmt->slots = taosMemoryCalloc(1, msgSize); + if (NULL == mgmt->slots) { + qError("calloc %d failed", (int32_t)msgSize); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { + int16_t widx = abs((int)(id % mgmt->slotNum)); + + SCtgRentSlot *slot = &mgmt->slots[widx]; + int32_t code = 0; + + CTG_LOCK(CTG_WRITE, &slot->lock); + if (NULL == slot->meta) { + slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size); + if (NULL == slot->meta) { + qError("taosArrayInit %d failed, id:%"PRIx64", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + } + + if (NULL == taosArrayPush(slot->meta, meta)) { + qError("taosArrayPush meta to rent failed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + slot->needSort = true; + + qDebug("add meta to rent, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + +_return: + + CTG_UNLOCK(CTG_WRITE, &slot->lock); + CTG_RET(code); +} + +int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, __compar_fn_t searchCompare) { + int16_t widx = abs((int)(id % mgmt->slotNum)); + + SCtgRentSlot *slot = &mgmt->slots[widx]; + int32_t code = 0; + + CTG_LOCK(CTG_WRITE, &slot->lock); + if (NULL == slot->meta) { + qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (slot->needSort) { + qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + taosArraySort(slot->meta, sortCompare); + slot->needSort = false; + qDebug("meta slot sorted, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + } + + void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ); + if (NULL == orig) { + qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + memcpy(orig, meta, size); + + qDebug("meta in rent updated, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + +_return: + + CTG_UNLOCK(CTG_WRITE, &slot->lock); + + if (code) { + qWarn("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); + CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size)); + } + + CTG_RET(code); +} + +int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare) { + int16_t widx = abs((int)(id % mgmt->slotNum)); + + SCtgRentSlot *slot = &mgmt->slots[widx]; + int32_t code = 0; + + CTG_LOCK(CTG_WRITE, &slot->lock); + if (NULL == slot->meta) { + qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (slot->needSort) { + taosArraySort(slot->meta, sortCompare); + slot->needSort = false; + qDebug("meta slot sorted, slot idx:%d, type:%d", widx, mgmt->type); + } + + int32_t idx = taosArraySearchIdx(slot->meta, &id, searchCompare, TD_EQ); + if (idx < 0) { + qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + taosArrayRemove(slot->meta, idx); + + qDebug("meta in rent removed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + +_return: + + CTG_UNLOCK(CTG_WRITE, &slot->lock); + + CTG_RET(code); +} + + +int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { + int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1); + if (ridx >= mgmt->slotNum) { + ridx %= mgmt->slotNum; + atomic_store_16(&mgmt->slotRIdx, ridx); + } + + SCtgRentSlot *slot = &mgmt->slots[ridx]; + int32_t code = 0; + + CTG_LOCK(CTG_READ, &slot->lock); + if (NULL == slot->meta) { + qDebug("empty meta in slot:%d, type:%d", ridx, mgmt->type); + *num = 0; + goto _return; + } + + size_t metaNum = taosArrayGetSize(slot->meta); + if (metaNum <= 0) { + qDebug("no meta in slot:%d, type:%d", ridx, mgmt->type); + *num = 0; + goto _return; + } + + size_t msize = metaNum * size; + *res = taosMemoryMalloc(msize); + if (NULL == *res) { + qError("malloc %d failed", (int32_t)msize); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + void *meta = taosArrayGet(slot->meta, 0); + + memcpy(*res, meta, msize); + + *num = (uint32_t)metaNum; + + qDebug("Got %d meta from rent, type:%d", (int32_t)metaNum, mgmt->type); + +_return: + + CTG_UNLOCK(CTG_READ, &slot->lock); + + CTG_RET(code); +} + +int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { + while (true) { + int64_t msec = taosGetTimestampMs(); + int64_t lsec = atomic_load_64(&mgmt->lastReadMsec); + if ((msec - lsec) < CTG_RENT_SLOT_SECOND * 1000) { + *res = NULL; + *num = 0; + qDebug("too short time period to get expired meta, type:%d", mgmt->type); + return TSDB_CODE_SUCCESS; + } + + if (lsec != atomic_val_compare_exchange_64(&mgmt->lastReadMsec, lsec, msec)) { + continue; + } + + break; + } + + CTG_ERR_RET(ctgMetaRentGetImpl(mgmt, res, num, size)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { + int32_t code = 0; + + SCtgDBCache newDBCache = {0}; + newDBCache.dbId = dbId; + + newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache.metaCache) { + ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache.stbCache) { + ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache)); + if (code) { + if (HASH_NODE_EXIST(code)) { + ctgDebug("db already in cache, dbFName:%s", dbFName); + goto _return; + } + + ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + CTG_CACHE_STAT_ADD(dbNum, 1); + + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + + ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); + + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgFreeDbCache(&newDBCache); + + CTG_RET(code); +} + + +void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { + CTG_LOCK(CTG_WRITE, &cache->stbLock); + if (cache->stbCache) { + void *pIter = taosHashIterate(cache->stbCache, NULL); + while (pIter) { + uint64_t *suid = NULL; + suid = taosHashGetKey(pIter, NULL); + + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { + ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); + } + + pIter = taosHashIterate(cache->stbCache, pIter); + } + } + CTG_UNLOCK(CTG_WRITE, &cache->stbLock); +} + + +int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { + uint64_t dbId = dbCache->dbId; + + ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + atomic_store_8(&dbCache->deleted, 1); + + ctgRemoveStbRent(pCtg, &dbCache->tbCache); + + ctgFreeDbCache(dbCache); + + CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + + ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { + ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + CTG_CACHE_STAT_SUB(dbNum, 1); + + ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, dbFName, &dbCache); + + if (dbCache) { + // TODO OPEN IT +#if 0 + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#else + if (0 == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } + + if (dbId && (dbCache->dbId == 0)) { + dbCache->dbId = dbId; + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#endif + CTG_ERR_RET(ctgRemoveDBFromCache(pCtg, dbCache, dbFName)); + } + + CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); + + ctgGetDBCache(pCtg, dbFName, &dbCache); + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgWriteDBVgInfoToCache(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { + int32_t code = 0; + SDBVgInfo* dbInfo = *pDbInfo; + + if (NULL == dbInfo->vgHash) { + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", + dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + bool newAdded = false; + SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; + + SCtgDBCache *dbCache = NULL; + CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SDBVgInfo *vgInfo = NULL; + CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); + + if (dbCache->vgInfo) { + if (dbInfo->vgVersion < dbCache->vgInfo->vgVersion) { + ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); + ctgWReleaseVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgVersion == dbCache->vgInfo->vgVersion && dbInfo->numOfTable == dbCache->vgInfo->numOfTable) { + ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); + ctgWReleaseVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + ctgFreeVgInfo(dbCache->vgInfo); + } + + dbCache->vgInfo = dbInfo; + + *pDbInfo = NULL; + + ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + + ctgWReleaseVgInfo(dbCache); + + dbCache = NULL; + + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + + CTG_RET(code); +} + + +int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { + SCtgTbMetaCache *tbCache = &dbCache->tbCache; + + CTG_LOCK(CTG_READ, &tbCache->metaLock); + if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + int8_t origType = 0; + uint64_t origSuid = 0; + bool isStb = meta->tableType == TSDB_SUPER_TABLE; + STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (orig) { + origType = orig->tableType; + + if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && orig->tversion >= meta->tversion) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + return TSDB_CODE_SUCCESS; + } + + if (origType == TSDB_SUPER_TABLE) { + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + } else { + CTG_CACHE_STAT_SUB(stblNum, 1); + } + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); + + origSuid = orig->suid; + } + } + + if (isStb) { + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + } + + if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { + if (isStb) { + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + } + + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + if (NULL == orig) { + CTG_CACHE_STAT_ADD(tblNum, 1); + } + + ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + ctgdShowTableMeta(pCtg, tbName, meta); + + if (!isStb) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + return TSDB_CODE_SUCCESS; + } + + STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + CTG_CACHE_STAT_ADD(stblNum, 1); + + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + + ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + + SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; + strcpy(metaRent.dbFName, dbFName); + strcpy(metaRent.stbName, tbName); + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq) { + STableMetaOutput* pOutput = NULL; + int32_t code = 0; + + CTG_ERR_RET(ctgCloneMetaOutput(pOut, &pOutput)); + CTG_ERR_JRET(ctgPutUpdateTbToQueue(pCtg, pOutput, syncReq)); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgFreeSTableMetaOutput(pOutput); + CTG_RET(code); +} + + +int32_t ctgActUpdateVg(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateVgMsg *msg = action->data; + + CTG_ERR_JRET(ctgWriteDBVgInfoToCache(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); + +_return: + + ctgFreeVgInfo(msg->dbInfo); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgActRemoveDB(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveDBMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + goto _return; + } + + if (dbCache->dbId != msg->dbId) { + ctgInfo("dbId already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); + goto _return; + } + + CTG_ERR_JRET(ctgRemoveDBFromCache(pCtg, dbCache, msg->dbFName)); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + +int32_t ctgActUpdateTb(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateTblMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + STableMetaOutput* output = msg->output; + SCtgDBCache *dbCache = NULL; + + if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { + ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { + ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { + int32_t metaSize = CTG_META_SIZE(output->tbMeta); + + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); + } + + if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); + } + +_return: + + if (output) { + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); + } + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + +int32_t ctgActRemoveStb(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveStbMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + if (msg->dbId && (dbCache->dbId != msg->dbId)) { + ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", stb:%s, suid:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { + ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + } else { + CTG_CACHE_STAT_SUB(stblNum, 1); + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { + ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + } else { + CTG_CACHE_STAT_SUB(tblNum, 1); + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + + ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + + CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); + + ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgActRemoveTb(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveTblMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId != msg->dbId) { + ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } else { + CTG_CACHE_STAT_SUB(tblNum, 1); + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgActUpdateUser(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateUserMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + if (NULL == pCtg->userCache) { + pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pCtg->userCache) { + ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); + if (NULL == pUser) { + SCtgUserAuth userAuth = {0}; + + userAuth.version = msg->userAuth.version; + userAuth.superUser = msg->userAuth.superAuth; + userAuth.createdDbs = msg->userAuth.createdDbs; + userAuth.readDbs = msg->userAuth.readDbs; + userAuth.writeDbs = msg->userAuth.writeDbs; + + if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { + ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + taosMemoryFreeClear(msg); + + return TSDB_CODE_SUCCESS; + } + + pUser->version = msg->userAuth.version; + + CTG_LOCK(CTG_WRITE, &pUser->lock); + + taosHashCleanup(pUser->createdDbs); + pUser->createdDbs = msg->userAuth.createdDbs; + msg->userAuth.createdDbs = NULL; + + taosHashCleanup(pUser->readDbs); + pUser->readDbs = msg->userAuth.readDbs; + msg->userAuth.readDbs = NULL; + + taosHashCleanup(pUser->writeDbs); + pUser->writeDbs = msg->userAuth.writeDbs; + msg->userAuth.writeDbs = NULL; + + CTG_UNLOCK(CTG_WRITE, &pUser->lock); + +_return: + + + taosHashCleanup(msg->userAuth.createdDbs); + taosHashCleanup(msg->userAuth.readDbs); + taosHashCleanup(msg->userAuth.writeDbs); + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + +void* ctgUpdateThreadFunc(void* param) { + setThreadName("catalog"); + + qInfo("catalog update thread started"); + + CTG_LOCK(CTG_READ, &gCtgMgmt.lock); + + while (true) { + if (tsem_wait(&gCtgMgmt.queue.reqSem)) { + qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + } + + if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + tsem_post(&gCtgMgmt.queue.rspSem); + break; + } + + SCtgMetaAction *action = NULL; + ctgPopAction(&action); + SCatalog *pCtg = ((SCtgUpdateMsgHeader *)action->data)->pCtg; + + ctgDebug("process [%s] action", gCtgAction[action->act].name); + + (*gCtgAction[action->act].func)(action); + + gCtgMgmt.queue.seqDone = action->seqId; + + if (action->syncReq) { + tsem_post(&gCtgMgmt.queue.rspSem); + } + + CTG_RUNTIME_STAT_ADD(qDoneNum, 1); + + ctgdShowClusterCache(pCtg); + } + + CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); + + qInfo("catalog update thread stopped"); + + return NULL; +} + + +int32_t ctgStartUpdateThread() { + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (taosThreadCreate(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + CTG_ERR_RET(terrno); + } + + taosThreadAttrDestroy(&thAttr); + return TSDB_CODE_SUCCESS; +} + + + diff --git a/source/libs/catalog/src/catalogDbg.c b/source/libs/catalog/src/ctgDbg.c similarity index 100% rename from source/libs/catalog/src/catalogDbg.c rename to source/libs/catalog/src/ctgDbg.c diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c new file mode 100644 index 0000000000..9e86b863f4 --- /dev/null +++ b/source/libs/catalog/src/ctgRemote.c @@ -0,0 +1,577 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trpc.h" +#include "query.h" +#include "tname.h" +#include "catalogInt.h" +#include "systable.h" +#include "ctgRemote.h" +#include "tref.h" + +int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target) { + int32_t code = 0; + + switch (reqType) { + case TDMT_MND_QNODE_LIST: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for qnode list, error:%s", tstrerror(rspCode)); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process qnode list rsp failed, error:%s", tstrerror(rspCode)); + CTG_ERR_RET(code); + } + + qDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(out)); + break; + } + case TDMT_MND_USE_DB: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for use db, error:%s, dbFName:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process use db rsp failed, error:%s, dbFName:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got db vgInfo from mnode, dbFName:%s", target); + break; + } + case TDMT_MND_GET_DB_CFG: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for get db cfg, error:%s, db:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process get db cfg rsp failed, error:%s, db:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got db cfg from mnode, dbFName:%s", target); + break; + } + case TDMT_MND_GET_INDEX: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for get index, error:%s, indexName:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process get index rsp failed, error:%s, indexName:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got index from mnode, indexName:%s", target); + break; + } + case TDMT_MND_RETRIEVE_FUNC: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for get udf, error:%s, funcName:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process get udf rsp failed, error:%s, funcName:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got udf from mnode, funcName:%s", target); + break; + } + case TDMT_MND_GET_USER_AUTH: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for get user auth, error:%s, user:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process get user auth rsp failed, error:%s, user:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got user auth from mnode, user:%s", target); + break; + } + case TDMT_MND_TABLE_META: { + if (TSDB_CODE_SUCCESS != rspCode) { + if (CTG_TABLE_NOT_EXIST(rspCode)) { + SET_META_TYPE_NULL(((STableMetaOutput*)out)->metaType); + qDebug("stablemeta not exist in mnode, tbFName:%s", target); + return TSDB_CODE_SUCCESS; + } + + qError("error rsp for stablemeta from mnode, error:%s, tbFName:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process mnode stablemeta rsp failed, error:%s, tbFName:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got table meta from mnode, tbFName:%s", target); + break; + } + case TDMT_VND_TABLE_META: { + if (TSDB_CODE_SUCCESS != rspCode) { + if (CTG_TABLE_NOT_EXIST(rspCode)) { + SET_META_TYPE_NULL(((STableMetaOutput*)out)->metaType); + qDebug("tablemeta not exist in vnode, tbFName:%s", target); + return TSDB_CODE_SUCCESS; + } + + qError("error rsp for table meta from vnode, code:%s, tbFName:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process vnode tablemeta rsp failed, code:%s, tbFName:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got table meta from vnode, tbFName:%s", target); + break; + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgHandleMsgCallback(void *param, const SDataBuf *pMsg, int32_t rspCode) { + SCtgTaskCallbackParam* cbParam = (SCtgTaskCallbackParam*)param; + int32_t code = 0; + + CTG_API_ENTER(); + + SCtgJob* pJob = taosAcquireRef(gCtgMgmt.jobPool, cbParam->refId); + if (NULL == pJob) { + qDebug("job refId %" PRIx64 " already dropped", cbParam->refId); + goto _return; + } + + SCtgTask *pTask = taosArrayGet(pJob->pTasks, cbParam->taskId); + + qDebug("QID:%" PRIx64 " task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); + + CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, cbParam->reqType, pMsg, rspCode)); + +_return: + + if (pJob) { + taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId); + } + + taosMemoryFree(param); + + CTG_API_LEAVE(code); +} + + +int32_t ctgMakeMsgSendInfo(SCtgTask* pTask, int32_t msgType, SMsgSendInfo **pMsgSendInfo) { + int32_t code = 0; + SMsgSendInfo *msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == msgSendInfo) { + qError("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); + CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SCtgTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SCtgTaskCallbackParam)); + if (NULL == param) { + qError("calloc %d failed", (int32_t)sizeof(SCtgTaskCallbackParam)); + CTG_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + param->reqType = msgType; + param->queryId = pTask->pJob->queryId; + param->refId = pTask->pJob->refId; + param->taskId = pTask->taskId; + + msgSendInfo->param = param; + msgSendInfo->fp = ctgHandleMsgCallback; + + *pMsgSendInfo = msgSendInfo; + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFree(param); + taosMemoryFree(msgSendInfo); + + CTG_RET(code); +} + +int32_t ctgAsyncSendMsg(CTG_PARAMS, SCtgTask* pTask, int32_t msgType, void *msg, uint32_t msgSize) { + int32_t code = 0; + SMsgSendInfo *pMsgSendInfo = NULL; + CTG_ERR_JRET(ctgMakeMsgSendInfo(pTask, msgType, &pMsgSendInfo)); + + pMsgSendInfo->msgInfo.pData = msg; + pMsgSendInfo->msgInfo.len = msgSize; + pMsgSendInfo->msgInfo.handle = NULL; + pMsgSendInfo->msgType = msgType; + + int64_t transporterId = 0; + code = asyncSendMsgToServer(pTrans, (SEpSet*)pMgmtEps, &transporterId, pMsgSendInfo); + if (code) { + ctgError("asyncSendMsgToSever failed, error: %s", tstrerror(code)); + CTG_ERR_JRET(code); + } + + ctgDebug("req msg sent, reqId:%" PRIx64 ", msg type:%d, %s", pTask->pJob->queryId, msgType, TMSG_INFO(msgType)); + return TSDB_CODE_SUCCESS; + +_return: + + if (pMsgSendInfo) { + taosMemoryFreeClear(pMsgSendInfo->param); + taosMemoryFreeClear(pMsgSendInfo); + } + + CTG_RET(code); +} + + + + +int32_t ctgGetQnodeListFromMnode(CTG_PARAMS, SArray *out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_QNODE_LIST; + + ctgDebug("try to get qnode list from mnode, mgmtEpInUse:%d", pMgmtEps->inUse); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](NULL, &msg, 0, &msgLen); + if (code) { + ctgError("Build qnode list msg failed, error:%s", tstrerror(code)); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, NULL)); + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetDBVgInfoFromMnode(CTG_PARAMS, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_USE_DB; + + ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](input, &msg, 0, &msgLen); + if (code) { + ctgError("Build use db msg failed, code:%x, db:%s", code, input->db); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(SUseDbOutput)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, input->db)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, input->db)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetDBCfgFromMnode(CTG_PARAMS, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_GET_DB_CFG; + + ctgDebug("try to get db cfg from mnode, dbFName:%s", dbFName); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void *)dbFName, &msg, 0, &msgLen); + if (code) { + ctgError("Build get db cfg msg failed, code:%x, db:%s", code, dbFName); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(SDbCfgInfo)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)dbFName)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_GET_DB_CFG, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)dbFName)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetIndexInfoFromMnode(CTG_PARAMS, const char *indexName, SIndexInfo *out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_GET_INDEX; + + ctgDebug("try to get index from mnode, indexName:%s", indexName); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void *)indexName, &msg, 0, &msgLen); + if (code) { + ctgError("Build get index msg failed, code:%x, db:%s", code, indexName); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(SIndexInfo)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)indexName)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)indexName)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetUdfInfoFromMnode(CTG_PARAMS, const char *funcName, SFuncInfo *out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_RETRIEVE_FUNC; + + ctgDebug("try to get udf info from mnode, funcName:%s", funcName); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void *)funcName, &msg, 0, &msgLen); + if (code) { + ctgError("Build get udf msg failed, code:%x, db:%s", code, funcName); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(SFuncInfo)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)funcName)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)funcName)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetUserDbAuthFromMnode(CTG_PARAMS, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_GET_USER_AUTH; + + ctgDebug("try to get user auth from mnode, user:%s", user); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void *)user, &msg, 0, &msgLen); + if (code) { + ctgError("Build get user auth msg failed, code:%x, db:%s", code, user); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(SGetUserAuthRsp)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)user)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)user)); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetTbMetaFromMnodeImpl(CTG_PARAMS, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask) { + SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_TABLE_META; + char tbFName[TSDB_TABLE_FNAME_LEN]; + sprintf(tbFName, "%s.%s", dbFName, tbName); + + ctgDebug("try to get table meta from mnode, tbFName:%s", tbFName); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](&bInput, &msg, 0, &msgLen); + if (code) { + ctgError("Build mnode stablemeta msg failed, code:%x", code); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(STableMetaOutput)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetTbMetaFromMnode(CTG_PARAMS, const SName* pTableName, STableMetaOutput* out, SCtgTask* pTask) { + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); + + return ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), dbFName, (char *)pTableName->tname, out, pTask); +} + +int32_t ctgGetTbMetaFromVnode(CTG_PARAMS, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTask* pTask) { + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); + int32_t reqType = TDMT_VND_TABLE_META; + char tbFName[TSDB_TABLE_FNAME_LEN]; + sprintf(tbFName, "%s.%s", dbFName, pTableName->tname); + + ctgDebug("try to get table meta from vnode, vgId:%d, tbFName:%s", vgroupInfo->vgId, tbFName); + + SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; + char *msg = NULL; + int32_t msgLen = 0; + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](&bInput, &msg, 0, &msgLen); + if (code) { + ctgError("Build vnode tablemeta msg failed, code:%x, tbFName:%s", code, tbFName); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(STableMetaOutput)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName)); + + return TSDB_CODE_SUCCESS; +} + + diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c new file mode 100644 index 0000000000..2d7fb8aa97 --- /dev/null +++ b/source/libs/catalog/src/ctgUtil.c @@ -0,0 +1,577 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trpc.h" +#include "query.h" +#include "tname.h" +#include "catalogInt.h" +#include "systable.h" + +void ctgFreeSMetaData(SMetaData* pData) { + taosArrayDestroy(pData->pTableMeta); + pData->pTableMeta = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pData->pDbVgroup); ++i) { + SArray** pArray = taosArrayGet(pData->pDbVgroup, i); + taosArrayDestroy(*pArray); + } + taosArrayDestroy(pData->pDbVgroup); + pData->pDbVgroup = NULL; + + taosArrayDestroy(pData->pTableHash); + pData->pTableHash = NULL; + + taosArrayDestroy(pData->pUdfList); + pData->pUdfList = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pData->pDbCfg); ++i) { + SDbCfgInfo* pInfo = taosArrayGet(pData->pDbCfg, i); + taosArrayDestroy(pInfo->pRetensions); + } + taosArrayDestroy(pData->pDbCfg); + pData->pDbCfg = NULL; + + taosArrayDestroy(pData->pIndex); + pData->pIndex = NULL; + + taosArrayDestroy(pData->pUser); + pData->pUser = NULL; + + taosArrayDestroy(pData->pQnodeList); + pData->pQnodeList = NULL; +} + +void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) { + taosHashCleanup(userCache->createdDbs); + taosHashCleanup(userCache->readDbs); + taosHashCleanup(userCache->writeDbs); +} + +void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { + if (NULL == mgmt->slots) { + return; + } + + for (int32_t i = 0; i < mgmt->slotNum; ++i) { + SCtgRentSlot *slot = &mgmt->slots[i]; + if (slot->meta) { + taosArrayDestroy(slot->meta); + slot->meta = NULL; + } + } + + taosMemoryFreeClear(mgmt->slots); +} + + +void ctgFreeTbMetaCache(SCtgTbMetaCache *cache) { + CTG_LOCK(CTG_WRITE, &cache->stbLock); + if (cache->stbCache) { + int32_t stblNum = taosHashGetSize(cache->stbCache); + taosHashCleanup(cache->stbCache); + cache->stbCache = NULL; + CTG_CACHE_STAT_SUB(stblNum, stblNum); + } + CTG_UNLOCK(CTG_WRITE, &cache->stbLock); + + CTG_LOCK(CTG_WRITE, &cache->metaLock); + if (cache->metaCache) { + int32_t tblNum = taosHashGetSize(cache->metaCache); + taosHashCleanup(cache->metaCache); + cache->metaCache = NULL; + CTG_CACHE_STAT_SUB(tblNum, tblNum); + } + CTG_UNLOCK(CTG_WRITE, &cache->metaLock); +} + +void ctgFreeVgInfo(SDBVgInfo *vgInfo) { + if (NULL == vgInfo) { + return; + } + + if (vgInfo->vgHash) { + taosHashCleanup(vgInfo->vgHash); + vgInfo->vgHash = NULL; + } + + taosMemoryFreeClear(vgInfo); +} + +void ctgFreeDbCache(SCtgDBCache *dbCache) { + if (NULL == dbCache) { + return; + } + + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + ctgFreeVgInfo (dbCache->vgInfo); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + + ctgFreeTbMetaCache(&dbCache->tbCache); +} + + +void ctgFreeHandle(SCatalog* pCtg) { + ctgFreeMetaRent(&pCtg->dbRent); + ctgFreeMetaRent(&pCtg->stbRent); + + if (pCtg->dbCache) { + int32_t dbNum = taosHashGetSize(pCtg->dbCache); + + void *pIter = taosHashIterate(pCtg->dbCache, NULL); + while (pIter) { + SCtgDBCache *dbCache = pIter; + + atomic_store_8(&dbCache->deleted, 1); + + ctgFreeDbCache(dbCache); + + pIter = taosHashIterate(pCtg->dbCache, pIter); + } + + taosHashCleanup(pCtg->dbCache); + + CTG_CACHE_STAT_SUB(dbNum, dbNum); + } + + if (pCtg->userCache) { + int32_t userNum = taosHashGetSize(pCtg->userCache); + + void *pIter = taosHashIterate(pCtg->userCache, NULL); + while (pIter) { + SCtgUserAuth *userCache = pIter; + + ctgFreeSCtgUserAuth(userCache); + + pIter = taosHashIterate(pCtg->userCache, pIter); + } + + taosHashCleanup(pCtg->userCache); + + CTG_CACHE_STAT_SUB(userNum, userNum); + } + + taosMemoryFree(pCtg); +} + + +void ctgFreeSUseDbOutput(SUseDbOutput* pOutput) { + if (NULL == pOutput || NULL == pOutput->dbVgroup) { + return; + } + + taosHashCleanup(pOutput->dbVgroup->vgHash); + taosMemoryFreeClear(pOutput->dbVgroup); + taosMemoryFree(pOutput); +} + +void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) { + taosMemoryFreeClear(pCtx->target); + if (NULL == pCtx->out) { + return; + } + + switch (pCtx->reqType) { + case TDMT_MND_GET_DB_CFG: { + SDbCfgInfo* pOut = (SDbCfgInfo*)pCtx->out; + taosArrayDestroy(pOut->pRetensions); + taosMemoryFreeClear(pCtx->out); + break; + } + case TDMT_MND_USE_DB:{ + SUseDbOutput* pOut = (SUseDbOutput*)pCtx->out; + ctgFreeSUseDbOutput(pOut); + pCtx->out = NULL; + break; + } + case TDMT_MND_GET_INDEX: { + SIndexInfo* pOut = (SIndexInfo*)pCtx->out; + taosMemoryFreeClear(pCtx->out); + break; + } + case TDMT_MND_QNODE_LIST: { + SArray* pOut = (SArray*)pCtx->out; + taosArrayDestroy(pOut); + pCtx->out = NULL; + break; + } + case TDMT_VND_TABLE_META: + case TDMT_MND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pCtx->out; + taosMemoryFree(pOut->tbMeta); + taosMemoryFreeClear(pCtx->out); + break; + } + case TDMT_MND_RETRIEVE_FUNC: { + SFuncInfo* pOut = (SFuncInfo*)pCtx->out; + taosMemoryFree(pOut->pCode); + taosMemoryFree(pOut->pComment); + taosMemoryFreeClear(pCtx->out); + break; + } + case TDMT_MND_GET_USER_AUTH: { + SGetUserAuthRsp* pOut = (SGetUserAuthRsp*)pCtx->out; + taosHashCleanup(pOut->createdDbs); + taosHashCleanup(pOut->readDbs); + taosHashCleanup(pOut->writeDbs); + taosMemoryFreeClear(pCtx->out); + break; + } + default: + qError("invalid reqType %d", pCtx->reqType); + break; + } +} + +void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput) { + if (NULL == pOutput) { + return; + } + + taosMemoryFree(pOutput->tbMeta); + taosMemoryFree(pOutput); +} + + +void ctgResetTbMetaTask(SCtgTask* pTask) { + SCtgTbMetaCtx* taskCtx = (SCtgTbMetaCtx*)pTask->taskCtx; + memset(&taskCtx->tbInfo, 0, sizeof(taskCtx->tbInfo)); + taskCtx->flag = CTG_FLAG_UNKNOWN_STB; + + if (pTask->msgCtx.lastOut) { + ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.lastOut); + pTask->msgCtx.lastOut = NULL; + } + if (pTask->msgCtx.out) { + ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.out); + pTask->msgCtx.out = NULL; + } + taosMemoryFreeClear(pTask->msgCtx.target); + taosMemoryFreeClear(pTask->res); +} + +void ctgFreeTask(SCtgTask* pTask) { + ctgFreeMsgCtx(&pTask->msgCtx); + + switch (pTask->type) { + case CTG_TASK_GET_QNODE: { + taosArrayDestroy((SArray*)pTask->res); + pTask->res = NULL; + break; + } + case CTG_TASK_GET_TB_META: { + SCtgTbMetaCtx* taskCtx = (SCtgTbMetaCtx*)pTask->taskCtx; + taosMemoryFreeClear(taskCtx->pName); + if (pTask->msgCtx.lastOut) { + ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.lastOut); + pTask->msgCtx.lastOut = NULL; + } + taosMemoryFreeClear(pTask->res); + break; + } + case CTG_TASK_GET_DB_VGROUP: { + taosArrayDestroy((SArray*)pTask->res); + pTask->res = NULL; + break; + } + case CTG_TASK_GET_DB_CFG: { + if (pTask->res) { + taosArrayDestroy(((SDbCfgInfo*)pTask->res)->pRetensions); + taosMemoryFreeClear(pTask->res); + } + break; + } + case CTG_TASK_GET_TB_HASH: { + SCtgTbHashCtx* taskCtx = (SCtgTbHashCtx*)pTask->taskCtx; + taosMemoryFreeClear(taskCtx->pName); + taosMemoryFreeClear(pTask->res); + break; + } + case CTG_TASK_GET_INDEX: { + taosMemoryFreeClear(pTask->taskCtx); + taosMemoryFreeClear(pTask->res); + break; + } + case CTG_TASK_GET_UDF: { + taosMemoryFreeClear(pTask->taskCtx); + taosMemoryFreeClear(pTask->res); + break; + } + case CTG_TASK_GET_USER: { + taosMemoryFreeClear(pTask->taskCtx); + taosMemoryFreeClear(pTask->res); + break; + } + default: + qError("invalid task type %d", pTask->type); + break; + } +} + +void ctgFreeTasks(SArray* pArray) { + if (NULL == pArray) { + return; + } + + int32_t num = taosArrayGetSize(pArray); + for (int32_t i = 0; i < num; ++i) { + SCtgTask* pTask = taosArrayGet(pArray, i); + ctgFreeTask(pTask); + } + + taosArrayDestroy(pArray); +} + +void ctgFreeJob(void* job) { + if (NULL == job) { + return; + } + + SCtgJob* pJob = (SCtgJob*)job; + + int64_t rid = pJob->refId; + uint64_t qid = pJob->queryId; + + ctgFreeTasks(pJob->pTasks); + + ctgFreeSMetaData(&pJob->jobRes); + + taosMemoryFree(job); + + qDebug("QID:%" PRIx64 ", job %" PRIx64 " freed", qid, rid); +} + +int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target) { + ctgFreeMsgCtx(pCtx); + + pCtx->reqType = reqType; + pCtx->out = out; + if (target) { + pCtx->target = strdup(target); + if (NULL == pCtx->target) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } else { + pCtx->target = NULL; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { + switch (hashMethod) { + default: + *fp = MurmurHash3_32; + break; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) { + SHashObj *vgroupHash = NULL; + SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; + int32_t code = 0; + int32_t vgNum = taosHashGetSize(vgHash); + + vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); + if (NULL == vgList) { + ctgError("taosArrayInit failed, num:%d", vgNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + void *pIter = taosHashIterate(vgHash, NULL); + while (pIter) { + vgInfo = pIter; + + if (NULL == taosArrayPush(vgList, vgInfo)) { + ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); + taosHashCancelIterate(vgHash, pIter); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + pIter = taosHashIterate(vgHash, pIter); + vgInfo = NULL; + } + + *pList = vgList; + + ctgDebug("Got vgList from cache, vgNum:%d", vgNum); + + return TSDB_CODE_SUCCESS; + +_return: + + if (vgList) { + taosArrayDestroy(vgList); + } + + CTG_RET(code); +} + + +int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { + int32_t code = 0; + + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + char db[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, db); + + if (vgNum <= 0) { + ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", db, vgNum); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } + + tableNameHashFp fp = NULL; + SVgroupInfo *vgInfo = NULL; + + CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pTableName, tbFullName); + + uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); + + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + vgInfo = pIter; + if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { + taosHashCancelIterate(dbInfo->vgHash, pIter); + break; + } + + pIter = taosHashIterate(dbInfo->vgHash, pIter); + vgInfo = NULL; + } + + if (NULL == vgInfo) { + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + *pVgroup = *vgInfo; + + CTG_RET(code); +} + +int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { + if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { + return -1; + } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { + return 1; + } else { + return 0; + } +} + +int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) { + if (*(int64_t *)key1 < ((SDbVgVersion*)key2)->dbId) { + return -1; + } else if (*(int64_t *)key1 > ((SDbVgVersion*)key2)->dbId) { + return 1; + } else { + return 0; + } +} + +int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { + if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { + return -1; + } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { + return 1; + } else { + return 0; + } +} + +int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2) { + if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) { + return -1; + } else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) { + return 1; + } else { + return 0; + } +} + + + + +int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { + *dst = taosMemoryMalloc(sizeof(SDBVgInfo)); + if (NULL == *dst) { + qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy(*dst, src, sizeof(SDBVgInfo)); + + size_t hashSize = taosHashGetSize(src->vgHash); + (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == (*dst)->vgHash) { + qError("taosHashInit %d failed", (int32_t)hashSize); + taosMemoryFreeClear(*dst); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + int32_t *vgId = NULL; + void *pIter = taosHashIterate(src->vgHash, NULL); + while (pIter) { + vgId = taosHashGetKey(pIter, NULL); + + if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) { + qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize); + taosHashCancelIterate(src->vgHash, pIter); + taosHashCleanup((*dst)->vgHash); + taosMemoryFreeClear(*dst); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + pIter = taosHashIterate(src->vgHash, pIter); + } + + + return TSDB_CODE_SUCCESS; +} + + + +int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) { + *pOutput = taosMemoryMalloc(sizeof(STableMetaOutput)); + if (NULL == *pOutput) { + qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy(*pOutput, output, sizeof(STableMetaOutput)); + + if (output->tbMeta) { + int32_t metaSize = CTG_META_SIZE(output->tbMeta); + (*pOutput)->tbMeta = taosMemoryMalloc(metaSize); + if (NULL == (*pOutput)->tbMeta) { + qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + taosMemoryFreeClear(*pOutput); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize); + } + + return TSDB_CODE_SUCCESS; +} + + + diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index cff0087d6c..6c7d1ac4ca 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -40,10 +40,8 @@ namespace { -extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, - bool *inCache, int32_t flag, uint64_t *dbId); extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); -extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); +extern "C" int32_t ctgActUpdateTb(SCtgMetaAction *action); extern "C" int32_t ctgdEnableDebug(char *option); extern "C" int32_t ctgdGetStatNum(char *option, void *res); @@ -52,7 +50,7 @@ void ctgTestSetRspCTableMeta(); void ctgTestSetRspSTableMeta(); void ctgTestSetRspMultiSTableMeta(); -extern "C" SCatalogMgmt gCtgMgmt; +//extern "C" SCatalogMgmt gCtgMgmt; enum { CTGT_RSP_VGINFO = 1, @@ -859,8 +857,12 @@ void *ctgTestGetCtableMetaThread(void *param) { strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); + SCtgTbMetaCtx ctx = {0}; + ctx.pName = &cn; + ctx.flag = CTG_FLAG_UNKNOWN_STB; + while (!ctgTestStop) { - code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &inCache, 0, NULL); + code = ctgReadTbMetaFromCache(pCtg, &ctx, &tbMeta); if (code || !inCache) { assert(0); } @@ -899,7 +901,7 @@ void *ctgTestSetCtableMetaThread(void *param) { msg->output = output; action.data = msg; - code = ctgActUpdateTbl(&action); + code = ctgActUpdateTb(&action); if (code) { assert(0); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b1b0e69084..d6ca5dc15c 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "filter.h" #include "function.h" #include "functionMgt.h" diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 588c3e90e7..9e073ec05b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1181,6 +1181,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); pInfo->invertible = allInvertible(pInfo->binfo.pCtx, numOfCols); + pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 3b1e66f2ad..80e777196e 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -66,22 +66,18 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) { } static int32_t getUdfInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) { - SFuncInfo* pInfo = NULL; - int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &pInfo); + SFuncInfo funcInfo = {0}; + int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &funcInfo); if (TSDB_CODE_SUCCESS != code) { return code; } - if (NULL == pInfo) { - snprintf(pParam->pErrBuf, pParam->errBufLen, "Invalid function name: %s", pFunc->functionName); - return TSDB_CODE_FUNC_INVALID_FUNTION; - } + pFunc->funcType = FUNCTION_TYPE_UDF; - pFunc->funcId = TSDB_FUNC_TYPE_AGGREGATE == pInfo->funcType ? FUNC_AGGREGATE_UDF_ID : FUNC_SCALAR_UDF_ID; - pFunc->node.resType.type = pInfo->outputType; - pFunc->node.resType.bytes = pInfo->outputLen; - pFunc->udfBufSize = pInfo->bufSize; - tFreeSFuncInfo(pInfo); - taosMemoryFree(pInfo); + pFunc->funcId = TSDB_FUNC_TYPE_AGGREGATE == funcInfo.funcType ? FUNC_AGGREGATE_UDF_ID : FUNC_SCALAR_UDF_ID; + pFunc->node.resType.type = funcInfo.outputType; + pFunc->node.resType.bytes = funcInfo.outputLen; + pFunc->udfBufSize = funcInfo.bufSize; + tFreeSFuncInfo(&funcInfo); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index d5dc967fed..441648e52b 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -72,12 +72,20 @@ static int32_t udfSpawnUdfd(SUdfdData* pData) { char path[PATH_MAX] = {0}; if (tsProcPath == NULL) { path[0] = '.'; + #ifdef WINDOWS + GetModuleFileName(NULL, path, PATH_MAX); + taosDirName(path); + #endif } else { strncpy(path, tsProcPath, strlen(tsProcPath)); taosDirName(path); } #ifdef WINDOWS - strcat(path, "udfd.exe"); + if (strlen(path)==0) { + strcat(path, "udfd.exe"); + } else { + strcat(path, "\\udfd.exe"); + } #else strcat(path, "/udfd"); #endif diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp index cc0dded570..784586dfb2 100644 --- a/source/libs/parser/test/parInitialATest.cpp +++ b/source/libs/parser/test/parInitialATest.cpp @@ -204,7 +204,7 @@ TEST_F(ParserInitialATest, alterTable) { } }; - auto setAlterTagFunc = [&](const char* pTbname, const char* pTagName, const uint8_t* pNewVal, uint32_t bytes) { + auto setAlterTagFunc = [&](const char* pTbname, const char* pTagName, uint8_t* pNewVal, uint32_t bytes) { memset(&expect, 0, sizeof(SVAlterTbReq)); expect.tbName = strdup(pTbname); expect.action = TSDB_ALTER_TABLE_UPDATE_TAG_VAL; @@ -215,7 +215,7 @@ TEST_F(ParserInitialATest, alterTable) { expect.pTagVal = pNewVal; }; - auto setAlterOptionsFunc = [&](const char* pTbname, int32_t ttl, const char* pComment = nullptr) { + auto setAlterOptionsFunc = [&](const char* pTbname, int32_t ttl, char* pComment = nullptr) { memset(&expect, 0, sizeof(SVAlterTbReq)); expect.tbName = strdup(pTbname); expect.action = TSDB_ALTER_TABLE_UPDATE_OPTIONS; @@ -240,7 +240,7 @@ TEST_F(ParserInitialATest, alterTable) { void* pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead)); SVAlterTbReq req = {0}; SDecoder coder = {0}; - tDecoderInit(&coder, (const uint8_t*)pBuf, pVgData->size); + tDecoderInit(&coder, (uint8_t*)pBuf, pVgData->size); ASSERT_EQ(tDecodeSVAlterTbReq(&coder, &req), TSDB_CODE_SUCCESS); ASSERT_EQ(std::string(req.tbName), std::string(expect.tbName)); @@ -274,7 +274,7 @@ TEST_F(ParserInitialATest, alterTable) { setAlterOptionsFunc("t1", 10, nullptr); run("ALTER TABLE t1 TTL 10"); - setAlterOptionsFunc("t1", -1, "test"); + setAlterOptionsFunc("t1", -1, (char*)"test"); run("ALTER TABLE t1 COMMENT 'test'"); setAlterColFunc("t1", TSDB_ALTER_TABLE_ADD_COLUMN, "cc1", TSDB_DATA_TYPE_BIGINT); @@ -290,7 +290,7 @@ TEST_F(ParserInitialATest, alterTable) { run("ALTER TABLE t1 RENAME COLUMN c1 cc1"); int32_t val = 10; - setAlterTagFunc("st1s1", "tag1", (const uint8_t*)&val, sizeof(val)); + setAlterTagFunc("st1s1", "tag1", (uint8_t*)&val, sizeof(val)); run("ALTER TABLE st1s1 SET TAG tag1=10"); // todo diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 6e184fec72..86bacc9220 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -14,6 +14,7 @@ */ #include "planTestUtil.h" +#include #include #include diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 5b358fb008..e7a680de3c 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -528,20 +528,18 @@ int32_t qwDropTask(QW_FPARAMS_DEF) { } int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { - qTaskInfo_t *taskHandle = &ctx->taskHandle; + qTaskInfo_t taskHandle = ctx->taskHandle; - if (TASK_TYPE_TEMP == ctx->taskType) { + if (TASK_TYPE_TEMP == ctx->taskType && taskHandle) { if (ctx->explain) { SExplainExecInfo *execInfo = NULL; int32_t resNum = 0; - QW_ERR_RET(qGetExplainExecInfo(ctx->taskHandle, &resNum, &execInfo)); + QW_ERR_RET(qGetExplainExecInfo(taskHandle, &resNum, &execInfo)); SRpcHandleInfo connInfo = ctx->ctrlConnInfo; connInfo.ahandle = NULL; QW_ERR_RET(qwBuildAndSendExplainRsp(&connInfo, execInfo, resNum)); } - - qwFreeTaskHandle(QW_FPARAMS(), taskHandle); } return TSDB_CODE_SUCCESS; @@ -554,16 +552,21 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { uint64_t useconds = 0; int32_t i = 0; int32_t execNum = 0; - qTaskInfo_t *taskHandle = &ctx->taskHandle; + qTaskInfo_t taskHandle = ctx->taskHandle; DataSinkHandle sinkHandle = ctx->sinkHandle; while (true) { QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); - code = qExecTask(*taskHandle, &pRes, &useconds); - if (code) { - QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); - QW_ERR_RET(code); + pRes = NULL; + + // if *taskHandle is NULL, it's killed right now + if (taskHandle) { + code = qExecTask(taskHandle, &pRes, &useconds); + if (code) { + QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_RET(code); + } } ++execNum; diff --git a/source/libs/stream/src/tstreamUpdate.c b/source/libs/stream/src/tstreamUpdate.c index 7921193fa2..d21dadfe55 100644 --- a/source/libs/stream/src/tstreamUpdate.c +++ b/source/libs/stream/src/tstreamUpdate.c @@ -19,18 +19,24 @@ #define DEFAULT_FALSE_POSITIVE 0.01 #define DEFAULT_BUCKET_SIZE 1024 #define ROWS_PER_MILLISECOND 1 -#define MAX_NUM_SCALABLE_BF 120 +#define MAX_NUM_SCALABLE_BF 100000 #define MIN_NUM_SCALABLE_BF 10 #define DEFAULT_PREADD_BUCKET 1 #define MAX_INTERVAL MILLISECOND_PER_MINUTE #define MIN_INTERVAL (MILLISECOND_PER_SECOND * 10) +#define DEFAULT_EXPECTED_ENTRIES 10000 + +static int64_t adjustExpEntries(int64_t entries) { + return TMIN(DEFAULT_EXPECTED_ENTRIES, entries); +} static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { if (pInfo->numSBFs < count) { count = pInfo->numSBFs; } for (uint64_t i = 0; i < count; ++i) { - SScalableBf *tsSBF = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND, DEFAULT_FALSE_POSITIVE); + int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); + SScalableBf *tsSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); taosArrayPush(pInfo->pTsSBFs, &tsSBF); } } @@ -38,9 +44,9 @@ static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { static void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) { if (count < pInfo->numSBFs - 1) { for (uint64_t i = 0; i < count; ++i) { - SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, i); + SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0); tScalableBfDestroy(pTsSBFs); - taosArrayRemove(pInfo->pTsSBFs, i); + taosArrayRemove(pInfo->pTsSBFs, 0); } } else { taosArrayClearP(pInfo->pTsSBFs, (FDelete)tScalableBfDestroy); @@ -66,7 +72,7 @@ static int64_t adjustInterval(int64_t interval, int32_t precision) { return val; } -static int64_t adjustWatermark(int64_t interval, int32_t watermark) { +static int64_t adjustWatermark(int64_t interval, int64_t watermark) { if (watermark <= 0 || watermark > MAX_NUM_SCALABLE_BF * interval) { watermark = MAX_NUM_SCALABLE_BF * interval; } else if (watermark < MIN_NUM_SCALABLE_BF * interval) { @@ -130,7 +136,8 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { } SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index); if (res == NULL) { - res = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND, DEFAULT_FALSE_POSITIVE); + int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); + res = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); taosArrayPush(pInfo->pTsSBFs, &res); } return res; diff --git a/source/libs/stream/test/tstreamUpdateTest.cpp b/source/libs/stream/test/tstreamUpdateTest.cpp index c1e4e2bec1..93e114db02 100644 --- a/source/libs/stream/test/tstreamUpdateTest.cpp +++ b/source/libs/stream/test/tstreamUpdateTest.cpp @@ -4,6 +4,7 @@ #include "ttime.h" using namespace std; +#define MAX_NUM_SCALABLE_BF 100000 TEST(TD_STREAM_UPDATE_TEST, update) { int64_t interval = 20 * 1000; @@ -91,11 +92,11 @@ TEST(TD_STREAM_UPDATE_TEST, update) { } SUpdateInfo *pSU4 = updateInfoInit(-1, TSDB_TIME_PRECISION_MILLI, -1); - GTEST_ASSERT_EQ(pSU4->watermark, 120 * pSU4->interval); + GTEST_ASSERT_EQ(pSU4->watermark, MAX_NUM_SCALABLE_BF * pSU4->interval); GTEST_ASSERT_EQ(pSU4->interval, MILLISECOND_PER_MINUTE); SUpdateInfo *pSU5 = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); - GTEST_ASSERT_EQ(pSU5->watermark, 120 * pSU4->interval); + GTEST_ASSERT_EQ(pSU5->watermark, MAX_NUM_SCALABLE_BF * pSU4->interval); GTEST_ASSERT_EQ(pSU5->interval, MILLISECOND_PER_MINUTE); diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index efefcbb3e7..04a989439a 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -411,7 +411,7 @@ SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen) { } uint32_t len; char* data = NULL; - if (tDecodeBinary(&decoder, (const uint8_t**)(&data), &len) < 0) { + if (tDecodeBinary(&decoder, (uint8_t**)(&data), &len) < 0) { return NULL; } assert(len = pMsg->dataLen); @@ -670,7 +670,7 @@ SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen) { } uint32_t len; char* data = NULL; - if (tDecodeBinary(&decoder, (const uint8_t**)(&data), &len) < 0) { + if (tDecodeBinary(&decoder, (uint8_t**)(&data), &len) < 0) { return NULL; } assert(len = pMsg->dataLen); diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 90b8e9dd8a..b6e131d4cc 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -18,14 +18,16 @@ if(USE_TD_MEMORY) add_definitions(-DUSE_TD_MEMORY) endif () if(BUILD_ADDR2LINE) - target_include_directories( - os - PUBLIC "${TD_SOURCE_DIR}/contrib/libdwarf/src/lib/libdwarf" - ) + if(NOT TD_WINDOWS) + target_include_directories( + os + PUBLIC "${TD_SOURCE_DIR}/contrib/libdwarf/src/lib/libdwarf" + ) + target_link_libraries( + os PUBLIC addr2line dl z + ) + endif() add_definitions(-DUSE_ADDR2LINE) - target_link_libraries( - os PUBLIC addr2line dl z - ) endif () if(CHECK_STR2INT_ERROR) add_definitions(-DTD_CHECK_STR_TO_INT_ERROR) diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 72654d0084..c4b7c9386e 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -91,7 +91,12 @@ void taosRemoveDir(const char *dirname) { bool taosDirExist(const char *dirname) { return taosCheckExistFile(dirname); } int32_t taosMkDir(const char *dirname) { + if (taosDirExist(dirname)) return 0; +#ifdef WINDOWS + int32_t code = _mkdir(dirname, 0755); +#else int32_t code = mkdir(dirname, 0755); +#endif if (code < 0 && errno == EEXIST) { return 0; } @@ -101,36 +106,48 @@ int32_t taosMkDir(const char *dirname) { int32_t taosMulMkDir(const char *dirname) { if (dirname == NULL) return -1; - char * temp = strdup(dirname); + char temp[1024]; +#ifdef WINDOWS + taosRealPath(dirname, temp, sizeof(temp)); +#else + strcpy(temp, dirname); +#endif char * pos = temp; int32_t code = 0; - if (strncmp(temp, "/", 1) == 0) { + if (taosDirExist(temp)) return code; + + if (strncmp(temp, TD_DIRSEP, 1) == 0) { pos += 1; - } else if (strncmp(temp, "./", 2) == 0) { + } else if (strncmp(temp, "." TD_DIRSEP, 2) == 0) { pos += 2; } for (; *pos != '\0'; pos++) { - if (*pos == '/') { + if (*pos == TD_DIRSEP[0]) { *pos = '\0'; + #ifdef WINDOWS + code = _mkdir(temp, 0755); + #else code = mkdir(temp, 0755); + #endif if (code < 0 && errno != EEXIST) { - free(temp); return code; } - *pos = '/'; + *pos = TD_DIRSEP[0]; } } - if (*(pos - 1) != '/') { + if (*(pos - 1) != TD_DIRSEP[0]) { + #ifdef WINDOWS + code = _mkdir(temp, 0755); + #else code = mkdir(temp, 0755); + #endif if (code < 0 && errno != EEXIST) { - free(temp); return code; } } - free(temp); // int32_t code = mkdir(dirname, 0755); if (code < 0 && errno == EEXIST) { @@ -233,7 +250,13 @@ char *taosDirName(char *name) { _splitpath(name, Drive1, Dir1, NULL, NULL); size_t dirNameLen = strlen(Drive1) + strlen(Dir1); if (dirNameLen > 0) { - name[dirNameLen] = 0; + if (name[dirNameLen - 1] == '/' || name[dirNameLen - 1] == '\\') { + name[dirNameLen - 1] = 0; + } else { + name[dirNameLen] = 0; + } + } else { + name[0] = 0; } return name; #else diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index aa64e65638..e08b668163 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -109,8 +109,11 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha int64_t taosCopyFile(const char *from, const char *to) { #ifdef WINDOWS - assert(0); - return -1; + if (CopyFile(from, to, 0)) { + return 1; + } else { + return -1; + } #else char buffer[4096]; int64_t size = 0; @@ -236,7 +239,7 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { void autoDelFileListAdd(const char *path) { return; } -TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { +TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { int fd = -1; FILE *fp = NULL; if (tdFileOptions & TD_FILE_STREAM) { @@ -343,7 +346,11 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { char *tbuf = (char *)buf; while (leftbytes > 0) { + #ifdef WINDOWS + readbytes = _read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); + #else readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); + #endif if (readbytes < 0) { if (errno == EINTR) { continue; @@ -379,10 +386,10 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #endif assert(pFile->fd >= 0); // Please check if you have closed the file. #ifdef WINDOWS - size_t pos = lseek(pFile->fd, 0, SEEK_CUR); - lseek(pFile->fd, offset, SEEK_SET); - int64_t ret = read(pFile->fd, buf, count); - lseek(pFile->fd, pos, SEEK_SET); + size_t pos = _lseek(pFile->fd, 0, SEEK_CUR); + _lseek(pFile->fd, offset, SEEK_SET); + int64_t ret = _read(pFile->fd, buf, count); + _lseek(pFile->fd, pos, SEEK_SET); #else int64_t ret = pread(pFile->fd, buf, count, offset); #endif @@ -428,7 +435,11 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); // Please check if you have closed the file. +#ifdef WINDOWS + int64_t ret = _lseek(pFile->fd, offset, whence); +#else int64_t ret = lseek(pFile->fd, offset, whence); +#endif #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); #endif @@ -567,12 +578,12 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in #ifdef WINDOWS - lseek(pFileIn->fd, (int32_t)(*offset), 0); + _lseek(pFileIn->fd, (int32_t)(*offset), 0); int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = {0}; for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - size_t rlen = read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_); + size_t rlen = _read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_); if (rlen <= 0) { return writeLen; } else if (rlen < _SEND_FILE_STEP_) { @@ -586,7 +597,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in int64_t remain = size - writeLen; if (remain > 0) { - size_t rlen = read(pFileIn->fd, (void *)buffer, (size_t)remain); + size_t rlen = _read(pFileIn->fd, (void *)buffer, (size_t)remain); if (rlen <= 0) { return writeLen; } else { diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index e3791af618..24bc9d0b4c 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -37,6 +37,49 @@ typedef struct TdMemoryInfo { #ifdef WINDOWS #define tstrdup(str) _strdup(str) + +int32_t taosBackTrace(void **buffer, int32_t size) { + int32_t frame = 0; + return frame; +} + +#ifdef USE_ADDR2LINE +#include +#pragma comment(lib, "dbghelp.lib") + +void taosPrintBackTrace() { + #define MAX_STACK_FRAMES 20 + + void *pStack[MAX_STACK_FRAMES]; + + HANDLE process = GetCurrentProcess(); + SymInitialize(process, NULL, TRUE); + WORD frames = CaptureStackBackTrace(1, MAX_STACK_FRAMES, pStack, NULL); + + char buf_tmp[1024]; + for (WORD i = 0; i < frames; ++i) { + DWORD64 address = (DWORD64)(pStack[i]); + + DWORD64 displacementSym = 0; + char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)]; + PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)buffer; + pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO); + pSymbol->MaxNameLen = MAX_SYM_NAME; + + DWORD displacementLine = 0; + IMAGEHLP_LINE64 line; + //SymSetOptions(SYMOPT_LOAD_LINES); + line.SizeOfStruct = sizeof(IMAGEHLP_LINE64); + + if (SymFromAddr(process, address, &displacementSym, pSymbol) && SymGetLineFromAddr64(process, address, &displacementLine, &line)) { + snprintf(buf_tmp,sizeof(buf_tmp),"BackTrace %08" PRId64 " %s:%d %s\n", taosGetSelfPthreadId(), line.FileName, line.LineNumber, pSymbol->Name); + } else { + snprintf(buf_tmp,sizeof(buf_tmp),"BackTrace error: %d\n",GetLastError()); + } + write(1,buf_tmp,strlen(buf_tmp)); + } +} +#endif #else #define tstrdup(str) strdup(str) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 7df4c26afd..d4cfe4fc39 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -68,9 +68,32 @@ int32_t tsem_wait(tsem_t* sem) { } int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { - int ret = 0; + struct timespec ts, rel; + FILETIME ft_before, ft_after; + int rc; - return ret; + rel.tv_sec = 0; + rel.tv_nsec = nanosecs; + + GetSystemTimeAsFileTime(&ft_before); + errno = 0; + rc = sem_timedwait(&sem, pthread_win32_getabstime_np(&ts, &rel)); + + /* This should have timed out */ + assert(errno == ETIMEDOUT); + assert(rc != 0); + GetSystemTimeAsFileTime(&ft_after); + // We specified a non-zero wait. Time must advance. + if (ft_before.dwLowDateTime == ft_after.dwLowDateTime && ft_before.dwHighDateTime == ft_after.dwHighDateTime) + { + printf("nanoseconds: %d, rc: %d, errno: %d. before filetime: %d, %d; after filetime: %d, %d\n", + nanosecs, rc, errno, + (int)ft_before.dwLowDateTime, (int)ft_before.dwHighDateTime, + (int)ft_after.dwLowDateTime, (int)ft_after.dwHighDateTime); + printf("time must advance during sem_timedwait."); + return 1; + } + return 0; } #elif defined(_TD_DARWIN_64) diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 105acb188a..572e2db6fd 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -718,7 +718,11 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { bzero((char *)&serverAdd, sizeof(serverAdd)); serverAdd.sin_family = AF_INET; +#ifdef WINDOWS + serverAdd.sin_addr.s_addr = INADDR_ANY; +#else serverAdd.sin_addr.s_addr = ip; +#endif serverAdd.sin_port = (uint16_t)htons(port); if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { @@ -882,6 +886,16 @@ void taosBlockSIGPIPE() { } uint32_t taosGetIpv4FromFqdn(const char *fqdn) { +#ifdef WINDOWS + // Initialize Winsock + WSADATA wsaData; + int iResult; + iResult = WSAStartup(MAKEWORD(2, 2), &wsaData); + if (iResult != 0) { + printf("WSAStartup failed: %d\n", iResult); + return 1; + } +#endif struct addrinfo hints = {0}; hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; @@ -899,12 +913,12 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) { } else { #ifdef EAI_SYSTEM if (ret == EAI_SYSTEM) { - // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno)); + printf("failed to get the ip address, fqdn:%s, errno:%d, since:%s", fqdn, errno, strerror(errno)); } else { - // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); + printf("failed to get the ip address, fqdn:%s, ret:%d, since:%s", fqdn, ret, gai_strerror(ret)); } #else - // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); + printf("failed to get the ip address, fqdn:%s, ret:%d, since:%s", fqdn, ret, gai_strerror(ret)); #endif return 0xFFFFFFFF; } diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index fd898984ed..185daf9e45 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -29,10 +29,10 @@ struct SEncoderNode { }; struct SDecoderNode { - SDecoderNode* pNext; - const uint8_t* data; - uint32_t size; - uint32_t pos; + SDecoderNode* pNext; + uint8_t* data; + uint32_t size; + uint32_t pos; }; void tEncoderInit(SEncoder* pEncoder, uint8_t* data, uint32_t size) { @@ -52,7 +52,7 @@ void tEncoderClear(SEncoder* pCoder) { memset(pCoder, 0, sizeof(*pCoder)); } -void tDecoderInit(SDecoder* pDecoder, const uint8_t* data, uint32_t size) { +void tDecoderInit(SDecoder* pDecoder, uint8_t* data, uint32_t size) { pDecoder->data = data; pDecoder->size = size; pDecoder->pos = 0; diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index 4b5be79a70..3adc31cc39 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -244,6 +244,7 @@ class TDTestCase: if user is None: user = self.root_user with taos_connect(user=user.name, passwd=user.passwd) as use: + time.sleep(2) use.query("use db") use.query("show tables") if check_priv == PRIVILEGES_ALL: @@ -398,6 +399,7 @@ class TDTestCase: tdLog.printNoPrefix("==========step 1.18: revoke all from all = nothing") self.revoke_user(user=self.users[2], priv=PRIVILEGES_ALL) + time.sleep(3) self.__user_check(user=self.users[2], check_priv=None) def __grant_err(self): diff --git a/tools/taos-tools b/tools/taos-tools index 0aad27d725..788929bdc4 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 0aad27d725f4ee6b18daf1db0c07d933aed16eea +Subproject commit 788929bdc475d264d8306ceff30f7df006fd18d8