diff --git a/cmake/cmake.options b/cmake/cmake.options index 3e655b1796..774c1531ec 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -9,61 +9,61 @@ option( ) IF(${TD_WINDOWS}) + IF(NOT TD_ASTRA) + MESSAGE("build pthread Win32") + option( + BUILD_PTHREAD + "If build pthread on Windows" + ON + ) - MESSAGE("build pthread Win32") - option( - BUILD_PTHREAD - "If build pthread on Windows" - ON - ) + MESSAGE("build gnu regex for Windows") + option( + BUILD_GNUREGEX + "If build gnu regex on Windows" + ON + ) - MESSAGE("build gnu regex for Windows") - option( - BUILD_GNUREGEX - "If build gnu regex on Windows" - ON - ) + MESSAGE("build iconv Win32") + option( + BUILD_WITH_ICONV + "If build iconv on Windows" + ON + ) - MESSAGE("build iconv Win32") - option( - BUILD_WITH_ICONV - "If build iconv on Windows" - ON - ) + MESSAGE("build msvcregex Win32") + option( + BUILD_MSVCREGEX + "If build msvcregex on Windows" + ON + ) - MESSAGE("build msvcregex Win32") - option( - BUILD_MSVCREGEX - "If build msvcregex on Windows" - ON - ) + MESSAGE("build wcwidth Win32") + option( + BUILD_WCWIDTH + "If build wcwidth on Windows" + ON + ) - MESSAGE("build wcwidth Win32") - option( - BUILD_WCWIDTH - "If build wcwidth on Windows" - ON - ) + MESSAGE("build wingetopt Win32") + option( + BUILD_WINGETOPT + "If build wingetopt on Windows" + ON + ) - MESSAGE("build wingetopt Win32") - option( - BUILD_WINGETOPT - "If build wingetopt on Windows" - ON - ) - - option( - TDENGINE_3 - "TDengine 3.x for taos-tools" - ON - ) - - option( - BUILD_CRASHDUMP - "If build crashdump on Windows" - ON - ) + option( + TDENGINE_3 + "TDengine 3.x for taos-tools" + ON + ) + option( + BUILD_CRASHDUMP + "If build crashdump on Windows" + ON + ) + ENDIF () ELSEIF (TD_DARWIN_64) IF(${BUILD_TEST}) add_definitions(-DCOMPILER_SUPPORTS_CXX13) @@ -71,58 +71,102 @@ ELSEIF (TD_DARWIN_64) ENDIF () option( - BUILD_GEOS - "If build with geos" + BUILD_WITH_LEMON + "If build with lemon" + ON +) + +option( + BUILD_WITH_UDF + "If build with UDF" + ON +) + +IF(NOT TD_ASTRA) + option( + BUILD_GEOS + "If build with geos" + ON + ) + + option( + BUILD_SHARED_LIBS + "" + OFF + ) + + option( + RUST_BINDINGS + "If build with rust-bindings" + ON + ) + + option( + BUILD_PCRE2 + "If build with pcre2" ON ) -option( - BUILD_SHARED_LIBS - "" - OFF + option( + JEMALLOC_ENABLED + "If build with jemalloc" + OFF + ) + + option( + BUILD_SANITIZER + "If build sanitizer" + OFF + ) + + option( + BUILD_ADDR2LINE + "If build addr2line" + OFF + ) + + option( + BUILD_WITH_LEVELDB + "If build with leveldb" + OFF ) -option( - RUST_BINDINGS - "If build with rust-bindings" - ON + option( + BUILD_WITH_ROCKSDB + "If build with rocksdb" + ON ) -option( - BUILD_PCRE2 - "If build with pcre2" - ON -) + option( + BUILD_WITH_LZ4 + "If build with lz4" + ON + ) +ELSE () -option( - JEMALLOC_ENABLED - "If build with jemalloc" - OFF + option( + BUILD_WITH_LZMA2 + "If build with lzma2" + ON ) -option( - BUILD_SANITIZER - "If build sanitizer" - OFF - ) +ENDIF () -option( - BUILD_ADDR2LINE - "If build addr2line" - OFF - ) +ADD_DEFINITIONS(-DUSE_AUDIT) +ADD_DEFINITIONS(-DUSE_GEOS) +ADD_DEFINITIONS(-DUSE_UDF) +ADD_DEFINITIONS(-DUSE_STREAM) +ADD_DEFINITIONS(-DUSE_PRCE2) +ADD_DEFINITIONS(-DUSE_RSMA) +ADD_DEFINITIONS(-DUSE_TSMA) +ADD_DEFINITIONS(-DUSE_TQ) +ADD_DEFINITIONS(-DUSE_TOPIC) +ADD_DEFINITIONS(-DUSE_MONITOR) +ADD_DEFINITIONS(-DUSE_REPORT) -option( - BUILD_WITH_LEVELDB - "If build with leveldb" - OFF -) - -option( - BUILD_WITH_ROCKSDB - "If build with rocksdb" - ON -) +IF(${TD_ASTRA_RPC}) + ADD_DEFINITIONS(-DTD_ASTRA_RPC) +ENDIF() IF(${TD_LINUX}) @@ -150,6 +194,12 @@ option( ON ) +option( + BUILD_WITH_LZMA2 + "If build with lzma2" + ON +) + ENDIF () IF(NOT TD_ENTERPRISE) @@ -191,6 +241,14 @@ option(BUILD_WITH_COS "If build with cos" OFF) ENDIF () +IF(${TAOSD_INTEGRATED}) + add_definitions(-DTAOSD_INTEGRATED) +ENDIF() + +IF(${TD_AS_LIB}) + add_definitions(-DTD_AS_LIB) +ENDIF() + option( BUILD_WITH_SQLITE "If build with sqlite" @@ -209,6 +267,14 @@ option( off ) +option( + BUILD_WITH_NURAFT + "If build with NuRaft" + OFF +) + +IF(NOT TD_ASTRA) + option( BUILD_WITH_UV "If build with libuv" @@ -242,6 +308,7 @@ option( "If use invertedIndex" ON ) +ENDIF () option( BUILD_RELEASE diff --git a/cmake/lzma_CMakeLists.txt.in b/cmake/lzma_CMakeLists.txt.in index b6e96fb89f..90a9efc2a7 100644 --- a/cmake/lzma_CMakeLists.txt.in +++ b/cmake/lzma_CMakeLists.txt.in @@ -1,7 +1,6 @@ # xz - -if (${TD_LINUX}) +if (${BUILD_WITH_LZMA2}) ExternalProject_Add(lzma2 GIT_REPOSITORY https://github.com/conor42/fast-lzma2.git SOURCE_DIR "${TD_CONTRIB_DIR}/lzma2" diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 745dc01cf6..48c753937d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -92,7 +92,9 @@ if(${BUILD_TEST}) endif(${BUILD_TEST}) # lz4 -cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +if(${BUILD_WITH_LZ4}) + cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif(${BUILD_WITH_LZ4}) # zlib cat("${TD_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -186,16 +188,22 @@ if(${BUILD_PCRE2}) cat("${TD_SUPPORT_DIR}/pcre2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() -find_program(C_COMPILER_LEMON NAMES gcc) -if(C_COMPILER_LEMON) - message(STATUS "LEMON C compiler: ${C_COMPILER_LEMON}") -else() - set(C_COMPILER_LEMON ${CMAKE_C_COMPILER}) - message(STATUS "LEMON C compiler: ${C_COMPILER_LEMON}") -endif() - # lemon -cat("${TD_SUPPORT_DIR}/lemon_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +if(${BUILD_WITH_LEMON}) + if(${TD_ACORE}) + set(C_COMPILER_LEMON ${CMAKE_C_COMPILER}) + else() + find_program(C_COMPILER_LEMON NAMES gcc) + endif() + if(C_COMPILER_LEMON) + message(STATUS "LEMON C compiler: ${C_COMPILER_LEMON}") + else() + set(C_COMPILER_LEMON ${CMAKE_C_COMPILER}) + message(STATUS "LEMON C compiler: ${C_COMPILER_LEMON}") + endif() + + cat("${TD_SUPPORT_DIR}/lemon_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() # Force specify CC=cc on MacOS. Because the default CC setting in the generated Makefile has issues finding standard library headers IF(${TD_DARWIN}) @@ -273,11 +281,13 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE) # endif() # lz4 -add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) -target_include_directories( - lz4_static - PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib -) +if(${BUILD_WITH_LZ4}) + add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) + target_include_directories( + lz4_static + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib + ) +endif(${BUILD_WITH_LZ4}) # zlib set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in") diff --git a/docs/en/06-advanced/03-stream.md b/docs/en/06-advanced/03-stream.md index 2fde52cc9a..3db3a3a18e 100644 --- a/docs/en/06-advanced/03-stream.md +++ b/docs/en/06-advanced/03-stream.md @@ -108,6 +108,13 @@ Under normal circumstances, stream computation tasks will not process data that By enabling the fill_history option, the created stream computation task will be capable of processing data written before, during, and after the creation of the stream. This means that data written either before or after the creation of the stream will be included in the scope of stream computation, thus ensuring data integrity and consistency. This setting provides users with greater flexibility, allowing them to flexibly handle historical and new data according to actual needs. +Tips: +- When enabling fill_ristory, creating a stream requires finding the boundary point of historical data. If there is a lot of historical data, it may cause the task of creating a stream to take a long time. In this case, the parameter streamRunHistorySync (supported since version 3.3.6.0) can be configured to 1 (default is 0), and the task of creating a stream can be processed in the background. The statement of creating a stream can be returned immediately without blocking subsequent operations. + +- Show streams can be used to view the progress of background stream creation (ready status indicates success, init status indicates stream creation in progress, failed status indicates that the stream creation has failed, and the message column can be used to view the reason for the failure. In the case of failed stream creation, the stream can be deleted and rebuilt). + +- Besides, do not create multiple streams asynchronously at the same time, as transaction conflicts may cause subsequent streams to fail. + For example, create a stream to count the number of data entries generated by all smart meters every 10s, and also calculate historical data. SQL as follows: ```sql diff --git a/docs/en/14-reference/01-components/02-taosc.md b/docs/en/14-reference/01-components/02-taosc.md index 83cb3d765b..69c06a5c23 100644 --- a/docs/en/14-reference/01-components/02-taosc.md +++ b/docs/en/14-reference/01-components/02-taosc.md @@ -72,6 +72,12 @@ The TDengine client driver provides all the APIs needed for application programm | tempDir | |Supported, effective immediately | Specifies the directory for generating temporary files during operation, default on Linux platform is /tmp | | minimalTmpDirGB | |Supported, effective immediately | Minimum space required to be reserved in the directory specified by tempDir, in GB, default value: 1 | +### Stream Related + +| Parameter Name |Supported Version|Dynamic Modification| Description | +|-----------------------|----------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| streamRunHistoryAsync | 3.3.6.0 |Supported, effective immediately | When creating a stream with the fill_history parameter, should the stream statement be executed asynchronously. Boolean value, async if true, sync if false. default is false | + ### Log Related |Parameter Name|Supported Version|Dynamic Modification|Description| diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 74e7e81675..bd112d29d8 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -101,6 +101,13 @@ PARTITION 子句中，为 tbname 定义了一个别名 tname， 在 PARTITION 通过启用 fill_history 选项，创建的流计算任务将具备处理创建前、创建过程中以及创建后写入的数据的能力。这意味着，无论数据是在流创建之前还是之后写入的，都将纳入流计算的范围，从而确保数据的完整性和一致性。这一设置为用户提供了更大的灵活性，使其能够根据实际需求灵活处理历史数据和新数据。 +注意： +- 开启 fill_history 时，创建流需要找到历史数据的分界点，如果历史数据很多，可能会导致创建流任务耗时较长，此时可以配置参数 streamRunHistoryAsync（3.3.6.0版本开始支持） 为 1 （默认为0），将创建流的任务放在后台处理，创建流的语句可立即返回，不阻塞后面的操作。 + +- 通过 show streams 可查看后台建流的进度（ready 状态表示成功，init 状态表示正在建流，failed 状态表示建流失败，失败时 message 列可以查看原因。对于建流失败的情况可以删除流重新建立）。 + +- 另外，不要同时异步创建多个流，可能由于事务冲突导致后面创建的流失败。 + 比如，创建一个流，统计所有智能电表每 10s 产生的数据条数，并且计算历史数据。SQL 如下： ```sql create stream if not exists count_history_s fill_history 1 into count_history as select count(*) from power.meters interval(10s) diff --git a/docs/zh/06-advanced/05-data-in/07-mqtt.mdx b/docs/zh/06-advanced/05-data-in/07-mqtt.mdx index c3242a80c2..5ac22a7423 100644 --- a/docs/zh/06-advanced/05-data-in/07-mqtt.mdx +++ b/docs/zh/06-advanced/05-data-in/07-mqtt.mdx @@ -37,6 +37,14 @@ TDengine 可以通过 MQTT 连接器从 MQTT 代理订阅数据并将其写入 T 在 **MQTT 端口** 中填写 MQTT 代理的端口，例如：`1883` +在 **TLS 校验** 中选择 TLS 证书的校验方式 + +1. 不开启：表示不进行 TLS 证书认证。在连接 MQTT 时，会先进行 TCP 连接，如果连接失败，会进行无证书认证模式的 TLS 连接。 + +2. 单向认证：开启 TLS 连接，并验证服务端证书，此时需要上传 CA 证书。 + +3. 双向认证：开启 TLS 连接，并与服务端进行双向认证，此时需要上传 CA 证书，客户端证书以及客户端密钥。 + 在 **用户** 中填写 MQTT 代理的用户名。 在 **密码** 中填写 MQTT 代理的密码。 @@ -44,13 +52,7 @@ TDengine 可以通过 MQTT 连接器从 MQTT 代理订阅数据并将其写入 T ![mqtt-03.png](./mqtt-03.png) -### 4. 配置 SSL 证书 - -如果 MQTT 代理使用了 SSL 证书，需要在 **SSL证书** 中上传证书文件。 - -![mqtt-04.png](./mqtt-04.png) - -### 5. 配置采集信息 +### 4. 配置采集信息 在 **采集配置** 区域填写采集任务相关的配置参数。 @@ -75,13 +77,13 @@ TDengine 可以通过 MQTT 连接器从 MQTT 代理订阅数据并将其写入 T ![mqtt-05.png](./mqtt-05.png) -### 6. 配置 MQTT Payload 解析 +### 5. 配置 MQTT Payload 解析 在 **MQTT Payload 解析** 区域填写 Payload 解析相关的配置参数。 taosX 可以使用 JSON 提取器解析数据，并允许用户在数据库中指定数据模型，包括，指定表名称和超级表名，设置普通列和标签列等。 -#### 6.1 解析 +#### 5.1 解析 有三种获取示例数据的方法： @@ -112,7 +114,7 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解 ![mqtt-07.png](./mqtt-07.png) -#### 6.2 字段拆分 +#### 5.2 字段拆分 在 **从列中提取或拆分** 中填写从消息体中提取或拆分的字段，例如：将 message 字段拆分成 `message_0` 和 `message_1` 这2 个字段，选择 split 提取器，seperator 填写 -, number 填写 2。 @@ -126,7 +128,7 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解 ![mqtt-09.png](./mqtt-09.png) -#### 6.3 数据过滤 +#### 5.3 数据过滤 在 **过滤** 中，填写过滤条件，例如：填写`id != 1`，则只有 id 不为 1 的数据才会被写入 TDengine。 @@ -138,7 +140,7 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解 ![mqtt-11.png](./mqtt-11.png) -#### 6.4 表映射 +#### 5.4 表映射 在 **目标超级表** 的下拉列表中选择一个目标超级表，也可以先点击右侧的 **创建超级表** 按钮创建新的超级表。 @@ -164,7 +166,7 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解 ![mqtt-19.png](./mqtt-19.png) -### 7. 高级选项 +### 6. 高级选项 在 **消息等待队列大小** 中填写接收 MQTT 消息的缓存队列大小，当队列满时，新到达的数据会直接丢弃。可设置为 0，即不缓存。 @@ -182,12 +184,12 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解 ![mqtt-14](./mqtt-14.png) -### 8. 异常处理策略 +### 7. 异常处理策略 import Contributing from './_03-exception-handling-strategy.mdx' -### 9. 创建完成 +### 8. 创建完成 点击 **提交** 按钮，完成创建 MQTT 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。 diff --git a/docs/zh/06-advanced/05-data-in/mqtt-03.png b/docs/zh/06-advanced/05-data-in/mqtt-03.png index 292fed0d1a..576db44905 100644 Binary files a/docs/zh/06-advanced/05-data-in/mqtt-03.png and b/docs/zh/06-advanced/05-data-in/mqtt-03.png differ diff --git a/docs/zh/06-advanced/05-data-in/mqtt-04.png b/docs/zh/06-advanced/05-data-in/mqtt-04.png deleted file mode 100644 index e099875679..0000000000 Binary files a/docs/zh/06-advanced/05-data-in/mqtt-04.png and /dev/null differ diff --git a/docs/zh/14-reference/01-components/02-taosc.md b/docs/zh/14-reference/01-components/02-taosc.md index 8952e35664..dfe840b575 100755 --- a/docs/zh/14-reference/01-components/02-taosc.md +++ b/docs/zh/14-reference/01-components/02-taosc.md @@ -305,6 +305,15 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在 - 动态修改：不支持 - 支持版本：从 v3.1.0.0 版本开始引入 +### 流相关 + +#### streamRunHistoryAsync +- 说明：创建流有 fill_history 参数时，是否异步执行建流语句 +- 类型：布尔；false：同步；true：异步 +- 默认值：false +- 动态修改：支持通过 SQL 修改，立即生效 +- 支持版本：从 v3.3.6.0 版本开始引入 + ### 日志相关 #### logDir diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 968568d2dc..c2043fba3a 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -204,8 +204,8 @@ typedef struct SPkInfo { typedef struct SDataBlockInfo { STimeWindow window; int32_t rowSize; - int64_t rows; // todo hide this attribute uint32_t capacity; + int64_t rows; // todo hide this attribute SBlockID id; int16_t hasVarCol; int16_t dataLoad; // denote if the data is loaded or not diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index aa07183592..519614b94e 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -174,17 +174,17 @@ static FORCE_INLINE void colDataSetInt32(SColumnInfoData* pColumnInfoData, uint3 static FORCE_INLINE void colDataSetInt64(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, int64_t* v) { int32_t type = pColumnInfoData->info.type; char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex; - *(int64_t*)p = *(int64_t*)v; + taosSetPInt64Aligned((int64_t*)p, v); } static FORCE_INLINE void colDataSetFloat(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, float* v) { char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex; - *(float*)p = *(float*)v; + taosSetPFloatAligned((float*)p, v); } static FORCE_INLINE void colDataSetDouble(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, double* v) { char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex; - *(double*)p = *(double*)v; + taosSetPDoubleAligned((double*)p, v); } int32_t getJsonValueLen(const char* data); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 313d1963fd..7b4b8c33ce 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -95,13 +95,13 @@ const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b001111 #define COL_VAL_IS_NULL(CV) ((CV)->flag == CV_FLAG_NULL) #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) -#define tRowGetKey(_pRow, _pKey) \ - do { \ - (_pKey)->ts = (_pRow)->ts; \ - (_pKey)->numOfPKs = 0; \ - if ((_pRow)->numOfPKs > 0) { \ - tRowGetPrimaryKey((_pRow), (_pKey)); \ - } \ +#define tRowGetKey(_pRow, _pKey) \ + do { \ + (_pKey)->ts = taosGetInt64Aligned(&((_pRow)->ts)); \ + (_pKey)->numOfPKs = 0; \ + if ((_pRow)->numOfPKs > 0) { \ + tRowGetPrimaryKey((_pRow), (_pKey)); \ + } \ } while (0) // SValueColumn ================================ diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 8e9367d7c2..4c51c97c5a 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -291,11 +291,13 @@ extern int32_t tsUptimeInterval; extern bool tsUpdateCacheBatch; extern bool tsDisableStream; extern int64_t tsStreamBufferSize; +extern int64_t tsStreamFailedTimeout; extern int tsStreamAggCnt; extern bool tsFilterScalarMode; extern int32_t tsMaxStreamBackendCache; extern int32_t tsPQSortMemThreshold; extern bool tsStreamCoverage; +extern bool tsStreamRunHistoryAsync; extern int8_t tsS3EpNum; extern int32_t tsStreamNotifyMessageSize; extern int32_t tsStreamNotifyFrameSize; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a068f19175..a702b1b3f2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -527,7 +527,6 @@ typedef struct SRetention { #define RETENTION_VALID(l, r) ((((l) == 0 && (r)->freq >= 0) || ((r)->freq > 0)) && ((r)->keep > 0)) #pragma pack(push, 1) - // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta typedef struct SEp { char fqdn[TSDB_FQDN_LEN]; @@ -4233,7 +4232,7 @@ typedef struct { int8_t rawData; int32_t minPollRows; int8_t enableBatchMeta; - SHashObj *uidHash; // to find if uid is duplicated + SHashObj *uidHash; // to find if uid is duplicated } SMqPollReq; int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 2b367939ff..37692a8cb9 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -130,6 +130,8 @@ TD_DEF_MSG_TYPE(TDMT_MND_UPDATE_ANODE, "update-anode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_ANODE, "drop-anode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_ANAL_ALGO, "retrieve-anal-algo", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_FAILED_STREAM, "create-stream-failed", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CHECK_STREAM_TIMER, "check-stream-status", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_DND_MSG) TD_NEW_MSG_SEG(TDMT_MND_MSG) // 1<<8 diff --git a/include/libs/function/function.h b/include/libs/function/function.h index b2bde42463..4cd946ee35 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -119,8 +119,8 @@ typedef struct SInputColumnInfoData { int32_t startRowIndex; // handle started row index int64_t numOfRows; // the number of rows needs to be handled bool blankFill; // fill blank data to block for empty table - int32_t numOfInputCols; // PTS is not included bool colDataSMAIsSet; // if agg is set or not + int32_t numOfInputCols; // PTS is not included SColumnInfoData *pPTS; // primary timestamp column SColumnInfoData *pPrimaryKey; // primary key column SColumnInfoData **pData; diff --git a/include/libs/geometry/geomFunc.h b/include/libs/geometry/geomFunc.h index 05627e71f5..e27b046951 100644 --- a/include/libs/geometry/geomFunc.h +++ b/include/libs/geometry/geomFunc.h @@ -20,6 +20,7 @@ extern "C" { #endif +#ifdef USE_GEOS #include "function.h" int32_t makePointFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); @@ -33,6 +34,7 @@ int32_t touchesFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pO int32_t coversFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t containsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t containsProperlyFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +#endif #ifdef __cplusplus } diff --git a/include/libs/geometry/geosWrapper.h b/include/libs/geometry/geosWrapper.h index d27d300b82..8a816ed81d 100644 --- a/include/libs/geometry/geosWrapper.h +++ b/include/libs/geometry/geosWrapper.h @@ -23,6 +23,7 @@ extern "C" { #include #include "os.h" +#ifdef USE_GEOS #include "tgeosctx.h" void geosFreeBuffer(void *buffer); @@ -55,6 +56,13 @@ int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, const GEOSPreparedGeometry **outputPreparedGeom); void destroyGeometry(GEOSGeometry **geom, const GEOSPreparedGeometry **preparedGeom); +#else +int32_t initCtxAsText(); +int32_t doAsText(const unsigned char *inputGeom, size_t size, char **outputWKT); +int32_t checkWKB(const unsigned char *wkb, size_t size); + +#endif + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 47906e301f..662c01a718 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -440,6 +440,7 @@ typedef struct SRangeAroundNode { typedef struct SSelectStmt { ENodeType type; // QUERY_NODE_SELECT_STMT bool isDistinct; + STimeWindow timeRange; SNodeList* pProjectionList; SNodeList* pProjectionBindList; SNode* pFromTable; @@ -457,7 +458,6 @@ typedef struct SSelectStmt { SNodeList* pOrderByList; // SOrderByExprNode SLimitNode* pLimit; SLimitNode* pSlimit; - STimeWindow timeRange; SNodeList* pHint; char stmtName[TSDB_TABLE_NAME_LEN]; uint8_t precision; @@ -638,23 +638,23 @@ typedef struct SQuery { ENodeType type; EQueryExecStage execStage; EQueryExecMode execMode; + int32_t msgType; + int32_t numOfResCols; + int32_t placeholderNum; + int8_t precision; bool haveResultSet; + bool showRewrite; + bool stableQuery; SNode* pPrevRoot; SNode* pRoot; SNode* pPostRoot; - int32_t numOfResCols; SSchema* pResSchema; - int8_t precision; SCmdMsgInfo* pCmdMsg; - int32_t msgType; SArray* pTargetTableList; SArray* pTableList; SArray* pDbList; - bool showRewrite; - int32_t placeholderNum; SArray* pPlaceholderValues; SNode* pPrepareRoot; - bool stableQuery; } SQuery; void nodesWalkSelectStmtImpl(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 4bdc2c4740..e8a3ef7601 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -103,6 +103,7 @@ typedef struct SParseContext { setQueryFn setQueryFp; timezone_t timezone; void *charsetCxt; + bool streamRunHistory; } SParseContext; int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); @@ -127,7 +128,7 @@ int32_t qInitKeywordsTable(); void qCleanupKeywordsTable(); int32_t qAppendStmtTableOutput(SQuery* pQuery, SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, - SStbInterlaceInfo* pBuildInfo); + SStbInterlaceInfo* pBuildInfo, SVCreateTbReq* ctbReq); int32_t qBuildStmtFinOutput(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDataBlocks); // int32_t qBuildStmtOutputFromTbList(SQuery* pQuery, SHashObj* pVgHash, SArray* pBlockList, STableDataCxt* pTbCtx, // int32_t tbNum); @@ -165,7 +166,8 @@ int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* b int32_t qBindStmt2RowValue(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, STSchema** pTSchema, SBindInfo2* pBindInfos, void* charsetCxt); int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, - TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt); + TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt, + SVCreateTbReq* pCreateTbReq); void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, @@ -195,7 +197,7 @@ int32_t serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap, SArray** pOut); void destoryCatalogReq(SCatalogReq* pCatalogReq); bool isPrimaryKeyImpl(SNode* pExpr); int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, - SStbInterlaceInfo* pBuildInfo); + SStbInterlaceInfo* pBuildInfo, SVCreateTbReq* ctbReq); #ifdef __cplusplus } diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 7658303f3f..f366963692 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -18,7 +18,9 @@ #include "tdatablock.h" +#ifdef USE_ROCKSDB #include "rocksdb/c.h" +#endif #include "tdbInt.h" #include "tsimplehash.h" #include "tstreamFileState.h" diff --git a/include/libs/transport/thttp.h b/include/libs/transport/thttp.h index f81895c416..661e1898b9 100644 --- a/include/libs/transport/thttp.h +++ b/include/libs/transport/thttp.h @@ -16,9 +16,7 @@ #ifndef _TD_UTIL_HTTP_H_ #define _TD_UTIL_HTTP_H_ -#include "os.h" -#include "tdef.h" -#include "tref.h" +#include #ifdef __cplusplus extern "C" { diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 7b3fb5bb79..09c8fc13ce 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -19,6 +19,7 @@ extern "C" { #endif +#ifndef TD_ASTRA_RPC #include #include #include "taosdef.h" @@ -190,6 +191,198 @@ int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf); int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf); int32_t rpcCvtErrCode(int32_t code); +#else +#include +#include +#include "taosdef.h" +#include "tmsg.h" +#include "ttrace.h" + +#define TAOS_CONN_SERVER 0 +#define TAOS_CONN_CLIENT 1 +#define IsReq(pMsg) (pMsg->msgType & 1U) + +extern int32_t tsRpcHeadSize; + +typedef struct { + uint32_t clientIp; + uint16_t clientPort; + int64_t applyIndex; + uint64_t applyTerm; + char user[TSDB_USER_LEN]; +} SRpcConnInfo; + +typedef enum { + TD_ASTRA_CLIENT = 1, + TD_ASTRA_DSVR_CLIENT = 2, + TD_ASTRA_DSVR_STA_CLIENT = 4, + TD_ASTRA_DSVR_SYNC_CLIENT = 8, + TD_ASTRA_DSVR = 16 +} RPC_TYPE; + +typedef struct SRpcHandleInfo { + // rpc info + void *handle; // rpc handle returned to app + int64_t refId; // refid, used by server + int8_t noResp; // has response or not(default 0, 0: resp, 1: no resp) + int8_t persistHandle; // persist handle or not + int8_t hasEpSet; + int32_t cliVer; + + // app info + void *ahandle; // app handle set by client + void *wrapper; // wrapper handle + void *node; // node mgmt handle +#ifdef TD_ASTRA_32 + void *ahandleEx; // app handle set by client +#endif + + // resp info + void *rsp; + int32_t rspLen; + + STraceId traceId; + + SRpcConnInfo conn; + int8_t forbiddenIp; + int8_t notFreeAhandle; + int8_t compressed; + int16_t connType; + int64_t seq; + int64_t qId; + int32_t msgType; + void *reqWithSem; + int32_t refIdMgt; +} SRpcHandleInfo; + +typedef struct SRpcMsg { + tmsg_t msgType; + void *pCont; + int32_t contLen; + int32_t code; + int32_t type; + void *parent; + SRpcHandleInfo info; + +} SRpcMsg; + +typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *epset); +typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); +typedef bool (*RpcTfp)(int32_t code, tmsg_t msgType); +typedef bool (*RpcFFfp)(tmsg_t msgType); +typedef bool (*RpcNoDelayfp)(tmsg_t msgType); +typedef void (*RpcDfp)(void *ahandle); + +typedef struct SRpcInit { + char localFqdn[TSDB_FQDN_LEN]; + uint16_t localPort; // local port + char *label; // for debug purpose + int32_t numOfThreads; // number of threads to handle connections + int32_t sessions; // number of sessions allowed + int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS + int32_t idleTime; // milliseconds, 0 means idle timer is disabled + int32_t compatibilityVer; + + int32_t retryMinInterval; // retry init interval + int32_t retryStepFactor; // retry interval factor + int32_t retryMaxInterval; // retry max interval + int64_t retryMaxTimeout; + + int32_t failFastThreshold; + int32_t failFastInterval; + + int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size + int8_t encryption; // encrypt or not + + // the following is for client app ecurity only + char *user; // user name + + // call back to process incoming msg + RpcCfp cfp; + + // retry not not for particular msg + RpcRfp rfp; + + // set up timeout for particular msg + RpcTfp tfp; + + // destroy client ahandle; + RpcDfp dfp; + // fail fast fp + RpcFFfp ffp; + + RpcNoDelayfp noDelayFp; + + int32_t connLimitNum; + int32_t connLimitLock; + int32_t timeToGetConn; + int8_t supportBatch; // 0: no batch, 1. batch + int32_t batchSize; + int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait + int32_t shareConnLimit; + int8_t shareConn; // 0: no share, 1. share + int8_t startReadTimer; + int64_t readTimeout; // s + void *parent; + +} SRpcInit; + +typedef struct { + void *val; + int32_t (*clone)(void *src, void **dst); +} SRpcCtxVal; + +typedef struct { + int32_t msgType; + void *val; + int32_t (*clone)(void *src, void **dst); +} SRpcBrokenlinkVal; + +typedef struct { + SHashObj *args; + SRpcBrokenlinkVal brokenVal; + void (*freeFunc)(const void *arg); + int64_t st; +} SRpcCtx; + +int32_t rpcInit(); +void rpcCleanup(); + +void *rpcOpen(const SRpcInit *pRpc); +void rpcClose(void *); +void rpcCloseImpl(void *); +void *rpcMallocCont(int64_t contLen); +void rpcFreeCont(void *pCont); +void *rpcReallocCont(void *ptr, int64_t contLen); + +// Because taosd supports multi-process mode +// These functions should not be used on the server side +// Please use tmsg functions, which are defined in tmsgcb.h +int32_t rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); +int32_t rpcSendResponse(SRpcMsg *pMsg); +int32_t rpcRegisterBrokenLinkArg(SRpcMsg *msg); +int32_t rpcReleaseHandle(void *handle, int8_t type, + int32_t status); // just release conn to rpc instance, no close sock + +// These functions will not be called in the child process +int32_t rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); +int32_t rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +int32_t rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int8_t *epUpdated, + int32_t timeoutMs); + +int32_t rpcFreeConnById(void *shandle, int64_t connId); + +int32_t rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); +int32_t rpcAllocHandle(int64_t *refId); +int32_t rpcSetIpWhite(void *thandl, void *arg); + +int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf); + +int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf); +int32_t rpcCvtErrCode(int32_t code); + +#endif + #ifdef __cplusplus } #endif diff --git a/include/os/os.h b/include/os/os.h index a9c5ce5679..94edcdad75 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -28,29 +28,34 @@ extern "C" { #if !defined(WINDOWS) #include -#if !defined(_ALPINE) +#if !defined(_ALPINE) && !defined(TD_ASTRA) #include #endif +#if !defined(TD_ASTRA) #include -#include -#include #include - -#include -#include #include #include -#include #include +#include +#else +#include +#endif + +#include +#include +#include +#include +#include #include #include #include #include -#include #if defined(DARWIN) #else +#if !defined(TD_ASTRA) #include #include #include @@ -58,6 +63,7 @@ extern "C" { #include #endif #endif +#endif #else #ifndef __func__ diff --git a/include/os/osDef.h b/include/os/osDef.h index be37c50264..e1583e986b 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -108,6 +108,9 @@ void wordfree(wordexp_t *pwordexp); #define LOG_ERR 0 #define LOG_INFO 1 void syslog(int unused, const char *format, ...); +#elif defined(TD_ASTRA) +char *stpncpy(char *dest, const char *src, int n); +char *strsep(char **stringp, const char *delim); #endif // WINDOWS #ifndef WINDOWS @@ -216,6 +219,9 @@ void syslog(int unused, const char *format, ...); // pthread_setname_np not defined #define setThreadName(name) #endif +#elif defined(TD_ASTRA) +#define setThreadName(name) +#define getThreadName(name) #else // Linux, length of name must <= 16 (the last '\0' included) #define setThreadName(name) \ @@ -237,12 +243,24 @@ void syslog(int unused, const char *format, ...); #if defined(_WIN32) #define TD_DIRSEP "\\" +#elif defined(TD_ASTRA) +#ifdef TD_ASTRA_TARGET +#define TD_DIRSEP "/" +#else +#define TD_DIRSEP "\\" +#endif #else #define TD_DIRSEP "/" #endif #if defined(_WIN32) #define TD_DIRSEP_CHAR '\\' +#elif defined(TD_ASTRA) +#ifdef TD_ASTRA_TARGET +#define TD_DIRSEP_CHAR '/' +#else +#define TD_DIRSEP_CHAR '\\' +#endif #else #define TD_DIRSEP_CHAR '/' #endif diff --git a/include/os/osDir.h b/include/os/osDir.h index 99d3ebccde..6d32ab36ce 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -60,6 +60,20 @@ extern "C" { #define TD_LOG_DIR_PATH "/var/log/taos/" #endif // CUS_PROMPT +#elif defined(TD_ASTRA) + +#ifdef CUS_NAME +#define TD_TMP_DIR_PATH "C:\\" CUS_NAME "\\Temp\\" +#define TD_CFG_DIR_PATH "C:\\" CUS_NAME "\\cfg\\" +#define TD_DATA_DIR_PATH "C:\\" CUS_NAME "\\data\\" +#define TD_LOG_DIR_PATH "C:\\" CUS_NAME "\\log\\" +#else +#define TD_TMP_DIR_PATH "C:\\TDengine\\Temp\\" +#define TD_CFG_DIR_PATH "C:\\TDengine\\cfg\\" +#define TD_DATA_DIR_PATH "C:\\TDengine\\data\\" +#define TD_LOG_DIR_PATH "C:\\TDengine\\log\\" +#endif // CUS_NAME + #else #define TD_TMP_DIR_PATH "/tmp/" diff --git a/include/os/osEndian.h b/include/os/osEndian.h index 8292b284d8..c27ef8e38f 100644 --- a/include/os/osEndian.h +++ b/include/os/osEndian.h @@ -20,6 +20,10 @@ extern "C" { #endif +#ifdef TD_ASTRA +#include +#endif + typedef enum { TD_LITTLE_ENDIAN = 0, TD_BIG_ENDIAN } td_endian_t; static const int32_t endian_test_var = 1; diff --git a/include/os/osFile.h b/include/os/osFile.h index 4df05b9ecf..46c9acf650 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -72,6 +72,7 @@ TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions); #define TD_FILE_ACCESS_EXIST_OK 0x1 #define TD_FILE_ACCESS_READ_OK 0x2 #define TD_FILE_ACCESS_WRITE_OK 0x4 +#define TD_FILE_ACCESS_EXEC_OK 0x8 #define TD_TMP_FILE_PREFIX "tdengine-" diff --git a/include/os/osMath.h b/include/os/osMath.h index edbfa935ce..9c27ecd033 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -29,13 +29,23 @@ extern "C" { #define TPOW2(x) ((x) * (x)) #define TABS(x) ((x) > 0 ? (x) : -(x)) -#define TSWAP(a, b) \ - do { \ - char *__tmp = (char*)alloca(sizeof(a)); \ - (void)memcpy(__tmp, &(a), sizeof(a)); \ - (void)memcpy(&(a), &(b), sizeof(a)); \ - (void)memcpy(&(b), __tmp, sizeof(a)); \ +#ifndef TD_ASTRA +#define TSWAP(a, b) \ + do { \ + char *__tmp = (char *)alloca(sizeof(a)); \ + (void)memcpy(__tmp, &(a), sizeof(a)); \ + (void)memcpy(&(a), &(b), sizeof(a)); \ + (void)memcpy(&(b), __tmp, sizeof(a)); \ } while (0) +#else +#define TSWAP(a, b) \ + do { \ + char __tmp[sizeof(a)]; \ + (void)memcpy(__tmp, &(a), sizeof(a)); \ + (void)memcpy(&(a), &(b), sizeof(a)); \ + (void)memcpy(&(b), __tmp, sizeof(a)); \ + } while (0) +#endif #ifdef WINDOWS diff --git a/include/os/osMemPool.h b/include/os/osMemPool.h index 78cdf15b49..5b01f46602 100644 --- a/include/os/osMemPool.h +++ b/include/os/osMemPool.h @@ -149,8 +149,9 @@ int32_t taosMemoryPoolInit(mpReserveFailFp, mpReserveReachFp); } while (0) -#ifndef BUILD_TEST extern void* gMemPoolHandle; + +#if !defined(BUILD_TEST) && !defined(TD_ASTRA) extern threadlocal void* threadPoolSession; extern threadlocal bool threadPoolEnabled; extern int8_t tsMemPoolFullFunc; @@ -173,10 +174,10 @@ extern int8_t tsMemPoolFullFunc; #define taosMemoryTrim(_size, _trimed) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolTrim(gMemPoolHandle, threadPoolSession, _size, (char*)__FILE__, __LINE__, _trimed)) : (taosMemTrim(_size, _trimed))) #define taosMemoryMallocAlign(_alignment, _size) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolMallocAlign(gMemPoolHandle, threadPoolSession, _alignment, _size, (char*)__FILE__, __LINE__)) : (taosMemMallocAlign(_alignment, _size))) #else -#define taosEnableMemoryPoolUsage(_pool, _session) -#define taosDisableMemoryPoolUsage() -#define taosSaveDisableMemoryPoolUsage() -#define taosRestoreEnableMemoryPoolUsage() +#define taosEnableMemPoolUsage(_session) +#define taosDisableMemPoolUsage() +#define taosSaveDisableMemPoolUsage(_enable, _randErr) +#define taosRestoreEnableMemPoolUsage(_enable, _randErr) #define taosMemoryMalloc(_size) taosMemMalloc(_size) #define taosMemoryCalloc(_num, _size) taosMemCalloc(_num, _size) diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 51e57820b9..d2ffb500b5 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -25,10 +25,22 @@ extern "C" { #if !defined(WINDOWS) #ifndef ALLOW_FORBID_FUNC -#define malloc MALLOC_FUNC_TAOS_FORBID -#define calloc CALLOC_FUNC_TAOS_FORBID +#ifdef malloc +#undef malloc +#endif +#define malloc MALLOC_FUNC_TAOS_FORBID +#ifdef calloc +#undef calloc +#endif +#define calloc CALLOC_FUNC_TAOS_FORBID +#ifdef realloc +#undef realloc +#endif #define realloc REALLOC_FUNC_TAOS_FORBID -#define free FREE_FUNC_TAOS_FORBID +#ifdef free +#undef free +#endif +#define free FREE_FUNC_TAOS_FORBID #ifdef strdup #undef strdup #define strdup STRDUP_FUNC_TAOS_FORBID diff --git a/include/os/osString.h b/include/os/osString.h index 63c7aca58c..d5b4687cfd 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -20,6 +20,10 @@ extern "C" { #endif +#ifdef TD_ASTRA +#define DISALLOW_NCHAR_WITHOUT_ICONV +#endif + typedef wchar_t TdWchar; typedef int32_t TdUcs4; #if !defined(DISALLOW_NCHAR_WITHOUT_ICONV)// && defined(DARWIN) @@ -101,8 +105,8 @@ void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetC int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt); int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt); -int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); -int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); +int32_t taosUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); +int32_t taosUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); bool taosValidateEncodec(const char *encodec); int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len, int32_t bufSize); int32_t taosHexDecode(const char *src, char *dst, int32_t len); @@ -131,6 +135,13 @@ int32_t taosAscii2Hex(const char *z, uint32_t n, void **data, uint32_t *size); bool isHex(const char *z, uint32_t n); bool isValidateHex(const char *z, uint32_t n); +#ifdef TD_ASTRA +static FORCE_INLINE size_t strnlen(const char *s, size_t maxlen) { + const char *end = (const char *)memchr(s, '\0', maxlen); + return end ? (size_t)(end - s) : maxlen; +} +#endif + #ifdef __cplusplus } #endif diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 5e1b889580..cdd70912ab 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -60,7 +60,7 @@ int32_t taosGetSystemUUIDLen(char *uid, int32_t uidlen); char *taosGetCmdlineByPID(int32_t pid); void taosSetCoreDump(bool enable); -#if !defined(LINUX) +#if !defined(LINUX) || defined(TD_ASTRA) #define _UTSNAME_LENGTH 65 #define _UTSNAME_MACHINE_LENGTH _UTSNAME_LENGTH diff --git a/include/os/osThread.h b/include/os/osThread.h index 73c499b1dc..9f5b6b3bdb 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -35,7 +35,7 @@ extern "C" { // #endif #endif -#if !defined(WINDOWS) && !defined(_ALPINE) +#if !defined(WINDOWS) && !defined(_ALPINE) && !defined(TD_ASTRA) #ifndef __USE_XOPEN2K #define TD_USE_SPINLOCK_AS_MUTEX typedef pthread_mutex_t pthread_spinlock_t; @@ -70,12 +70,19 @@ typedef pthread_cond_t TdThreadCond; typedef pthread_condattr_t TdThreadCondAttr; typedef pthread_key_t TdThreadKey; #endif - +#ifdef TD_ASTRA +#define STACK_SIZE_DEFAULT (1048576 << 1) +#define STACK_SIZE_SMALL (1048576) +#else +#define STACK_SIZE_DEFAULT (10485760) +#endif #define taosThreadCleanupPush pthread_cleanup_push #define taosThreadCleanupPop pthread_cleanup_pop #if !defined(WINDOWS) #if defined(_TD_DARWIN_64) // MACOS #define taosThreadRwlockAttrSetKindNP(A, B) ((void)0) +#elif defined(TD_ASTRA) +#define taosThreadRwlockAttrSetKindNP(A, B) ((void)0) #else // LINUX #if _XOPEN_SOURCE >= 500 || _POSIX_C_SOURCE >= 200809L #define taosThreadRwlockAttrSetKindNP(A, B) pthread_rwlockattr_setkind_np(A, B) @@ -94,7 +101,7 @@ typedef pthread_key_t TdThreadKey; #else #define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER #endif - +#if 0 // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC @@ -200,6 +207,7 @@ typedef pthread_key_t TdThreadKey; #define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID #define sigwait SIGWAIT_FUNC_TAOS_FORBID #endif +#endif int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void *(*start)(void *), void *arg); int32_t taosThreadAttrDestroy(TdThreadAttr *attr); diff --git a/include/util/cus_name.h b/include/util/cus_name.h index fe05491742..adaab6fc48 100644 --- a/include/util/cus_name.h +++ b/include/util/cus_name.h @@ -20,7 +20,7 @@ // support OEM // #ifndef TD_PRODUCT_NAME -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA) #define TD_PRODUCT_NAME "TDengine Enterprise Edition" #else #define TD_PRODUCT_NAME "TDengine Community Edition" diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 66031f284a..674fbfa772 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -58,6 +58,11 @@ int32_t taosGetErrSize(); #define SET_ERROR_MSG(MSG, ...) \ (void)snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__) +#ifndef TD_ASTRA_RTP +#define SET_ERRNO(_code) (errno = (_code)) +#define ERRNO errno +#endif + #define TSDB_CODE_SUCCESS 0 #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error diff --git a/include/util/tconv.h b/include/util/tconv.h index c886b38f51..38d9204b81 100644 --- a/include/util/tconv.h +++ b/include/util/tconv.h @@ -31,8 +31,8 @@ void taosConvDestroy(); //int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt); //int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv); //bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt); -//int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); -//int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); +//int32_t taosUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); +//int32_t taosUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); #ifdef __cplusplus } diff --git a/include/util/tdef.h b/include/util/tdef.h index fea057dca8..a77ad5dbf0 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -25,6 +25,20 @@ extern "C" { #define TSDB__packed +#if defined(TD_ASTRA_32) +#define PACK_PUSH_MIN _Pragma("pack(push, 4)") +#elif defined(WINDOWS) +#define PACK_PUSH_MIN __pragma(pack(push, 1)) +#else +#define PACK_PUSH_MIN _Pragma("pack(push, 1)") +#endif + +#if defined(WINDOWS) +#define PACK_POP __pragma(pack(pop)) +#else +#define PACK_POP _Pragma("pack(pop)") +#endif + #define TSKEY int64_t #define TSKEY_MIN INT64_MIN #define TSKEY_MAX INT64_MAX @@ -339,6 +353,8 @@ typedef enum ELogicConditionType { #define TSDB_DNODE_CONFIG_LEN 128 #define TSDB_DNODE_VALUE_LEN 256 +#define TSDB_RESERVE_VALUE_LEN 256 + #define TSDB_CLUSTER_VALUE_LEN 1000 #define TSDB_GRANT_LOG_COL_LEN 15600 diff --git a/include/util/tgeosctx.h b/include/util/tgeosctx.h index 6c27d24dac..fb73140715 100644 --- a/include/util/tgeosctx.h +++ b/include/util/tgeosctx.h @@ -20,6 +20,7 @@ extern "C" { #endif +#ifdef USE_GEOS #include #include @@ -41,6 +42,7 @@ typedef struct SGeosContext { SGeosContext *acquireThreadLocalGeosCtx(); int32_t getThreadLocalGeosCtx(SGeosContext **ppCtx); const char *getGeosErrMsg(int32_t code); +#endif #ifdef __cplusplus } diff --git a/include/util/tlog.h b/include/util/tlog.h index 07b11bb16e..2201afddc6 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -34,6 +34,11 @@ typedef enum { DEBUG_FILE = 128 } ELogLevel; +typedef enum { + LOG_MODE_TAOSC = 1, + LOG_MODE_TAOSD = 2 +} ELogMode; + typedef void (*LogFp)(int64_t ts, ELogLevel level, const char *content); extern bool tsLogEmbedded; diff --git a/include/util/tpcre2.h b/include/util/tpcre2.h index 4fd461cfc8..1a90544bed 100644 --- a/include/util/tpcre2.h +++ b/include/util/tpcre2.h @@ -16,6 +16,7 @@ #ifndef _TD_ULIT_PCRE2_H_ #define _TD_ULIT_PCRE2_H_ +#ifdef USE_PRCE2 #define PCRE2_CODE_UNIT_WIDTH 8 #include "pcre2.h" @@ -31,4 +32,5 @@ void destroyRegexes(pcre2_code* pWktRegex, pcre2_match_data* pWktMatchData); } #endif +#endif // USE_PRCE2 #endif // _TD_UTIL_PAGEDBUF_H_ diff --git a/include/util/tutil.h b/include/util/tutil.h index b17bdab1ac..09c53981ce 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -54,6 +54,136 @@ int32_t parseCfgReal(const char *str, float *out); bool tIsValidFileName(const char *fileName, const char *pattern); bool tIsValidFilePath(const char *filePath, const char *pattern); +#ifdef TD_ASTRA +static FORCE_INLINE int32_t taosStrcasecmp(const char *s1, const char *s2) { + if (s1[0] == 0 && s2[0] == 0) return 0; + return strcasecmp(s1, s2); +} + +static FORCE_INLINE int32_t taosStrncasecmp(const char *s1, const char *s2, size_t n) { + if (s1[0] == 0 && s2[0] == 0) return 0; + return strncasecmp(s1, s2, n); +} +#else +#define taosStrcasecmp strcasecmp +#define taosStrncasecmp strncasecmp +#endif + +#ifdef NO_UNALIGNED_ACCESS +#define CHECK_ALIGNMENT +static FORCE_INLINE int64_t taosGetInt64Aligned(int64_t *pVal) { +#ifdef CHECK_ALIGNMENT + if ((((uintptr_t)pVal) & 7) == 0) return *pVal; +#endif + int64_t val; + memcpy(&val, pVal, sizeof(int64_t)); + return val; +} + +static FORCE_INLINE uint64_t taosGetUInt64Aligned(uint64_t *pVal) { +#ifdef CHECK_ALIGNMENT + if ((((uintptr_t)pVal) & 7) == 0) return *pVal; +#endif + uint64_t val; + memcpy(&val, pVal, sizeof(uint64_t)); + return val; +} + +static FORCE_INLINE float taosGetFloatAligned(float *pVal) { +#ifdef CHECK_ALIGNMENT + if ((((uintptr_t)pVal) & 7) == 0) return *pVal; +#endif + float val; + memcpy(&val, pVal, sizeof(float)); + return val; +} + +static FORCE_INLINE double taosGetDoubleAligned(double *pVal) { +#ifdef CHECK_ALIGNMENT + if ((((uintptr_t)pVal) & 7) == 0) return *pVal; +#endif + double val; + memcpy(&val, pVal, sizeof(double)); + return val; +} + +static FORCE_INLINE void taosSetInt64Aligned(int64_t *p, int64_t val) { +#ifdef CHECK_ALIGNMENT + if ((((uintptr_t)p) & 7) == 0) { + *p = val; + return; + } +#endif + memcpy(p, &val, sizeof(int64_t)); +} + +static FORCE_INLINE void taosSetUInt64Aligned(uint64_t *p, uint64_t val) { +#ifdef CHECK_ALIGNMENT + if ((((uintptr_t)p) & 7) == 0) { + *p = val; + return; + } +#endif + memcpy(p, &val, sizeof(uint64_t)); +} + +static FORCE_INLINE void taosSetPInt64Aligned(int64_t *to, int64_t *from) { +#ifdef CHECK_ALIGNMENT + if ((((uintptr_t)from) & 7) == 0 && ((uintptr_t)to & 7) == 0) { + *to = *from; + return; + } +#endif + memcpy(to, from, sizeof(int64_t)); +} + +static FORCE_INLINE void taosSetPFloatAligned(float *to, float *from) { +#ifdef CHECK_ALIGNMENT + if ((((uintptr_t)from) & 7) == 0 && ((uintptr_t)to & 7) == 0) { + *to = *from; + return; + } +#endif + memcpy(to, from, sizeof(float)); +} + +static FORCE_INLINE void taosSetPDoubleAligned(double *to, double *from) { +#ifdef CHECK_ALIGNMENT + if ((((uintptr_t)from) & 7) == 0 && ((uintptr_t)to & 7) == 0) { + *to = *from; + return; + } +#endif + memcpy(to, from, sizeof(double)); +} + +static FORCE_INLINE void taosSetPUInt64Aligned(uint64_t *to, uint64_t *from) { +#ifdef CHECK_ALIGNMENT + if ((((uintptr_t)from) & 7) == 0 && ((uintptr_t)to & 7) == 0) { + *to = *from; + return; + } +#endif + memcpy(to, from, sizeof(uint64_t)); +} + +#define TAOS_SET_OBJ_ALIGNED(pTo, vFrom) memcpy((pTo), &(vFrom), sizeof(*(pTo))) +#define TAOS_SET_POBJ_ALIGNED(pTo, pFrom) memcpy((pTo), (pFrom), sizeof(*(pTo))) +#else +static FORCE_INLINE int64_t taosGetInt64Aligned(int64_t *pVal) { return *pVal; } +static FORCE_INLINE uint64_t taosGetUInt64Aligned(uint64_t *pVal) { return *pVal; } +static FORCE_INLINE float taosGetFloatAligned(float *pVal) { return *pVal; } +static FORCE_INLINE double taosGetDoubleAligned(double *pVal) { return *pVal; } +static FORCE_INLINE void taosSetInt64Aligned(int64_t *p, int64_t val) { *p = val; } +static FORCE_INLINE void taosSetUInt64Aligned(uint64_t *p, uint64_t val) { *p = val; } +static FORCE_INLINE void taosSetPInt64Aligned(int64_t *to, int64_t *from) { *to = *from; } +static FORCE_INLINE void taosSetPFloatAligned(float *to, float *from) { *to = *from; } +static FORCE_INLINE void taosSetPDoubleAligned(double *to, double *from) { *to = *from; } +static FORCE_INLINE void taosSetPUInt64Aligned(uint64_t *to, uint64_t *from) { *to = *from; } +#define TAOS_SET_OBJ_ALIGNED(pTo, vFrom) *(pTo) = (vFrom) +#define TAOS_SET_POBJ_ALIGNED(pTo, pFrom) *(pTo) = *(pFrom) +#endif + static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { T_MD5_CTX context; tMD5Init(&context); @@ -148,20 +278,20 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, #define QUERY_CHECK_CODE TSDB_CHECK_CODE -#define TSDB_CHECK_CONDITION(condition, CODE, LINO, LABEL, ERRNO) \ - if (UNLIKELY(!(condition))) { \ - (CODE) = (ERRNO); \ - (LINO) = __LINE__; \ - goto LABEL; \ +#define TSDB_CHECK_CONDITION(condition, CODE, LINO, LABEL, _ERRNO) \ + if (UNLIKELY(!(condition))) { \ + (CODE) = (_ERRNO); \ + (LINO) = __LINE__; \ + goto LABEL; \ } #define QUERY_CHECK_CONDITION TSDB_CHECK_CONDITION -#define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, ERRNO) \ - if (UNLIKELY((ptr) == NULL)) { \ - (CODE) = (ERRNO); \ - (LINO) = __LINE__; \ - goto LABEL; \ +#define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, _ERRNO) \ + if (UNLIKELY((ptr) == NULL)) { \ + (CODE) = (_ERRNO); \ + (LINO) = __LINE__; \ + goto LABEL; \ } #define QUERY_CHECK_NULL TSDB_CHECK_NULL @@ -170,7 +300,7 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, #define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL) -#define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member))) +#define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) #define TAOS_GET_TERRNO(code) (terrno == 0 ? code : terrno) @@ -187,13 +317,13 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } \ } while (0) -#define TAOS_CHECK_RETURN_SET_CODE(CMD, CODE, ERRNO) \ - do { \ - int32_t __c = (CMD); \ - if (__c != TSDB_CODE_SUCCESS) { \ - (CODE) = (ERRNO); \ - TAOS_RETURN(__c); \ - } \ +#define TAOS_CHECK_RETURN_SET_CODE(CMD, CODE, _ERRNO) \ + do { \ + int32_t __c = (CMD); \ + if (__c != TSDB_CODE_SUCCESS) { \ + (CODE) = (_ERRNO); \ + TAOS_RETURN(__c); \ + } \ } while (0) #define TAOS_CHECK_RETURN_WITH_RELEASE(CMD, PTR1, PTR2) \ @@ -234,14 +364,14 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } \ } while (0) -#define TAOS_CHECK_EXIT_SET_CODE(CMD, CODE, ERRNO) \ - do { \ - code = (CMD); \ - if (code < TSDB_CODE_SUCCESS) { \ - (CODE) = (ERRNO); \ - lino = __LINE__; \ - goto _exit; \ - } \ +#define TAOS_CHECK_EXIT_SET_CODE(CMD, CODE, _ERRNO) \ + do { \ + code = (CMD); \ + if (code < TSDB_CODE_SUCCESS) { \ + (CODE) = (_ERRNO); \ + lino = __LINE__; \ + goto _exit; \ + } \ } while (0) #define TAOS_UNUSED(expr) (void)(expr) diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 2113aa7921..2519893a4b 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -21,10 +21,14 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include/client" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) +set(TAOSD_MODULE "") +if(${TAOSD_INTEGRATED}) + set(TAOSD_MODULE "taosd") +endif() target_link_libraries( ${TAOS_LIB} INTERFACE api - PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry + PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry ${TAOSD_MODULE} ) if(TD_WINDOWS) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index ed37b45bd7..8cffc6c056 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -119,17 +119,17 @@ typedef struct { } SAppInstServerCFG; struct SAppInstInfo { int64_t numOfConns; - SCorEpSet mgmtEp; int32_t totalDnodes; int32_t onlineDnodes; - TdThreadMutex qnodeMutex; - SArray* pQnodeList; - SAppClusterSummary summary; - SList* pConnList; // STscObj linked list int64_t clusterId; + SAppClusterSummary summary; + SArray* pQnodeList; + SList* pConnList; // STscObj linked list void* pTransporter; SAppHbMgr* pAppHbMgr; char* instKey; + TdThreadMutex qnodeMutex; + SCorEpSet mgmtEp; SAppInstServerCFG serverCfg; }; @@ -265,8 +265,9 @@ typedef struct SReqRelInfo { typedef struct SRequestObj { int8_t resType; // query or tmq - uint64_t requestId; int32_t type; // request type + uint64_t requestId; + SQuery* pQuery; STscObj* pTscObj; char* pDb; // current database string char* sqlstr; // sql string @@ -294,13 +295,13 @@ typedef struct SRequestObj { uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog uint32_t retry; int64_t allocatorRefId; - SQuery* pQuery; void* pPostPlan; SReqRelInfo relation; void* pWrapper; SMetaData parseMeta; char* effectiveUser; int8_t source; + bool streamRunHistory; } SRequestObj; typedef struct SSyncQueryParam { diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index ec61b2ff57..662ffc33ee 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -123,6 +123,7 @@ typedef struct SStmtStatInfo { typedef struct SStmtQNode { bool restoreTbCols; STableColsData tblData; + SVCreateTbReq *pCreateTbReq; struct SStmtQNode* next; } SStmtQNode; diff --git a/source/client/inc/clientStmt2.h b/source/client/inc/clientStmt2.h index 02cf9d1708..0d17885768 100644 --- a/source/client/inc/clientStmt2.h +++ b/source/client/inc/clientStmt2.h @@ -102,6 +102,8 @@ typedef struct { SHashObj *pVgHash; SBindInfo2 *pBindInfo; bool bindRowFormat; + bool fixValueTags; + SVCreateTbReq *fixValueTbReq; SStbInterlaceInfo siInfo; } SStmtSQLInfo2; @@ -234,8 +236,9 @@ int stmtClose2(TAOS_STMT2 *stmt); int stmtExec2(TAOS_STMT2 *stmt, int *affected_rows); int stmtPrepare2(TAOS_STMT2 *stmt, const char *sql, unsigned long length); int stmtSetTbName2(TAOS_STMT2 *stmt, const char *tbName); -int stmtSetTbTags2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *tags); -int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx); +int stmtSetTbTags2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *tags, SVCreateTbReq **pCreateTbReq); +int stmtCheckTags2(TAOS_STMT2 *stmt, SVCreateTbReq **pCreateTbReq); +int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx, SVCreateTbReq *pCreateTbReq); int stmtGetStbColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_ALL **fields); int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums); int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 986874c5bb..0cea3a0185 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -54,9 +54,9 @@ #define ENV_ERR_RET(c, info) \ do { \ - int32_t _code = c; \ + int32_t _code = (c); \ if (_code != TSDB_CODE_SUCCESS) { \ - errno = _code; \ + terrno = _code; \ tscInitRes = _code; \ tscError(info); \ return; \ @@ -128,7 +128,7 @@ static void concatStrings(SArray *list, char *buf, int size) { } } } - +#ifdef USE_REPORT static int32_t generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_t reqType, int64_t duration) { cJSON *json = cJSON_CreateObject(); int32_t code = TSDB_CODE_SUCCESS; @@ -217,7 +217,7 @@ _end: cJSON_Delete(json); return code; } - +#endif static bool checkSlowLogExceptDb(SRequestObj *pRequest, char *exceptDb) { if (pRequest->pDb != NULL) { return strcmp(pRequest->pDb, exceptDb) != 0; @@ -283,6 +283,7 @@ static void deregisterRequest(SRequestObj *pRequest) { } } +#ifdef USE_REPORT if (pTscObj->pAppInfo->serverCfg.monitorParas.tsEnableMonitor) { if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) { sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT); @@ -308,6 +309,7 @@ static void deregisterRequest(SRequestObj *pRequest) { } } } +#endif releaseTscObj(pTscObj->id); } @@ -789,7 +791,7 @@ void stopAllQueries(SRequestObj *pRequest) { } } } - +#ifdef USE_REPORT void crashReportThreadFuncUnexpectedStopped(void) { atomic_store_32(&clientStop, -1); } static void *tscCrashReportThreadFp(void *param) { @@ -888,15 +890,15 @@ int32_t tscCrashReportInit() { TSC_ERR_JRET(taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE)); TdThread crashReportThread; if (taosThreadCreate(&crashReportThread, &thAttr, tscCrashReportThreadFp, NULL) != 0) { - tscError("failed to create crashReport thread since %s", strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - TSC_ERR_RET(errno); + tscError("failed to create crashReport thread since %s", strerror(ERRNO)); + terrno = TAOS_SYSTEM_ERROR(ERRNO); + TSC_ERR_RET(terrno); } (void)taosThreadAttrDestroy(&thAttr); _return: if (code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); TSC_ERR_RET(terrno); } @@ -921,6 +923,113 @@ void tscStopCrashReport() { void tscWriteCrashInfo(int signum, void *sigInfo, void *context) { writeCrashLogToFile(signum, sigInfo, CUS_PROMPT, lastClusterId, appInfo.startTime); } +#endif + +#ifdef TAOSD_INTEGRATED +typedef struct { + TdThread pid; + int32_t stat; // < 0: start failed, 0: init(not start), 1: start successfully +} SDaemonObj; + +extern int dmStartDaemon(int argc, char const *argv[]); +extern void dmStopDaemon(); + +SDaemonObj daemonObj = {0}; + +typedef struct { + int32_t argc; + char **argv; +} SExecArgs; + +static void *dmStartDaemonFunc(void *param) { + int32_t code = 0; + SExecArgs *pArgs = (SExecArgs *)param; + int32_t argc = pArgs->argc; + char **argv = pArgs->argv; + + code = dmStartDaemon(argc, (const char **)argv); + if (code != 0) { + printf("failed to start taosd since %s\r\n", tstrerror(code)); + goto _exit; + } + +_exit: + if (code != 0) { + atomic_store_32(&daemonObj.stat, code); + } + return NULL; +} + +static int32_t shellStartDaemon(int argc, char *argv[]) { + int32_t code = 0, lino = 0; + SExecArgs *pArgs = NULL; + int64_t startMs = taosGetTimestampMs(), endMs = startMs; + + TdThreadAttr thAttr; + (void)taosThreadAttrInit(&thAttr); + (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL); +#endif + pArgs = (SExecArgs *)taosMemoryCalloc(1, sizeof(SExecArgs)); + if (pArgs == NULL) { + code = terrno; + TAOS_CHECK_EXIT(code); + } + pArgs->argc = argc; + pArgs->argv = argv; + +#ifndef TD_AS_LIB + tsLogEmbedded = 1; +#endif + + TAOS_CHECK_EXIT(taosThreadCreate(&daemonObj.pid, &thAttr, dmStartDaemonFunc, pArgs)); + + while (true) { + if (atomic_load_64(&tsDndStart)) { + atomic_store_32(&daemonObj.stat, 1); + break; + } + int32_t daemonstat = atomic_load_32(&daemonObj.stat); + if (daemonstat < 0) { + code = daemonstat; + TAOS_CHECK_EXIT(code); + } + + if (daemonstat > 1) { + code = TSDB_CODE_APP_ERROR; + TAOS_CHECK_EXIT(code); + } + taosMsleep(1000); + } + +_exit: + endMs = taosGetTimestampMs(); + (void)taosThreadAttrDestroy(&thAttr); + taosMemoryFreeClear(pArgs); + if (code) { + printf("\r\n The daemon start failed at line %d since %s, cost %" PRIi64 " ms\r\n", lino, tstrerror(code), + endMs - startMs); + } else { + printf("\r\n The daemon started successfully, cost %" PRIi64 " ms\r\n", endMs - startMs); + } +#ifndef TD_AS_LIB + tsLogEmbedded = 0; +#endif + return code; +} + +void shellStopDaemon() { +#ifndef TD_AS_LIB + tsLogEmbedded = 1; +#endif + dmStopDaemon(); + if (taosCheckPthreadValid(daemonObj.pid)) { + (void)taosThreadJoin(daemonObj.pid, NULL); + taosThreadClear(&daemonObj.pid); + } +} +#endif void taos_init_imp(void) { #if defined(LINUX) @@ -938,7 +1047,8 @@ void taos_init_imp(void) { // In the APIs of other program language, taos_cleanup is not available yet. // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. (void)atexit(taos_cleanup); - errno = TSDB_CODE_SUCCESS; + SET_ERRNO(TSDB_CODE_SUCCESS); + terrno = TSDB_CODE_SUCCESS; taosSeedRand(taosGetTimestampSec()); appInfo.pid = taosGetPId(); @@ -956,20 +1066,26 @@ void taos_init_imp(void) { const char *logName = CUS_PROMPT "log"; ENV_ERR_RET(taosInitLogOutput(&logName), "failed to init log output"); if (taosCreateLog(logName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { - (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logName, strerror(errno), configDir); + (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logName, strerror(ERRNO), configDir); tscInitRes = terrno; return; } +#ifdef TAOSD_INTEGRATED + ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0), "failed to init cfg"); +#else ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1), "failed to init cfg"); +#endif initQueryModuleMsgHandle(); +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL) { tscInitRes = terrno; tscError("failed to init conv"); return; } -#ifndef WINDOWS +#endif +#if !defined(WINDOWS) && !defined(TD_ASTRA) ENV_ERR_RET(tzInit(), "failed to init timezone"); #endif ENV_ERR_RET(monitorInit(), "failed to init monitor"); @@ -997,9 +1113,13 @@ void taos_init_imp(void) { ENV_ERR_RET(taosGetAppName(appInfo.appName, NULL), "failed to get app name"); ENV_ERR_RET(taosThreadMutexInit(&appInfo.mutex, NULL), "failed to init thread mutex"); +#ifdef USE_REPORT ENV_ERR_RET(tscCrashReportInit(), "failed to init crash report"); +#endif ENV_ERR_RET(qInitKeywordsTable(), "failed to init parser keywords table"); - +#ifdef TAOSD_INTEGRATED + ENV_ERR_RET(shellStartDaemon(0, NULL), "failed to start taosd daemon"); +#endif tscInfo("TAOS driver is initialized successfully"); } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 1395bed989..770dacb43e 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -187,7 +187,7 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { for (int32_t j = 0; j < rsp->vgNum; ++j) { SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { - tscError("hash push failed, errno:%d", errno); + tscError("hash push failed, terrno:%d", terrno); code = terrno; goto _return; } @@ -1401,16 +1401,19 @@ static int32_t hbCreateThread() { TdThreadAttr thAttr; TSC_ERR_JRET(taosThreadAttrInit(&thAttr)); TSC_ERR_JRET(taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE)); +#ifdef TD_COMPACT_OS + TSC_ERR_JRET(taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL)); +#endif if (taosThreadCreate(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); TSC_ERR_RET(terrno); } (void)taosThreadAttrDestroy(&thAttr); _return: if (code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); TSC_ERR_RET(terrno); } @@ -1431,12 +1434,12 @@ static void hbStopThread() { if (clientHbMgr.quitByKill) { code = taosThreadKill(clientHbMgr.thread, 0); if (TSDB_CODE_SUCCESS != code) { - tscError("taosThreadKill failed since %s", tstrerror(TAOS_SYSTEM_ERROR(code))); + tscError("taosThreadKill failed since %s", tstrerror(code)); } } else { code = taosThreadJoin(clientHbMgr.thread, NULL); if (TSDB_CODE_SUCCESS != code) { - tscError("taosThreadJoin failed since %s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + tscError("taosThreadJoin failed since %s", tstrerror(code)); } } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6c0ddbb672..7f8d8f4193 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1126,13 +1126,13 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock* } } - tscDebug("lastKey:%" PRId64 " vgId:%d, vgVer:%" PRId64, ts, *(int32_t*)pRow[1], *(int64_t*)pRow[2]); + tscInfo("[create stream with histroy] lastKey:%" PRId64 " vgId:%d, vgVer:%" PRId64, ts, *(int32_t*)pRow[1], *(int64_t*)pRow[2]); } (*pBlock)->info.window.ekey = lastTs; (*pBlock)->info.rows = numOfRows; - tscDebug("lastKey:%" PRId64 " numOfRows:%d from all vgroups", lastTs, numOfRows); + tscInfo("[create stream with histroy] lastKey:%" PRId64 " numOfRows:%d from all vgroups", lastTs, numOfRows); return TSDB_CODE_SUCCESS; } @@ -2413,7 +2413,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { int32_t hasColumnSeg = *(int32_t*)p; p += sizeof(int32_t); - uint64_t groupId = *(uint64_t*)p; + uint64_t groupId = taosGetUInt64Aligned((uint64_t*)p); p += sizeof(uint64_t); // check fields @@ -2469,10 +2469,11 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { return TSDB_CODE_TSC_INTERNAL_ERROR; } +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV if (convertUcs4) { code = doConvertUCS4(pResultInfo, colLength); } - +#endif return code; } @@ -3115,7 +3116,9 @@ void doRequestCallback(SRequestObj* pRequest, int32_t code) { code = TSDB_CODE_SUCCESS; pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; } - pRequest->body.queryFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, code); + if (pRequest->body.queryFp != NULL) { + pRequest->body.queryFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, code); + } SRequestObj* pReq = acquireRequest(this); if (pReq != NULL) { pReq->inCallback = false; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 82c509cf0a..0873bd5619 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -35,6 +35,10 @@ #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 +#ifdef TAOSD_INTEGRATED +extern void shellStopDaemon(); +#endif + static int32_t sentinel = TSC_VAR_NOT_RELEASE; static int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SSqlCallbackWrapper *pWrapper); @@ -55,7 +59,7 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { return ret; } -#ifndef WINDOWS +#if !defined(WINDOWS) && !defined(TD_ASTRA) static void freeTz(void *p) { timezone_t tz = *(timezone_t *)p; tzfree(tz); @@ -95,7 +99,7 @@ static timezone_t setConnnectionTz(const char *val) { tz = tzalloc("UTC"); if (tz == NULL) { tscError("%s set timezone UTC error", __func__); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); goto END; } } @@ -153,6 +157,7 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co val = NULL; } +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV if (option == TSDB_OPTION_CONNECTION_CHARSET || option == TSDB_OPTION_CONNECTION_CLEAR) { if (val != NULL) { if (!taosValidateEncodec(val)) { @@ -169,9 +174,9 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co pObj->optionInfo.charsetCxt = NULL; } } - +#endif if (option == TSDB_OPTION_CONNECTION_TIMEZONE || option == TSDB_OPTION_CONNECTION_CLEAR) { -#ifndef WINDOWS +#if !defined(WINDOWS) && !defined(TD_ASTRA) if (val != NULL) { if (val[0] == 0) { val = "UTC"; @@ -239,7 +244,7 @@ void taos_cleanup(void) { tscWarn("failed to cleanup task queue"); } -#ifndef WINDOWS +#if !defined(WINDOWS) && !defined(TD_ASTRA) tzCleanup(); #endif tmqMgmtClose(); @@ -259,10 +264,14 @@ void taos_cleanup(void) { taosConvDestroy(); DestroyRegexCache(); - +#ifdef TAOSD_INTEGRATED + shellStopDaemon(); +#endif tscInfo("all local resources released"); taosCleanupCfg(); +#ifndef TAOSD_INTEGRATED taosCloseLog(); +#endif } static setConfRet taos_set_config_imp(const char *config) { @@ -1418,7 +1427,8 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SS .parseSqlParam = pWrapper, .setQueryFp = setQueryRequest, .timezone = pTscObj->optionInfo.timezone, - .charsetCxt = pTscObj->optionInfo.charsetCxt}; + .charsetCxt = pTscObj->optionInfo.charsetCxt, + .streamRunHistory = pRequest->streamRunHistory}; int8_t biMode = atomic_load_8(&((STscObj *)pTscObj)->biMode); (*pCxt)->biMode = biMode; return TSDB_CODE_SUCCESS; @@ -2170,7 +2180,7 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col } STscStmt2 *pStmt = (STscStmt2 *)stmt; - if( atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum)>1) { + if (atomic_load_8((int8_t *)&pStmt->asyncBindParam.asyncBindNum) > 1) { tscError("async bind param is still working, please try again later"); return TSDB_CODE_TSC_STMT_API_ERROR; } @@ -2210,16 +2220,17 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col } } + SVCreateTbReq *pCreateTbReq = NULL; if (bindv->tags && bindv->tags[i]) { - code = stmtSetTbTags2(stmt, bindv->tags[i]); - if (code) { - goto out; - } - } else if (pStmt->bInfo.tbType == TSDB_CHILD_TABLE && pStmt->sql.autoCreateTbl) { - code = stmtSetTbTags2(stmt, NULL); - if (code) { - return code; - } + code = stmtSetTbTags2(stmt, bindv->tags[i], &pCreateTbReq); + } else if (pStmt->sql.autoCreateTbl || pStmt->bInfo.needParse) { + code = stmtCheckTags2(stmt, &pCreateTbReq); + } else { + pStmt->sql.autoCreateTbl = false; + } + + if (code) { + goto out; } if (bindv->bind_cols && bindv->bind_cols[i]) { @@ -2239,7 +2250,7 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col goto out; } - code = stmtBindBatch2(stmt, bind, col_idx); + code = stmtBindBatch2(stmt, bind, col_idx, pCreateTbReq); if (TSDB_CODE_SUCCESS != code) { goto out; } @@ -2283,7 +2294,7 @@ int taos_stmt2_bind_param_a(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t c (void)taosThreadCondSignal(&(pStmt->asyncBindParam.waitCond)); (void)atomic_sub_fetch_8(&pStmt->asyncBindParam.asyncBindNum, 1); (void)taosThreadMutexUnlock(&(pStmt->asyncBindParam.mutex)); - tscError("async bind failed, code:%d , %s", code_s, tstrerror(code_s)); + tscError("async bind failed, code:%d , %s", code_s, tstrerror(code_s)); } return code_s; diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 51ef93ba8f..87b755bb44 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -669,8 +669,8 @@ static void monitorSendAllSlowLog() { int64_t size = getFileSize(pClient->path); if (size <= 0) { if (size < 0) { - tscError("monitor failed to get file size:%s, err:%d", pClient->path, errno); - if (errno == ENOENT) { + tscError("monitor failed to get file size:%s, err:%d", pClient->path, ERRNO); + if (ERRNO == ENOENT) { processFileRemoved(pClient); } } @@ -810,16 +810,16 @@ static void* monitorThreadFunc(void* param) { static int32_t tscMonitortInit() { TdThreadAttr thAttr; if (taosThreadAttrInit(&thAttr) != 0) { - tscError("failed to init thread attr since %s", strerror(errno)); + tscError("failed to init thread attr since %s", strerror(ERRNO)); return TSDB_CODE_TSC_INTERNAL_ERROR; } if (taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE) != 0) { - tscError("failed to set thread attr since %s", strerror(errno)); + tscError("failed to set thread attr since %s", strerror(ERRNO)); return TSDB_CODE_TSC_INTERNAL_ERROR; } if (taosThreadCreate(&monitorThread, &thAttr, monitorThreadFunc, NULL) != 0) { - tscError("failed to create monitor thread since %s", strerror(errno)); + tscError("failed to create monitor thread since %s", strerror(ERRNO)); return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -873,7 +873,7 @@ int32_t monitorInit() { if (tsem2_init(&monitorSem, 0, 0) != 0) { tscError("sem init error since %s", terrstr()); - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } code = taosOpenQueue(&monitorQueue); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 50202e1139..7e47adedbe 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -151,12 +151,14 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { POINTER_BYTES) != 0) { tscError("failed to put appInfo into appInfo.pInstMapByClusterId"); } else { +#ifdef USE_MONITOR MonitorSlowLogData data = {0}; data.clusterId = pTscObj->pAppInfo->clusterId; data.type = SLOW_LOG_READ_BEGINNIG; (void)monitorPutData2MonitorQueue(data); // ignore monitorClientSlowQueryInit(connectRsp.clusterId); monitorClientSQLReqInit(connectRsp.clusterId); +#endif } } @@ -856,6 +858,119 @@ int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) { return code; } +static int32_t setCreateStreamFailedRsp(void* param, SDataBuf* pMsg, int32_t code) { + if (pMsg) { + taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pMsg->pData); + } + if (code != 0){ + tscError("setCreateStreamFailedRsp since %s", tstrerror(code)); + } else{ + tscInfo("setCreateStreamFailedRsp success"); + } + return code; +} + +void sendCreateStreamFailedMsg(SRequestObj* pRequest, char* streamName){ + int32_t code = 0; + tscInfo("send failed stream name to mgmt: %s", streamName); + int32_t size = INT_BYTES + strlen(streamName); + void *buf = taosMemoryMalloc(size); + if (buf == NULL) { + tscError("failed to strdup stream name: %s", terrstr()); + return; + } + *(int32_t*)buf = pRequest->code; + memcpy(POINTER_SHIFT(buf, INT_BYTES), streamName, strlen(streamName)); + + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + taosMemoryFree(buf); + tscError("failed to calloc msgSendInfo: %s", terrstr()); + return; + } + sendInfo->msgInfo = (SDataBuf){.pData = buf, .len = size, .handle = NULL}; + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->msgType = TDMT_MND_FAILED_STREAM; + sendInfo->fp = setCreateStreamFailedRsp; + + SEpSet epSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + code = asyncSendMsgToServer(pRequest->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); + if (code != 0) { + tscError("failed to send failed stream name to mgmt since %s", tstrerror(code)); + } +} + +static void processCreateStreamSecondPhaseRsp(void* param, void* res, int32_t code) { + SRequestObj* pRequest = res; + if (code != 0 && param != NULL){ + sendCreateStreamFailedMsg(pRequest, param); + } + taosMemoryFree(param); + destroyRequest(pRequest); +} + +static char* getStreamName(SRequestObj* pRequest){ + SCreateStreamStmt* pStmt = (SCreateStreamStmt*)(pRequest->pQuery->pRoot); + SName name; + int32_t code = tNameSetDbName(&name, pRequest->pTscObj->acctId, pStmt->streamName, strlen(pStmt->streamName)); + if (TSDB_CODE_SUCCESS != code) { + tscError("failed to set db name for stream since %s", tstrerror(code)); + return NULL; + } else{ + char *streamName = taosMemoryCalloc(1, TSDB_STREAM_FNAME_LEN); + (void)tNameGetFullDbName(&name, streamName); + return streamName; + } +} + +void processCreateStreamSecondPhase(SRequestObj* pRequest){ + tscInfo("[create stream with histroy] create in second phase"); + char *streamName = getStreamName(pRequest); + size_t sqlLen = strlen(pRequest->sqlstr); + SRequestObj* pRequestNew = NULL; + int32_t code = buildRequest(pRequest->pTscObj->id, pRequest->sqlstr, sqlLen, streamName, false, &pRequestNew, 0); + if (code != TSDB_CODE_SUCCESS) { + tscError("[create stream with histroy] create in second phase, build request failed since %s", tstrerror(code)); + return; + } + pRequestNew->source = pRequest->source; + pRequestNew->body.queryFp = processCreateStreamSecondPhaseRsp; + pRequestNew->streamRunHistory = true; + doAsyncQuery(pRequestNew, false); +} + +int32_t processCreateStreamFirstPhaseRsp(void* param, SDataBuf* pMsg, int32_t code) { + SRequestObj* pRequest = param; + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + } + + if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) { + if (removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)) != 0) { + tscError("failed to remove meta data for table"); + } + } + + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + + if (code == 0 && !pRequest->streamRunHistory && tsStreamRunHistoryAsync){ + processCreateStreamSecondPhase(pRequest); + } + + if (pRequest->body.queryFp != NULL) { + pRequest->body.queryFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, code); + } else { + if (tsem_post(&pRequest->body.rspSem) != 0) { + tscError("failed to post semaphore"); + } + } + + return code; +} + __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) { switch (msgType) { case TDMT_MND_CONNECT: @@ -872,9 +987,12 @@ __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) { return processAlterStbRsp; case TDMT_MND_SHOW_VARIABLES: return processShowVariablesRsp; + case TDMT_MND_CREATE_STREAM: + return processCreateStreamFirstPhaseRsp; case TDMT_MND_COMPACT_DB: return processCompactDbRsp; default: return genericRspCallback; } } + diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 74489ca00f..f271c3f282 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -37,9 +37,9 @@ kvVal->f = (float)result; #define SET_BIGINT \ - errno = 0; \ + SET_ERRNO(0); \ int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ - if (errno == ERANGE) { \ + if (ERRNO == ERANGE) { \ smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ return false; \ } \ @@ -63,9 +63,9 @@ kvVal->i = result; #define SET_UBIGINT \ - errno = 0; \ + SET_ERRNO(0); \ uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ - if (errno == ERANGE || result < 0) { \ + if (ERRNO == ERANGE || result < 0) { \ smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ return false; \ } \ @@ -1247,7 +1247,9 @@ void freeSSmlKv(void *data) { SSmlKv *kv = (SSmlKv *)data; if (kv->keyEscaped) taosMemoryFreeClear(kv->key); if (kv->valueEscaped) taosMemoryFreeClear(kv->value); +#ifdef USE_GEOS if (kv->type == TSDB_DATA_TYPE_GEOMETRY) geosFreeBuffer((void *)(kv->value)); +#endif if (kv->type == TSDB_DATA_TYPE_VARBINARY) taosMemoryFreeClear(kv->value); } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 4ae02bd7f7..4ff12888ce 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -97,13 +97,14 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } if (pVal->value[0] == 'g' || pVal->value[0] == 'G') { // geometry +#ifdef USE_GEOS if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') { int32_t code = initCtxGeomFromText(); if (code != TSDB_CODE_SUCCESS) { return code; } - char* tmp = taosMemoryCalloc(pVal->length, 1); - if (tmp == NULL){ + char *tmp = taosMemoryCalloc(pVal->length, 1); + if (tmp == NULL) { return terrno; } (void)memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); @@ -115,12 +116,15 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { pVal->type = TSDB_DATA_TYPE_GEOMETRY; if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - geosFreeBuffer((void*)(pVal->value)); + geosFreeBuffer((void *)(pVal->value)); return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } return TSDB_CODE_SUCCESS; } return TSDB_CODE_TSC_INVALID_VALUE; +#else + return TSDB_CODE_OPS_NOT_SUPPORT; +#endif } if (pVal->value[0] == 'b' || pVal->value[0] == 'B') { // varbinary diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 14815b009a..5187a76e82 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -762,7 +762,7 @@ int32_t stmtAsyncOutput(STscStmt* pStmt, void* param) { atomic_store_8((int8_t*)&pStmt->sql.siInfo.tableColsReady, true); } else { STMT_ERR_RET(qAppendStmtTableOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, &pParam->tblData, pStmt->exec.pCurrBlock, - &pStmt->sql.siInfo)); + &pStmt->sql.siInfo, NULL)); // taosMemoryFree(pParam->pTbData); @@ -809,7 +809,7 @@ int32_t stmtStartBindThread(STscStmt* pStmt) { } if (taosThreadCreate(&pStmt->bindThread, &thAttr, stmtBindThreadFunc, pStmt) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); STMT_ERR_RET(terrno); } diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index e47f9dd2d8..ab5fd52ee3 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -183,7 +183,7 @@ static int32_t stmtGetTbName(TAOS_STMT2* stmt, char** tbName) { pStmt->sql.type = STMT_TYPE_MULTI_INSERT; if ('\0' == pStmt->bInfo.tbName[0]) { - tscError("no table name set"); + tscWarn("no table name set, OK if it is a stmt get fields"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR); } @@ -240,9 +240,6 @@ static int32_t stmtUpdateInfo(TAOS_STMT2* stmt, STableMeta* pTableMeta, void* ta STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash)); pStmt->sql.autoCreateTbl = autoCreateTbl; - if (pStmt->sql.autoCreateTbl) { - pStmt->sql.stbInterlaceMode = false; - } return TSDB_CODE_SUCCESS; } @@ -294,18 +291,18 @@ static int32_t stmtParseSql(STscStmt2* pStmt) { } STableDataCxt* pTableCtx = *pSrc; - if (pStmt->sql.stbInterlaceMode) { - int16_t lastIdx = -1; + // if (pStmt->sql.stbInterlaceMode) { + // int16_t lastIdx = -1; - for (int32_t i = 0; i < pTableCtx->boundColsInfo.numOfBound; ++i) { - if (pTableCtx->boundColsInfo.pColIndex[i] < lastIdx) { - pStmt->sql.stbInterlaceMode = false; - break; - } + // for (int32_t i = 0; i < pTableCtx->boundColsInfo.numOfBound; ++i) { + // if (pTableCtx->boundColsInfo.pColIndex[i] < lastIdx) { + // pStmt->sql.stbInterlaceMode = false; + // break; + // } - lastIdx = pTableCtx->boundColsInfo.pColIndex[i]; - } - } + // lastIdx = pTableCtx->boundColsInfo.pColIndex[i]; + // } + // } if (NULL == pStmt->sql.pBindInfo) { pStmt->sql.pBindInfo = taosMemoryMalloc(pTableCtx->boundColsInfo.numOfBound * sizeof(*pStmt->sql.pBindInfo)); @@ -319,7 +316,6 @@ static int32_t stmtParseSql(STscStmt2* pStmt) { static int32_t stmtCleanBindInfo(STscStmt2* pStmt) { pStmt->bInfo.tbUid = 0; - pStmt->bInfo.tbSuid = 0; pStmt->bInfo.tbVgId = -1; pStmt->bInfo.tbType = 0; pStmt->bInfo.needParse = true; @@ -331,7 +327,10 @@ static int32_t stmtCleanBindInfo(STscStmt2* pStmt) { qDestroyBoundColInfo(pStmt->bInfo.boundTags); taosMemoryFreeClear(pStmt->bInfo.boundTags); } - pStmt->bInfo.stbFName[0] = 0; + if (!pStmt->sql.autoCreateTbl) { + pStmt->bInfo.stbFName[0] = 0; + pStmt->bInfo.tbSuid = 0; + } return TSDB_CODE_SUCCESS; } @@ -439,6 +438,9 @@ static int32_t stmtCleanSQLInfo(STscStmt2* pStmt) { taosArrayDestroy(pStmt->sql.nodeList); taosHashCleanup(pStmt->sql.pVgHash); pStmt->sql.pVgHash = NULL; + if (pStmt->sql.fixValueTags) { + tdDestroySVCreateTbReq(pStmt->sql.fixValueTbReq); + } void* pIter = taosHashIterate(pStmt->sql.pTableCache, NULL); while (pIter) { @@ -691,12 +693,11 @@ static int32_t stmtAsyncOutput(STscStmt2* pStmt, void* param) { atomic_store_8((int8_t*)&pStmt->sql.siInfo.tableColsReady, true); } else { - STMT_ERR_RET(qAppendStmtTableOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, &pParam->tblData, pStmt->exec.pCurrBlock, - &pStmt->sql.siInfo)); - + int code = qAppendStmtTableOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, &pParam->tblData, pStmt->exec.pCurrBlock, + &pStmt->sql.siInfo, pParam->pCreateTbReq); // taosMemoryFree(pParam->pTbData); - (void)atomic_sub_fetch_64(&pStmt->sql.siInfo.tbRemainNum, 1); + STMT_ERR_RET(code); } return TSDB_CODE_SUCCESS; } @@ -738,7 +739,7 @@ static int32_t stmtStartBindThread(STscStmt2* pStmt) { } if (taosThreadCreate(&pStmt->bindThread, &thAttr, stmtBindThreadFunc, pStmt) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); STMT_ERR_RET(terrno); } @@ -852,7 +853,7 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) { pStmt->sql.siInfo.tableColsReady = true; if (pStmt->options.asyncExecFn) { if (tsem_init(&pStmt->asyncExecSem, 0, 1) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); (void)stmtClose2(pStmt); return NULL; } @@ -1019,7 +1020,7 @@ int stmtSetTbName2(TAOS_STMT2* stmt, const char* tbName) { return TSDB_CODE_SUCCESS; } -int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) { +int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags, SVCreateTbReq** pCreateTbReq) { STscStmt2* pStmt = (STscStmt2*)stmt; STMT_DLOG_E("start to set tbTags"); @@ -1048,22 +1049,125 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) { // tscWarn("no tags or cols bound in sql, will not bound tags"); // return TSDB_CODE_SUCCESS; // } - - STableDataCxt** pDataBlock = - (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - if (NULL == pDataBlock) { - tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_APP_ERROR); + if (pStmt->sql.autoCreateTbl && pStmt->sql.stbInterlaceMode) { + STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, pStmt->bInfo.tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, + pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + STMT_ERR_RET(tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName)); } + STableDataCxt** pDataBlock = NULL; + if (pStmt->exec.pCurrBlock) { + pDataBlock = &pStmt->exec.pCurrBlock; + } else { + pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if (NULL == pDataBlock) { + tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_CACHE_ERROR); + } + // pStmt->exec.pCurrBlock = *pDataBlock; + // if (pStmt->sql.stbInterlaceMode) { + // taosArrayDestroy(pStmt->exec.pCurrBlock->pData->aCol); + // (*pDataBlock)->pData->aCol = NULL; + // } + } if (pStmt->bInfo.inExecCache && !pStmt->sql.autoCreateTbl) { return TSDB_CODE_SUCCESS; } tscDebug("start to bind stmt tag values"); - STMT_ERR_RET(qBindStmtTagsValue2(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName, + + void* boundTags = NULL; + if (pStmt->sql.stbInterlaceMode) { + boundTags = pStmt->sql.siInfo.boundTags; + *pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pCreateTbReq) { + return terrno; + } + int32_t vgId = -1; + STMT_ERR_RET(stmtTryAddTableVgroupInfo(pStmt, &vgId)); + (*pCreateTbReq)->uid = vgId; + } else { + boundTags = pStmt->bInfo.boundTags; + } + + STMT_ERR_RET(qBindStmtTagsValue2(*pDataBlock, boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt)); + pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt, *pCreateTbReq)); + + return TSDB_CODE_SUCCESS; +} + +int stmtCheckTags2(TAOS_STMT2* stmt, SVCreateTbReq** pCreateTbReq) { + STscStmt2* pStmt = (STscStmt2*)stmt; + + STMT_DLOG_E("start to set tbTags"); + + if (pStmt->errCode != TSDB_CODE_SUCCESS) { + return pStmt->errCode; + } + + if (!pStmt->sql.stbInterlaceMode) { + return TSDB_CODE_SUCCESS; + } + + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); + + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && + STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; + } + STMT_ERR_RET(stmtCreateRequest(pStmt)); + + if (pStmt->bInfo.needParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + if (!pStmt->sql.autoCreateTbl) { + return TSDB_CODE_SUCCESS; + } + } + + if (pStmt->sql.stbInterlaceMode && NULL == pStmt->sql.siInfo.pDataCtx) { + STMT_ERR_RET(stmtInitStbInterlaceTableInfo(pStmt)); + } + + STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, pStmt->bInfo.tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, + pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + STMT_ERR_RET(tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName)); + + STableDataCxt** pDataBlock = NULL; + if (pStmt->exec.pCurrBlock) { + pDataBlock = &pStmt->exec.pCurrBlock; + } else { + pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if (NULL == pDataBlock) { + tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_CACHE_ERROR); + } + } + + if (!((*pDataBlock)->pData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE)) { + return TSDB_CODE_SUCCESS; + } + + if (pStmt->sql.fixValueTags) { + STMT_ERR_RET(cloneSVreateTbReq(pStmt->sql.fixValueTbReq, pCreateTbReq)); + if ((*pCreateTbReq)->name) { + taosMemoryFree((*pCreateTbReq)->name); + } + (*pCreateTbReq)->name = taosStrdup(pStmt->bInfo.tbName); + int32_t vgId = -1; + STMT_ERR_RET(stmtTryAddTableVgroupInfo(pStmt, &vgId)); + (*pCreateTbReq)->uid = vgId; + return TSDB_CODE_SUCCESS; + } + + if ((*pDataBlock)->pData->pCreateTbReq) { + pStmt->sql.fixValueTags = true; + STMT_ERR_RET(cloneSVreateTbReq((*pDataBlock)->pData->pCreateTbReq, &pStmt->sql.fixValueTbReq)); + STMT_ERR_RET(cloneSVreateTbReq(pStmt->sql.fixValueTbReq, pCreateTbReq)); + (*pCreateTbReq)->uid = (*pDataBlock)->pMeta->vgId; + } return TSDB_CODE_SUCCESS; } @@ -1110,22 +1214,27 @@ static int stmtFetchStbColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIEL } STableDataCxt** pDataBlock = NULL; + bool cleanStb = false; - if (pStmt->sql.stbInterlaceMode) { + if (pStmt->sql.stbInterlaceMode && pStmt->sql.siInfo.pDataCtx != NULL) { pDataBlock = &pStmt->sql.siInfo.pDataCtx; } else { + cleanStb = true; pDataBlock = (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - if (NULL == pDataBlock) { - tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERRI_JRET(TSDB_CODE_APP_ERROR); - } } - STMT_ERRI_JRET(qBuildStmtStbColFields(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.preCtbname, fieldNum, fields)); - if (pStmt->bInfo.tbType == TSDB_SUPER_TABLE) { + if (NULL == pDataBlock || NULL == *pDataBlock) { + tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); + STMT_ERRI_JRET(TSDB_CODE_APP_ERROR); + } + + STMT_ERRI_JRET( + qBuildStmtStbColFields(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.preCtbname, fieldNum, fields)); + if (pStmt->bInfo.tbType == TSDB_SUPER_TABLE && cleanStb) { pStmt->bInfo.needParse = true; qDestroyStmtDataBlock(*pDataBlock); + *pDataBlock = NULL; if (taosHashRemove(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)) != 0) { tscError("get fileds %s remove exec blockHash fail", pStmt->bInfo.tbFName); STMT_ERRI_JRET(TSDB_CODE_APP_ERROR); @@ -1281,6 +1390,10 @@ static int stmtAddBatch2(TAOS_STMT2* stmt) { param->restoreTbCols = true; param->next = NULL; + if (pStmt->sql.autoCreateTbl) { + pStmt->bInfo.tagsCached = true; + } + stmtEnqueue(pStmt, param); return TSDB_CODE_SUCCESS; @@ -1334,7 +1447,7 @@ static int32_t stmtRestoreQueryFields(STscStmt2* pStmt) { return TSDB_CODE_SUCCESS; } */ -int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { +int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx, SVCreateTbReq* pCreateTbReq) { STscStmt2* pStmt = (STscStmt2*)stmt; int32_t code = 0; @@ -1443,6 +1556,8 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { param->restoreTbCols = false; tstrncpy(param->tblData.tbName, pStmt->bInfo.tbName, TSDB_TABLE_NAME_LEN); + + param->pCreateTbReq = pCreateTbReq; } int64_t startUs3 = taosGetTimestampUs(); @@ -1452,7 +1567,8 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { if (colIdx < 0) { if (pStmt->sql.stbInterlaceMode) { - (*pDataBlock)->pData->flags = 0; + // (*pDataBlock)->pData->flags = 0; + (*pDataBlock)->pData->flags &= ~SUBMIT_REQ_COLUMN_DATA_FORMAT; code = qBindStmtStbColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo, pStmt->taos->optionInfo.charsetCxt); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index d19bd9e009..ad1f7f028e 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1346,13 +1346,13 @@ static void doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) static int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; if (pParam == NULL) { - goto FAIL; + goto _ERR; } tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); if (tmq == NULL) { code = TSDB_CODE_TMQ_CONSUMER_CLOSED; - goto FAIL; + goto _ERR; } if (code != TSDB_CODE_SUCCESS) { @@ -1408,7 +1408,7 @@ static int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { } } - FAIL: + _ERR: if (pParam && pParam->sync) { SAskEpInfo* pInfo = pParam->pParam; if (pInfo) { @@ -1760,7 +1760,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { // init semaphore if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) { tqErrorC("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, - tstrerror(TAOS_SYSTEM_ERROR(errno)), pTmq->groupId); + tstrerror(TAOS_SYSTEM_ERROR(ERRNO)), pTmq->groupId); SET_ERROR_MSG_TMQ("init t_sem failed") goto _failed; } diff --git a/source/client/test/stmt2Test.cpp b/source/client/test/stmt2Test.cpp index 8b6419b4c5..c7ae20b096 100644 --- a/source/client/test/stmt2Test.cpp +++ b/source/client/test/stmt2Test.cpp @@ -43,8 +43,7 @@ void checkError(TAOS_STMT2* stmt, int code) { if (pStmt == nullptr || pStmt->sql.sqlStr == nullptr || pStmt->exec.pRequest == nullptr) { printf("stmt api error\n stats : %d\n errstr : %s\n", pStmt->sql.status, taos_stmt_errstr(stmt)); } else { - printf("stmt api error\n sql : %s\n stats : %d\n errstr : %s\n", pStmt->sql.sqlStr, pStmt->sql.status, - taos_stmt_errstr(stmt)); + printf("stmt api error\n sql : %s\n stats : %d\n", pStmt->sql.sqlStr, pStmt->sql.status); } ASSERT_EQ(code, TSDB_CODE_SUCCESS); } @@ -133,9 +132,9 @@ void do_query(TAOS* taos, const char* sql) { taos_free_result(result); } -void do_stmt(TAOS* taos, TAOS_STMT2_OPTION* option, const char* sql, int CTB_NUMS, int ROW_NUMS, int CYC_NUMS, - bool hastags, bool createTable) { - printf("test sql : %s\n", sql); +void do_stmt(const char* msg, TAOS* taos, TAOS_STMT2_OPTION* option, const char* sql, int CTB_NUMS, int ROW_NUMS, + int CYC_NUMS, bool hastags, bool createTable) { + printf("stmt2 [%s] : %s\n", msg, sql); do_query(taos, "drop database if exists stmt2_testdb_1"); do_query(taos, "create database IF NOT EXISTS stmt2_testdb_1"); do_query(taos, "create stable stmt2_testdb_1.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))"); @@ -903,14 +902,14 @@ TEST(stmt2Case, stmt2_stb_insert) { TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0); ASSERT_NE(taos, nullptr); // normal - TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL}; + TAOS_STMT2_OPTION option = {0, false, true, NULL, NULL}; { - do_stmt(taos, &option, "insert into `stmt2_testdb_1`.`stb` (tbname,ts,b,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, - true); + do_stmt("no-interlcace", taos, &option, "insert into `stmt2_testdb_1`.`stb` (tbname,ts,b,t1,t2) values(?,?,?,?,?)", + 3, 3, 3, true, true); } { - do_stmt(taos, &option, "insert into `stmt2_testdb_1`.? using `stmt2_testdb_1`.`stb` tags(?,?) values(?,?)", 3, 3, 3, - true, true); + do_stmt("no-interlcace", taos, &option, + "insert into `stmt2_testdb_1`.? using `stmt2_testdb_1`.`stb` tags(?,?) values(?,?)", 3, 3, 3, true, true); } // async @@ -918,28 +917,61 @@ TEST(stmt2Case, stmt2_stb_insert) { aa->async_affected_rows = 0; ASSERT_EQ(tsem_init(&aa->sem, 0, 0), TSDB_CODE_SUCCESS); void* param = aa; - option = {0, true, true, stmtAsyncQueryCb, param}; + option = {0, false, true, stmtAsyncQueryCb, param}; { - do_stmt(taos, &option, "insert into stmt2_testdb_1.stb (ts,b,tbname,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, true); + do_stmt("no-interlcace & aync exec", taos, &option, + "insert into stmt2_testdb_1.stb (ts,b,tbname,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, true); } { - do_stmt(taos, &option, "insert into stmt2_testdb_1.? using stmt2_testdb_1.stb (t1,t2)tags(?,?) (ts,b)values(?,?)", - 3, 3, 3, true, true); + do_stmt("no-interlcace & aync exec", taos, &option, + "insert into stmt2_testdb_1.? using stmt2_testdb_1.stb (t1,t2)tags(?,?) (ts,b)values(?,?)", 3, 3, 3, true, + true); } - // { do_stmt(taos, &option, "insert into db.? values(?,?)", 3, 3, 3, false, true); } // interlace = 0 & use db] do_query(taos, "use stmt2_testdb_1"); option = {0, false, false, NULL, NULL}; - { do_stmt(taos, &option, "insert into stb (tbname,ts,b) values(?,?,?)", 3, 3, 3, false, true); } - { do_stmt(taos, &option, "insert into ? using stb (t1,t2)tags(?,?) (ts,b)values(?,?)", 3, 3, 3, true, true); } - { do_stmt(taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); } + { + do_stmt("no-interlcace & no-db", taos, &option, "insert into stb (tbname,ts,b) values(?,?,?)", 3, 3, 3, false, + true); + } + { + do_stmt("no-interlcace & no-db", taos, &option, "insert into ? using stb (t1,t2)tags(?,?) (ts,b)values(?,?)", 3, 3, + 3, true, true); + } + { do_stmt("no-interlcace & no-db", taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); } // interlace = 1 option = {0, true, true, stmtAsyncQueryCb, param}; - { do_stmt(taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); } + { do_stmt("interlcace & preCreateTB", taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); } option = {0, true, true, NULL, NULL}; - { do_stmt(taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); } + { do_stmt("interlcace & preCreateTB", taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); } + + // auto create table + // interlace = 1 + option = {0, true, true, NULL, NULL}; + { + do_stmt("interlcace & no-preCreateTB", taos, &option, "insert into ? using stb (t1,t2)tags(?,?) (ts,b)values(?,?)", + 3, 3, 3, true, false); + } + { + do_stmt("interlcace & no-preCreateTB", taos, &option, + "insert into stmt2_testdb_1.? using stb (t1,t2)tags(1,'abc') (ts,b)values(?,?)", 3, 3, 3, false, false); + } + { + do_stmt("interlcace & no-preCreateTB", taos, &option, + "insert into stmt2_testdb_1.stb (ts,b,tbname,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, false); + } + // interlace = 0 + option = {0, false, false, NULL, NULL}; + { + do_stmt("no-interlcace & no-preCreateTB", taos, &option, + "insert into ? using stb (t1,t2)tags(?,?) (ts,b)values(?,?)", 3, 3, 3, true, false); + } + { + do_stmt("no-interlcace & no-preCreateTB", taos, &option, + "insert into stmt2_testdb_1.stb (ts,b,tbname,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, false); + } do_query(taos, "drop database if exists stmt2_testdb_1"); (void)tsem_destroy(&aa->sem); @@ -967,7 +999,8 @@ TEST(stmt2Case, stmt2_insert_non_statndard) { TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); ASSERT_NE(stmt, nullptr); const char* sql = "INSERT INTO stmt2_testdb_6.? using stmt2_testdb_6.stb1 (int_tag)tags(1) (ts) VALUES (?)"; - int code = taos_stmt2_prepare(stmt, sql, 0); + printf("stmt2 [%s] : %s\n", "less params", sql); + int code = taos_stmt2_prepare(stmt, sql, 0); checkError(stmt, code); int total_affect_rows = 0; @@ -987,7 +1020,7 @@ TEST(stmt2Case, stmt2_insert_non_statndard) { TAOS_STMT2_BIND* tagv[2] = {&tags1, &tags2}; TAOS_STMT2_BIND* paramv[2] = {¶ms1, ¶ms2}; char* tbname[2] = {"tb1", "tb2"}; - TAOS_STMT2_BINDV bindv = {2, &tbname[0], NULL, ¶mv[0]}; + TAOS_STMT2_BINDV bindv = {2, &tbname[0], tagv, ¶mv[0]}; code = taos_stmt2_bind_param(stmt, &bindv, -1); checkError(stmt, code); @@ -1004,10 +1037,12 @@ TEST(stmt2Case, stmt2_insert_non_statndard) { // less cols and tags { - TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + TAOS_STMT2_OPTION option = {0, false, false, NULL, NULL}; ASSERT_NE(stmt, nullptr); const char* sql = "INSERT INTO stmt2_testdb_6.stb1 (ts,int_tag,tbname) VALUES (?,?,?)"; - int code = taos_stmt2_prepare(stmt, sql, 0); + printf("stmt2 [%s] : %s\n", "less params", sql); + int code = taos_stmt2_prepare(stmt, sql, 0); checkError(stmt, code); int total_affect_rows = 0; @@ -1026,7 +1061,7 @@ TEST(stmt2Case, stmt2_insert_non_statndard) { TAOS_STMT2_BIND* tagv[2] = {&tags1, &tags2}; TAOS_STMT2_BIND* paramv[2] = {¶ms1, ¶ms2}; - char* tbname[2] = {"tb3", "tb4"}; + char* tbname[2] = {"tb1", "tb2"}; TAOS_STMT2_BINDV bindv = {2, &tbname[0], &tagv[0], ¶mv[0]}; code = taos_stmt2_bind_param(stmt, &bindv, -1); checkError(stmt, code); @@ -1044,10 +1079,15 @@ TEST(stmt2Case, stmt2_insert_non_statndard) { // disorder cols and tags { - TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + TAOS_STMT2_OPTION option = {0, false, false, NULL, NULL}; + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); ASSERT_NE(stmt, nullptr); + do_query(taos, + "INSERT INTO stmt2_testdb_6.stb1 (ts, int_tag, tbname) VALUES (1591060627000, 5, 'tb5')(1591060627000, " + "6,'tb6')"); const char* sql = "INSERT INTO stmt2_testdb_6.stb1 (binary_tag,int_col,tbname,ts,int_tag) VALUES (?,?,?,?,?)"; - int code = taos_stmt2_prepare(stmt, sql, 0); + printf("stmt2 [%s] : %s\n", "disorder params", sql); + int code = taos_stmt2_prepare(stmt, sql, 0); checkError(stmt, code); int tag_i = 0; @@ -1096,6 +1136,7 @@ TEST(stmt2Case, stmt2_insert_non_statndard) { ASSERT_NE(stmt, nullptr); const char* sql = "INSERT INTO stmt2_testdb_6.? using stmt2_testdb_6.stb1 (int_tag)tags(1) (int_col,ts)VALUES (?,?)"; + printf("stmt2 [%s] : %s\n", "PK error", sql); int code = taos_stmt2_prepare(stmt, sql, 0); checkError(stmt, code); @@ -1131,11 +1172,10 @@ TEST(stmt2Case, stmt2_insert_db) { ASSERT_NE(taos, nullptr); do_query(taos, "drop database if exists stmt2_testdb_12"); do_query(taos, "create database IF NOT EXISTS stmt2_testdb_12"); + do_query(taos, "create stable `stmt2_testdb_12`.`stb1`(ts timestamp, int_col int) tags(int_tag int)"); do_query(taos, - "create stable `stmt2_testdb_12`.`stb1` (ts timestamp, int_col int,long_col bigint,double_col " - "double,bool_col bool,binary_col binary(20),nchar_col nchar(20),varbinary_col varbinary(20),geometry_col " - "geometry(200)) tags(int_tag int,long_tag bigint,double_tag double,bool_tag bool,binary_tag " - "binary(20),nchar_tag nchar(20),varbinary_tag varbinary(20),geometry_tag geometry(200));"); + "INSERT INTO `stmt2_testdb_12`.`stb1` (ts,int_tag,tbname) VALUES " + "(1591060627000,1,'tb1')(1591060627000,2,'tb2')"); TAOS_STMT2_OPTION option = {0, false, false, NULL, NULL}; diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index 3469e8ecec..bae9b852ed 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -217,7 +217,7 @@ static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) { } if (taosFsyncFile(fp) < 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(ERRNO), &lino, _exit); } _exit: diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index 77c115339c..ad1ac5cbd8 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -11866,6 +11866,7 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm // auto create table if (pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { if (!(pSubmitTbData->pCreateTbReq)) { + uError("auto create table but request is NULL"); return TSDB_CODE_INVALID_MSG; } TAOS_CHECK_EXIT(tEncodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq)); @@ -12005,7 +12006,7 @@ int32_t tEncodeSubmitReq(SEncoder *pCoder, const SSubmitReq2 *pReq) { for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) { SSubmitTbData *pSubmitTbData = taosArrayGet(pReq->aSubmitTbData, i); if ((pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) && pSubmitTbData->pCreateTbReq == NULL) { - pSubmitTbData->flags = 0; + pSubmitTbData->flags &= ~SUBMIT_REQ_AUTO_CREATE_TABLE; } TAOS_CHECK_EXIT(tEncodeSSubmitTbData(pCoder, pSubmitTbData)); } diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index 8b453f7d82..6b7ed436ff 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -6,7 +6,7 @@ #include "tglobal.h" #define ERRNO_ERR_FORMAT "errno:%d,msg:%s" -#define ERRNO_ERR_DATA errno, strerror(errno) +#define ERRNO_ERR_DATA ERRNO, strerror(ERRNO) // deleteRsync function produce empty directories, traverse base directory to remove them static void removeEmptyDir() { @@ -150,7 +150,7 @@ int32_t startRsync() { if (taosMulMkDir(tsCheckpointBackupDir) != 0) { uError("[rsync] build checkpoint backup dir failed, path:%s," ERRNO_ERR_FORMAT, tsCheckpointBackupDir, ERRNO_ERR_DATA); - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); return code; } @@ -170,10 +170,10 @@ int32_t startRsync() { code = system(cmd); if (code != 0) { uError("[rsync] cmd:%s start server failed, code:%d," ERRNO_ERR_FORMAT, cmd, code, ERRNO_ERR_DATA); - if (errno == 0) { + if (ERRNO == 0) { return 0; } else { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); } } else { uInfo("[rsync] cmd:%s start server successful", cmd); @@ -223,7 +223,7 @@ int32_t uploadByRsync(const char* id, const char* path, int64_t checkpointId) { if (code != 0) { uError("[rsync] s-task:%s prepare checkpoint dir in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path, tsSnodeAddress, code, ERRNO_ERR_DATA); - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); } else { int64_t el = (taosGetTimestampMs() - st); uDebug("[rsync] s-task:%s prepare checkpoint dir in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path, @@ -267,7 +267,7 @@ int32_t uploadByRsync(const char* id, const char* path, int64_t checkpointId) { if (code != 0) { uError("[rsync] s-task:%s upload checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path, tsSnodeAddress, code, ERRNO_ERR_DATA); - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); } else { int64_t el = (taosGetTimestampMs() - st); uDebug("[rsync] s-task:%s upload checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path, @@ -355,7 +355,7 @@ int32_t deleteRsync(const char* id) { int32_t code = taosMkDir(tmp); if (code != 0) { uError("[rsync] make tmp dir failed. code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } char command[PATH_MAX] = {0}; @@ -367,7 +367,7 @@ int32_t deleteRsync(const char* id) { taosRemoveDir(tmp); if (code != 0) { uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } uDebug("[rsync] delete data:%s successful", id); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 10c5eb71c1..4cde045678 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -182,6 +182,7 @@ static const SSysDbTableSchema streamSchema[] = { {.name = "checkpoint_interval", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "checkpoint_backup", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "history_scan_idle", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "message", .bytes = TSDB_RESERVE_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysDbTableSchema streamTaskSchema[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4a9a5ac274..7b995b62bf 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -98,7 +98,7 @@ static int32_t getDataLen(int32_t type, const char* pData) { } static int32_t colDataSetValHelp(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) { - if (isNull || pData == NULL) { + if (isNull || pData == NULL) { // There is a placehold for each NULL value of binary or nchar type. if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { pColumnInfoData->varmeta.offset[rowIndex] = -1; // it is a null value of VAR type. @@ -1472,7 +1472,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0); SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0); if (pColInfoData == NULL || pOrder == NULL) { - return errno; + return terrno; } int64_t p0 = taosGetTimestampUs(); @@ -1861,14 +1861,14 @@ int32_t createSpecialDataBlock(EStreamType type, SSDataBlock** pBlock) { // window start ts void* px = taosArrayPush(p->pDataBlock, &infoData); if (px == NULL) { - code = errno; + code = terrno; goto _err; } // window end ts px = taosArrayPush(p->pDataBlock, &infoData); if (px == NULL) { - code = errno; + code = terrno; goto _err; } @@ -1878,14 +1878,14 @@ int32_t createSpecialDataBlock(EStreamType type, SSDataBlock** pBlock) { // uid px = taosArrayPush(p->pDataBlock, &infoData); if (px == NULL) { - code = errno; + code = terrno; goto _err; } // group id px = taosArrayPush(p->pDataBlock, &infoData); if (px == NULL) { - code = errno; + code = terrno; goto _err; } @@ -1895,14 +1895,14 @@ int32_t createSpecialDataBlock(EStreamType type, SSDataBlock** pBlock) { // calculate start ts px = taosArrayPush(p->pDataBlock, &infoData); if (px == NULL) { - code = errno; + code = terrno; goto _err; } // calculate end ts px = taosArrayPush(p->pDataBlock, &infoData); if (px == NULL) { - code = errno; + code = terrno; goto _err; } @@ -1911,7 +1911,7 @@ int32_t createSpecialDataBlock(EStreamType type, SSDataBlock** pBlock) { infoData.info.bytes = VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN; px = taosArrayPush(p->pDataBlock, &infoData); if (px == NULL) { - code = errno; + code = terrno; goto _err; } @@ -3260,7 +3260,11 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, size_t dataBuflen, in data += sizeof(bool); *actualLen = dataLen; +#ifndef NO_UNALIGNED_ACCESS *groupId = pBlock->info.id.groupId; +#else + taosSetPUInt64Aligned(groupId, &pBlock->info.id.groupId); +#endif if (dataLen > dataBuflen) goto _exit; return dataLen; @@ -3300,7 +3304,11 @@ int32_t blockDecode(SSDataBlock* pBlock, const char* pData, const char** pEndPos pStart += sizeof(int32_t); // group id sizeof(uint64_t) +#ifndef NO_UNALIGNED_ACCESS pBlock->info.id.groupId = *(uint64_t*)pStart; +#else + taosSetPUInt64Aligned(&pBlock->info.id.groupId, (uint64_t*)pStart); +#endif pStart += sizeof(uint64_t); if (pBlock->pDataBlock == NULL) { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index c3ca29c607..32b8ac3074 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1372,7 +1372,7 @@ int32_t tValueCompare(const SValue *tv1, const SValue *tv2) { return ret ? ret : (tv1->nData < tv2->nData ? -1 : (tv1->nData > tv2->nData ? 1 : 0)); } case TSDB_DATA_TYPE_NCHAR: { - int32_t ret = tasoUcs4Compare((TdUcs4 *)tv1->pData, (TdUcs4 *)tv2->pData, + int32_t ret = taosUcs4Compare((TdUcs4 *)tv1->pData, (TdUcs4 *)tv2->pData, tv1->nData < tv2->nData ? tv1->nData : tv2->nData); return ret ? ret : (tv1->nData < tv2->nData ? -1 : (tv1->nData > tv2->nData ? 1 : 0)); } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 756d03fd8a..e30a647066 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -61,8 +61,12 @@ int32_t tsMaxShellConns = 50000; int32_t tsShellActivityTimer = 3; // second // memory pool -int8_t tsMemPoolFullFunc = 0; -int8_t tsQueryUseMemoryPool = 1; +int8_t tsMemPoolFullFunc = 0; +#ifndef TD_ASTRA +int8_t tsQueryUseMemoryPool = 1; +#else +int8_t tsQueryUseMemoryPool = 0; +#endif int32_t tsQueryBufferPoolSize = 0; // MB int32_t tsSingleQueryMaxMemorySize = 0; // MB int32_t tsMinReservedMemorySize = 0; // MB @@ -133,23 +137,38 @@ int8_t tsGrant = 1; bool tsCompareAsStrInGreatest = true; // monitor -bool tsEnableMonitor = true; +#ifdef USE_MONITOR +bool tsEnableMonitor = true; +#else +bool tsEnableMonitor = false; +#endif int32_t tsMonitorInterval = 30; char tsMonitorFqdn[TSDB_FQDN_LEN] = {0}; uint16_t tsMonitorPort = 6043; int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; bool tsMonitorLogProtocol = false; -bool tsMonitorForceV2 = true; +#ifdef USE_MONITOR +bool tsMonitorForceV2 = true; +#else +bool tsMonitorForceV2 = false; +#endif // audit +#ifdef USE_AUDIT bool tsEnableAudit = true; bool tsEnableAuditCreateTable = true; bool tsEnableAuditDelete = true; int32_t tsAuditInterval = 5000; +#else +bool tsEnableAudit = false; +bool tsEnableAuditCreateTable = false; +bool tsEnableAuditDelete = false; +int32_t tsAuditInterval = 200000; +#endif // telem -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) || !defined(USE_REPORT) bool tsEnableTelem = false; #else bool tsEnableTelem = true; @@ -159,7 +178,7 @@ char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.tdengine.com"; uint16_t tsTelemPort = 80; char *tsTelemUri = "/report"; -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) || !defined(USE_REPORT) bool tsEnableCrashReport = false; #else bool tsEnableCrashReport = true; @@ -288,7 +307,7 @@ bool tsIfAdtFse = false; // ADT-FSE algorithom or origina char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR // udf -#ifdef WINDOWS +#if defined(WINDOWS) || !defined(USE_UDF) bool tsStartUdfd = false; #else bool tsStartUdfd = true; @@ -322,11 +341,17 @@ int32_t tsGrantHBInterval = 60; int32_t tsUptimeInterval = 300; // seconds char tsUdfdResFuncs[512] = ""; // taosudf resident funcs that teardown when taosudf exits char tsUdfdLdLibPath[512] = ""; -bool tsDisableStream = false; +#ifdef USE_STREAM +bool tsDisableStream = false; +#else +bool tsDisableStream = true; +#endif int64_t tsStreamBufferSize = 128 * 1024 * 1024; +int64_t tsStreamFailedTimeout = 30 * 60 * 1000; bool tsFilterScalarMode = false; int tsStreamAggCnt = 100000; bool tsStreamCoverage = false; +bool tsStreamRunHistoryAsync = false; bool tsUpdateCacheBatch = true; @@ -382,7 +407,7 @@ int32_t taosSetTfsCfg(SConfig *pCfg) { tsDiskCfg[0].disable = 0; tstrncpy(tsDataDir, pItem->str, PATH_MAX); if (taosMulMkDir(tsDataDir) != 0) { - int32_t code = TAOS_SYSTEM_ERROR(errno); + int32_t code = TAOS_SYSTEM_ERROR(ERRNO); uError("failed to create dataDir:%s, since:%s", tsDataDir, tstrerror(code)); TAOS_RETURN(code); } @@ -751,6 +776,9 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddBool(pCfg, "compareAsStrInGreatest", tsCompareAsStrInGreatest, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "streamRunHistoryAsync", tsStreamRunHistoryAsync, CFG_DYN_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -941,6 +969,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamFailedTimeout", tsStreamFailedTimeout, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1800, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); @@ -1358,7 +1387,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minimalTmpDirGB"); tsTempSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024); if (taosMulMkDir(tsTempDir) != 0) { - int32_t code = TAOS_SYSTEM_ERROR(errno); + int32_t code = TAOS_SYSTEM_ERROR(ERRNO); uError("failed to create tempDir:%s since %s", tsTempDir, tstrerror(code)); TAOS_RETURN(code); } @@ -1484,6 +1513,9 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamCoverage"); tsStreamCoverage = pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamRunHistoryAsync"); + tsStreamRunHistoryAsync = pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "compareAsStrInGreatest"); tsCompareAsStrInGreatest = pItem->bval; @@ -1615,6 +1647,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(taosSetSlowLogScope(pItem->str, &scope)); tsSlowLogScope = scope; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "queryRspPolicy"); + tsQueryRspPolicy = pItem->i32; +#ifdef USE_MONITOR TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "monitor"); tsEnableMonitor = pItem->bval; @@ -1634,15 +1669,13 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "monitorComp"); tsMonitorComp = pItem->bval; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "queryRspPolicy"); - tsQueryRspPolicy = pItem->i32; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "monitorLogProtocol"); tsMonitorLogProtocol = pItem->bval; TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "monitorForceV2"); tsMonitorForceV2 = pItem->i32; - +#endif +#ifdef USE_AUDIT TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "audit"); tsEnableAudit = pItem->bval; @@ -1654,29 +1687,31 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "auditInterval"); tsAuditInterval = pItem->i32; - +#endif +#ifdef USE_REPORT TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "telemetryReporting"); tsEnableTelem = pItem->bval; TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "crashReporting"); tsEnableCrashReport = pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "telemetryInterval"); + tsTelemInterval = pItem->i32; + + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "telemetryServer"); + TAOS_CHECK_RETURN(taosCheckCfgStrValueLen(pItem->name, pItem->str, TSDB_FQDN_LEN)); + tstrncpy(tsTelemServer, pItem->str, TSDB_FQDN_LEN); +#endif + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "ttlChangeOnWrite"); tsTtlChangeOnWrite = pItem->bval; TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "ttlFlushThreshold"); tsTtlFlushThreshold = pItem->i32; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "telemetryInterval"); - tsTelemInterval = pItem->i32; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "rsyncPort"); tsRsyncPort = pItem->i32; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "telemetryServer"); - TAOS_CHECK_RETURN(taosCheckCfgStrValueLen(pItem->name, pItem->str, TSDB_FQDN_LEN)); - tstrncpy(tsTelemServer, pItem->str, TSDB_FQDN_LEN); - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "snodeAddress"); TAOS_CHECK_RETURN(taosCheckCfgStrValueLen(pItem->name, pItem->str, TSDB_FQDN_LEN)); tstrncpy(tsSnodeAddress, pItem->str, TSDB_FQDN_LEN); @@ -1768,7 +1803,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableWhiteList"); tsEnableWhiteList = pItem->bval; - +#ifdef USE_UDF TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "udf"); tsStartUdfd = pItem->bval; @@ -1782,7 +1817,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } - +#endif TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "cacheLazyLoadThreshold"); tsCacheLazyLoadThreshold = pItem->i32; @@ -1811,6 +1846,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamBufferSize"); tsStreamBufferSize = pItem->i64; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamFailedTimeout"); + tsStreamFailedTimeout = pItem->i64; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamAggCnt"); tsStreamAggCnt = pItem->i32; @@ -1834,7 +1872,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minDiskFreeSize"); tsMinDiskFreeSize = pItem->i64; - +#ifdef USE_S3 TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "s3MigrateIntervalSec"); tsS3MigrateIntervalSec = pItem->i32; @@ -1846,7 +1884,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "s3UploadDelaySec"); tsS3UploadDelaySec = pItem->i32; - +#endif TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "experimental"); tsExperimental = pItem->bval; @@ -1883,24 +1921,31 @@ int32_t taosSetReleaseCfg(SConfig *pCfg); static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag); +static int8_t tsLogCreated = 0; + int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; + int32_t mode = tsc ? LOG_MODE_TAOSC : LOG_MODE_TAOSD; SConfig *pCfg = NULL; + if (atomic_val_compare_exchange_8(&tsLogCreated, 0, 1) != 0) return 0; + if (tsCfg == NULL) { TAOS_CHECK_GOTO(osDefaultInit(), &lino, _exit); } TAOS_CHECK_GOTO(cfgInit(&pCfg), &lino, _exit); - if (tsc) { - tsLogEmbedded = 0; - TAOS_CHECK_GOTO(taosAddClientLogCfg(pCfg), &lino, _exit); - } else { - tsLogEmbedded = 1; - TAOS_CHECK_GOTO(taosAddClientLogCfg(pCfg), &lino, _exit); +#ifdef TAOSD_INTEGRATED + mode |= LOG_MODE_TAOSD; + tsLogEmbedded = 1; +#else + tsLogEmbedded = (mode & LOG_MODE_TAOSC) ? 0 : 1; +#endif + TAOS_CHECK_GOTO(taosAddClientLogCfg(pCfg), &lino, _exit); + if (mode & LOG_MODE_TAOSD) { TAOS_CHECK_GOTO(taosAddServerLogCfg(pCfg), &lino, _exit); } @@ -1914,10 +1959,8 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi goto _exit; } - if (tsc) { - TAOS_CHECK_GOTO(taosSetClientLogCfg(pCfg), &lino, _exit); - } else { - TAOS_CHECK_GOTO(taosSetClientLogCfg(pCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetClientLogCfg(pCfg), &lino, _exit); + if (mode & LOG_MODE_TAOSD) { TAOS_CHECK_GOTO(taosSetServerLogCfg(pCfg), &lino, _exit); } @@ -1930,7 +1973,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi goto _exit; } - if ((code = taosInitLog(logname, logFileNum, tsc)) != 0) { + if ((code = taosInitLog(logname, logFileNum, mode)) != 0) { (void)printf("failed to init log file since %s\n", tstrerror(code)); goto _exit; } @@ -2163,6 +2206,10 @@ int32_t readCfgFile(const char *path, bool isGlobal) { uError("failed to stat file:%s , since %s", filename, tstrerror(code)); TAOS_RETURN(code); } + if (fileSize == 0) { + uInfo("config file:%s is empty", filename); + TAOS_RETURN(TSDB_CODE_SUCCESS); + } TdFilePtr pFile = taosOpenFile(filename, TD_FILE_READ); if (pFile == NULL) { code = terrno; @@ -2309,7 +2356,7 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, char *name = pItem->name; for (int32_t d = 0; d < optionSize; ++d) { const char *optName = pOptions[d].optionName; - if (strcasecmp(name, optName) != 0) continue; + if (taosStrcasecmp(name, optName) != 0) continue; code = TSDB_CODE_SUCCESS; switch (pItem->dtype) { case CFG_DTYPE_BOOL: { @@ -2742,7 +2789,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { tstrncpy(tsTempDir, pItem->str, PATH_MAX); TAOS_CHECK_GOTO(taosExpandDir(tsTempDir, tsTempDir, PATH_MAX), &lino, _out); if (taosMulMkDir(tsTempDir) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); uError("failed to create tempDir:%s since %s", tsTempDir, tstrerror(code)); goto _out; } @@ -2790,6 +2837,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { {"numOfRpcSessions", &tsNumOfRpcSessions}, {"bypassFlag", &tsBypassFlag}, {"safetyCheckLevel", &tsSafetyCheckLevel}, + {"streamRunHistoryAsync", &tsStreamRunHistoryAsync}, {"streamCoverage", &tsStreamCoverage}, {"compareAsStrInGreatest", &tsCompareAsStrInGreatest}}; @@ -2828,7 +2876,7 @@ int32_t taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVa static int taosLogVarComp(void const *lp, void const *rp) { SLogVar *lpVar = (SLogVar *)lp; SLogVar *rpVar = (SLogVar *)rp; - return strcasecmp(lpVar->name, rpVar->name); + return taosStrcasecmp(lpVar->name, rpVar->name); } static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag, SArray *noNeedToSetVars) { @@ -3063,7 +3111,7 @@ int32_t taosPersistGlobalConfig(SArray *array, const char *path, int32_t version taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pConfigFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); uError("failed to open file:%s since %s", filename, tstrerror(code)); TAOS_RETURN(code); } @@ -3072,7 +3120,7 @@ int32_t taosPersistGlobalConfig(SArray *array, const char *path, int32_t version if (taosWriteFile(pConfigFile, serialized, strlen(serialized)) < 0) { lino = __LINE__; - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); uError("failed to write file:%s since %s", filename, tstrerror(code)); goto _exit; } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 6bd64fb5e6..ab8b34021b 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -222,7 +222,7 @@ static int compareKv(const void* p1, const void* p2) { SSmlKv* kv2 = (SSmlKv*)p2; int32_t kvLen1 = kv1->keyLen; int32_t kvLen2 = kv2->keyLen; - int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); + int32_t res = taosStrncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); if (res != 0) { return res; } else { diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 22ab106bb3..17ada4231b 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -234,13 +234,13 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch } /* mktime will be affected by TZ, set by using taos_options */ -#ifdef WINDOWS +#if defined(WINDOWS) || defined(TD_ASTRA) int64_t seconds = user_mktime64(tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, 0); // int64_t seconds = gmtime(&tm); #else int64_t seconds = taosTimeGm(&tm); if (seconds == -1){ - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } #endif @@ -591,13 +591,13 @@ int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecisi */ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) { - errno = 0; + SET_ERRNO(0); char* endPtr = NULL; /* get the basic numeric value */ int64_t timestamp = taosStr2Int64(token, &endPtr, 10); - if ((timestamp == 0 && token[0] != '0') || errno != 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + if ((timestamp == 0 && token[0] != '0') || ERRNO != 0) { + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } /* natual month/year are not allowed in absolute duration */ @@ -611,12 +611,12 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow) { - errno = 0; + SET_ERRNO(0); /* get the basic numeric value */ *duration = taosStr2Int64(token, NULL, 10); - if ((*duration < 0 && !negativeAllow) || errno != 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + if ((*duration < 0 && !negativeAllow) || ERRNO != 0) { + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } *unit = token[tokenLen - 1]; @@ -648,7 +648,7 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision, t struct tm tm; time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); if(taosLocalTime(&tt, &tm, NULL, 0, tz) == NULL) { - uError("failed to convert time to gm time, code:%d", errno); + uError("failed to convert time to gm time, code:%d", ERRNO); return t; } int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)numOfMonth; @@ -664,7 +664,7 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision, t tt = taosMktime(&tm, tz); if (tt == -1){ - uError("failed to convert gm time to time, code:%d", errno); + uError("failed to convert gm time to time, code:%d", ERRNO); return t; } return (int64_t)(tt * TSDB_TICK_PER_SECOND(precision) + fraction); @@ -718,14 +718,14 @@ int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interva struct tm tm; time_t t = (time_t)skey; if (taosLocalTime(&t, &tm, NULL, 0, NULL) == NULL) { - uError("%s failed to convert time to local time, code:%d", __FUNCTION__, errno); + uError("%s failed to convert time to local time, code:%d", __FUNCTION__, ERRNO); return ret; } int32_t smon = tm.tm_year * 12 + tm.tm_mon; t = (time_t)ekey; if (taosLocalTime(&t, &tm, NULL, 0, NULL) == NULL) { - uError("%s failed to convert time to local time, code:%d", __FUNCTION__, errno); + uError("%s failed to convert time to local time, code:%d", __FUNCTION__, ERRNO); return ret; } int32_t emon = tm.tm_year * 12 + tm.tm_mon; @@ -752,7 +752,7 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { struct tm tm; time_t tt = (time_t)start; if (taosLocalTime(&tt, &tm, NULL, 0, pInterval->timezone) == NULL){ - uError("%s failed to convert time to local time, code:%d", __FUNCTION__, errno); + uError("%s failed to convert time to local time, code:%d", __FUNCTION__, ERRNO); return ts; } tm.tm_sec = 0; @@ -772,7 +772,7 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { tt = taosMktime(&tm, pInterval->timezone); if (tt == -1){ - uError("%s failed to convert local time to time, code:%d", __FUNCTION__, errno); + uError("%s failed to convert local time to time, code:%d", __FUNCTION__, ERRNO); return ts; } start = (int64_t)(tt * TSDB_TICK_PER_SECOND(precision)); @@ -990,7 +990,7 @@ int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precisio } if (NULL == taosLocalTime(", &ptm, buf, bufLen, NULL)) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } int32_t length = (int32_t)taosStrfTime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); length += tsnprintf(ts + length, fractionLen, format, mod); @@ -1032,7 +1032,7 @@ int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm, timezone_t tz) tm->fsec = ts % TICK_PER_SECOND[precision] * (TICK_PER_SECOND[TSDB_TIME_PRECISION_NANO] / TICK_PER_SECOND[precision]); time_t t = ts / TICK_PER_SECOND[precision]; if (NULL == taosLocalTime(&t, &tm->tm, NULL, 0, tz)) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } return TSDB_CODE_SUCCESS; } @@ -1508,6 +1508,8 @@ static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int case TSFKW_TZH:{ #ifdef WINDOWS int32_t gmtoff = -_timezone; +#elif defined(TD_ASTRA) + int32_t gmtoff = -timezone; #else int32_t gmtoff = tm->tm.tm_gmtoff; #endif @@ -1594,7 +1596,7 @@ static const char* tsFormatStr2Int32(int32_t* dest, const char* str, int32_t len // no integers found return NULL; } - if (errno == ERANGE || res > INT32_MAX || res < INT32_MIN) { + if (ERRNO == ERANGE || res > INT32_MAX || res < INT32_MIN) { // out of range return NULL; } @@ -1937,6 +1939,8 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec if (tzHour != 0) { #ifdef WINDOWS int32_t gmtoff = -_timezone; +#elif defined(TD_ASTRA) + int32_t gmtoff = -timezone; #else int32_t gmtoff = tm.tm.tm_gmtoff; #endif @@ -2136,7 +2140,7 @@ int64_t taosGetTimestampToday(int32_t precision, timezone_t tz) { } struct tm tm; if (taosLocalTime(&t, &tm, NULL, 0, tz) == NULL){ - uError("%s failed to get local time, code:%d", __FUNCTION__, errno); + uError("%s failed to get local time, code:%d", __FUNCTION__, ERRNO); return t; } tm.tm_hour = 0; @@ -2145,7 +2149,7 @@ int64_t taosGetTimestampToday(int32_t precision, timezone_t tz) { time_t tmp = taosMktime(&tm, tz); if (tmp == (time_t)-1) { - uError("%s failed to get timestamp of today, code:%d", __FUNCTION__, errno); + uError("%s failed to get timestamp of today, code:%d", __FUNCTION__, ERRNO); return t; } return (int64_t)tmp * factor; diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 60cf7963a8..2e587d58d9 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -64,7 +64,7 @@ static int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint n--; } - errno = 0; + SET_ERRNO(0); char *endPtr = NULL; if (z[0] == '0' && n > 2) { if (z[1] == 'b' || z[1] == 'B') { @@ -75,7 +75,7 @@ static int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint if (z[1] == 'x' || z[1] == 'X') { // parsing as hex *value = taosStr2UInt64(z, &endPtr, 16); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + if (ERRNO == ERANGE || ERRNO == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; @@ -85,11 +85,11 @@ static int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint if (parseFloat) { // parsing as double double val = taosStr2Double(z, &endPtr); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + if (ERRNO == ERANGE || ERRNO == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } if (val > (double)UINT64_MAX) { - errno = ERANGE; + SET_ERRNO(ERANGE); return TSDB_CODE_FAILED; } *value = round(val); @@ -101,14 +101,14 @@ static int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value) { if (n == 0) { - errno = EINVAL; + SET_ERRNO(EINVAL); return TSDB_CODE_FAILED; } - errno = 0; + SET_ERRNO(0); char *endPtr = NULL; *value = taosStr2Double(z, &endPtr); // 0x already converted here - if (errno == ERANGE || errno == EINVAL) return TSDB_CODE_FAILED; + if (ERRNO == ERANGE || ERRNO == EINVAL) return TSDB_CODE_FAILED; if (endPtr - z == n) return TSDB_CODE_SUCCESS; if (type == TK_NK_BIN || type == TK_NK_STRING) { @@ -124,12 +124,12 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value) { } int32_t toIntegerPure(const char *z, int32_t n, int32_t base, int64_t *value) { - errno = 0; + SET_ERRNO(0); char *endPtr = NULL; *value = taosStr2Int64(z, &endPtr, base); if (endPtr - z == n) { - if (errno == ERANGE || errno == EINVAL) { + if (ERRNO == ERANGE || ERRNO == EINVAL) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; @@ -159,12 +159,12 @@ int32_t toIntegerPure(const char *z, int32_t n, int32_t base, int64_t *value) { } int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { - errno = 0; + SET_ERRNO(0); char *endPtr = NULL; switch (type) { case TK_NK_INTEGER: { *value = taosStr2Int64(z, &endPtr, 10); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + if (ERRNO == ERANGE || ERRNO == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; @@ -174,7 +174,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { if (val < (double)INT64_MIN || val > (double)INT64_MAX) { return TSDB_CODE_FAILED; } - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + if (ERRNO == ERANGE || ERRNO == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } *value = val; @@ -185,18 +185,18 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } if (n == 0) { - errno = EINVAL; + SET_ERRNO(EINVAL); return TSDB_CODE_FAILED; } // 1. try to parse as integer *value = taosStr2Int64(z, &endPtr, 10); if (endPtr - z == n) { - if (errno == ERANGE || errno == EINVAL) { + if (ERRNO == ERANGE || ERRNO == EINVAL) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; - } else if (errno == 0 && *endPtr == '.') { + } else if (ERRNO == 0 && *endPtr == '.') { // pure decimal part const char *s = endPtr + 1; const char *end = z + n; @@ -254,7 +254,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { - errno = 0; + SET_ERRNO(0); char *endPtr = NULL; const char *p = z; switch (type) { @@ -263,7 +263,7 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { if (*p == '-' && *value) { return TSDB_CODE_FAILED; } - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + if (ERRNO == ERANGE || ERRNO == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; @@ -273,7 +273,7 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { if (val < 0 || val > (double)UINT64_MAX) { return TSDB_CODE_FAILED; } - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + if (ERRNO == ERANGE || ERRNO == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } *value = val; @@ -284,7 +284,7 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { } if (n == 0) { - errno = EINVAL; + SET_ERRNO(EINVAL); return TSDB_CODE_FAILED; } @@ -292,11 +292,11 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { *value = taosStr2UInt64(p, &endPtr, 10); if (endPtr - z == n) { - if (errno == ERANGE || errno == EINVAL || (*p == '-' && *value)) { + if (ERRNO == ERANGE || ERRNO == EINVAL || (*p == '-' && *value)) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; - } else if (errno == 0 && *endPtr == '.') { + } else if (ERRNO == 0 && *endPtr == '.') { const char *s = endPtr + 1; const char *end = z + n; bool pure = true; @@ -331,12 +331,12 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { } int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { - errno = 0; + SET_ERRNO(0); char *endPtr = NULL; *value = taosStr2Int64(z, &endPtr, base); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - errno = 0; + if (ERRNO == ERANGE || ERRNO == EINVAL || endPtr - z != n) { + SET_ERRNO(0); return TSDB_CODE_FAILED; } @@ -344,7 +344,7 @@ int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { } int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { - errno = 0; + SET_ERRNO(0); char *endPtr = NULL; const char *p = z; @@ -354,8 +354,8 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { } *value = taosStr2UInt64(z, &endPtr, base); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - errno = 0; + if (ERRNO == ERANGE || ERRNO == EINVAL || endPtr - z != n) { + SET_ERRNO(0); return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index ac89c35db5..422f1e4855 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -8,7 +8,11 @@ add_subdirectory(mgmt_dnode) add_subdirectory(test) aux_source_directory(exe EXEC_SRC) -add_executable(taosd ${EXEC_SRC}) +if(${TAOSD_INTEGRATED}) + add_library(taosd ${EXEC_SRC}) +else() + add_executable(taosd ${EXEC_SRC}) +endif() target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index bd7da3f4d6..65a8f5a24e 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -75,6 +75,7 @@ static void dmSetAssert(int32_t signum, void *sigInfo, void *context) { tsAssert static void dmStopDnode(int signum, void *sigInfo, void *context) { // taosIgnSignal(SIGUSR1); // taosIgnSignal(SIGUSR2); +#ifndef TD_ASTRA if (taosIgnSignal(SIGTERM) != 0) { dWarn("failed to ignore signal SIGTERM"); } @@ -90,16 +91,20 @@ static void dmStopDnode(int signum, void *sigInfo, void *context) { if (taosIgnSignal(SIGBREAK) != 0) { dWarn("failed to ignore signal SIGBREAK"); } - +#endif dInfo("shut down signal is %d", signum); -#ifndef WINDOWS - dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, +#if !defined(WINDOWS) && !defined(TD_ASTRA) + if (sigInfo != NULL) { + dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); + } #endif dmStop(); } +void dmStopDaemon() { dmStopDnode(SIGTERM, NULL, NULL); } + void dmLogCrash(int signum, void *sigInfo, void *context) { // taosIgnSignal(SIGTERM); // taosIgnSignal(SIGHUP); @@ -120,8 +125,9 @@ void dmLogCrash(int signum, void *sigInfo, void *context) { if (taosIgnSignal(SIGSEGV) != 0) { dWarn("failed to ignore signal SIGABRT"); } +#ifdef USE_REPORT writeCrashLogToFile(signum, sigInfo, CUS_PROMPT "d", dmGetClusterId(), global.startTime); - +#endif #ifdef _TD_DARWIN_64 exit(signum); #elif defined(WINDOWS) @@ -297,10 +303,12 @@ static void dmPrintArgs(int32_t argc, char const *argv[]) { char path[1024] = {0}; taosGetCwd(path, sizeof(path)); - char args[1024] = {0}; - int32_t arglen = tsnprintf(args, sizeof(args), "%s", argv[0]); - for (int32_t i = 1; i < argc; ++i) { - arglen = arglen + tsnprintf(args + arglen, sizeof(args) - arglen, " %s", argv[i]); + char args[1024] = {0}; + if (argc > 0) { + int32_t arglen = tsnprintf(args, sizeof(args), "%s", argv[0]); + for (int32_t i = 1; i < argc; ++i) { + arglen = arglen + tsnprintf(args + arglen, sizeof(args) - arglen, " %s", argv[i]); + } } dInfo("startup path:%s args:%s", path, args); @@ -366,7 +374,11 @@ static void taosCleanupArgs() { if (global.envCmd != NULL) taosMemoryFreeClear(global.envCmd); } +#ifdef TAOSD_INTEGRATED +int dmStartDaemon(int argc, char const *argv[]) { +#else int main(int argc, char const *argv[]) { +#endif int32_t code = 0; #ifdef TD_JEMALLOC_ENABLED bool jeBackgroundThread = true; @@ -469,13 +481,14 @@ int mainWindows(int argc, char **argv) { return code; } +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL) { dError("failed to init conv"); taosCloseLog(); taosCleanupArgs(); return code; } - +#endif if (global.checkS3) { code = dmCheckS3(); taosCleanupCfg(); @@ -546,8 +559,6 @@ int mainWindows(int argc, char **argv) { dInfo("start to init service"); dmSetSignalHandle(); - tsDndStart = taosGetTimestampMs(); - tsDndStartOsUptime = taosGetOsUptime(); code = dmRun(); dInfo("shutting down the service"); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 291c30580a..dc99cecbd8 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -95,6 +95,7 @@ static void *dmStatusInfoThreadFp(void *param) { return NULL; } +#if defined(TD_ENTERPRISE) SDmNotifyHandle dmNotifyHdl = {.state = 0}; #define TIMESERIES_STASH_NUM 5 static void *dmNotifyThreadFp(void *param) { @@ -196,7 +197,9 @@ static void *dmNotifyThreadFp(void *param) { return NULL; } +#endif +#ifdef USE_MONITOR static void *dmMonitorThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); @@ -231,7 +234,8 @@ static void *dmMonitorThreadFp(void *param) { return NULL; } - +#endif +#ifdef USE_AUDIT static void *dmAuditThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); @@ -252,7 +256,8 @@ static void *dmAuditThreadFp(void *param) { return NULL; } - +#endif +#ifdef USE_REPORT static void *dmCrashReportThreadFp(void *param) { int32_t code = 0; SDnodeMgmt *pMgmt = param; @@ -330,14 +335,18 @@ static void *dmCrashReportThreadFp(void *param) { return NULL; } +#endif int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { int32_t code = 0; TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL); +#endif if (taosThreadCreate(&pMgmt->statusThread, &thAttr, dmStatusThreadFp, pMgmt) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); dError("failed to create status thread since %s", tstrerror(code)); return code; } @@ -352,8 +361,11 @@ int32_t dmStartConfigThread(SDnodeMgmt *pMgmt) { TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL); +#endif if (taosThreadCreate(&pMgmt->configThread, &thAttr, dmConfigThreadFp, pMgmt) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); dError("failed to create config thread since %s", tstrerror(code)); return code; } @@ -368,8 +380,11 @@ int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt) { TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL); +#endif if (taosThreadCreate(&pMgmt->statusInfoThread, &thAttr, dmStatusInfoThreadFp, pMgmt) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); dError("failed to create status Info thread since %s", tstrerror(code)); return code; } @@ -399,14 +414,14 @@ void dmStopStatusInfoThread(SDnodeMgmt *pMgmt) { taosThreadClear(&pMgmt->statusInfoThread); } } - +#ifdef TD_ENTERPRISE int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) { int32_t code = 0; TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); if (taosThreadCreate(&pMgmt->notifyThread, &thAttr, dmNotifyThreadFp, pMgmt) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); dError("failed to create notify thread since %s", tstrerror(code)); return code; } @@ -429,55 +444,64 @@ void dmStopNotifyThread(SDnodeMgmt *pMgmt) { dError("failed to destroy notify sem"); } } - +#endif int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) { int32_t code = 0; +#ifdef USE_MONITOR TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); if (taosThreadCreate(&pMgmt->monitorThread, &thAttr, dmMonitorThreadFp, pMgmt) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); dError("failed to create monitor thread since %s", tstrerror(code)); return code; } (void)taosThreadAttrDestroy(&thAttr); tmsgReportStartup("dnode-monitor", "initialized"); +#endif return 0; } int32_t dmStartAuditThread(SDnodeMgmt *pMgmt) { int32_t code = 0; +#ifdef USE_AUDIT TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); if (taosThreadCreate(&pMgmt->auditThread, &thAttr, dmAuditThreadFp, pMgmt) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); dError("failed to create audit thread since %s", tstrerror(code)); return code; } (void)taosThreadAttrDestroy(&thAttr); tmsgReportStartup("dnode-audit", "initialized"); +#endif return 0; } void dmStopMonitorThread(SDnodeMgmt *pMgmt) { +#ifdef USE_MONITOR if (taosCheckPthreadValid(pMgmt->monitorThread)) { (void)taosThreadJoin(pMgmt->monitorThread, NULL); taosThreadClear(&pMgmt->monitorThread); } +#endif } void dmStopAuditThread(SDnodeMgmt *pMgmt) { +#ifdef USE_AUDIT if (taosCheckPthreadValid(pMgmt->auditThread)) { (void)taosThreadJoin(pMgmt->auditThread, NULL); taosThreadClear(&pMgmt->auditThread); } +#endif } int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) { int32_t code = 0; +#ifdef USE_REPORT if (!tsEnableCrashReport) { return 0; } @@ -486,17 +510,19 @@ int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) { (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); if (taosThreadCreate(&pMgmt->crashReportThread, &thAttr, dmCrashReportThreadFp, pMgmt) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); dError("failed to create crashReport thread since %s", tstrerror(code)); return code; } (void)taosThreadAttrDestroy(&thAttr); tmsgReportStartup("dnode-crashReport", "initialized"); +#endif return 0; } void dmStopCrashReportThread(SDnodeMgmt *pMgmt) { +#ifdef USE_REPORT if (!tsEnableCrashReport) { return; } @@ -505,6 +531,7 @@ void dmStopCrashReportThread(SDnodeMgmt *pMgmt) { (void)taosThreadJoin(pMgmt->crashReportThread, NULL); taosThreadClear(&pMgmt->crashReportThread); } +#endif } static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index a95ec42f7e..41aa0f27dd 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -212,7 +212,7 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { } if (taosCloseFile(&pFile) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _OVER; } TAOS_CHECK_GOTO(taosRenameFile(file, realfile), NULL, _OVER); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index f9fdbff4bd..4819d66c53 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -174,6 +174,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_FAILED_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_PAUSE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RESUME_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_snode/src/smInt.c b/source/dnode/mgmt/mgmt_snode/src/smInt.c index eea3585c72..e3a67e512c 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smInt.c +++ b/source/dnode/mgmt/mgmt_snode/src/smInt.c @@ -18,6 +18,8 @@ #include "libs/function/function.h" #include "libs/function/tudf.h" +#ifdef USE_STREAM + static int32_t smRequire(const SMgmtInputOpt *pInput, bool *required) { return dmReadFile(pInput->path, pInput->name, required); } @@ -98,3 +100,4 @@ SMgmtFunc smGetMgmtFunc() { return mgmtFunc; } +#endif diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index cb14155b1c..058b03554d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -350,14 +350,14 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { goto _OVER1; } if (taosFsyncFile(pFile) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); lino = __LINE__; goto _OVER1; } code = taosCloseFile(&pFile); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); lino = __LINE__; goto _OVER1; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 8a18727b99..ac16854076 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -183,7 +183,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { if ((pRetention->freq >= 0 && pRetention->keep > 0)) pCfg->isRsma = 1; } } -#if defined(TD_ENTERPRISE) +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) pCfg->tsdbCfg.encryptAlgorithm = pCreate->encryptAlgorithm; if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) { tstrncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); @@ -199,7 +199,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->walCfg.retentionSize = pCreate->walRetentionSize; pCfg->walCfg.segSize = pCreate->walSegmentSize; pCfg->walCfg.level = pCreate->walLevel; -#if defined(TD_ENTERPRISE) +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) pCfg->walCfg.encryptAlgorithm = pCreate->encryptAlgorithm; if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) { tstrncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); @@ -208,7 +208,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->walCfg.encryptAlgorithm = 0; #endif -#if defined(TD_ENTERPRISE) +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) pCfg->tdbEncryptAlgorithm = pCreate->encryptAlgorithm; if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) { tstrncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 5da3b2ce9a..cce238eb58 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -584,9 +584,10 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { SWrapperCfg *pCfgs = NULL; int32_t numOfVnodes = 0; - if (vmGetVnodeListFromFile(pMgmt, &pCfgs, &numOfVnodes) != 0) { - dInfo("failed to get vnode list from disk since %s", terrstr()); - return -1; + int32_t code = 0; + if ((code = vmGetVnodeListFromFile(pMgmt, &pCfgs, &numOfVnodes)) != 0) { + dInfo("failed to get vnode list from disk since %s", tstrerror(code)); + return code; } pMgmt->state.totalVnodes = numOfVnodes; @@ -623,8 +624,11 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL); +#endif if (taosThreadCreate(&pThread->thread, &thAttr, vmOpenVnodeInThread, pThread) != 0) { - dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); + dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(ERRNO)); } (void)taosThreadAttrDestroy(&thAttr); @@ -646,13 +650,12 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { if ((pMgmt->state.openVnodes + pMgmt->state.dropVnodes) != pMgmt->state.totalVnodes) { dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes); - terrno = TSDB_CODE_VND_INIT_FAILED; - return -1; + return terrno = TSDB_CODE_VND_INIT_FAILED; } - if (updateVnodesList && vmWriteVnodeListToFile(pMgmt) != 0) { - dError("failed to write vnode list since %s", terrstr()); - return -1; + if (updateVnodesList && (code = vmWriteVnodeListToFile(pMgmt)) != 0) { + dError("failed to write vnode list since %s", tstrerror(code)); + return code; } dInfo("successfully opened %d vnodes", pMgmt->state.totalVnodes); @@ -741,9 +744,11 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) { TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL); +#endif if (taosThreadCreate(&pThread->thread, &thAttr, vmCloseVnodeInThread, pThread) != 0) { - dError("thread:%d, failed to create thread to close vnode since %s", pThread->threadIndex, strerror(errno)); + dError("thread:%d, failed to create thread to close vnode since %s", pThread->threadIndex, strerror(ERRNO)); } (void)taosThreadAttrDestroy(&thAttr); @@ -852,8 +857,11 @@ static int32_t vmInitTimer(SVnodeMgmt *pMgmt) { TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL); +#endif if (taosThreadCreate(&pMgmt->thread, &thAttr, vmThreadFp, pMgmt) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); dError("failed to create vnode timer thread since %s", tstrerror(code)); return code; } @@ -889,19 +897,19 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { code = taosThreadRwlockInit(&pMgmt->hashLock, NULL); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _OVER; } code = taosThreadMutexInit(&pMgmt->mutex, NULL); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _OVER; } code = taosThreadMutexInit(&pMgmt->fileLock, NULL); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _OVER; } @@ -1049,7 +1057,7 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) { (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); if (taosThreadCreate(&pThread->thread, &thAttr, vmRestoreVnodeInThread, pThread) != 0) { - dError("thread:%d, failed to create thread to restore vnode since %s", pThread->threadIndex, strerror(errno)); + dError("thread:%d, failed to create thread to restore vnode since %s", pThread->threadIndex, strerror(ERRNO)); } (void)taosThreadAttrDestroy(&thAttr); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 41f3c64e7d..d4d209ceef 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -120,6 +120,7 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { } static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { +#ifdef USE_STREAM SVnodeObj *pVnode = pInfo->ahandle; const STraceId *trace = &pMsg->info.traceId; @@ -135,9 +136,11 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); +#endif } static void vmProcessStreamCtrlQueue(SQueueInfo *pInfo, STaosQall* pQall, int32_t numOfItems) { +#ifdef USE_STREAM SVnodeObj *pVnode = pInfo->ahandle; void *pItem = NULL; int32_t code = 0; @@ -163,6 +166,7 @@ static void vmProcessStreamCtrlQueue(SQueueInfo *pInfo, STaosQall* pQall, int32_ rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } +#endif } static void vmProcessStreamChkptQueue(SQueueInfo *pInfo, STaosQall* pQall, int32_t numOfItems) { diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 5196987f28..bcedcdefa6 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -17,7 +17,9 @@ #define _TD_DND_MGMT_H_ // tobe deleted +#ifdef USE_UDF #include "uv.h" +#endif #include "dmInt.h" #include "tfs.h" @@ -58,6 +60,7 @@ typedef struct { char desc[TSDB_STEP_DESC_LEN]; } SStartupInfo; +#ifdef USE_UDF typedef struct SUdfdData { bool startCalled; bool needCleanUp; @@ -71,6 +74,7 @@ typedef struct SUdfdData { int32_t stopCalled; int32_t dnodeId; } SUdfdData; +#endif typedef struct SDnode { int8_t once; @@ -78,7 +82,9 @@ typedef struct SDnode { EDndRunStatus status; SStartupInfo startup; SDnodeData data; +#ifdef USE_UDF SUdfdData udfdData; +#endif TdThreadMutex mutex; TdFilePtr lockfile; STfs *pTfs; diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 1708a0c8d9..8e1b50a789 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -101,8 +101,8 @@ static int32_t dmCheckDiskSpace() { int32_t code = 0; code = osUpdate(); if (code != 0) { - code = 0; // ignore the error, just log it dError("failed to update os info since %s", tstrerror(code)); + code = 0; // ignore the error, just log it } if (!dmDataSpaceAvailable()) { code = TSDB_CODE_NO_DISKSPACE; diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 85d891c8d8..836bd1877c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -169,7 +169,7 @@ int32_t dmInitVars(SDnode *pDnode) { return code; } -#if defined(TD_ENTERPRISE) +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) tsiEncryptAlgorithm = pData->encryptAlgorigthm; tsiEncryptScope = pData->encryptScope; /* diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 3cb9030f60..6ce8a24844 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -154,6 +154,9 @@ int32_t dmRunDnode(SDnode *pDnode) { dmCloseNodes(pDnode); return code; } + tsDndStart = taosGetTimestampMs(); + if (tsDndStart == 0) ++tsDndStart; // avoid 0 + tsDndStartOsUptime = taosGetOsUptime(); while (1) { if (pDnode->stop) { diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 49c1bdf3df..63fb39356b 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -64,7 +64,7 @@ static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) { if (code < 0) return -1; tjsonGetInt32ValueFromDouble(pJson, "dropped", pData->dropped, code); if (code < 0) return -1; -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) tjsonGetInt32ValueFromDouble(pJson, "encryptAlgor", pData->encryptAlgorigthm, code); if (code < 0) return -1; tjsonGetInt32ValueFromDouble(pJson, "encryptScope", pData->encryptScope, code); @@ -131,7 +131,7 @@ int32_t dmReadEps(SDnodeData *pData) { if (taosStatFile(file, NULL, NULL, NULL) < 0) { dInfo("dnode file:%s not exist", file); -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) if (strlen(tsEncryptAlgorithm) > 0) { if (strcmp(tsEncryptAlgorithm, "sm4") == 0) { pData->encryptAlgorigthm = DND_CA_SM4; @@ -289,7 +289,7 @@ static int32_t dmEncodeEps(SJson *pJson, SDnodeData *pData) { if (tjsonAddIntegerToObject(pJson, "engineVer", pData->engineVer) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "clusterId", pData->clusterId) < 0) return -1; if (tjsonAddDoubleToObject(pJson, "dropped", pData->dropped) < 0) return -1; -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) if (tjsonAddDoubleToObject(pJson, "encryptAlgor", pData->encryptAlgorigthm) < 0) return -1; if (tjsonAddDoubleToObject(pJson, "encryptScope", pData->encryptScope) < 0) return -1; #endif @@ -657,7 +657,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) { pData->oldDnodeEps = taosArrayInit(1, sizeof(SDnodeEpPair)); if (pData->oldDnodeEps == NULL) { code = terrno; - dError("failed to calloc dnodeEp array since %s", strerror(errno)); + dError("failed to calloc dnodeEp array since %s", strerror(ERRNO)); goto _OVER; } diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index d5006e1e4d..36de50ed7e 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -163,7 +163,7 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) { } if (taosCloseFile(&pFile) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _OVER; } TAOS_CHECK_GOTO(taosRenameFile(file, realfile), NULL, _OVER); @@ -206,7 +206,7 @@ int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile) { } while (retryTimes < 12); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); (void)taosCloseFile(pFile); *pFile = NULL; return code; @@ -249,12 +249,12 @@ static int32_t dmWriteCheckCodeFile(char *file, char *realfile, char *key, bool } if (taosFsyncFile(pFile) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _OVER; } if (taosCloseFile(&pFile) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _OVER; } @@ -291,7 +291,7 @@ static int32_t dmWriteEncryptCodeFile(char *file, char *realfile, char *encryptC } if (taosCloseFile(&pFile) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _OVER; } @@ -373,7 +373,7 @@ _OVER: } int32_t dmUpdateEncryptKey(char *key, bool toLogFile) { -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) int32_t code = -1; int32_t lino = 0; char *machineId = NULL; @@ -499,7 +499,7 @@ _OVER: } int32_t dmGetEncryptKey() { -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) int32_t code = -1; char encryptFile[PATH_MAX] = {0}; char checkFile[PATH_MAX] = {0}; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 050c722167..38c6984578 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -833,7 +833,7 @@ typedef struct { int32_t indexForMultiAggBalance; int8_t subTableWithoutMd5; - char reserve[256]; + char reserve[TSDB_RESERVE_VALUE_LEN]; } SStreamObj; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f4226d78a3..0b70541d55 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -925,7 +925,7 @@ static int32_t mndCheckDbEncryptKey(SMnode *pMnode, SCreateDbReq *pReq) { SDnodeObj *pDnode = NULL; void *pIter = NULL; -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) if (pReq->encryptAlgorithm == TSDB_ENCRYPT_ALGO_NONE) goto _exit; TAOS_CHECK_GOTO(grantCheck(TSDB_GRANT_DB_ENCRYPTION), NULL, _exit); if (tsEncryptionKeyStat != ENCRYPT_KEY_STAT_LOADED) { diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index a6602b392b..598746dd6a 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -18,6 +18,7 @@ #include "taoserror.h" #include "tunit.h" +#ifdef USE_STREAM static void *freeStreamTasks(SArray *pTaskLevel); int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { @@ -198,6 +199,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { } void *freeStreamTasks(SArray *pTaskLevel) { + if (pTaskLevel == NULL) return NULL; int32_t numOfLevel = taosArrayGetSize(pTaskLevel); for (int32_t i = 0; i < numOfLevel; i++) { @@ -233,6 +235,7 @@ void tFreeStreamObj(SStreamObj *pStream) { taosMemoryFree(pStream->tagSchema.pSchema); } } +#endif SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 6a3b6e1cd8..48bf36687b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -294,7 +294,7 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) { static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew) { mTrace("dnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); pOld->updateTime = pNew->updateTime; -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) tstrncpy(pOld->machineId, pNew->machineId, TSDB_MACHINE_ID_LEN + 1); #endif return 0; @@ -350,7 +350,7 @@ static SDnodeObj *mndAcquireDnodeByEp(SMnode *pMnode, char *pEpStr) { pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); if (pIter == NULL) break; - if (strncasecmp(pEpStr, pDnode->ep, TSDB_EP_LEN) == 0) { + if (taosStrncasecmp(pEpStr, pDnode->ep, TSDB_EP_LEN) == 0) { sdbCancelFetch(pSdb, pIter); return pDnode; } @@ -372,7 +372,7 @@ static SDnodeObj *mndAcquireDnodeAllStatusByEp(SMnode *pMnode, char *pEpStr) { pIter = sdbFetchAll(pSdb, SDB_DNODE, pIter, (void **)&pDnode, &objStatus, true); if (pIter == NULL) break; - if (strncasecmp(pEpStr, pDnode->ep, TSDB_EP_LEN) == 0) { + if (taosStrncasecmp(pEpStr, pDnode->ep, TSDB_EP_LEN) == 0) { sdbCancelFetch(pSdb, pIter); return pDnode; } @@ -478,7 +478,7 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S CHECK_MONITOR_PARA(tsSlowLogMaxLen, DND_REASON_STATUS_MONITOR_SLOW_LOG_SQL_MAX_LEN_NOT_MATCH); CHECK_MONITOR_PARA(tsSlowLogScope, DND_REASON_STATUS_MONITOR_SLOW_LOG_SCOPE_NOT_MATCH); - if (0 != strcasecmp(pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb)) { + if (0 != taosStrcasecmp(pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb)) { mError("dnode:%d, tsSlowLogExceptDb:%s inconsistent with cluster:%s", pDnode->id, pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb); terrno = TSDB_CODE_DNODE_INVALID_MONITOR_PARAS; @@ -492,20 +492,20 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S return DND_REASON_STATUS_INTERVAL_NOT_MATCH; } - if ((0 != strcasecmp(pCfg->timezone, tsTimezoneStr)) && (pMnode->checkTime != pCfg->checkTime)) { + if ((0 != taosStrcasecmp(pCfg->timezone, tsTimezoneStr)) && (pMnode->checkTime != pCfg->checkTime)) { mError("dnode:%d, timezone:%s checkTime:%" PRId64 " inconsistent with cluster %s %" PRId64, pDnode->id, pCfg->timezone, pCfg->checkTime, tsTimezoneStr, pMnode->checkTime); terrno = TSDB_CODE_DNODE_INVALID_TIMEZONE; return DND_REASON_TIME_ZONE_NOT_MATCH; } - if (0 != strcasecmp(pCfg->locale, tsLocale)) { + if (0 != taosStrcasecmp(pCfg->locale, tsLocale)) { mError("dnode:%d, locale:%s inconsistent with cluster:%s", pDnode->id, pCfg->locale, tsLocale); terrno = TSDB_CODE_DNODE_INVALID_LOCALE; return DND_REASON_LOCALE_NOT_MATCH; } - if (0 != strcasecmp(pCfg->charset, tsCharset)) { + if (0 != taosStrcasecmp(pCfg->charset, tsCharset)) { mError("dnode:%d, charset:%s inconsistent with cluster:%s", pDnode->id, pCfg->charset, tsCharset); terrno = TSDB_CODE_DNODE_INVALID_CHARSET; return DND_REASON_CHARSET_NOT_MATCH; @@ -1424,7 +1424,7 @@ _exit: static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) { int32_t code = 0; -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) SMnode *pMnode = pReq->info.node; SMCfgDnodeReq cfgReq = {0}; TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index f5dac9df65..abc2e7bdcf 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -81,7 +81,7 @@ int32_t mndInitIdx(SMnode *pMnode) { static int32_t mndFindSuperTableTagId(const SStbObj *pStb, const char *tagName, int8_t *hasIdx) { for (int32_t tag = 0; tag < pStb->numOfTags; tag++) { - if (strcasecmp(pStb->pTags[tag].name, tagName) == 0) { + if (taosStrcasecmp(pStb->pTags[tag].name, tagName) == 0) { if (IS_IDX_ON(&pStb->pTags[tag])) { *hasIdx = 1; } @@ -936,7 +936,7 @@ int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxOb pIter = sdbFetch(pSdb, SDB_IDX, pIter, (void **)&pIdx); if (pIter == NULL) break; - if (pIdx->stbUid == pStb->uid && strcasecmp(pIdx->colName, tagName) == 0) { + if (pIdx->stbUid == pStb->uid && taosStrcasecmp(pIdx->colName, tagName) == 0) { memcpy((char *)idx, (char *)pIdx, sizeof(SIdxObj)); sdbRelease(pSdb, pIdx); sdbCancelFetch(pSdb, pIter); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 09314c9e63..c8e5de06ac 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -204,6 +204,18 @@ static void mndStreamCheckNode(SMnode *pMnode) { } } +static void mndStreamCheckStatus(SMnode *pMnode) { + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + if (pReq != NULL) { + SRpcMsg rpcMsg = {.msgType = TDMT_MND_CHECK_STREAM_TIMER, .pCont = pReq, .contLen = contLen}; + // TODO check return value + if (tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg) < 0) { + mError("failed to put into write-queue since %s, line:%d", terrstr(), __LINE__); + } + } +} + static void mndStreamConsensusChkpt(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); @@ -375,6 +387,7 @@ static int32_t minCronTime() { } void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { int32_t code = 0; +#ifndef TD_ASTRA if (sec % tsTtlPushIntervalSec == 0) { mndPullupTtl(pMnode); } @@ -382,11 +395,12 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { if (sec % tsTrimVDbIntervalSec == 0) { mndPullupTrimDb(pMnode); } - +#endif +#ifdef USE_S3 if (tsS3MigrateEnabled && sec % tsS3MigrateIntervalSec == 0) { mndPullupS3MigrateDb(pMnode); } - +#endif if (sec % tsTransPullupInterval == 0) { mndPullupTrans(pMnode); } @@ -394,11 +408,12 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { if (sec % tsCompactPullupInterval == 0) { mndPullupCompacts(pMnode); } - +#ifdef USE_TOPIC if (sec % tsMqRebalanceInterval == 0) { mndCalMqRebalance(pMnode); } - +#endif +#ifdef USE_STREAM if (sec % 30 == 0) { // send the checkpoint info every 30 sec mndStreamCheckpointTimer(pMnode); } @@ -407,22 +422,28 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { mndStreamCheckNode(pMnode); } + if (sec % (tsStreamFailedTimeout/1000) == 0) { + mndStreamCheckStatus(pMnode); + } + if (sec % 5 == 0) { mndStreamConsensusChkpt(pMnode); } - +#endif +#ifdef USE_REPORT if (sec % tsTelemInterval == (TMIN(86400, (tsTelemInterval - 1)))) { mndPullupTelem(pMnode); } - +#endif +#ifndef TD_ASTRA if (sec % tsGrantHBInterval == 0) { mndPullupGrant(pMnode); } - +#endif if (sec % tsUptimeInterval == 0) { mndIncreaseUpTime(pMnode); } - +#ifndef TD_ASTRA if (sec % (tsArbHeartBeatIntervalSec) == 0) { if ((code = mndPullupArbHeartbeat(pMnode)) != 0) { mError("failed to pullup arb heartbeat, since:%s", tstrerror(code)); @@ -434,6 +455,7 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { mError("failed to pullup arb check sync, since:%s", tstrerror(code)); } } +#endif } void mndDoTimerCheckTask(SMnode *pMnode, int64_t sec) { if (sec % (tsStatusInterval * 5) == 0) { @@ -476,6 +498,9 @@ static int32_t mndInitTimer(SMnode *pMnode) { TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL); +#endif if ((code = taosThreadCreate(&pMnode->thread, &thAttr, mndThreadFp, pMnode)) != 0) { mError("failed to create timer thread since %s", tstrerror(code)); TAOS_RETURN(code); @@ -524,7 +549,7 @@ static int32_t mndInitWal(SMnode *pMnode) { .encryptAlgorithm = 0, .encryptKey = {0}}; -#if defined(TD_ENTERPRISE) +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL) { cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL) ? tsiEncryptAlgorithm : 0; if (tsEncryptKey[0] == '\0') { @@ -897,7 +922,7 @@ _OVER: pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER || pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT || pMsg->msgType == TDMT_MND_S3MIGRATE_DB_TIMER || pMsg->msgType == TDMT_MND_ARB_HEARTBEAT_TIMER || - pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) { + pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER || pMsg->msgType == TDMT_MND_CHECK_STREAM_TIMER) { mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, pMnode->stopped, state.restored, syncStr(state.state)); TAOS_RETURN(code); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index e318529cf8..8ea00dc0cb 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -103,7 +103,7 @@ END: if (pPlan) nodesDestroyNode((SNode*)pPlan); TAOS_RETURN(code); } - +#ifdef USE_STREAM int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) { STaskOutputInfo* pInfo = &pTask->outputInfo; @@ -814,7 +814,9 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t skey, SAr TAOS_RETURN(code); } +#endif +#ifdef USE_TOPIC int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) { int32_t code = 0; SSdb* pSdb = pMnode->pSdb; @@ -911,3 +913,4 @@ END: qDestroyQueryPlan(pPlan); return code; } +#endif diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 05b9826ae8..7ec8c0777f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -846,7 +846,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) { for (int32_t col = 0; col < pStb->numOfColumns; ++col) { SSchema *pSchema = &pStb->pColumns[col]; - if (strncasecmp(pSchema->name, colName, TSDB_COL_NAME_LEN) == 0) { + if (taosStrncasecmp(pSchema->name, colName, TSDB_COL_NAME_LEN) == 0) { return pSchema; } } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7fe5c5fb80..483108e8f4 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -40,6 +40,8 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); +static int32_t mndProcessFailedStreamReq(SRpcMsg *pReq); +static int32_t mndProcessCheckStreamStatusReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq); @@ -95,6 +97,8 @@ int32_t mndInitStream(SMnode *pMnode) { }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); + mndSetMsgHandle(pMnode, TDMT_MND_FAILED_STREAM, mndProcessFailedStreamReq); + mndSetMsgHandle(pMnode, TDMT_MND_CHECK_STREAM_TIMER, mndProcessCheckStreamStatusReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq); mndSetMsgHandle(pMnode, TDMT_MND_NODECHECK_TIMER, mndProcessNodeCheck); @@ -229,7 +233,7 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream) { } static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream) { - mTrace("stream:%s, perform delete action", pStream->name); + mInfo("stream:%s, perform delete action", pStream->name); taosWLockLatch(&pStream->lock); tFreeStreamObj(pStream); taosWUnLockLatch(&pStream->lock); @@ -246,7 +250,14 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream pOldStream->updateTime = pNewStream->updateTime; pOldStream->checkpointId = pNewStream->checkpointId; pOldStream->checkpointFreq = pNewStream->checkpointFreq; - + if (pOldStream->tasks == NULL){ + pOldStream->tasks = pNewStream->tasks; + pNewStream->tasks = NULL; + } + if (pOldStream->pHTasksList == NULL){ + pOldStream->pHTasksList = pNewStream->pHTasksList; + pNewStream->pHTasksList = NULL; + } taosWUnLockLatch(&pOldStream->lock); return 0; } @@ -346,7 +357,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, snprintf(p, tListLen(p), "%s_%s", pObj->name, "fillhistory"); pObj->hTaskUid = mndGenerateUid(pObj->name, strlen(pObj->name)); - pObj->status = 0; + pObj->status = STREAM_STATUS__NORMAL; pObj->conf.igExpired = pCreate->igExpired; pObj->conf.trigger = pCreate->triggerType; @@ -362,7 +373,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, code = terrno; mInfo("stream:%s failed to create, source db %s not exist since %s", pCreate->name, pObj->sourceDb, tstrerror(code)); - goto FAIL; + goto _ERR; } pObj->sourceDbUid = pSourceDb->uid; @@ -375,7 +386,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, code = terrno; mError("stream:%s failed to create, target db %s not exist since %s", pCreate->name, pObj->targetDb, tstrerror(code)); - goto FAIL; + goto _ERR; } tstrncpy(pObj->targetDb, pTargetDb->name, TSDB_DB_FNAME_LEN); @@ -396,12 +407,12 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, // deserialize ast if ((code = nodesStringToNode(pObj->ast, &pAst)) < 0) { - goto FAIL; + goto _ERR; } // create output schema if ((code = createSchemaByFields(pCreate->pCols, &pObj->outputSchema)) != TSDB_CODE_SUCCESS) { - goto FAIL; + goto _ERR; } int32_t numOfNULL = taosArrayGetSize(pCreate->fillNullCols); @@ -410,7 +421,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SSchema *pFullSchema = taosMemoryCalloc(pObj->outputSchema.nCols, sizeof(SSchema)); if (!pFullSchema) { code = terrno; - goto FAIL; + goto _ERR; } int32_t nullIndex = 0; @@ -471,12 +482,12 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, // using ast and param to build physical plan if ((code = qCreateQueryPlan(&cxt, &pPlan, NULL)) < 0) { - goto FAIL; + goto _ERR; } // save physcial plan if ((code = nodesNodeToString((SNode *)pPlan, false, &pObj->physicalPlan, NULL)) != 0) { - goto FAIL; + goto _ERR; } pObj->tagSchema.nCols = pCreate->numOfTags; @@ -484,7 +495,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, pObj->tagSchema.pSchema = taosMemoryCalloc(pCreate->numOfTags, sizeof(SSchema)); if (pObj->tagSchema.pSchema == NULL) { code = terrno; - goto FAIL; + goto _ERR; } } @@ -502,7 +513,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, memcpy(pObj->tagSchema.pSchema[i].name, pField->name, TSDB_COL_NAME_LEN); } -FAIL: +_ERR: if (pAst != NULL) nodesDestroyNode(pAst); if (pPlan != NULL) qDestroyQueryPlan(pPlan); return code; @@ -732,11 +743,11 @@ static int32_t doStreamCheck(SMnode *pMnode, SStreamObj *pStreamObj) { ++numOfStream; } - sdbRelease(pMnode->pSdb, pStream); if (numOfStream > MND_STREAM_MAX_NUM) { mError("too many streams, no more than %d for each database, failed to create stream:%s", MND_STREAM_MAX_NUM, pStreamObj->name); + sdbRelease(pMnode->pSdb, pStream); sdbCancelFetch(pMnode->pSdb, pIter); return TSDB_CODE_MND_TOO_MANY_STREAMS; } @@ -744,9 +755,11 @@ static int32_t doStreamCheck(SMnode *pMnode, SStreamObj *pStreamObj) { if (pStream->targetStbUid == pStreamObj->targetStbUid) { mError("Cannot write the same stable as other stream:%s, failed to create stream:%s", pStream->name, pStreamObj->name); + sdbRelease(pMnode->pSdb, pStream); sdbCancelFetch(pMnode->pSdb, pIter); return TSDB_CODE_MND_INVALID_TARGET_TABLE; } + sdbRelease(pMnode->pSdb, pStream); } return TSDB_CODE_SUCCESS; @@ -823,6 +836,58 @@ _end: return code; } +static int32_t mndProcessCheckStreamStatusReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SStreamObj *pStream = NULL; + void *pIter = NULL; + + while ((pIter = sdbFetch(pMnode->pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) { + taosWLockLatch(&pStream->lock); + if (pStream->status == STREAM_STATUS__INIT && (taosGetTimestampMs() - pStream->createTime > tsStreamFailedTimeout || + taosGetTimestampMs() - pStream->createTime < 0)){ + pStream->status = STREAM_STATUS__FAILED; + tstrncpy(pStream->reserve, "timeout", sizeof(pStream->reserve)); + mInfo("stream:%s, set status to failed success because of timeout", pStream->name); + } + taosWUnLockLatch(&pStream->lock); + sdbRelease(pMnode->pSdb, pStream); + } + + return 0; +} + +static int32_t mndProcessFailedStreamReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SStreamObj *pStream = NULL; + int32_t code = TSDB_CODE_SUCCESS; + int32_t errCode = *(int32_t*)pReq->pCont; + char streamName[TSDB_STREAM_FNAME_LEN] = {0}; + memcpy(streamName, POINTER_SHIFT(pReq->pCont,INT_BYTES), TMIN(pReq->contLen - INT_BYTES, TSDB_STREAM_FNAME_LEN - 1)); + +#ifdef WINDOWS + code = TSDB_CODE_MND_INVALID_PLATFORM; + return code; +#endif + + mInfo("stream:%s, start to set stream failed", streamName); + + code = mndAcquireStream(pMnode, streamName, &pStream); + if (pStream == NULL) { + mError("stream:%s, failed to get stream when failed stream since %s", streamName, tstrerror(code)); + return code; + } + + taosWLockLatch(&pStream->lock); + pStream->status = STREAM_STATUS__FAILED; + tstrncpy(pStream->reserve, tstrerror(errCode), sizeof(pStream->reserve)); + taosWUnLockLatch(&pStream->lock); + mndReleaseStream(pMnode, pStream); + + mInfo("stream:%s, end to set stream failed success", streamName); + + return code; +} + static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; @@ -851,14 +916,16 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { code = mndAcquireStream(pMnode, createReq.name, &pStream); if (pStream != NULL && code == 0) { - if (createReq.igExists) { - mInfo("stream:%s, already exist, ignore exist is set", createReq.name); - mndReleaseStream(pMnode, pStream); - tFreeSCMCreateStreamReq(&createReq); - return code; - } else { - code = TSDB_CODE_MND_STREAM_ALREADY_EXIST; - goto _OVER; + if (pStream->tasks != NULL){ + if (createReq.igExists) { + mInfo("stream:%s, already exist, ignore exist is set", createReq.name); + mndReleaseStream(pMnode, pStream); + tFreeSCMCreateStreamReq(&createReq); + return code; + } else { + code = TSDB_CODE_MND_STREAM_ALREADY_EXIST; + goto _OVER; + } } } else if (code != TSDB_CODE_MND_STREAM_NOT_EXIST) { goto _OVER; @@ -900,88 +967,82 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } + bool buildEmptyStream = false; + if (createReq.lastTs == 0 && createReq.fillHistory != STREAM_FILL_HISTORY_OFF){ + streamObj.status = STREAM_STATUS__INIT; + buildEmptyStream = true; + } + + if ((code = mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, streamObj.sourceDb)) != 0) { + goto _OVER; + } + + if ((code = mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, streamObj.targetDb)) != 0) { + goto _OVER; + } + code = doStreamCheck(pMnode, &streamObj); TSDB_CHECK_CODE(code, lino, _OVER); + // schedule stream task for stream obj + if (!buildEmptyStream) { + code = mndScheduleStream(pMnode, &streamObj, createReq.lastTs, createReq.pVgroupVerList); + if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("stream:%s, failed to schedule since %s", createReq.name, tstrerror(code)); + goto _OVER; + } + // add notify info into all stream tasks + code = addStreamNotifyInfo(&createReq, &streamObj); + if (code != TSDB_CODE_SUCCESS) { + mError("stream:%s failed to add stream notify info since %s", createReq.name, tstrerror(code)); + goto _OVER; + } + + // add into buffer firstly + // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already. + streamMutexLock(&execInfo.lock); + mDebug("stream stream:%s start to register tasks into task nodeList and set initial checkpointId", createReq.name); + saveTaskAndNodeInfoIntoBuf(&streamObj, &execInfo); + streamMutexUnlock(&execInfo.lock); + } + code = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, pMsg, &pTrans); if (pTrans == NULL || code) { goto _OVER; } // create stb for stream - if (createReq.createStb == STREAM_CREATE_STABLE_TRUE) { + if (createReq.createStb == STREAM_CREATE_STABLE_TRUE && !buildEmptyStream) { if ((code = mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user)) < 0) { mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createReq.name, tstrerror(code)); - mndTransDrop(pTrans); goto _OVER; } } else { mDebug("stream:%s no need create stable", createReq.name); } - // schedule stream task for stream obj - code = mndScheduleStream(pMnode, &streamObj, createReq.lastTs, createReq.pVgroupVerList); - if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("stream:%s, failed to schedule since %s", createReq.name, tstrerror(code)); - mndTransDrop(pTrans); - goto _OVER; - } - - // add notify info into all stream tasks - code = addStreamNotifyInfo(&createReq, &streamObj); - if (code != TSDB_CODE_SUCCESS) { - mError("stream:%s failed to add stream notify info since %s", createReq.name, tstrerror(code)); - mndTransDrop(pTrans); - goto _OVER; - } - // add stream to trans code = mndPersistStream(pTrans, &streamObj); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stream:%s, failed to persist since %s", createReq.name, tstrerror(code)); - mndTransDrop(pTrans); goto _OVER; } - if ((code = mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, streamObj.sourceDb)) != 0) { - mndTransDrop(pTrans); - goto _OVER; - } - - if ((code = mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, streamObj.targetDb)) != 0) { - mndTransDrop(pTrans); - goto _OVER; - } - - // add into buffer firstly - // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already. - streamMutexLock(&execInfo.lock); - mDebug("stream stream:%s start to register tasks into task nodeList and set initial checkpointId", createReq.name); - saveTaskAndNodeInfoIntoBuf(&streamObj, &execInfo); - streamMutexUnlock(&execInfo.lock); - // execute creation code = mndTransPrepare(pMnode, pTrans); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code)); - mndTransDrop(pTrans); goto _OVER; } - mndTransDrop(pTrans); - SName dbname = {0}; - code = tNameFromString(&dbname, createReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - if (code) { + if (tNameFromString(&dbname, createReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE) != 0) { mError("invalid source dbname:%s in create stream, code:%s", createReq.sourceDB, tstrerror(code)); - goto _OVER; } SName name = {0}; - code = tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_TABLE); - if (code) { + if (tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_TABLE) != 0) { mError("invalid stream name:%s in create strem, code:%s", createReq.name, tstrerror(code)); - goto _OVER; } // reuse this function for stream @@ -1001,6 +1062,7 @@ _OVER: code = TSDB_CODE_ACTION_IN_PROGRESS; } + mndTransDrop(pTrans); mndReleaseStream(pMnode, pStream); tFreeSCMCreateStreamReq(&createReq); tFreeStreamObj(&streamObj); @@ -2352,6 +2414,23 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) { return tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } +static int32_t mndProcessStatusCheck(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + if (sdbGetSize(pSdb, SDB_STREAM) <= 0) { + return 0; + } + + int32_t size = sizeof(SMStreamNodeCheckMsg); + SMStreamNodeCheckMsg *pMsg = rpcMallocCont(size); + if (pMsg == NULL) { + return terrno; + } + + SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_NODECHANGE_CHECK, .pCont = pMsg, .contLen = size}; + return tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); +} + void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { SStreamTaskIter *pIter = NULL; int32_t code = createStreamTaskIter(pStream, &pIter); diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 0caaccc28d..214736d75e 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -995,6 +995,8 @@ static void mndShowStreamStatus(char *dst, int8_t status) { tstrncpy(dst, "recover", MND_STREAM_TRIGGER_NAME_SIZE); } else if (status == STREAM_STATUS__PAUSE) { tstrncpy(dst, "paused", MND_STREAM_TRIGGER_NAME_SIZE); + } else if (status == STREAM_STATUS__INIT) { + tstrncpy(dst, "init", MND_STREAM_TRIGGER_NAME_SIZE); } } @@ -1108,7 +1110,7 @@ int32_t setStreamAttrInResBlock(SStreamObj *pStream, SSDataBlock *pBlock, int32_ TSDB_CHECK_CODE(code, lino, _end); int8_t streamStatus = atomic_load_8(&pStream->status); - if (isPaused) { + if (isPaused && pStream->tasks != NULL) { streamStatus = STREAM_STATUS__PAUSE; } mndShowStreamStatus(status2, streamStatus); @@ -1207,6 +1209,17 @@ int32_t setStreamAttrInResBlock(SStreamObj *pStream, SSDataBlock *pBlock, int32_ TSDB_CHECK_NULL(pColInfo, code, lino, _end, terrno); code = colDataSetVal(pColInfo, numOfRows, (const char *)dstStr, false); + TSDB_CHECK_CODE(code, lino, _end); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TSDB_CHECK_NULL(pColInfo, code, lino, _end, terrno); + char msg[TSDB_RESERVE_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; + if (streamStatus == STREAM_STATUS__FAILED){ + STR_TO_VARSTR(msg, pStream->reserve) + } else { + STR_TO_VARSTR(msg, " ") + } + code = colDataSetVal(pColInfo, numOfRows, (const char *)msg, false); _end: if (code) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 83fad45919..57e1e01f2c 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -795,7 +795,7 @@ int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, c if (pIter == NULL) break; if (pTrans->oper == oper) { - if (strcasecmp(dbname, pTrans->dbname) == 0) { + if (taosStrcasecmp(dbname, pTrans->dbname) == 0) { mInfo("trans:%d, db:%s oper:%d matched with input", pTrans->id, dbname, oper); taosWLockLatch(&pTrans->lockRpcArray); if (pTrans->pRpcArray == NULL) { @@ -874,13 +874,13 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { static bool mndCheckDbConflict(const char *conflict, STrans *pTrans) { if (conflict[0] == 0) return false; - if (strcasecmp(conflict, pTrans->dbname) == 0) return true; + if (taosStrcasecmp(conflict, pTrans->dbname) == 0) return true; return false; } static bool mndCheckStbConflict(const char *conflict, STrans *pTrans) { if (conflict[0] == 0) return false; - if (strcasecmp(conflict, pTrans->stbname) == 0) return true; + if (taosStrcasecmp(conflict, pTrans->stbname) == 0) return true; return false; } @@ -996,7 +996,7 @@ int32_t mndTransCheckConflictWithCompact(SMnode *pMnode, STrans *pTrans) { thisConflict = true; } if (pTrans->conflict == TRN_CONFLICT_DB || pTrans->conflict == TRN_CONFLICT_DB_INSIDE) { - if (strcasecmp(pTrans->dbname, pCompact->dbname) == 0) thisConflict = true; + if (taosStrcasecmp(pTrans->dbname, pCompact->dbname) == 0) thisConflict = true; } if (thisConflict) { @@ -1292,10 +1292,14 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { int32_t mndTransProcessRsp(SRpcMsg *pRsp) { int32_t code = 0; SMnode *pMnode = pRsp->info.node; +#ifndef TD_ASTRA_32 int64_t signature = (int64_t)(pRsp->info.ahandle); int32_t transId = (int32_t)(signature >> 32); int32_t action = (int32_t)((signature << 32) >> 32); - +#else + int32_t transId = (int32_t)(pRsp->info.ahandle); + int32_t action = (int32_t)(pRsp->info.ahandleEx); +#endif STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans == NULL) { code = TSDB_CODE_MND_RETURN_VALUE_NULL; @@ -1418,11 +1422,18 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio TAOS_RETURN(TSDB_CODE_MND_TRANS_CTX_SWITCH); } +#ifndef TD_ASTRA_32 int64_t signature = pTrans->id; signature = (signature << 32); signature += pAction->id; SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .info.ahandle = (void *)signature}; +#else + SRpcMsg rpcMsg = {.msgType = pAction->msgType, + .contLen = pAction->contLen, + .info.ahandle = (void *)pTrans->id, + .info.ahandleEx = (void *)pAction->id}; +#endif rpcMsg.pCont = rpcMallocCont(pAction->contLen); if (rpcMsg.pCont == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 0e9d2dd1a1..5cfa8bd82c 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -15,7 +15,9 @@ #define _DEFAULT_SOURCE // clang-format off +#ifndef TD_ASTRA #include +#endif #include "mndUser.h" #include "audit.h" #include "mndDb.h" @@ -688,11 +690,12 @@ void mndCleanupUser(SMnode *pMnode) { ipWhiteMgtCleanup(); } static void ipRangeToStr(SIpV4Range *range, char *buf) { struct in_addr addr; addr.s_addr = range->ip; - +#ifndef TD_ASTRA (void)uv_inet_ntop(AF_INET, &addr, buf, 32); if (range->mask != 32) { (void)tsnprintf(buf + strlen(buf), 36 - strlen(buf), "/%d", range->mask); } +#endif return; } static bool isDefaultRange(SIpV4Range *pRange) { @@ -839,12 +842,13 @@ static int32_t createDefaultIpWhiteList(SIpWhiteList **ppWhiteList) { } (*ppWhiteList)->num = 1; SIpV4Range *range = &((*ppWhiteList)->pIpRange[0]); - +#ifndef TD_ASTRA struct in_addr addr; if (uv_inet_pton(AF_INET, "127.0.0.1", &addr) == 0) { range->ip = addr.s_addr; range->mask = 32; } +#endif return 0; } @@ -2330,6 +2334,7 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode TAOS_CHECK_GOTO(mndRemoveTablePriviledge(pMnode, pAlterTbs, pNewUser->useDbs, pAlterReq, pSdb), &lino, _OVER); } +#ifdef USE_TOPIC if (ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) { int32_t len = strlen(pAlterReq->objname) + 1; SMqTopicObj *pTopic = NULL; @@ -2357,7 +2362,7 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode } mndReleaseTopic(pMnode, pTopic); } - +#endif _OVER: if (code < 0) { mError("user:%s, failed to alter user privileges at line %d since %s", pAlterReq->user, lino, tstrerror(code)); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index ff3d87819b..f1c57f7030 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -154,13 +154,13 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { static int32_t sdbCreateDir(SSdb *pSdb) { int32_t code = 0; if (taosMulMkDir(pSdb->currDir) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); mError("failed to create dir:%s since %s", pSdb->currDir, tstrerror(code)); TAOS_RETURN(code); } if (taosMkDir(pSdb->tmpDir) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); mError("failed to create dir:%s since %s", pSdb->tmpDir, tstrerror(code)); TAOS_RETURN(code); } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 4b1404e41d..6e7a0a450c 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -182,7 +182,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { int64_t maxId = 0; ret = taosReadFile(pFile, &maxId, sizeof(int64_t)); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); TAOS_RETURN(code); } if (ret != sizeof(int64_t)) { @@ -196,7 +196,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { int64_t ver = 0; ret = taosReadFile(pFile, &ver, sizeof(int64_t)); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); TAOS_RETURN(code); } if (ret != sizeof(int64_t)) { @@ -556,7 +556,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb, int32_t skip_type) { int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen); if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { - code = errno; + code = terrno; taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); break; @@ -576,7 +576,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb, int32_t skip_type) { if (code == 0) { code = taosFsyncFile(pFile); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); mError("failed to sync sdb file:%s since %s", tmpfile, tstrerror(code)); } } @@ -822,13 +822,13 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i } if (taosFsyncFile(pIter->file) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); mError("sdbiter:%p, failed to fasync file %s since %s", pIter, pIter->name, tstrerror(code)); goto _OVER; } if (taosCloseFile(&pIter->file) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _OVER; } pIter->file = NULL; diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index 37131300c6..7df1fcd485 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -136,7 +136,7 @@ int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val) { TAOS_RETURN(code); } - *(int64_t *)(pRaw->pData + dataPos) = val; + taosSetInt64Aligned((int64_t *)(pRaw->pData + dataPos), val); return 0; } @@ -297,7 +297,7 @@ int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val) { TAOS_RETURN(code); } - *val = *(int64_t *)(pRaw->pData + dataPos); + taosSetPInt64Aligned(val, (int64_t *)(pRaw->pData + dataPos)); return 0; } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index dcdc70da68..991205629f 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -72,7 +72,7 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { code = startRsync(); if (code != 0) { terrno = code; - goto FAIL; + goto _ERR; } pSnode->msgCb = pOption->msgCb; @@ -80,13 +80,13 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { taosGetTimestampMs(), tqStartTaskCompleteCallback, &pSnode->pMeta); if (code != TSDB_CODE_SUCCESS) { terrno = code; - goto FAIL; + goto _ERR; } streamMetaLoadAllTasks(pSnode->pMeta); return pSnode; -FAIL: +_ERR: taosMemoryFree(pSnode); return NULL; } diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 6cc907f152..33b9e87427 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -164,7 +164,7 @@ int32_t metaFilterTableName(void* pVnode, SMetaFltParam* param, SArray* pUids); int32_t metaFilterTtl(void* pVnode, SMetaFltParam* param, SArray* pUids); int32_t metaGetColCmpr(SMeta* pMeta, tb_uid_t uid, SHashObj** colCmprObj); -#ifndef META_REFACT +#if !defined(META_REFACT) && !defined(TD_ASTRA) // SMetaDB int metaOpenDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 1778b7cda8..0c8333a85f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -334,6 +334,7 @@ struct STsdbFS { }; typedef struct { +#ifdef USE_ROCKSDB rocksdb_t *db; rocksdb_comparator_t *my_comparator; rocksdb_block_based_table_options_t *tableoptions; @@ -343,6 +344,7 @@ typedef struct { rocksdb_readoptions_t *readoptions; rocksdb_writebatch_t *writebatch; TdThreadMutex writeBatchMutex; +#endif int32_t sver; tb_uid_t suid; tb_uid_t uid; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 635f9946d5..8658e2df4c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -19,7 +19,9 @@ #include "executor.h" #include "filter.h" #include "qworker.h" +#ifdef USE_ROCKSDB #include "rocksdb/c.h" +#endif #include "sync.h" #include "tRealloc.h" #include "tchecksum.h" @@ -273,7 +275,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgL int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessSeekReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessPollPush(STQ* pTq); int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg); @@ -466,12 +468,12 @@ typedef struct { } SVATaskID; struct SVnode { - char* path; - SVnodeCfg config; SVState state; SVStatis statis; + char* path; STfs* pTfs; int32_t diskPrimary; + SVnodeCfg config; SMsgCb msgCb; bool disableWrite; diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index b23059079a..5a768f2e3e 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -999,7 +999,7 @@ static void metaTimeSeriesNotifyCheck(SMeta *pMeta) { if (deltaTS > tsTimeSeriesThreshold) { if (0 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) { if (tsem_post(&dmNotifyHdl.sem) != 0) { - metaError("vgId:%d, failed to post semaphore, errno:%d", TD_VID(pMeta->pVnode), errno); + metaError("vgId:%d, failed to post semaphore, errno:%d", TD_VID(pMeta->pVnode), ERRNO); } } } diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index e0d8c66f0c..0cad2d60db 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -16,6 +16,21 @@ #include "meta.h" #include "vnd.h" +#ifndef NO_UNALIGNED_ACCESS +#define TDB_KEY_ALIGN(k1, k2, kType) +#else +#define TDB_KEY_ALIGN(k1, k2, kType) \ + kType _k1, _k2; \ + if (((uintptr_t)(k1) & 7)) { \ + memcpy(&_k1, (k1), sizeof(kType)); \ + (k1) = &_k1; \ + } \ + if (((uintptr_t)(k2) & 7)) { \ + memcpy(&_k2, (k2), sizeof(kType)); \ + (k2) = &_k2; \ + } +#endif + static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); @@ -248,7 +263,7 @@ _exit: metaDebug("vgId:%d %s success", TD_VID(pVnode), __func__); *ppMeta = pMeta; } - return code; + TAOS_RETURN(code); } void vnodeGetMetaPath(SVnode *pVnode, const char *metaDir, char *fname) { @@ -424,14 +439,14 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { || (metaBackupExists && metaExists && metaTempExists) // ) { metaError("vgId:%d, invalid meta state, please check!", TD_VID(pVnode)); - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_FAILED); } else if (!metaBackupExists && metaExists && metaTempExists) { taosRemoveDir(metaTempDir); } else if (metaBackupExists && !metaExists && metaTempExists) { code = taosRenameFile(metaTempDir, metaDir); if (code) { metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code)); - return code; + TAOS_RETURN(code); } taosRemoveDir(metaBackupDir); } else if (metaBackupExists && metaExists && !metaTempExists) { @@ -442,14 +457,14 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { code = metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, rollback); if (code) { metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code)); - return code; + TAOS_RETURN(code); } if (generateNewMeta) { code = metaGenerateNewMeta(ppMeta); if (code) { metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code)); - return code; + TAOS_RETURN(code); } } @@ -551,6 +566,8 @@ static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen STbDbKey *pTbDbKey1 = (STbDbKey *)pKey1; STbDbKey *pTbDbKey2 = (STbDbKey *)pKey2; + TDB_KEY_ALIGN(pTbDbKey1, pTbDbKey2, STbDbKey); + if (pTbDbKey1->version > pTbDbKey2->version) { return 1; } else if (pTbDbKey1->version < pTbDbKey2->version) { @@ -570,6 +587,8 @@ static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLe SSkmDbKey *pSkmDbKey1 = (SSkmDbKey *)pKey1; SSkmDbKey *pSkmDbKey2 = (SSkmDbKey *)pKey2; + TDB_KEY_ALIGN(pSkmDbKey1, pSkmDbKey2, SSkmDbKey); + if (pSkmDbKey1->uid > pSkmDbKey2->uid) { return 1; } else if (pSkmDbKey1->uid < pSkmDbKey2->uid) { @@ -586,8 +605,8 @@ static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLe } static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { - tb_uid_t uid1 = *(tb_uid_t *)pKey1; - tb_uid_t uid2 = *(tb_uid_t *)pKey2; + tb_uid_t uid1 = taosGetInt64Aligned((int64_t*)pKey1); + tb_uid_t uid2 = taosGetInt64Aligned((int64_t*)pKey2); if (uid1 > uid2) { return 1; @@ -602,6 +621,8 @@ static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL SCtbIdxKey *pCtbIdxKey1 = (SCtbIdxKey *)pKey1; SCtbIdxKey *pCtbIdxKey2 = (SCtbIdxKey *)pKey2; + TDB_KEY_ALIGN(pCtbIdxKey1, pCtbIdxKey2, SCtbIdxKey); + if (pCtbIdxKey1->suid > pCtbIdxKey2->suid) { return 1; } else if (pCtbIdxKey1->suid < pCtbIdxKey2->suid) { @@ -623,6 +644,8 @@ int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { tb_uid_t uid1 = 0, uid2 = 0; int c; + TDB_KEY_ALIGN(pTagIdxKey1, pTagIdxKey2, STagIdxKey); + // compare suid if (pTagIdxKey1->suid > pTagIdxKey2->suid) { return 1; @@ -682,6 +705,9 @@ int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { static int btimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { SBtimeIdxKey *pBtimeIdxKey1 = (SBtimeIdxKey *)pKey1; SBtimeIdxKey *pBtimeIdxKey2 = (SBtimeIdxKey *)pKey2; + + TDB_KEY_ALIGN(pBtimeIdxKey1, pBtimeIdxKey2, SBtimeIdxKey); + if (pBtimeIdxKey1->btime > pBtimeIdxKey2->btime) { return 1; } else if (pBtimeIdxKey1->btime < pBtimeIdxKey2->btime) { @@ -701,6 +727,8 @@ static int ncolIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen SNcolIdxKey *pNcolIdxKey1 = (SNcolIdxKey *)pKey1; SNcolIdxKey *pNcolIdxKey2 = (SNcolIdxKey *)pKey2; + TDB_KEY_ALIGN(pNcolIdxKey1, pNcolIdxKey2, SNcolIdxKey); + if (pNcolIdxKey1->ncol > pNcolIdxKey2->ncol) { return 1; } else if (pNcolIdxKey1->ncol < pNcolIdxKey2->ncol) { @@ -720,6 +748,8 @@ static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL SSmaIdxKey *pSmaIdxKey1 = (SSmaIdxKey *)pKey1; SSmaIdxKey *pSmaIdxKey2 = (SSmaIdxKey *)pKey2; + TDB_KEY_ALIGN(pSmaIdxKey1, pSmaIdxKey2, SSmaIdxKey); + if (pSmaIdxKey1->uid > pSmaIdxKey2->uid) { return 1; } else if (pSmaIdxKey1->uid < pSmaIdxKey2->uid) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 25c98d0e56..378d5cd7a4 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -171,14 +171,15 @@ int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchem indexMultiTermDestroy(terms); taosArrayDestroy(pTagVals); -#endif return code; _exception: indexMultiTermDestroy(terms); taosArrayDestroy(pTagVals); +#endif return code; } int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) { +int32_t code = 0; #ifdef USE_INVERTED_INDEX if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { return TSDB_CODE_INVALID_PARA; @@ -192,7 +193,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche int32_t nTagData = 0; SArray *pTagVals = NULL; - int32_t code = tTagToValArray((const STag *)data, &pTagVals); + code = tTagToValArray((const STag *)data, &pTagVals); if (code) { return code; } @@ -253,11 +254,11 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche code = indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); indexMultiTermDestroy(terms); taosArrayDestroy(pTagVals); -#endif return code; _exception: indexMultiTermDestroy(terms); taosArrayDestroy(pTagVals); +#endif return code; } @@ -730,7 +731,7 @@ int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_ return terrno; } - (*ppTagIdxKey)->suid = suid; + taosSetInt64Aligned(&((*ppTagIdxKey)->suid), suid); (*ppTagIdxKey)->cid = cid; (*ppTagIdxKey)->isNull = (pTagData == NULL) ? 1 : 0; (*ppTagIdxKey)->type = type; @@ -739,10 +740,10 @@ int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_ if (IS_VAR_DATA_TYPE(type)) { memcpy((*ppTagIdxKey)->data, (uint16_t *)&nTagData, VARSTR_HEADER_SIZE); if (pTagData != NULL) memcpy((*ppTagIdxKey)->data + VARSTR_HEADER_SIZE, pTagData, nTagData); - *(tb_uid_t *)((*ppTagIdxKey)->data + VARSTR_HEADER_SIZE + nTagData) = uid; + taosSetInt64Aligned((tb_uid_t *)((*ppTagIdxKey)->data + VARSTR_HEADER_SIZE + nTagData), uid); } else { if (pTagData != NULL) memcpy((*ppTagIdxKey)->data, pTagData, nTagData); - *(tb_uid_t *)((*ppTagIdxKey)->data + nTagData) = uid; + taosSetInt64Aligned((tb_uid_t *)((*ppTagIdxKey)->data + nTagData), uid); } return 0; diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 45912b5ae1..ba486bece5 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -423,7 +423,7 @@ static int32_t tdRsmaStartExecutor(const SSma *pSma) { for (int32_t i = 0; i < tsNumOfVnodeRsmaThreads; ++i) { if (taosThreadCreate(&pthread[i], &thAttr, tdRSmaExecutorFunc, (void *)pSma) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); smaError("vgId:%d, failed to create pthread for rsma since %s", SMA_VID(pSma), tstrerror(code)); TAOS_RETURN(code); } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ad3e4d015a..05564ffdea 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -288,7 +288,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat TAOS_RETURN(terrno); } if (taosMulMkDir(s) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); taosMemoryFree(s); TAOS_RETURN(code); } diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index c9868f0398..f3208bd88e 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -36,9 +36,13 @@ int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) { } int32_t tdProcessTSmaCreate(SSma *pSma, int64_t ver, const char *msg) { +#ifdef USE_TSMA int32_t code = tdProcessTSmaCreateImpl(pSma, ver, msg); TAOS_RETURN(code); +#else + return TSDB_CODE_SUCCESS; +#endif } int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c80edd070d..6e61df2511 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -366,7 +366,7 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { return 0; } -int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg) { +int32_t tqProcessPollPush(STQ* pTq) { if (pTq == NULL) { return TSDB_CODE_INVALID_PARA; } diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 00e6104bd0..2dd4bec173 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -138,7 +138,7 @@ int32_t streamStateSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS pWriter->ever = ever; if (taosMkDir(pTq->pStreamMeta->path) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); tqError("vgId:%d, vnode %s snapshot writer failed to create directory %s since %s", TD_VID(pTq->pVnode), STREAM_STATE_TRANSFER, pTq->pStreamMeta->path, tstrerror(code)); goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 4dff1d08d9..637e07e618 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -29,7 +29,8 @@ void tsdbLRUCacheRelease(SLRUCache *cache, LRUHandle *handle, bool eraseIfLastRe } static int32_t tsdbOpenBCache(STsdb *pTsdb) { - int32_t code = 0, lino = 0; + int32_t code = 0, lino = 0; +#ifdef USE_S3 int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; int64_t szBlock = tsS3BlockSize <= 1024 ? 1024 : tsS3BlockSize; SLRUCache *pCache = taosLRUCacheInit((int64_t)tsS3BlockCacheSize * szBlock * szPage, 0, .5); @@ -48,11 +49,12 @@ _err: tsdbError("tsdb/bcache: vgId:%d, %s failed at line %d since %s.", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } - +#endif TAOS_RETURN(code); } static void tsdbCloseBCache(STsdb *pTsdb) { +#ifdef USE_S3 SLRUCache *pCache = pTsdb->bCache; if (pCache) { int32_t elems = taosLRUCacheGetElems(pCache); @@ -65,10 +67,12 @@ static void tsdbCloseBCache(STsdb *pTsdb) { (void)taosThreadMutexDestroy(&pTsdb->bMutex); } +#endif } static int32_t tsdbOpenPgCache(STsdb *pTsdb) { int32_t code = 0, lino = 0; +#ifdef USE_S3 int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; SLRUCache *pCache = taosLRUCacheInit((int64_t)tsS3PageCacheSize * szPage, 0, .5); @@ -86,11 +90,12 @@ _err: if (code) { tsdbError("tsdb/pgcache: vgId:%d, open failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); } - +#endif TAOS_RETURN(code); } static void tsdbClosePgCache(STsdb *pTsdb) { +#ifdef USE_S3 SLRUCache *pCache = pTsdb->pgCache; if (pCache) { int32_t elems = taosLRUCacheGetElems(pCache); @@ -103,6 +108,7 @@ static void tsdbClosePgCache(STsdb *pTsdb) { (void)taosThreadMutexDestroy(&pTsdb->bMutex); } +#endif } #define ROCKS_KEY_LEN (sizeof(tb_uid_t) + sizeof(int16_t) + sizeof(int8_t)) @@ -166,7 +172,7 @@ static int myCmp(void *state, const char *a, size_t alen, const char *b, size_t static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { int32_t code = 0, lino = 0; - +#ifdef USE_ROCKSDB rocksdb_comparator_t *cmp = rocksdb_comparator_create(NULL, myCmpDestroy, myCmp, myCmpName); if (NULL == cmp) { TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); @@ -252,11 +258,12 @@ _err2: rocksdb_block_based_options_destroy(tableoptions); _err: rocksdb_comparator_destroy(cmp); - +#endif TAOS_RETURN(code); } static void tsdbCloseRocksCache(STsdb *pTsdb) { +#ifdef USE_ROCKSDB rocksdb_close(pTsdb->rCache.db); (void)taosThreadMutexDestroy(&pTsdb->rCache.writeBatchMutex); rocksdb_flushoptions_destroy(pTsdb->rCache.flushoptions); @@ -268,9 +275,11 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) { rocksdb_comparator_destroy(pTsdb->rCache.my_comparator); taosMemoryFree(pTsdb->rCache.pTSchema); taosArrayDestroy(pTsdb->rCache.ctxArray); +#endif } static void rocksMayWrite(STsdb *pTsdb, bool force) { +#ifdef USE_ROCKSDB rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; int count = rocksdb_writebatch_count(wb); @@ -286,6 +295,7 @@ static void rocksMayWrite(STsdb *pTsdb, bool force) { rocksdb_writebatch_clear(wb); } +#endif } typedef struct { @@ -769,23 +779,24 @@ int32_t tsdbCacheCommit(STsdb *pTsdb) { char *err = NULL; SLRUCache *pCache = pTsdb->lruCache; - rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; + // rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; (void)taosThreadMutexLock(&pTsdb->lruMutex); taosLRUCacheApply(pCache, tsdbCacheFlushDirty, pTsdb); +#ifdef USE_ROCKSDB rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); - +#endif (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - +#ifdef USE_ROCKSDB if (NULL != err) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); rocksdb_free(err); code = TSDB_CODE_FAILED; } - +#endif TAOS_RETURN(code); } @@ -895,7 +906,7 @@ static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, i int32_t code = 0, lino = 0; SLRUCache *pCache = pTsdb->lruCache; - rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; + // rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; SRowKey emptyRowKey = {.ts = TSKEY_MIN, .numOfPKs = 0}; SLastCol emptyCol = { .rowKey = emptyRowKey, .colVal = COL_VAL_NONE(cid, col_type), .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_VALID}; @@ -914,24 +925,24 @@ int32_t tsdbCacheCommitNoLock(STsdb *pTsdb) { char *err = NULL; SLRUCache *pCache = pTsdb->lruCache; - rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; + // rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; taosLRUCacheApply(pCache, tsdbCacheFlushDirty, pTsdb); - +#ifdef USE_ROCKSDB rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); - if (NULL != err) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); rocksdb_free(err); code = TSDB_CODE_FAILED; } - +#endif TAOS_RETURN(code); } static int32_t tsdbCacheGetValuesFromRocks(STsdb *pTsdb, size_t numKeys, const char *const *ppKeysList, size_t *pKeysListSizes, char ***pppValuesList, size_t **ppValuesListSizes) { +#ifdef USE_ROCKSDB char **valuesList = taosMemoryCalloc(numKeys, sizeof(char *)); if (!valuesList) return terrno; size_t *valuesListSizes = taosMemoryCalloc(numKeys, sizeof(size_t)); @@ -954,6 +965,7 @@ static int32_t tsdbCacheGetValuesFromRocks(STsdb *pTsdb, size_t numKeys, const c *pppValuesList = valuesList; *ppValuesListSizes = valuesListSizes; +#endif TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -995,9 +1007,11 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, TAOS_CHECK_GOTO(tsdbCacheGetValuesFromRocks(pTsdb, 2, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes), NULL, _exit); - +#ifdef USE_ROCKSDB rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; +#endif { +#ifdef USE_ROCKSDB SLastCol *pLastCol = NULL; if (values_list[0] != NULL) { code = tsdbCacheDeserialize(values_list[0], values_list_sizes[0], &pLastCol); @@ -1028,6 +1042,7 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, rocksdb_free(values_list[0]); rocksdb_free(values_list[1]); +#endif for (int i = 0; i < 2; i++) { LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[i], klen); @@ -1346,6 +1361,7 @@ static void tsdbCacheUpdateLastColToNone(SLastCol *pLastCol, ELastCacheStatus ca static int32_t tsdbCachePutToRocksdb(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLastCol) { int32_t code = 0; +#ifdef USE_ROCKSDB char *rocks_value = NULL; size_t vlen = 0; @@ -1361,7 +1377,7 @@ static int32_t tsdbCachePutToRocksdb(STsdb *pTsdb, SLastKey *pLastKey, SLastCol (void)taosThreadMutexUnlock(&pTsdb->rCache.writeBatchMutex); taosMemoryFree(rocks_value); - +#endif TAOS_RETURN(code); } @@ -1482,7 +1498,7 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray goto _exit; } - rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; + // rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; for (int i = 0; i < num_keys; ++i) { SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, idxKey->idx); @@ -1553,9 +1569,11 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray taosMemoryFree(keys_list); taosMemoryFree(keys_list_sizes); if (values_list) { +#ifdef USE_ROCKSDB for (int i = 0; i < num_keys; ++i) { rocksdb_free(values_list[i]); } +#endif taosMemoryFree(values_list); } taosMemoryFree(values_list_sizes); @@ -1754,7 +1772,7 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SArray *remainCols, SCacheRowsReader *pr, int8_t ltype) { int32_t code = 0, lino = 0; - rocksdb_writebatch_t *wb = NULL; + // rocksdb_writebatch_t *wb = NULL; SArray *pTmpColArray = NULL; bool extraTS = false; @@ -2007,9 +2025,11 @@ _exit: taosMemoryFree(keys_list); taosMemoryFree(keys_list_sizes); if (values_list) { + #ifdef USE_ROCKSDB for (int i = 0; i < num_keys; ++i) { rocksdb_free(values_list[i]); } + #endif taosMemoryFree(values_list); } taosMemoryFree(values_list_sizes); @@ -2707,7 +2727,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE &values_list, &values_list_sizes), NULL, _exit); - rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; + // rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; for (int i = 0; i < numKeys; ++i) { SLastCol *pLastCol = NULL; if (values_list[i] != NULL) { @@ -2757,9 +2777,11 @@ _exit: taosMemoryFree(keys_list); taosMemoryFree(keys_list_sizes); if (values_list) { + #if USE_ROCKSDB for (int i = 0; i < numKeys; ++i) { rocksdb_free(values_list[i]); } +#endif taosMemoryFree(values_list); } taosMemoryFree(values_list_sizes); @@ -4083,6 +4105,7 @@ int32_t tsdbCacheGetElems(SVnode *pVnode) { return elems; } +#ifdef USE_S3 // block cache static void getBCacheKey(int32_t fid, int64_t commitID, int64_t blkno, char *key, int *len) { struct { @@ -4202,3 +4225,4 @@ void tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_t * tsdbCacheRelease(pFD->pTsdb->pgCache, handle); } +#endif diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index a949fd69f0..a26165c4f0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -122,7 +122,7 @@ static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) { } if (taosFsyncFile(pFD) < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(ERRNO), lino, _exit); } _exit: @@ -744,7 +744,7 @@ static int32_t tsdbFSRollback(STsdb *pTsdb) { _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(errno)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(ERRNO)); } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index ae78511e8b..54eb2e4fad 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -286,7 +286,7 @@ static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) { tsdbRemoveFile(fobj->fname); return; } - +#ifdef USE_S3 if (!remove_all) { // remove local last chunk file char lc_path[TSDB_FILENAME_LEN]; @@ -330,6 +330,7 @@ static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) { tsdbRemoveFile(lc_path); } +#endif } int32_t tsdbTFileObjRemove(STFileObj *fobj) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 3d50c81a5d..81531e7ec8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -342,7 +342,7 @@ _exit: } return code; } - +#ifdef USE_S3 static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { int32_t code = 0; int32_t lino; @@ -409,8 +409,9 @@ _exit: } return code; } - +#endif static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) { +#ifdef USE_S3 int32_t code = 0; int32_t lino; int64_t n = 0; @@ -513,6 +514,9 @@ _exit: TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code); } return code; +#else + return TSDB_CODE_INTERNAL_ERROR; +#endif } int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint, @@ -568,7 +572,7 @@ int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) TSDB_CHECK_CODE(code, lino, _exit); if (taosFsyncFile(pFD->pFD) < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(ERRNO), lino, _exit); } _exit: diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 60fab24218..3732cbab96 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -356,7 +356,9 @@ static int32_t tsdbRetention(void *arg) { // do retention if (rtner.fset) { if (rtnArg->s3Migrate) { +#ifdef USE_S3 TAOS_CHECK_GOTO(tsdbDoS3Migrate(&rtner), &lino, _exit); +#endif } else { TAOS_CHECK_GOTO(tsdbDoRetention(&rtner), &lino, _exit); } @@ -426,6 +428,7 @@ int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now) { return code; } +#ifdef USE_S3 static int32_t tsdbS3FidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int32_t s3KeepLocal, int64_t nowSec) { int32_t localFid; TSKEY key; @@ -777,10 +780,14 @@ static int32_t tsdbGetS3SizeImpl(STsdb *tsdb, int64_t *size) { return code; } +#endif + int32_t tsdbGetS3Size(STsdb *tsdb, int64_t *size) { int32_t code = 0; +#ifdef USE_S3 (void)taosThreadMutexLock(&tsdb->mutex); code = tsdbGetS3SizeImpl(tsdb, size); (void)taosThreadMutexUnlock(&tsdb->mutex); +#endif return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c index cf0eee62f6..073476f01b 100644 --- a/source/dnode/vnode/src/vnd/vnodeAsync.c +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -438,6 +438,12 @@ static int32_t vnodeAsyncDestroy(SVAsync **async) { } static void vnodeAsyncLaunchWorker(SVAsync *async) { + TdThreadAttr thAttr; + (void)taosThreadAttrInit(&thAttr); + (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL); +#endif for (int32_t i = 0; i < async->numWorkers; i++) { if (async->workers[i].state == EVA_WORKER_STATE_ACTIVE) { continue; @@ -446,7 +452,7 @@ static void vnodeAsyncLaunchWorker(SVAsync *async) { async->workers[i].state = EVA_WORKER_STATE_UINIT; } - int32_t ret = taosThreadCreate(&async->workers[i].thread, NULL, vnodeAsyncLoop, &async->workers[i]); + int32_t ret = taosThreadCreate(&async->workers[i].thread, &thAttr, vnodeAsyncLoop, &async->workers[i]); if (ret) { vError("failed to create worker thread since %s", tstrerror(ret)); } else { @@ -455,6 +461,9 @@ static void vnodeAsyncLaunchWorker(SVAsync *async) { } break; } +#ifdef TD_ASTRA + (void)taosThreadAttrDestroy(&thAttr); +#endif } int32_t vnodeAsyncOpen() { diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 5dd5f56a8d..5d2669ce01 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -48,7 +48,7 @@ static int32_t vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBu if (taosThreadSpinInit(pPool->lock, 0) != 0) { taosMemoryFree((void *)pPool->lock); taosMemoryFree(pPool); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } } else { pPool->lock = NULL; @@ -227,7 +227,7 @@ void vnodeBufPoolAddToFreeList(SVBufPool *pPool) { SVBufPool *pNewPool = NULL; if (vnodeBufPoolCreate(pVnode, pPool->id, size, &pNewPool) < 0) { vWarn("vgId:%d, failed to change buffer pool of id %d size from %" PRId64 " to %" PRId64 " since %s", - TD_VID(pVnode), pPool->id, pPool->node.size, size, tstrerror(errno)); + TD_VID(pVnode), pPool->id, pPool->node.size, size, tstrerror(ERRNO)); } else { vInfo("vgId:%d, buffer pool of id %d size changed from %" PRId64 " to %" PRId64, TD_VID(pVnode), pPool->id, pPool->node.size, size); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 653f525e09..21a274abf1 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -260,7 +260,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { } tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code); if (code) return code; -#if defined(TD_ENTERPRISE) +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) { if (tsEncryptKey[0] == 0) { return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; @@ -287,7 +287,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (code) return code; tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code); if (code) return code; -#if defined(TD_ENTERPRISE) +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) { if (tsEncryptKey[0] == 0) { return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; @@ -298,7 +298,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { #endif tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code); if (code) return code; -#if defined(TD_ENTERPRISE) +#if defined(TD_ENTERPRISE) || defined(TD_ASTRA_TODO) if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) { if (tsEncryptKey[0] == 0) { return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 7fd3f9a8b6..eeb90a41b1 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -206,7 +206,7 @@ _exit: vError("vgId:%d, failed to close file", pInfo->config.vgId); } taosMemoryFree(data); - return code; + TAOS_RETURN(code); } int vnodeCommitInfo(const char *dir) { diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index e8e176c4f1..06582fa7d2 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -115,6 +115,7 @@ void initMetadataAPI(SStoreMeta* pMeta) { } void initTqAPI(SStoreTqReader* pTq) { +#ifdef USE_TQ pTq->tqReaderOpen = tqReaderOpen; pTq->tqReaderSetColIdList = tqReaderSetColIdList; @@ -146,9 +147,11 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqGetResultBlockTime = tqGetResultBlockTime; pTq->tqGetStreamExecProgress = tqGetStreamExecInfo; +#endif } void initStateStoreAPI(SStateStore* pStore) { +#ifdef USE_STREAM pStore->streamFileStateInit = streamFileStateInit; pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; @@ -246,6 +249,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateDestroy = streamStateDestroy; pStore->streamStateReloadInfo = streamStateReloadInfo; pStore->streamStateCopyBackend = streamStateCopyBackend; +#endif } void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { @@ -268,8 +272,10 @@ void initMetaFilterAPI(SMetaDataFilterAPI* pFilter) { } void initFunctionStateStore(SFunctionStateStore* pStore) { +#ifdef USE_STREAM pStore->streamStateFuncPut = streamStateFuncPut; pStore->streamStateFuncGet = streamStateFuncGet; +#endif } void initCacheFn(SStoreCacheReader* pCache) { diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 26b8d310e8..3301585c45 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -51,8 +51,8 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs // create vnode env if (vnodeMkDir(pTfs, path)) { - vError("vgId:%d, failed to prepare vnode dir since %s, path: %s", pCfg->vgId, strerror(errno), path); - return TAOS_SYSTEM_ERROR(errno); + vError("vgId:%d, failed to prepare vnode dir since %s, path: %s", pCfg->vgId, strerror(ERRNO), path); + return TAOS_SYSTEM_ERROR(ERRNO); } vnodeGetPrimaryDir(path, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); @@ -341,11 +341,13 @@ void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId) { } // int32_t nlevel = tfsGetLevel(pTfs); +#ifdef USE_S3 if (nodeId > 0 && vgId > 0 /*&& nlevel > 1*/ && tsS3Enabled) { char vnode_prefix[TSDB_FILENAME_LEN]; snprintf(vnode_prefix, TSDB_FILENAME_LEN, "%d/v%df", nodeId, vgId); tcsDeleteObjectsByPrefix(vnode_prefix); } +#endif } static int32_t vnodeCheckDisk(int32_t diskPrimary, STfs *pTfs) { @@ -386,7 +388,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } if (vnodeMkDir(pTfs, path)) { - vError("vgId:%d, failed to prepare vnode dir since %s, path: %s", info.config.vgId, strerror(errno), path); + vError("vgId:%d, failed to prepare vnode dir since %s, path: %s", info.config.vgId, strerror(ERRNO), path); return NULL; } // save vnode info on dnode ep changed diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 34894825f2..5625b12901 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -115,9 +115,9 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { // query meta metaReaderDoInit(&mer1, pVnode->pMeta, META_READER_LOCK); if (reqTbUid) { - errno = 0; + SET_ERRNO(0); uint64_t tbUid = taosStr2UInt64(infoReq.tbName, NULL, 10); - if (errno == ERANGE || tbUid == 0) { + if (ERRNO == ERANGE || tbUid == 0) { code = TSDB_CODE_TDB_TABLE_NOT_EXIST; goto _exit3; } diff --git a/source/dnode/vnode/src/vnd/vnodeRetention.c b/source/dnode/vnode/src/vnd/vnodeRetention.c index 6dca7a9a60..9bcaae14c6 100644 --- a/source/dnode/vnode/src/vnd/vnodeRetention.c +++ b/source/dnode/vnode/src/vnd/vnodeRetention.c @@ -25,5 +25,9 @@ int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now) { int32_t vnodeAsyncS3Migrate(SVnode *pVnode, int64_t now) { // async migration +#ifdef USE_S3 return tsdbAsyncS3Migrate(pVnode->pTsdb, now); +#else + return TSDB_CODE_INTERNAL_ERROR; +#endif } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 9e3c6861c0..feec2404b9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -217,10 +217,11 @@ static void vnodeSnapReaderDestroyTsdbRanges(SVSnapReader *pReader) { void vnodeSnapReaderClose(SVSnapReader *pReader) { vInfo("vgId:%d, close vnode snapshot reader", TD_VID(pReader->pVnode)); vnodeSnapReaderDestroyTsdbRanges(pReader); - +#ifdef USE_RSMA if (pReader->pRsmaReader) { rsmaSnapReaderClose(&pReader->pRsmaReader); } +#endif if (pReader->pTsdbReader) { tsdbSnapReaderClose(&pReader->pTsdbReader); @@ -233,7 +234,7 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) { if (pReader->pMetaReader) { metaSnapReaderClose(&pReader->pMetaReader); } - +#ifdef USE_TQ if (pReader->pTqSnapReader) { tqSnapReaderClose(&pReader->pTqSnapReader); } @@ -245,7 +246,7 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) { if (pReader->pTqCheckInfoReader) { tqSnapReaderClose(&pReader->pTqCheckInfoReader); } - +#endif taosMemoryFree(pReader); } @@ -363,6 +364,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) } // TQ ================ +#ifdef USE_TQ vInfo("vgId:%d tq transform start", vgId); if (!pReader->tqHandleDone) { if (pReader->pTqSnapReader == NULL) { @@ -412,8 +414,9 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) tqSnapReaderClose(&pReader->pTqOffsetReader); } } - +#endif // STREAM ============ +#ifdef USE_STREAM vInfo("vgId:%d stream task start to take snapshot", vgId); if (!pReader->streamTaskDone) { if (pReader->pStreamTaskReader == NULL) { @@ -454,8 +457,9 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) pReader->pStreamStateReader = NULL; } } - +#endif // RSMA ============== +#ifdef USE_RSMA if (VND_IS_RSMA(pReader->pVnode) && !pReader->rsmaDone) { // open if not if (pReader->pRsmaReader == NULL) { @@ -472,7 +476,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) rsmaSnapReaderClose(&pReader->pRsmaReader); } } - +#endif *ppData = NULL; *nData = 0; @@ -674,12 +678,12 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * code = tsdbSnapRAWWriterPrepareClose(pWriter->pTsdbSnapRAWWriter); if (code) goto _exit; } - +#ifdef USE_RSMA if (pWriter->pRsmaSnapWriter) { code = rsmaSnapWriterPrepareClose(pWriter->pRsmaSnapWriter, rollback); if (code) goto _exit; } - +#endif // commit json if (!rollback) { pWriter->info.state.committed = pWriter->ever; @@ -715,7 +719,7 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * code = tsdbSnapRAWWriterClose(&pWriter->pTsdbSnapRAWWriter, rollback); if (code) goto _exit; } - +#ifdef USE_TQ if (pWriter->pTqSnapHandleWriter) { code = tqSnapWriterClose(&pWriter->pTqSnapHandleWriter, rollback); if (code) goto _exit; @@ -730,7 +734,8 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * code = tqSnapWriterClose(&pWriter->pTqSnapOffsetWriter, rollback); if (code) goto _exit; } - +#endif +#ifdef USE_STREAM if (pWriter->pStreamTaskWriter) { code = streamTaskSnapWriterClose(pWriter->pStreamTaskWriter, rollback, pWriter->pStreamStateWriter == NULL ? 1 : 0); @@ -745,12 +750,13 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * pWriter->pStreamStateWriter = NULL; if (code) goto _exit; } - +#endif +#ifdef USE_RSMA if (pWriter->pRsmaSnapWriter) { code = rsmaSnapWriterClose(&pWriter->pRsmaSnapWriter, rollback); if (code) goto _exit; } - +#endif code = vnodeBegin(pVnode); if (code) goto _exit; @@ -857,6 +863,7 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { code = tsdbSnapRAWWrite(pWriter->pTsdbSnapRAWWriter, pHdr); TSDB_CHECK_CODE(code, lino, _exit); } break; +#ifdef USE_TQ case SNAP_DATA_TQ_HANDLE: { // tq handle if (pWriter->pTqSnapHandleWriter == NULL) { @@ -887,6 +894,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { code = tqSnapOffsetWrite(pWriter->pTqSnapOffsetWriter, pData, nData); TSDB_CHECK_CODE(code, lino, _exit); } break; +#endif +#ifdef USE_STREAM case SNAP_DATA_STREAM_TASK: case SNAP_DATA_STREAM_TASK_CHECKPOINT: { if (pWriter->pStreamTaskWriter == NULL) { @@ -905,6 +914,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { TSDB_CHECK_CODE(code, lino, _exit); } break; +#endif +#ifdef USE_RSMA case SNAP_DATA_RSMA1: case SNAP_DATA_RSMA2: case SNAP_DATA_QTASK: { @@ -918,6 +929,7 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { code = rsmaSnapWrite(pWriter->pRsmaSnapWriter, pData, nData); TSDB_CHECK_CODE(code, lino, _exit); } break; +#endif default: break; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index ea4ca896ab..df6e377261 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -87,10 +87,10 @@ static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, i if (uid == 0) { uid = tGenIdPI64(); } - *(int64_t *)(pCoder->data + pCoder->pos) = uid; + taosSetInt64Aligned((int64_t *)(pCoder->data + pCoder->pos), uid); // btime - *(int64_t *)(pCoder->data + pCoder->pos + 8) = btime; + taosSetInt64Aligned((int64_t *)(pCoder->data + pCoder->pos + 8), btime); tEndDecode(pCoder); @@ -274,7 +274,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int } if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { - *(int64_t *)(pCoder->data + pCoder->pos) = uid; + taosSetInt64Aligned((int64_t *)(pCoder->data + pCoder->pos), uid); pCoder->pos += sizeof(int64_t); } else { if (tDecodeI64(pCoder, &submitTbData.uid) < 0) { @@ -353,8 +353,12 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int for (int32_t iRow = 0; iRow < nRow; ++iRow) { SRow *pRow = (SRow *)(pCoder->data + pCoder->pos); pCoder->pos += pRow->len; - +#ifndef NO_UNALIGNED_ACCESS if (pRow->ts < minKey || pRow->ts > maxKey) { +#else + TSKEY ts = taosGetInt64Aligned(&pRow->ts); + if (ts < minKey || ts > maxKey) { +#endif code = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; TSDB_CHECK_CODE(code, lino, _exit); } @@ -362,7 +366,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int } if (!tDecodeIsEnd(pCoder)) { - *(int64_t *)(pCoder->data + pCoder->pos) = ctimeMs; + taosSetInt64Aligned((int64_t *)(pCoder->data + pCoder->pos), ctimeMs); pCoder->pos += sizeof(int64_t); } @@ -687,6 +691,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg if (vnodeProcessBatchDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; /* TQ */ +#if defined(USE_TQ) || defined(USE_STREAM) case TDMT_VND_TMQ_SUBSCRIBE: if (tqProcessSubscribeReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; @@ -757,6 +762,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg } } break; +#endif case TDMT_VND_ALTER_CONFIRM: needCommit = pVnode->config.hashChange; if (vnodeProcessAlterConfirmReq(pVnode, ver, pReq, len, pRsp) < 0) { @@ -882,7 +888,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { case TDMT_VND_TMQ_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_VND_TMQ_CONSUME_PUSH: - return tqProcessPollPush(pVnode->pTq, pMsg); + return tqProcessPollPush(pVnode->pTq); default: vError("unknown msg type:%d in query queue", pMsg->msgType); return TSDB_CODE_APP_ERROR; @@ -925,19 +931,20 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { #endif // case TDMT_VND_TMQ_CONSUME: // return tqProcessPollReq(pVnode->pTq, pMsg); +#ifdef USE_TQ case TDMT_VND_TMQ_VG_WALINFO: return tqProcessVgWalInfoReq(pVnode->pTq, pMsg); case TDMT_VND_TMQ_VG_COMMITTEDINFO: return tqProcessVgCommittedInfoReq(pVnode->pTq, pMsg); case TDMT_VND_TMQ_SEEK: return tqProcessSeekReq(pVnode->pTq, pMsg); - +#endif default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_APP_ERROR; } } - +#ifdef USE_STREAM int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("vgId:%d, msg:%p in stream queue is processing", pVnode->config.vgId, pMsg); @@ -1035,6 +1042,7 @@ int32_t vnodeProcessStreamChkptMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pI return TSDB_CODE_APP_ERROR; } } +#endif void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { int32_t code = tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); @@ -1794,7 +1802,7 @@ static int32_t vnodeCellValConvertToColVal(STColumn *pCol, SCellVal *pCellVal, S float f = GET_FLOAT_VAL(pCellVal->val); memcpy(&pColVal->value.val, &f, sizeof(f)); } else if (TSDB_DATA_TYPE_DOUBLE == pCol->type) { - pColVal->value.val = *(int64_t *)pCellVal->val; + taosSetPInt64Aligned(&pColVal->value.val, (int64_t *)pCellVal->val); } else { GET_TYPED_DATA(pColVal->value.val, int64_t, pCol->type, pCellVal->val); } @@ -1974,7 +1982,12 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in SRow **aRow = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP); SRowKey lastRowKey; for (int32_t iRow = 0; iRow < nRow; ++iRow) { +#ifndef NO_UNALIGNED_ACCESS if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey) { +#else + TSKEY ts = taosGetInt64Aligned(&(aRow[iRow]->ts)); + if (ts < minKey || ts > maxKey) { +#endif code = TSDB_CODE_INVALID_MSG; vError("vgId:%d %s failed 2 since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), ver); goto _exit; @@ -2183,6 +2196,7 @@ _exit: } static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { +#ifdef USE_TSMA SVCreateTSmaReq req = {0}; SDecoder coder = {0}; @@ -2217,6 +2231,9 @@ _err: vError("vgId:%d, failed to create tsma %s:%" PRIi64 " version %" PRIi64 "for table %" PRIi64 " since %s", TD_VID(pVnode), req.indexName, req.indexUid, ver, req.tableUid, terrstr()); return terrno; +#else + return TSDB_CODE_INTERNAL_ERROR; +#endif } /** @@ -2300,10 +2317,10 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe } if (pVnode->config.szCache != req.pages) { - if (metaAlterCache(pVnode->pMeta, req.pages) < 0) { + if ((terrno = metaAlterCache(pVnode->pMeta, req.pages)) < 0) { vError("vgId:%d, failed to change vnode pages from %d to %d failed since %s", TD_VID(pVnode), - pVnode->config.szCache, req.pages, tstrerror(errno)); - return errno; + pVnode->config.szCache, req.pages, tstrerror(terrno)); + return terrno; } else { vInfo("vgId:%d, vnode pages is changed from %d to %d", TD_VID(pVnode), pVnode->config.szCache, req.pages); pVnode->config.szCache = req.pages; @@ -2373,7 +2390,7 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe int32_t ret = tsdbDisableAndCancelAllBgTask(pVnode->pTsdb); if (ret != 0) { - vError("vgId:%d, failed to disable bg task since %s", TD_VID(pVnode), tstrerror(errno)); + vError("vgId:%d, failed to disable bg task since %s", TD_VID(pVnode), tstrerror(ERRNO)); } pVnode->config.sttTrigger = req.sttTrigger; @@ -2394,7 +2411,7 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe if (walChanged) { if (walAlter(pVnode->pWal, &pVnode->config.walCfg) != 0) { - vError("vgId:%d, failed to alter wal config since %s", TD_VID(pVnode), tstrerror(errno)); + vError("vgId:%d, failed to alter wal config since %s", TD_VID(pVnode), tstrerror(ERRNO)); } } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 317fd3307e..0fa12ada99 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -573,6 +573,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) walApplyVer(pVnode->pWal, commitIdx); pVnode->restored = true; +#ifdef USE_STREAM SStreamMeta *pMeta = pVnode->pTq->pStreamMeta; streamMetaWLock(pMeta); @@ -609,6 +610,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) } streamMetaWUnLock(pMeta); +#endif } static void vnodeBecomeFollower(const SSyncFSM *pFsm) { @@ -625,12 +627,14 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) { } (void)taosThreadMutexUnlock(&pVnode->lock); +#ifdef USE_TQ if (pVnode->pTq) { tqUpdateNodeStage(pVnode->pTq, false); if (tqStopStreamAllTasksAsync(pVnode->pTq->pStreamMeta, &pVnode->msgCb) != 0) { vError("vgId:%d, failed to stop stream tasks", pVnode->config.vgId); } } +#endif } static void vnodeBecomeLearner(const SSyncFSM *pFsm) { @@ -651,17 +655,21 @@ static void vnodeBecomeLearner(const SSyncFSM *pFsm) { static void vnodeBecomeLeader(const SSyncFSM *pFsm) { SVnode *pVnode = pFsm->data; vDebug("vgId:%d, become leader", pVnode->config.vgId); +#ifdef USE_TQ if (pVnode->pTq) { tqUpdateNodeStage(pVnode->pTq, true); } +#endif } static void vnodeBecomeAssignedLeader(const SSyncFSM *pFsm) { SVnode *pVnode = pFsm->data; vDebug("vgId:%d, become assigned leader", pVnode->config.vgId); +#ifdef USE_TQ if (pVnode->pTq) { tqUpdateNodeStage(pVnode->pTq, true); } +#endif } static bool vnodeApplyQueueEmpty(const SSyncFSM *pFsm) { diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 6a92c46f32..5f9236716d 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -339,8 +339,10 @@ typedef struct SCtgTSMACache { } SCtgTSMACache; typedef struct SCtgDBCache { - SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads uint64_t dbId; + uint64_t dbCacheNum[CTG_CI_MAX_VALUE]; + uint64_t dbCacheSize; + SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads int8_t deleted; SCtgVgCache vgCache; SCtgCfgCache cfgCache; @@ -349,8 +351,6 @@ typedef struct SCtgDBCache { SHashObj* stbCache; // key:suid, value:char* SHashObj* tsmaCache; // key:tbname, value: SCtgTSMACache int32_t tsmaVersion; - uint64_t dbCacheNum[CTG_CI_MAX_VALUE]; - uint64_t dbCacheSize; } SCtgDBCache; typedef struct SCtgRentSlot { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 3025fd91ad..4712a942e9 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -3307,9 +3307,12 @@ int32_t ctgStartUpdateThread() { TdThreadAttr thAttr; CTG_ERR_JRET(taosThreadAttrInit(&thAttr)); CTG_ERR_JRET(taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE)); +#ifdef TD_COMPACT_OS + CTG_ERR_JRET(taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL)); +#endif if (taosThreadCreate(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); CTG_ERR_RET(terrno); } @@ -3318,7 +3321,7 @@ int32_t ctgStartUpdateThread() { _return: if (code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); CTG_ERR_RET(terrno); } diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 17674aae82..766c40c358 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -232,15 +232,15 @@ void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache) { return; } - int32_t code = TSDB_CODE_SUCCESS; - void *pIter = taosHashIterate(dbCache->stbCache, NULL); + int32_t code = TSDB_CODE_SUCCESS; + void *pIter = taosHashIterate(dbCache->stbCache, NULL); while (pIter) { - uint64_t *suid = NULL; - suid = taosHashGetKey(pIter, NULL); + uint64_t *pSuid = taosHashGetKey(pIter, NULL); + uint64_t suid = taosGetUInt64Aligned(pSuid); - code = ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); + code = ctgMetaRentRemove(&pCtg->stbRent, suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); if (TSDB_CODE_SUCCESS == code) { - ctgDebug("suid:0x%" PRIx64 ", stb removed from rent", *suid); + ctgDebug("suid:0x%" PRIx64 ", stb removed from rent", suid); } pIter = taosHashIterate(dbCache->stbCache, pIter); diff --git a/source/libs/crypt/src/crypt.c b/source/libs/crypt/src/crypt.c index 055c2f57cd..1cd740663f 100644 --- a/source/libs/crypt/src/crypt.c +++ b/source/libs/crypt/src/crypt.c @@ -24,7 +24,7 @@ int32_t CBC_Decrypt(SCryptOpts *opts) { return CBC_DecryptImpl(opts); } -#ifndef TD_ENTERPRISE +#if !defined(TD_ENTERPRISE) && !defined(TD_ASTRA) int32_t CBC_EncryptImpl(SCryptOpts *opts) { memcpy(opts->result, opts->source, opts->len); return opts->len; diff --git a/source/libs/executor/inc/groupcache.h b/source/libs/executor/inc/groupcache.h index f50947ead7..c0ca19f330 100755 --- a/source/libs/executor/inc/groupcache.h +++ b/source/libs/executor/inc/groupcache.h @@ -24,7 +24,7 @@ extern "C" { #define GROUP_CACHE_MAX_FILE_FDS 10 #define GROUP_CACHE_DEFAULT_VGID 0 -#pragma pack(push, 1) +#pragma pack(push, 1) typedef struct SGcBlkBufBasic { int32_t fileId; int64_t blkId; diff --git a/source/libs/executor/inc/hashjoin.h b/source/libs/executor/inc/hashjoin.h index 542763ffd3..8958f833c0 100755 --- a/source/libs/executor/inc/hashjoin.h +++ b/source/libs/executor/inc/hashjoin.h @@ -30,8 +30,7 @@ extern "C" { typedef int32_t (*hJoinImplFp)(SOperatorInfo*); - -#pragma pack(push, 1) +#pragma pack(push, 1) typedef struct SBufRowInfo { void* next; uint16_t pageId; diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 1ef8317232..26e1abeee5 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -120,7 +120,7 @@ static int32_t allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, pBuf->pData = taosMemoryMalloc(pBuf->allocSize); if (pBuf->pData == NULL) { - qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno)); + qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(ERRNO)); return terrno; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index ac6b089de1..c310cfe53a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2497,7 +2497,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI SResultRow* pRow = getResultRowByPos(pBuf, &pResultRowInfo->cur, false); if (pRow) { - w = pRow->win; + TAOS_SET_OBJ_ALIGNED(&w, pRow->win); } // in case of typical time window, we can calculate time window directly. diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 7f4335282f..61547f5fb3 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1339,7 +1339,7 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s return terrno; } - pCond->twindows = TSWINDOW_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(&pCond->twindows, TSWINDOW_INITIALIZER); pCond->suid = pMtInfo->suid; pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index bdcdeb2c32..5bc4cc9e21 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -641,7 +641,7 @@ static FORCE_INLINE void initGcVgroupCtx(SOperatorInfo* pOperator, SGcVgroupCtx* pVgCtx->pTbList = pTbList; pVgCtx->id = vgId; (void)snprintf(pVgCtx->fileCtx.baseFilename, sizeof(pVgCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d_%d", - tsTempDir, getpid(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, downstreamId, vgId); + tsTempDir, taosGetPId(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, downstreamId, vgId); pVgCtx->fileCtx.baseFilename[sizeof(pVgCtx->fileCtx.baseFilename) - 1] = 0; pVgCtx->fileCtx.baseNameLen = strlen(pVgCtx->fileCtx.baseFilename); @@ -1453,7 +1453,7 @@ static int32_t initGroupCacheDownstreamCtx(SOperatorInfo* pOperator) { } (void)snprintf(pCtx->fileCtx.baseFilename, sizeof(pCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d", - tsTempDir, getpid(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, pCtx->id); + tsTempDir, taosGetPId(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, pCtx->id); pCtx->fileCtx.baseFilename[sizeof(pCtx->fileCtx.baseFilename) - 1] = 0; pCtx->fileCtx.baseNameLen = strlen(pCtx->fileCtx.baseFilename); } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 56060b0061..5ac8977377 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1012,6 +1012,7 @@ int32_t convertTagDataToStr(char* str, int32_t strBuffLen, int type, void* buf, } static int32_t sysTableGetGeomText(char* iGeom, int32_t nGeom, char** output, int32_t* nOutput) { +#ifdef USE_GEOS int32_t code = 0; char* outputWKT = NULL; @@ -1033,6 +1034,9 @@ static int32_t sysTableGetGeomText(char* iGeom, int32_t nGeom, char** output, in *nOutput = strlen(outputWKT); return code; +#else + TAOS_RETURN(TSDB_CODE_OPS_NOT_SUPPORT); +#endif } static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 6fbce065f4..956598eb6b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -59,8 +59,7 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo } // set time window for current result - pResultRow->win = (*win); - + TAOS_SET_POBJ_ALIGNED(&pResultRow->win, win); *pResult = pResultRow; return setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset); } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 17c390e239..e9178f7e0e 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -1316,7 +1316,7 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize); int32_t ret = taosReadFromCFile(pRegion->buf, readBytes, 1, pMemFile->pTdFile); if (ret != 1) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } pRegion->bufLen = readBytes; @@ -1342,7 +1342,7 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i int32_t ret = taosReadFromCFile(pRegion->buf, readBytes, 1, pMemFile->pTdFile); if (ret != 1) { taosMemoryFreeClear(*ppRow); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } memcpy(*ppRow + szThisBlock, pRegion->buf, rowLen - szThisBlock); @@ -1366,7 +1366,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) { taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath); pMemFile->pTdFile = taosOpenCFile(pMemFile->memFilePath, "w+b"); if (pMemFile->pTdFile == NULL) { - code = terrno = TAOS_SYSTEM_ERROR(errno); + code = terrno = TAOS_SYSTEM_ERROR(ERRNO); } } if (code == TSDB_CODE_SUCCESS) { @@ -1490,7 +1490,7 @@ static int32_t tsortCloseRegion(SSortHandle* pHandle) { if (writeBytes > 0) { int32_t ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile); if (ret != 1) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } pMemFile->bRegionDirty = false; @@ -1540,7 +1540,7 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p int32_t writeBytes = pMemFile->currRegionOffset - (pMemFile->writeFileOffset - pRegion->fileOffset); int32_t ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile); if (ret != 1) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } pMemFile->writeFileOffset = pRegion->fileOffset + pMemFile->currRegionOffset; diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 62989bb293..f31b70325c 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -33,6 +33,7 @@ target_link_libraries( PUBLIC uv_a ) +if(${BUILD_WITH_UDF}) add_executable(taosudf src/udfd.c) if(${TD_DARWIN}) @@ -60,3 +61,5 @@ target_link_libraries( if(${BUILD_TEST}) add_subdirectory(test) endif() + +endif(${BUILD_WITH_UDF}) \ No newline at end of file diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 8233b2d450..c2979e85a3 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -736,7 +736,7 @@ static int32_t checkFixedValue(SNode* pNode, const SParamInfo* paramPattern, int } } else { for (int32_t k = 0; k < paramPattern->fixedValueSize; k++) { - if (strcasecmp(pVal->literal, paramPattern->fixedStrValue[k]) == 0) { + if (taosStrcasecmp(pVal->literal, paramPattern->fixedStrValue[k]) == 0) { code = TSDB_CODE_SUCCESS; *isMatch = true; break; @@ -4417,6 +4417,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = NULL, .finalizeFunc = NULL }, + #ifdef USE_GEOS { .name = "st_geomfromtext", .type = FUNCTION_TYPE_GEOM_FROM_TEXT, @@ -4606,6 +4607,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = containsProperlyFunction, .finalizeFunc = NULL }, +#endif { .name = "_tbuid", .type = FUNCTION_TYPE_TBUID, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3590e3c209..9c1eb8ba11 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -566,20 +566,22 @@ int32_t countFunction(SqlFunctionCtx* pCtx) { int32_t type = pInput->pData[0]->info.type; - char* buf = GET_ROWCELL_INTERBUF(pResInfo); + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + int64_t val = *((int64_t*)buf); if (IS_NULL_TYPE(type)) { // select count(NULL) returns 0 numOfElem = 1; - *((int64_t*)buf) += 0; + val += 0; } else { numOfElem = getNumOfElems(pCtx); - *((int64_t*)buf) += numOfElem; + val += numOfElem; } + taosSetInt64Aligned((int64_t*)buf, val); if (tsCountAlwaysReturnValue) { pResInfo->numOfRes = 1; } else { - SET_VAL(pResInfo, *((int64_t*)buf), 1); + SET_VAL(pResInfo, val, 1); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 03935bf1a0..755f37ac02 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -148,6 +148,7 @@ int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) { } int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) { +#ifdef USE_UDF if (!fmIsUserDefinedFunc(funcId)) { return TSDB_CODE_FAILED; } @@ -156,6 +157,9 @@ int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) { pFpSet->process = udfAggProcess; pFpSet->finalize = udfAggFinalize; return TSDB_CODE_SUCCESS; +#else + TAOS_RETURN(TSDB_CODE_OPS_NOT_SUPPORT); +#endif } int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 747152b8c3..f03bec5a32 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -12,6 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#ifdef USE_UDF #include "uv.h" #include "os.h" @@ -2323,3 +2324,15 @@ _exit: return code; } +#else +#include "tudf.h" + +int32_t cleanUpUdfs() { return 0; } +int32_t udfcOpen() { return 0; } +int32_t udfcClose() { return 0; } +int32_t udfStartUdfd(int32_t startDnodeId) { return 0; } +void udfStopUdfd() { return; } +int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output) { + return TSDB_CODE_OPS_NOT_SUPPORT; +} +#endif diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 4bdb67a750..576e893cd5 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#ifdef USE_UDF // clang-format off #include "uv.h" #include "os.h" @@ -1078,7 +1079,7 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) { TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); if (file == NULL) { - fnError("taosudf write udf shared library: %s failed, error: %d %s", path, errno, strerror(terrno)); + fnError("taosudf write udf shared library: %s failed, error: %d %s", path, ERRNO, strerror(ERRNO)); return TSDB_CODE_FILE_CORRUPTED; } int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); @@ -1865,3 +1866,4 @@ _exit: return code; } +#endif diff --git a/source/libs/geometry/src/geomFunc.c b/source/libs/geometry/src/geomFunc.c index 2ce2d92fa7..baeb5878cc 100644 --- a/source/libs/geometry/src/geomFunc.c +++ b/source/libs/geometry/src/geomFunc.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#ifdef USE_GEOS #include #include "geosWrapper.h" #include "geomFunc.h" @@ -385,3 +386,4 @@ int32_t containsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t containsProperlyFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return geomRelationFunction(pInput, pOutput, false, doContainsProperly); } +#endif diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index 8c94dfc0da..57c6535244 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#ifdef USE_GEOS #include "geosWrapper.h" #include "tutil.h" #include "types.h" @@ -479,3 +480,4 @@ void destroyGeometry(GEOSGeometry **geom, const GEOSPreparedGeometry **preparedG *geom = NULL; } } +#endif diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index b065f0c91e..0b7b7cc8b9 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -195,21 +195,21 @@ TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t d return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_FLOAT) { float va = taosStr2Float(a, NULL); - if (errno == ERANGE && va == -1) { + if (ERRNO == ERANGE && va == -1) { return CONTINUE; } float vb = taosStr2Float(b, NULL); - if (errno == ERANGE && va == -1) { + if (ERRNO == ERANGE && va == -1) { return CONTINUE; } return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_DOUBLE) { double va = taosStr2Double(a, NULL); - if (errno == ERANGE && va == -1) { + if (ERRNO == ERANGE && va == -1) { return CONTINUE; } double vb = taosStr2Double(b, NULL); - if (errno == ERANGE && va == -1) { + if (ERRNO == ERANGE && va == -1) { return CONTINUE; } return tDoCompare(func, cmptype, &va, &vb); diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 2351273975..0219e586ca 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -230,7 +230,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int #endif } if (ctx->file.pFile == NULL) { - indexError("failed to open file, error %d", errno); + indexError("failed to open file, error %d", ERRNO); goto END; } } else if (ctx->type == TMEMORY) { @@ -307,7 +307,7 @@ int32_t idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len) { IFileCtx* ctx = write->wrt; int nWrite = ctx->write(ctx, buf, len); if (nWrite != len) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); return code; } write->count += len; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index c6c091c630..b67873a8b4 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -224,7 +224,7 @@ int32_t tfileReaderCreate(IFileCtx* ctx, TFileReader** pReader) { if ((code = tfileReaderLoadFst(reader)) != 0) { indexError("failed to load index fst, suid:%" PRIu64 ", colName:%s, code:0x%x", reader->header.suid, - reader->header.colName, errno); + reader->header.colName, ERRNO); TAOS_CHECK_GOTO(code, NULL, _End); } @@ -598,7 +598,7 @@ int32_t tfileReaderOpen(SIndex* idx, uint64_t suid, int64_t version, const char* IFileCtx* wc = idxFileCtxCreate(TFILE, fullname, true, 1024 * 1024 * 1024); if (wc == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); indexError("failed to open readonly file: %s, reason: %s", fullname, tstrerror(code)); return code; } @@ -987,7 +987,7 @@ static int tfileWriteHeader(TFileWriter* writer) { indexInfo("tfile pre write header size: %d", writer->ctx->size(writer->ctx)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); if (sizeof(buf) != nwrite) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); indexError("failed to write header, code:0x%x, filename: %s", code, writer->ctx->file.buf); return code; } @@ -1016,7 +1016,7 @@ static int tfileWriteFooter(TFileWriter* write) { indexInfo("tfile write footer size: %d", write->ctx->size(write->ctx)); if (nwrite != sizeof(FILE_MAGIC_NUMBER)) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } else { return nwrite; } @@ -1028,7 +1028,7 @@ static int tfileReaderLoadHeader(TFileReader* reader) { int64_t nread = reader->ctx->readFrom(reader->ctx, (uint8_t*)buf, sizeof(buf), 0); if (nread < 0) { - indexError("actual Read: %d, to read: %d, code:0x%x, filename: %s", (int)(nread), (int)sizeof(buf), errno, + indexError("actual Read: %d, to read: %d, code:0x%x, filename: %s", (int)(nread), (int)sizeof(buf), ERRNO, reader->ctx->file.buf); } else { indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), reader->ctx->file.buf); diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 090ff02ffb..8c2e34c997 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -285,7 +285,7 @@ static void initLog() { if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); - } + } } class IndexEnv : public ::testing::Test { protected: diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 493101eb27..0e70393496 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -40,7 +40,7 @@ static void initLog() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, 0) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 3d4aed3bc8..18553b1575 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -381,7 +381,12 @@ static int32_t tlvDecodeI64(STlv* pTlv, int64_t* pValue) { static int32_t tlvDecodeValueI64(STlvDecoder* pDecoder, int64_t* pValue) { int32_t code = tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); if (TSDB_CODE_SUCCESS == code) { +#ifndef NO_UNALIGNED_ACCESS *pValue = ntohll(*pValue); +#else + int64_t value = taosGetInt64Aligned(pValue); + taosSetInt64Aligned(pValue, ntohll(value)); +#endif } return code; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 23f647aa16..faf4dfa309 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -145,7 +145,11 @@ static int32_t nodesCallocImpl(int32_t size, void** pOut) { return TSDB_CODE_SUCCESS; } - if (g_pNodeAllocator->pCurrChunk->usedSize + size > g_pNodeAllocator->pCurrChunk->availableSize) { + int32_t alignedSize = size; +#ifdef NO_UNALIGNED_ACCESS + alignedSize = (size + 3) & (~3); +#endif + if (g_pNodeAllocator->pCurrChunk->usedSize + alignedSize > g_pNodeAllocator->pCurrChunk->availableSize) { int32_t code = callocNodeChunk(g_pNodeAllocator, NULL); if (TSDB_CODE_SUCCESS != code) { *pOut = NULL; @@ -153,25 +157,31 @@ static int32_t nodesCallocImpl(int32_t size, void** pOut) { } } void* p = g_pNodeAllocator->pCurrChunk->pBuf + g_pNodeAllocator->pCurrChunk->usedSize; - g_pNodeAllocator->pCurrChunk->usedSize += size; + g_pNodeAllocator->pCurrChunk->usedSize += alignedSize; *pOut = p; return TSDB_CODE_SUCCESS; ; } +#ifdef NO_UNALIGNED_ACCESS +#define NODE_ALLOCATOR_HEAD_LEN 4 +#else +#define NODE_ALLOCATOR_HEAD_LEN 1 +#endif + static int32_t nodesCalloc(int32_t num, int32_t size, void** pOut) { void* p = NULL; - int32_t code = nodesCallocImpl(num * size + 1, &p); + int32_t code = nodesCallocImpl(num * size + NODE_ALLOCATOR_HEAD_LEN, &p); if (TSDB_CODE_SUCCESS != code) { return code; } *(char*)p = (NULL != g_pNodeAllocator) ? 1 : 0; - *pOut = (char*)p + 1; + *pOut = (char*)p + NODE_ALLOCATOR_HEAD_LEN; return TSDB_CODE_SUCCESS; } void nodesFree(void* p) { - char* ptr = (char*)p - 1; + char* ptr = (char*)p - NODE_ALLOCATOR_HEAD_LEN; if (0 == *ptr) { taosMemoryFree(ptr); } @@ -190,7 +200,7 @@ static int32_t createNodeAllocator(int32_t chunkSize, SNodeAllocator** pAllocato return code; } if (0 != taosThreadMutexInit(&(*pAllocator)->mutex, NULL)) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 20f2ab81e0..dae98e7dff 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -14,7 +14,9 @@ * along with this program. If not, see . */ #include +#ifndef TD_ASTRA #include +#endif #include "nodes.h" #include "parAst.h" @@ -1497,7 +1499,7 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode interval->pOffset = pOffset; interval->pSliding = pSliding; interval->pFill = pFill; - interval->timeRange = TSWINDOW_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(&interval->timeRange, TSWINDOW_INITIALIZER); interval->timezone = pCxt->pQueryCxt->timezone; return (SNode*)interval; _err: @@ -3064,7 +3066,8 @@ _err: static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange) { int32_t code = TSDB_CODE_SUCCESS; - char* ipCopy = taosStrdup(ipRange); +#ifndef TD_ASTRA + char* ipCopy = taosStrdup(ipRange); if (!ipCopy) return terrno; char* slash = strchr(ipCopy, '/'); if (slash) { @@ -3097,6 +3100,7 @@ static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange } taosMemoryFreeClear(ipCopy); +#endif return code; } @@ -3794,7 +3798,7 @@ static bool validateNotifyUrl(const char* url) { if (!url || *url == '\0') return false; for (int32_t i = 0; i < ARRAY_SIZE(prefix); ++i) { - if (strncasecmp(url, prefix[i], strlen(prefix[i])) == 0) { + if (taosStrncasecmp(url, prefix[i], strlen(prefix[i])) == 0) { host = url + strlen(prefix[i]); break; } @@ -3840,9 +3844,9 @@ SNode* createStreamNotifyOptions(SAstCreateContext* pCxt, SNodeList* pAddrUrls, FOREACH(pNode, pEventTypes) { char *eventStr = ((SValueNode *)pNode)->literal; - if (strncasecmp(eventStr, eWindowOpenStr, strlen(eWindowOpenStr) + 1) == 0) { + if (taosStrncasecmp(eventStr, eWindowOpenStr, strlen(eWindowOpenStr) + 1) == 0) { BIT_FLAG_SET_MASK(eventTypes, SNOTIFY_EVENT_WINDOW_OPEN); - } else if (strncasecmp(eventStr, eWindowCloseStr, strlen(eWindowCloseStr) + 1) == 0) { + } else if (taosStrncasecmp(eventStr, eWindowCloseStr, strlen(eWindowCloseStr) + 1) == 0) { BIT_FLAG_SET_MASK(eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE); } else { pCxt->errCode = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 7487b07004..930ec66795 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -47,7 +47,7 @@ static FORCE_INLINE bool isNullValue(int8_t dataType, SToken* pToken) { } static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPtr) { - errno = 0; + SET_ERRNO(0); *value = taosStr2Double(pToken->z, endPtr); // not a valid integer number, return error @@ -449,6 +449,7 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t // need to call geosFreeBuffer(*output) later static int parseGeometry(SToken* pToken, unsigned char** output, size_t* size) { +#ifdef USE_GEOS int32_t code = TSDB_CODE_FAILED; //[ToDo] support to parse WKB as well as WKT @@ -465,6 +466,9 @@ static int parseGeometry(SToken* pToken, unsigned char** output, size_t* size) { } return code; +#else + TAOS_RETURN(TSDB_CODE_OPS_NOT_SUPPORT); +#endif } static int32_t parseVarbinary(SToken* pToken, uint8_t** pData, uint32_t* nData, int32_t bytes) { @@ -652,7 +656,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && ERRNO == ERANGE) || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); } @@ -717,7 +721,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s %d %d", strerror(terrno), errno, EILSEQ); + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s %d %d", strerror(terrno), ERRNO, EILSEQ); taosMemoryFree(p); return buildSyntaxErrMsg(pMsgBuf, buf, pToken->z); } @@ -1933,7 +1937,6 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return buildInvalidOperationMsg(&pCxt->msg, "not support mixed bind and non-bind values"); } pCxt->isStmtBind = true; - pStmt->usingTableProcessing = true; if (pCols->pColIndex[i] == tbnameIdx) { *bFoundTbName = true; char* tbName = NULL; @@ -1968,6 +1971,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* if (!(tag_index < numOfTags)) { return buildInvalidOperationMsg(&pCxt->msg, "not expected numOfTags"); } + pStmt->usingTableProcessing = true; pCxt->tags.pColIndex[tag_index++] = pCols->pColIndex[i] - numOfCols; pCxt->tags.mixTagsCols = true; pCxt->tags.numOfBound++; diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index c61b5e611c..5306d483ca 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -66,9 +66,9 @@ int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData) { } int32_t qAppendStmtTableOutput(SQuery* pQuery, SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, - SStbInterlaceInfo* pBuildInfo) { + SStbInterlaceInfo* pBuildInfo, SVCreateTbReq* ctbReq) { // merge according to vgId - return insAppendStmtTableDataCxt(pAllVgHash, pTbData, pTbCtx, pBuildInfo); + return insAppendStmtTableDataCxt(pAllVgHash, pTbData, pTbCtx, pBuildInfo, ctbReq); } int32_t qBuildStmtFinOutput(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDataBlocks) { @@ -487,7 +487,8 @@ _return: } int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, - TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) { + TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt, + SVCreateTbReq* pCreateTbReq) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; int32_t code = TSDB_CODE_SUCCESS; @@ -607,6 +608,13 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c goto end; } + if (pCreateTbReq){ + code = insBuildCreateTbReq(pCreateTbReq, tName, pTag, suid, sTableName, tagName, + pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); + pTag = NULL; + goto end; + } + if (NULL == pDataBlock->pData->pCreateTbReq) { pDataBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); if (NULL == pDataBlock->pData->pCreateTbReq) { diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 32edd5b8c3..80be418353 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -610,7 +610,7 @@ int32_t qBuildStmtFinOutput1(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDa } int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, - SStbInterlaceInfo* pBuildInfo) { + SStbInterlaceInfo* pBuildInfo, SVCreateTbReq* ctbReq) { int32_t code = TSDB_CODE_SUCCESS; uint64_t uid; int32_t vgId; @@ -618,14 +618,27 @@ int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, pTbCtx->pData->aRowP = pTbData->aCol; code = insGetStmtTableVgUid(pAllVgHash, pBuildInfo, pTbData, &uid, &vgId); - if (TSDB_CODE_SUCCESS != code) { - return code; + if (ctbReq !=NULL && code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + pTbCtx->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE; + vgId = (int32_t)ctbReq->uid; + uid = 0; + pTbCtx->pMeta->vgId=(int32_t)ctbReq->uid; + ctbReq->uid=0; + pTbCtx->pData->pCreateTbReq = ctbReq; + code = TSDB_CODE_SUCCESS; + } else { + if (TSDB_CODE_SUCCESS != code) { + return code; + } + pTbCtx->pMeta->vgId = vgId; + pTbCtx->pMeta->uid = uid; + pTbCtx->pData->uid = uid; + pTbCtx->pData->pCreateTbReq = NULL; + if (ctbReq != NULL) { + tdDestroySVCreateTbReq(ctbReq); + } } - pTbCtx->pMeta->vgId = vgId; - pTbCtx->pMeta->uid = uid; - pTbCtx->pData->uid = uid; - if (!pTbCtx->ordered) { code = tRowSort(pTbCtx->pData->aRowP); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9d472fab80..4a1a2c0b07 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -76,7 +76,7 @@ typedef struct SSysTableShowAdapter { const char* pDbName; const char* pTableName; int32_t numOfShowCols; - const char* pShowCols[2]; + const char* pShowCols[3]; } SSysTableShowAdapter; typedef struct SCollectJoinCondsContext { @@ -187,8 +187,8 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = { .showType = QUERY_NODE_SHOW_STREAMS_STMT, .pDbName = TSDB_INFORMATION_SCHEMA_DB, .pTableName = TSDB_INS_TABLE_STREAMS, - .numOfShowCols = 1, - .pShowCols = {"stream_name"} + .numOfShowCols = 3, + .pShowCols = {"stream_name","status","message"} }, { .showType = QUERY_NODE_SHOW_TABLES_STMT, @@ -2154,7 +2154,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_DOUBLE: { pVal->datum.d = taosStr2Double(pVal->literal, NULL); - if (strict && (((pVal->datum.d == HUGE_VAL || pVal->datum.d == -HUGE_VAL) && errno == ERANGE) || + if (strict && (((pVal->datum.d == HUGE_VAL || pVal->datum.d == -HUGE_VAL) && ERRNO == ERANGE) || isinf(pVal->datum.d) || isnan(pVal->datum.d))) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } @@ -3751,7 +3751,7 @@ static bool isWindowJoinSubTbTag(SSelectStmt* pSelect, SNode* pNode) { } SRealTableNode* pTargetTable = pProbeTable; - if (0 != strcasecmp(pCol->tableAlias, pProbeTable->table.tableAlias)) { + if (0 != taosStrcasecmp(pCol->tableAlias, pProbeTable->table.tableAlias)) { pTargetTable = pBuildTable; } @@ -3791,7 +3791,7 @@ static bool isWindowJoinSubTbname(SSelectStmt* pSelect, SNode* pNode) { SRealTableNode* pTargetTable = pProbeTable; bool isProbeTable = true; SValueNode* pVal = (SValueNode*)nodesListGetNode(pFuncNode->pParameterList, 0); - if (NULL != pVal && 0 != strcasecmp(pVal->literal, pProbeTable->table.tableAlias)) { + if (NULL != pVal && 0 != taosStrcasecmp(pVal->literal, pProbeTable->table.tableAlias)) { pTargetTable = pBuildTable; isProbeTable = false; } @@ -5966,7 +5966,7 @@ static int32_t getTimeRange(SNode** pPrimaryKeyCond, STimeWindow* pTimeRange, bo if (TSDB_CODE_SUCCESS == code) { *pPrimaryKeyCond = pNew; if (nodeType(pNew) == QUERY_NODE_VALUE) { - *pTimeRange = TSWINDOW_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(pTimeRange, TSWINDOW_INITIALIZER); } else { code = filterGetTimeRange(*pPrimaryKeyCond, pTimeRange, pIsStrict); } @@ -5976,7 +5976,7 @@ static int32_t getTimeRange(SNode** pPrimaryKeyCond, STimeWindow* pTimeRange, bo static int32_t getQueryTimeRange(STranslateContext* pCxt, SNode* pWhere, STimeWindow* pTimeRange) { if (NULL == pWhere) { - *pTimeRange = TSWINDOW_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(pTimeRange, TSWINDOW_INITIALIZER); return TSDB_CODE_SUCCESS; } @@ -5994,7 +5994,7 @@ static int32_t getQueryTimeRange(STranslateContext* pCxt, SNode* pWhere, STimeWi bool isStrict = false; code = getTimeRange(&pPrimaryKeyCond, pTimeRange, &isStrict); } else { - *pTimeRange = TSWINDOW_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(pTimeRange, TSWINDOW_INITIALIZER); } } nodesDestroyNode(pCond); @@ -8184,10 +8184,9 @@ static int32_t translateSetOperator(STranslateContext* pCxt, SSetOperator* pSetO static int32_t partitionDeleteWhere(STranslateContext* pCxt, SDeleteStmt* pDelete) { if (NULL == pDelete->pWhere) { - pDelete->timeRange = TSWINDOW_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(&pDelete->timeRange, TSWINDOW_INITIALIZER); return TSDB_CODE_SUCCESS; } - SNode* pPrimaryKeyCond = NULL; SNode* pOtherCond = NULL; int32_t code = filterPartitionCond(&pDelete->pWhere, &pPrimaryKeyCond, NULL, &pDelete->pTagCond, &pOtherCond); @@ -8202,7 +8201,7 @@ static int32_t partitionDeleteWhere(STranslateContext* pCxt, SDeleteStmt* pDelet code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DELETE_WHERE); } } else { - pDelete->timeRange = TSWINDOW_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(&pDelete->timeRange, TSWINDOW_INITIALIZER); } } nodesDestroyNode(pPrimaryKeyCond); @@ -10820,7 +10819,7 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt const char* validConfigs[] = { "encrypt_key", }; - if (0 == strncasecmp(cfgReq.config, validConfigs[0], strlen(validConfigs[0]) + 1)) { + if (0 == taosStrncasecmp(cfgReq.config, validConfigs[0], strlen(validConfigs[0]) + 1)) { int32_t klen = strlen(cfgReq.value); if (klen > ENCRYPT_KEY_LEN || klen < ENCRYPT_KEY_LEN_MIN) { tFreeSMCfgDnodeReq(&cfgReq); @@ -12635,6 +12634,7 @@ static int32_t createLastTsSelectStmt(char* pDb, const char* pTable, const char* return code; } + parserInfo("[create stream with histroy] create select last ts query, db:%s, table:%s", pDb, pTable); tstrncpy(col->tableAlias, pTable, tListLen(col->tableAlias)); tstrncpy(col->colName, pkColName, tListLen(col->colName)); SNodeList* pParameterList = NULL; @@ -12839,7 +12839,7 @@ static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt getSourceDatabase(pStmt->pQuery, pCxt->pParseCxt->acctId, pReq->sourceDB); code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); } - if (TSDB_CODE_SUCCESS == code && pStmt->pOptions->fillHistory) { + if (TSDB_CODE_SUCCESS == code && pStmt->pOptions->fillHistory && (pCxt->pParseCxt->streamRunHistory || !tsStreamRunHistoryAsync)) { SRealTableNode* pTable = (SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable); code = createLastTsSelectStmt(pTable->table.dbName, pTable->table.tableName, pTable->pMeta->schema[0].name, &pStmt->pPrevQuery); @@ -13061,6 +13061,7 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, SSDa code = buildCmdMsg(&cxt, TDMT_MND_CREATE_STREAM, (FSerializeFunc)tSerializeSCMCreateStreamReq, pStmt->pReq); } + qInfo("[create stream with histroy] post create stream, lastTs:%" PRId64, pStmt->pReq->lastTs); if (TSDB_CODE_SUCCESS == code) { code = setQuery(&cxt, pQuery); } @@ -16021,7 +16022,7 @@ static int32_t prepareReadCsvFile(STranslateContext* pCxt, SCreateSubTableFromFi return code; _ERR: - taosCloseFile(&fp); + TAOS_UNUSED(taosCloseFile(&fp)); taosMemoryFreeClear(pCreateInfo); destructParseFileContext(&pParFileCxt); return code; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index a5758a17c3..2b02d18193 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -840,7 +840,7 @@ int32_t createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* select->timeLineResMode = select->isDistinct ? TIME_LINE_NONE : TIME_LINE_GLOBAL; select->timeLineCurMode = TIME_LINE_GLOBAL; select->onlyHasKeepOrderFunc = true; - select->timeRange = TSWINDOW_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(&select->timeRange, TSWINDOW_INITIALIZER); select->pHint = pHint; select->lastProcessByRowFuncId = -1; *ppSelect = (SNode*)select; @@ -1205,9 +1205,9 @@ int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFNam int32_t code = getMetaDataFromHash(pDbFName, strlen(pDbFName), pMetaCache->pDbInfo, (void**)&pDbInfo); if (TSDB_CODE_SUCCESS == code) { *pVersion = pDbInfo->vgVer; - *pDbId = pDbInfo->dbId; + taosSetInt64Aligned(pDbId, pDbInfo->dbId); *pTableNum = pDbInfo->tbNum; - *pStateTs = pDbInfo->stateTs; + taosSetInt64Aligned(pStateTs, pDbInfo->stateTs); } return code; } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 3cabd94c95..4b3170a480 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -121,8 +121,10 @@ void generatePerformanceSchema(MockCatalogService* mcs) { .addColumn("id", TSDB_DATA_TYPE_INT) .addColumn("create_time", TSDB_DATA_TYPE_TIMESTAMP) .done(); - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STREAMS, TSDB_SYSTEM_TABLE, 2) + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STREAMS, TSDB_SYSTEM_TABLE, 4) .addColumn("stream_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) + .addColumn("status", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) + .addColumn("message", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) .addColumn("create_time", TSDB_DATA_TYPE_TIMESTAMP) .done(); mcs->createTableBuilder(TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_CONSUMERS, TSDB_SYSTEM_TABLE, 2) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index d88bcb6f4b..70e660075a 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -390,7 +390,7 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT pScan->tableType = pRealTable->pMeta->tableType; pScan->scanSeq[0] = hasRepeatScanFuncs ? 2 : 1; pScan->scanSeq[1] = 0; - pScan->scanRange = TSWINDOW_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(&pScan->scanRange, TSWINDOW_INITIALIZER); pScan->tableName.type = TSDB_TABLE_NAME_T; pScan->tableName.acctId = pCxt->pPlanCxt->acctId; tstrncpy(pScan->tableName.dbname, pRealTable->table.dbName, TSDB_DB_NAME_LEN); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 3378202274..bc8566cc0c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -5968,7 +5968,7 @@ static int32_t hashJoinOptRewriteJoin(SOptimizeContext* pCxt, SLogicNode* pNode, } #endif } else { - pJoin->timeRange = TSWINDOW_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(&pJoin->timeRange, TSWINDOW_INITIALIZER); } #if 0 @@ -6013,7 +6013,7 @@ static int32_t hashJoinOptRewriteJoin(SOptimizeContext* pCxt, SLogicNode* pNode, if (TSWINDOW_IS_EQUAL(pScan->scanRange, TSWINDOW_INITIALIZER)) { continue; } else if (pJoin->timeRange.ekey < pScan->scanRange.skey || pJoin->timeRange.skey > pScan->scanRange.ekey) { - pJoin->timeRange = TSWINDOW_DESC_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(&pJoin->timeRange, TSWINDOW_DESC_INITIALIZER); break; } else { pJoin->timeRange.skey = TMAX(pJoin->timeRange.skey, pScan->scanRange.skey); @@ -6159,8 +6159,8 @@ int32_t stbJoinOptRewriteToTagScan(SLogicNode* pJoin, SNode* pNode) { SColumnNode* pTargetCol = (SColumnNode*)pTarget; FOREACH(pTag, pTags) { SColumnNode* pTagCol = (SColumnNode*)pTag; - if (0 == strcasecmp(pTargetCol->node.aliasName, pTagCol->colName) && - 0 == strcasecmp(pTargetCol->tableAlias, pTagCol->tableAlias)) { + if (0 == taosStrcasecmp(pTargetCol->node.aliasName, pTagCol->colName) && + 0 == taosStrcasecmp(pTargetCol->tableAlias, pTagCol->tableAlias)) { found = true; break; } diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 3d41b6ef5e..eeefd58388 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -87,7 +87,7 @@ int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t clientId, int32_t rwType) { if (!HASH_NODE_EXIST(code)) { QW_UNLOCK(QW_WRITE, &mgmt->schLock); - QW_SCH_ELOG("taosHashPut new sch to scheduleHash failed, errno:%d", errno); + QW_SCH_ELOG("taosHashPut new sch to scheduleHash failed, errno:%d", ERRNO); taosHashCleanup(newSch.tasksHash); QW_ERR_RET(code); } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 6689e12edd..a711b78868 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4468,7 +4468,7 @@ int32_t filterGetTimeRangeImpl(SFilterInfo *info, STimeWindow *win, bool *isStri FLT_CHK_JMP(emptyGroups == info->groupNum); if (prev->notnull) { - *win = TSWINDOW_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(win, TSWINDOW_INITIALIZER); } else { FLT_ERR_JRET(filterGetRangeNum(prev, &num)); @@ -4481,8 +4481,8 @@ int32_t filterGetTimeRangeImpl(SFilterInfo *info, STimeWindow *win, bool *isStri SFilterRange tra; FLT_ERR_JRET(filterGetRangeRes(prev, &tra)); - win->skey = tra.s; - win->ekey = tra.e; + taosSetInt64Aligned(&win->skey, tra.s); + taosSetInt64Aligned(&win->ekey, tra.e); if (FILTER_GET_FLAG(tra.sflag, RANGE_FLG_EXCLUDE)) { win->skey++; } @@ -4494,17 +4494,19 @@ int32_t filterGetTimeRangeImpl(SFilterInfo *info, STimeWindow *win, bool *isStri (void)filterFreeRangeCtx(prev); // No need to handle the return value. (void)filterFreeRangeCtx(tmpc); // No need to handle the return value. - qDebug("qFilter time range:[%" PRId64 "]-[%" PRId64 "]", win->skey, win->ekey); + qDebug("qFilter time range:[%" PRId64 "]-[%" PRId64 "]", taosGetInt64Aligned(&win->skey), + taosGetInt64Aligned(&win->ekey)); return TSDB_CODE_SUCCESS; _return: - *win = TSWINDOW_DESC_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(win, TSWINDOW_DESC_INITIALIZER); (void)filterFreeRangeCtx(prev); // No need to handle the return value. (void)filterFreeRangeCtx(tmpc); // No need to handle the return value. - qDebug("qFilter time range:[%" PRId64 "]-[%" PRId64 "]", win->skey, win->ekey); + qDebug("qFilter time range:[%" PRId64 "]-[%" PRId64 "]", taosGetInt64Aligned(&win->skey), + taosGetInt64Aligned(&win->ekey)); return code; } @@ -4572,7 +4574,7 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { } SArray *points = colRange->points; if (taosArrayGetSize(points) == 2) { - *win = TSWINDOW_DESC_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(win, TSWINDOW_DESC_INITIALIZER); SFltSclPoint *startPt = taosArrayGet(points, 0); SFltSclPoint *endPt = taosArrayGet(points, 1); if (NULL == startPt || NULL == endPt) { @@ -4582,16 +4584,16 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { SFltSclDatum end; FLT_ERR_JRET(fltSclGetTimeStampDatum(startPt, &start)); FLT_ERR_JRET(fltSclGetTimeStampDatum(endPt, &end)); - win->skey = start.i; - win->ekey = end.i; + taosSetInt64Aligned(&win->skey, start.i); + taosSetInt64Aligned(&win->ekey, end.i); *isStrict = info->isStrict; goto _return; } else if (taosArrayGetSize(points) == 0) { - *win = TSWINDOW_DESC_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(win, TSWINDOW_DESC_INITIALIZER); goto _return; } } - *win = TSWINDOW_INITIALIZER; + TAOS_SET_OBJ_ALIGNED(win, TSWINDOW_INITIALIZER); *isStrict = false; goto _return; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index ff45baa1e0..742759db07 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -428,6 +428,7 @@ _return: } static FORCE_INLINE int32_t varToGeometry(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { +#ifdef USE_GEOS //[ToDo] support to parse WKB as well as WKT int32_t code = TSDB_CODE_SUCCESS; size_t len = 0; @@ -464,6 +465,9 @@ _return: VarDataLenT dummyHeader = 0; SCL_ERR_RET(colDataSetVal(pOut->columnData, rowIndex, (const char *)&dummyHeader, false)); SCL_RET(code); +#else + TAOS_RETURN(TSDB_CODE_OPS_NOT_SUPPORT); +#endif } // TODO opt performance, tmp is not needed. diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index caec0a3776..0b90c7b7ff 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -914,7 +914,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { } if (tsem_init(&pJob->rspSem, 0, 0)) { - SCH_JOB_ELOG("tsem_init failed, errno:%d", errno); + SCH_JOB_ELOG("tsem_init failed, errno:%d", ERRNO); SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index a4edda8d60..a81ca1becd 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -115,7 +115,7 @@ int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad SSchNodeInfo nodeInfo = {.addr = *addr, .handle = SCH_GET_TASK_HANDLE(pTask)}; if (taosHashPut(pTask->execNodes, &execId, sizeof(execId), &nodeInfo, sizeof(nodeInfo))) { - SCH_TASK_ELOG("taosHashPut nodeInfo to execNodes failed, errno:%d", errno); + SCH_TASK_ELOG("taosHashPut nodeInfo to execNodes failed, errno:%d", ERRNO); SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -629,7 +629,7 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - SCH_TASK_ELOG("taosHashPut task to succTask list failed, errno:%d", errno); + SCH_TASK_ELOG("taosHashPut task to succTask list failed, errno:%d", ERRNO); SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -656,7 +656,7 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - SCH_TASK_ELOG("taosHashPut task to failTask list failed, errno:%d", errno); + SCH_TASK_ELOG("taosHashPut task to failTask list failed, errno:%d", ERRNO); SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -681,7 +681,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); + SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", ERRNO); SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -803,7 +803,7 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { SQueryNodeAddr *naddr = &nload->addr; if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { - SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); + SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, ERRNO); SCH_ERR_RET(terrno); } @@ -836,7 +836,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (pTask->plan->execNode.epSet.numOfEps > 0) { if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) { - SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", errno); + SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", ERRNO); SCH_ERR_RET(terrno); } diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index aa1923a033..71a3f03683 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -317,7 +317,7 @@ uint64_t schGenUUID(void) { if (hashId == 0) { int32_t code = taosGetSystemUUID32(&hashId); if (code != TSDB_CODE_SUCCESS) { - qError("Failed to get the system uid, reason:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + qError("Failed to get the system uid, reason:%s", tstrerror(TAOS_SYSTEM_ERROR(ERRNO))); } } diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 2b5312c0a0..31c23c604e 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -13,8 +13,9 @@ * along with this program. If not, see . */ -#ifndef _STREAM_BACKEDN_ROCKSDB_H_ -#define _STREAM_BACKEDN_ROCKSDB_H_ +#ifdef USE_STREAM +#ifndef _STREAM_BACKEND_ROCKSDB_H_ +#define _STREAM_BACKEND_ROCKSDB_H_ #include "rocksdb/c.h" // #include "streamInt.h" @@ -283,4 +284,5 @@ uint32_t nextPow2(uint32_t x); } #endif -#endif +#endif /* _STREAM_BACKEND_ROCKSDB_H_ */ +#endif /* USE_STREAM */ diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index dbaddd15a8..4a6e7a0522 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +#ifdef USE_STREAM #ifndef _STREAM_INC_H_ #define _STREAM_INC_H_ @@ -265,3 +265,4 @@ int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t checkpointId #endif #endif /* ifndef _STREAM_INC_H_ */ +#endif /* USE_STREAM */ diff --git a/source/libs/stream/inc/streamsm.h b/source/libs/stream/inc/streamsm.h index 47e0ce1b55..acc1246b38 100644 --- a/source/libs/stream/inc/streamsm.h +++ b/source/libs/stream/inc/streamsm.h @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +#ifdef USE_STREAM #ifndef TDENGINE_STREAMSM_H #define TDENGINE_STREAMSM_H @@ -61,3 +61,4 @@ struct SStreamTaskSM { #endif #endif // TDENGINE_STREAMSM_H +#endif // USE_STREAM diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 6031710bf9..a5052b65a7 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -249,7 +249,7 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { tstrerror(terrno), state); code = taosMkDir(state); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); } } @@ -515,7 +515,7 @@ int32_t rebuildFromRemoteChkp_s3(const char* key, char* chkpPath, int64_t chkpId } else { code = taosMkDir(defaultPath); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _EXIT; } } @@ -563,8 +563,8 @@ int32_t copyFiles_create(char* src, char* dst, int8_t type) { // create and copy file int32_t err = taosCopyFile(src, dst); - if (errno == EXDEV || errno == ENOTSUP) { - errno = 0; + if (ERRNO == EXDEV || ERRNO == ENOTSUP) { + SET_ERRNO(0); return 0; } return 0; @@ -602,7 +602,7 @@ int32_t backendFileCopyFilesImpl(const char* src, const char* dst) { goto _ERROR; } - errno = 0; + SET_ERRNO(0); TdDirEntryPtr de = NULL; while ((de = taosReadDir(pDir)) != NULL) { char* name = taosGetDirEntryName(de); @@ -625,14 +625,14 @@ int32_t backendFileCopyFilesImpl(const char* src, const char* dst) { if (strncmp(name, current, strlen(name) <= currLen ? strlen(name) : currLen) == 0) { code = copyFiles_create(srcName, dstName, 0); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); stError("failed to copy file, detail: %s to %s reason:%s", srcName, dstName, tstrerror(code)); goto _ERROR; } } else if (strncmp(name, info, strlen(name) <= infoLen ? strlen(name) : infoLen) == 0) { code = copyFiles_create(srcName, dstName, 0); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); stError("failed to copy file, detail: %s to %s reason:%s", srcName, dstName, tstrerror(code)); goto _ERROR; } @@ -640,7 +640,7 @@ int32_t backendFileCopyFilesImpl(const char* src, const char* dst) { } else { code = copyFiles_hardlink(srcName, dstName, 0); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); stError("failed to hard link file, detail:%s to %s, reason:%s", srcName, dstName, tstrerror(code)); goto _ERROR; } else { @@ -722,7 +722,7 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId code = createDirIfNotExist(prefixPath); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _EXIT; } @@ -734,7 +734,7 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId code = createDirIfNotExist(defaultPath); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _EXIT; } @@ -746,7 +746,7 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId code = createDirIfNotExist(checkpointRoot); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _EXIT; } @@ -2757,7 +2757,7 @@ int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64 code = taosMkDir(temp); if (code != 0) { taosMemoryFree(temp); - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } } @@ -5029,7 +5029,7 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { } if (taosCopyFile(srcBuf, dstBuf) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); stError("failed to copy file from %s to %s, reason:%s", srcBuf, dstBuf, tstrerror(code)); goto _ERROR; } @@ -5066,7 +5066,7 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { } if (taosCopyFile(srcBuf, dstBuf) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); stError("failed to copy file from %s to %s, reason:%s", srcBuf, dstBuf, tstrerror(code)); goto _ERROR; } @@ -5158,7 +5158,7 @@ int32_t bkdMgtCreate(char* path, SBkdMgt** mgt) { } if (taosThreadRwlockInit(&p->rwLock, NULL) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); bkdMgtDestroy(p); return code; } diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 29da3fc88e..00e3596af9 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -326,8 +326,6 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { streamTmrStop(pInfo->checkRspTmr); pInfo->checkRspTmr = NULL; } - - streamMutexDestroy(&pInfo->checkInfoLock); } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index afda288b90..87746bc899 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -1483,7 +1483,7 @@ int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char int32_t code = tcsGetObjectToFile(buf, dstName); if (code != 0) { taosMemoryFree(buf); - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } taosMemoryFree(buf); return 0; @@ -1509,7 +1509,7 @@ int32_t streamTaskUploadCheckpoint(const char* id, const char* path, int64_t che if (strlen(tsSnodeAddress) != 0) { code = uploadByRsync(id, path, checkpointId); if (code != 0) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } } else if (tsS3StreamEnabled) { return uploadCheckpointToS3(id, path); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index f08b4a7922..1c84396909 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1320,13 +1320,13 @@ int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, in int32_t tlen; tEncodeSize(tEncodeStreamDispatchReq, pReq, tlen, code); if (code < 0) { - goto FAIL; + goto _ERR; } buf = rpcMallocCont(sizeof(SMsgHead) + tlen); if (buf == NULL) { code = terrno; - goto FAIL; + goto _ERR; } ((SMsgHead*)buf)->vgId = htonl(vgId); @@ -1336,7 +1336,7 @@ int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, in tEncoderInit(&encoder, abuf, tlen); if ((code = tEncodeStreamDispatchReq(&encoder, pReq)) < 0) { tEncoderClear(&encoder); - goto FAIL; + goto _ERR; } tEncoderClear(&encoder); @@ -1345,7 +1345,7 @@ int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, in return tmsgSendReq(pEpSet, &msg); -FAIL: +_ERR: if (buf) { rpcFreeCont(buf); } diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 88495ac760..0f322f48c5 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -734,7 +734,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t if (pSnapFile->fd == 0) { pSnapFile->fd = streamOpenFile(pSnapFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pSnapFile->fd == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); stError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pHandle->metaPath, TD_DIRSEP, pHdr->name, tstrerror(code)); } @@ -774,7 +774,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t SBackendFileItem* pItem2 = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); pSnapFile->fd = streamOpenFile(pSnapFile->path, pItem2->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pSnapFile->fd == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); stError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pSnapFile->path, TD_DIRSEP, pHdr->name, tstrerror(code)); return code; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index f7ae4915ec..ebe6042f0a 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -151,10 +151,6 @@ int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL; pTask->taskCheckInfo.pList = taosArrayInit(4, sizeof(SDownstreamStatusInfo)); - code = taosThreadMutexInit(&pTask->taskCheckInfo.checkInfoLock, NULL); - if (code) { - return code; - } if (fillHistory && !hasFillhistory) { stError("s-task:0x%x create task failed, due to inconsistent fill-history flag", pTask->id.taskId); @@ -226,13 +222,13 @@ void tFreeStreamTask(void* pParam) { STaskExecStatisInfo* pStatis = &pTask->execInfo; ETaskStatus status1 = TASK_STATUS__UNINIT; - streamMutexLock(&pTask->lock); if (pTask->status.pSM != NULL) { + streamMutexLock(&pTask->lock); SStreamTaskState status = streamTaskGetStatus(pTask); p = status.name; status1 = status.state; + streamMutexUnlock(&pTask->lock); } - streamMutexUnlock(&pTask->lock); stDebug("start to free s-task:0x%x %p, state:%s, refId:%" PRId64, taskId, pTask, p, pTask->id.refId); @@ -304,17 +300,21 @@ void tFreeStreamTask(void* pParam) { tSimpleHashCleanup(pTask->pNameMap); } + if (pTask->status.pSM != NULL) { + streamMutexDestroy(&pTask->lock); + streamMutexDestroy(&pTask->msgInfo.lock); + streamMutexDestroy(&pTask->taskCheckInfo.checkInfoLock); + } + streamDestroyStateMachine(pTask->status.pSM); pTask->status.pSM = NULL; streamTaskDestroyUpstreamInfo(&pTask->upstreamInfo); taosMemoryFree(pTask->outputInfo.pTokenBucket); - streamMutexDestroy(&pTask->lock); taosArrayDestroy(pTask->msgInfo.pSendInfo); pTask->msgInfo.pSendInfo = NULL; - streamMutexDestroy(&pTask->msgInfo.lock); taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList); pTask->outputInfo.pNodeEpsetUpdateList = NULL; @@ -529,6 +529,11 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i return terrno; } + code = taosThreadMutexInit(&pTask->taskCheckInfo.checkInfoLock, NULL); + if (code) { + return code; + } + if (pTask->chkInfo.pActiveInfo == NULL) { code = streamTaskCreateActiveChkptInfo(&pTask->chkInfo.pActiveInfo); if (code) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index bc5c2c1510..fef6f016d3 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -855,7 +855,7 @@ int32_t syncIsCatchUp(int64_t rid) { if (pSyncNode == NULL) { code = TSDB_CODE_SYN_RETURN_VALUE_NULL; if (terrno != 0) code = terrno; - sError("sync Node Acquire error since %d", errno); + sError("sync Node Acquire error since %d", ERRNO); TAOS_RETURN(code); } @@ -883,7 +883,7 @@ ESyncRole syncGetRole(int64_t rid) { if (pSyncNode == NULL) { code = TSDB_CODE_SYN_RETURN_VALUE_NULL; if (terrno != 0) code = terrno; - sError("sync Node Acquire error since %d", errno); + sError("sync Node Acquire error since %d", ERRNO); TAOS_RETURN(code); } @@ -899,7 +899,7 @@ int64_t syncGetTerm(int64_t rid) { if (pSyncNode == NULL) { code = TSDB_CODE_SYN_RETURN_VALUE_NULL; if (terrno != 0) code = terrno; - sError("sync Node Acquire error since %d", errno); + sError("sync Node Acquire error since %d", ERRNO); TAOS_RETURN(code); } @@ -1080,7 +1080,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion) { if (!taosDirExist((char*)(pSyncInfo->path))) { if (taosMkDir(pSyncInfo->path) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); sError("vgId:%d, failed to create dir:%s since %s", pSyncInfo->vgId, pSyncInfo->path, terrstr()); goto _error; } @@ -1339,7 +1339,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion) { // init peer heartbeat timer for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { if ((code = syncHbTimerInit(pSyncNode, &(pSyncNode->peerHeartbeatTimerArr[i]), (pSyncNode->replicasId)[i])) != 0) { - errno = code; + terrno = code; goto _error; } } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 34f48d5960..0759b3f97c 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -1095,11 +1095,11 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn SyncTerm term = -1; SyncIndex firstVer = pNode->pLogStore->syncLogBeginIndex(pNode->pLogStore); SyncIndex index = TMIN(pMsg->matchIndex, pNode->pLogBuf->matchIndex); - errno = 0; + SET_ERRNO(0); if (pMsg->matchIndex < pNode->pLogBuf->matchIndex) { code = syncLogReplGetPrevLogTerm(pMgr, pNode, index + 1, &term); - if (term < 0 && (errno == ENFILE || errno == EMFILE || errno == ENOENT)) { + if (term < 0 && (ERRNO == ENFILE || ERRNO == EMFILE || ERRNO == ENOENT)) { sError("vgId:%d, failed to get prev log term since %s. index:%" PRId64, pNode->vgId, tstrerror(code), index + 1); TAOS_RETURN(code); } @@ -1381,19 +1381,19 @@ int32_t syncLogBufferCreate(SSyncLogBuffer** ppBuf) { } if (taosThreadMutexAttrInit(&pBuf->attr) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); sError("failed to init log buffer mutexattr due to %s", tstrerror(code)); goto _exit; } if (taosThreadMutexAttrSetType(&pBuf->attr, PTHREAD_MUTEX_RECURSIVE) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); sError("failed to set log buffer mutexattr type due to %s", tstrerror(code)); goto _exit; } if (taosThreadMutexInit(&pBuf->mutex, &pBuf->attr) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); sError("failed to init log buffer mutex due to %s", tstrerror(code)); goto _exit; } diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 5d938afa67..733bc21190 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -160,7 +160,7 @@ int32_t syncWriteCfgFile(SSyncNode *pNode) { } if (taosFsyncFile(pFile) < 0) { - TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(errno)); + TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(ERRNO)); } TAOS_CHECK_EXIT(taosCloseFile(&pFile)); @@ -175,7 +175,8 @@ _exit: if (pFile != NULL) taosCloseFile(&pFile); if (code != 0) { - sError("vgId:%d, failed to write sync cfg file:%s since %s", pNode->vgId, realfile, tstrerror(code)); + sError("vgId:%d, failed at line %d to write sync cfg file:%s since %s", pNode->vgId, lino, realfile, + tstrerror(code)); } TAOS_RETURN(code); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index e7ecb67e69..657282efe1 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -126,8 +126,8 @@ static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncI if (code != 0) { int32_t err = code; const char* errStr = tstrerror(err); - int32_t sysErr = errno; - const char* sysErrStr = strerror(errno); + int32_t sysErr = ERRNO; + const char* sysErrStr = strerror(ERRNO); sNError(pData->pSyncNode, "wal restore from snapshot error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", snapshotIndex, @@ -233,8 +233,8 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr if (TSDB_CODE_SUCCESS != code) { int32_t err = terrno; const char* errStr = tstrerror(err); - int32_t sysErr = errno; - const char* sysErrStr = strerror(errno); + int32_t sysErr = ERRNO; + const char* sysErrStr = strerror(ERRNO); sNError(pData->pSyncNode, "wal write error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", pEntry->index, err, errStr, sysErr, sysErrStr); @@ -283,8 +283,8 @@ int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncR if (code != 0) { int32_t err = code; const char* errStr = tstrerror(err); - int32_t sysErr = errno; - const char* sysErrStr = strerror(errno); + int32_t sysErr = ERRNO; + const char* sysErrStr = strerror(ERRNO); if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { sNTrace(pData->pSyncNode, "wal read not exist, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", index, @@ -347,8 +347,8 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn if (code != 0) { int32_t err = code; const char* errStr = tstrerror(err); - int32_t sysErr = errno; - const char* sysErrStr = strerror(errno); + int32_t sysErr = ERRNO; + const char* sysErrStr = strerror(ERRNO); sError("vgId:%d, wal truncate error, from-index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, fromIndex, err, errStr, sysErr, sysErrStr); } @@ -399,8 +399,8 @@ int32_t raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { if (code != 0) { int32_t err = code; const char* errStr = tstrerror(err); - int32_t sysErr = errno; - const char* sysErrStr = strerror(errno); + int32_t sysErr = ERRNO; + const char* sysErrStr = strerror(ERRNO); sError("vgId:%d, wal update commit index error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, index, err, errStr, sysErr, sysErrStr); diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index ca39bd1a23..9b31fd0ebe 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -188,12 +188,12 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (TDB_FD_INVALID(pPager->fd)) { // if (pPager->fd < 0) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } ret = tdbGnrtFileID(pPager->fd, pPager->fid, false); if (ret < 0) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } // pPager->jfd = -1; @@ -265,8 +265,8 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { (void)tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (TDB_FD_INVALID(pTxn->jfd)) { - tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); - return terrno = TAOS_SYSTEM_ERROR(errno); + tdbError("failed to open file due to %s. jFileName:%s", strerror(ERRNO), pPager->jFileName); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } pTxn->jPageSet = hashset_create(); @@ -308,8 +308,8 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { // sync the journal file ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { - tdbError("failed to fsync: %s. jFileName:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - return terrno = TAOS_SYSTEM_ERROR(errno); + tdbError("failed to fsync: %s. jFileName:%s, %" PRId64, strerror(ERRNO), pPager->jFileName, pTxn->txnId); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } // loop to write the dirty pages to file @@ -356,8 +356,8 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { // sync the db file if (tdbOsFSync(pPager->fd) < 0) { - tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); - return terrno = TAOS_SYSTEM_ERROR(errno); + tdbError("failed to fsync fd due to %s. file:%s", strerror(ERRNO), pPager->dbFileName); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } return 0; @@ -369,13 +369,13 @@ int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { // remove the journal file if (tdbOsClose(pTxn->jfd) < 0) { - tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - return terrno = TAOS_SYSTEM_ERROR(errno); + tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(ERRNO), pPager->jFileName, pTxn->txnId); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } - if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { - tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); - return terrno = TAOS_SYSTEM_ERROR(errno); + if (tdbOsRemove(jTxnFileName) < 0 && ERRNO != ENOENT) { + tdbError("failed to remove file due to %s. file:%s", strerror(ERRNO), jTxnFileName); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } // pPager->inTran = 0; @@ -393,8 +393,8 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { // sync the journal file ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { - tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - return terrno = TAOS_SYSTEM_ERROR(errno); + tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(ERRNO), pPager->jFileName, pTxn->txnId); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } // loop to write the dirty pages to file @@ -493,8 +493,8 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { // sync the journal file ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { - tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - return terrno = TAOS_SYSTEM_ERROR(errno); + tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(ERRNO), pPager->jFileName, pTxn->txnId); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } tdb_fd_t jfd = pTxn->jfd; @@ -505,8 +505,8 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { } if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) { - tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName); - return terrno = TAOS_SYSTEM_ERROR(errno); + tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(ERRNO), pPager->dbFileName); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); @@ -523,7 +523,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); if (ret < 0) { tdbOsFree(pageBuf); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } tdbTrace("pager/abort: restore pgno:%d,", pgno); @@ -533,14 +533,14 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); if (ret < 0) { tdbOsFree(pageBuf); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset); + tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(ERRNO), pPager->dbFileName, offset); tdbOsFree(pageBuf); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } char *buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); @@ -550,20 +550,20 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize); if (ret < 0) { - tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, + tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(ERRNO), pPager->dbFileName, pPager->pageSize); tdbFreeEncryptBuf(pPager, buf); tdbOsFree(pageBuf); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } tdbFreeEncryptBuf(pPager, buf); } if (tdbOsFSync(pPager->fd) < 0) { - tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); + tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(ERRNO), pPager->dbFileName); tdbOsFree(pageBuf); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } tdbOsFree(pageBuf); @@ -590,16 +590,16 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { // 4, remove the journal file if (tdbOsClose(pTxn->jfd) < 0) { - tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - return terrno = TAOS_SYSTEM_ERROR(errno); + tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(ERRNO), pPager->jFileName, pTxn->txnId); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } char jTxnFileName[TDB_FILENAME_LEN]; (void)tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); - if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { - tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); - return terrno = TAOS_SYSTEM_ERROR(errno); + if (tdbOsRemove(jTxnFileName) < 0 && ERRNO != ENOENT) { + tdbError("failed to remove file due to %s. file:%s", strerror(ERRNO), jTxnFileName); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } // pPager->inTran = 0; @@ -906,7 +906,7 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32 " unlock page failed.", pPager, pgno, nRead, pPage->pageSize); } - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; @@ -994,16 +994,16 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { ret = tdbOsWrite(pPager->pActiveTxn->jfd, &pgno, sizeof(pgno)); if (ret < 0) { - tdbError("failed to write pgno due to %s. file:%s, pgno:%u, txnId:%" PRId64, strerror(errno), pPager->jFileName, + tdbError("failed to write pgno due to %s. file:%s, pgno:%u, txnId:%" PRId64, strerror(ERRNO), pPager->jFileName, pgno, pPager->pActiveTxn->txnId); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } ret = tdbOsWrite(pPager->pActiveTxn->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { - tdbError("failed to write page data due to %s. file:%s, pageSize:%d, txnId:%" PRId64, strerror(errno), + tdbError("failed to write page data due to %s. file:%s, pageSize:%d, txnId:%" PRId64, strerror(ERRNO), pPager->jFileName, pPage->pageSize, pPager->pActiveTxn->txnId); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } return 0; @@ -1015,16 +1015,16 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - tdbError("failed to lseek due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset); - terrno = TAOS_SYSTEM_ERROR(errno); + tdbError("failed to lseek due to %s. file:%s, offset:%" PRId64, strerror(ERRNO), pPager->dbFileName, offset); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return -1; } ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize); if (ret < 0) { - tdbError("failed to write page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName, + tdbError("failed to write page data due to %s. file:%s, pageSize:%d", strerror(ERRNO), pPager->dbFileName, pPage->pageSize); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return -1; } @@ -1042,9 +1042,9 @@ static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) { ret = tdbOsPWrite(pPager->fd, buf, pPage->pageSize, offset); if (ret < 0) { tdbFreeEncryptBuf(pPager, buf); - tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName, + tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(ERRNO), pPager->dbFileName, pPage->pageSize); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } tdbFreeEncryptBuf(pPager, buf); @@ -1064,12 +1064,12 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); if (ret < 0) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) { - tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName); - return terrno = TAOS_SYSTEM_ERROR(errno); + tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(ERRNO), pPager->dbFileName); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } pageBuf = tdbOsCalloc(1, pPager->pageSize); @@ -1086,7 +1086,7 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); if (ret < 0) { tdbOsFree(pageBuf); - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } tdbTrace("pager/restore: restore pgno:%d,", pgno); @@ -1094,14 +1094,14 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); if (ret < 0) { tdbOsFree(pageBuf); - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset); + tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(ERRNO), pPager->dbFileName, offset); tdbOsFree(pageBuf); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } char *buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); @@ -1111,32 +1111,32 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize); if (ret < 0) { - tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, + tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(ERRNO), pPager->dbFileName, pPager->pageSize); tdbFreeEncryptBuf(pPager, buf); tdbOsFree(pageBuf); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } tdbFreeEncryptBuf(pPager, buf); } if (tdbOsFSync(pPager->fd) < 0) { - tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); + tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(ERRNO), pPager->dbFileName); tdbOsFree(pageBuf); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } tdbOsFree(pageBuf); if (tdbOsClose(jfd) < 0) { - tdbError("failed to close jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); - return terrno = TAOS_SYSTEM_ERROR(errno); + tdbError("failed to close jfd due to %s. jFileName:%s", strerror(ERRNO), pPager->jFileName); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } - if (tdbOsRemove(jFileName) < 0 && errno != ENOENT) { - tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); - return terrno = TAOS_SYSTEM_ERROR(errno); + if (tdbOsRemove(jFileName) < 0 && ERRNO != ENOENT) { + tdbError("failed to remove file due to %s. jFileName:%s", strerror(ERRNO), pPager->jFileName); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } return 0; @@ -1153,7 +1153,7 @@ int tdbPagerRestoreJournals(SPager *pPager) { tdbDirEntryPtr pDirEntry; tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); if (pDir == NULL) { - tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); + tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(ERRNO)); return terrno; } @@ -1199,7 +1199,7 @@ int tdbPagerRollback(SPager *pPager) { tdbDirEntryPtr pDirEntry; tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); if (pDir == NULL) { - tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); + tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(ERRNO)); return terrno; } @@ -1212,11 +1212,11 @@ int tdbPagerRollback(SPager *pPager) { memcpy(jname, pPager->pEnv->dbName, dirLen); jname[dirLen] = '/'; memcpy(jname + dirLen + 1, name, strlen(name)); - if (tdbOsRemove(jname) < 0 && errno != ENOENT) { + if (tdbOsRemove(jname) < 0 && ERRNO != ENOENT) { tdbCloseDir(&pDir); - tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name); - return terrno = TAOS_SYSTEM_ERROR(errno); + tdbError("failed to remove file due to %s. jFileName:%s", strerror(ERRNO), name); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } } } diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index 14845ed8ee..2d3a0b1efe 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -60,7 +60,7 @@ int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size) { ret = tdbOsFileSize(fd, &szBytes); if (ret < 0) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } *size = szBytes / szPage; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 84696e6cdb..7a7f4d12cb 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -39,7 +39,7 @@ int32_t tfsOpen(SDiskCfg *pCfg, int32_t ndisk, STfs **ppTfs) { } if (taosThreadSpinInit(&pTfs->lock, 0) != 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(ERRNO), &lino, _exit); } for (int32_t level = 0; level < TFS_MAX_TIERS; level++) { @@ -271,7 +271,7 @@ int32_t tfsMkdirAt(STfs *pTfs, const char *rname, SDiskID diskId) { } (void)snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname); if (taosMkDir(aname) != 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } TAOS_RETURN(0); @@ -283,7 +283,7 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { char *s = NULL; char *dir = NULL; if ((code = tfsMkdirAt(pTfs, rname, diskId)) < 0) { - if (errno == ENOENT) { + if (ERRNO == ENOENT) { // Try to create upper if ((s = taosStrdup(rname)) == NULL) { TAOS_CHECK_GOTO(terrno, &lino, _exit); @@ -384,8 +384,8 @@ static int32_t tfsRenameAt(STfs *pTfs, SDiskID diskId, const char *orname, const (void)snprintf(oaname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, orname); (void)snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname); - if (taosRenameFile(oaname, naname) != 0 && errno != ENOENT) { - int32_t code = TAOS_SYSTEM_ERROR(errno); // TODO: use return value of taosRenameFile directly + if (taosRenameFile(oaname, naname) != 0 && ERRNO != ENOENT) { + int32_t code = TAOS_SYSTEM_ERROR(ERRNO); // TODO: use return value of taosRenameFile directly fError("%s failed to rename %s to %s since %s", __func__, oaname, naname, tstrerror(code)); TAOS_RETURN(code); } @@ -565,10 +565,17 @@ int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); } +#ifndef TD_ASTRA if (!taosCheckAccessFile(dirName, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) { fError("failed to mount %s to FS since no R/W access rights", pCfg->dir); TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); } +#else // TD_ASTRA_TODO + if (!taosCheckAccessFile(dirName, TD_FILE_ACCESS_EXIST_OK)) { + fError("failed to mount %s to FS since no access rights", pCfg->dir); + TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); + } +#endif if (!taosIsDir(dirName)) { fError("failed to mount %s to FS since not a directory", pCfg->dir); @@ -581,6 +588,7 @@ int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { } static int32_t tfsFormatDir(char *idir, char *odir) { +#ifndef TD_ASTRA int32_t code = 0, lino = 0; wordexp_t wep = {0}; int32_t dirLen = 0; @@ -607,6 +615,10 @@ _exit: dirLen); } TAOS_RETURN(code); +#else + tstrncpy(odir, idir, TSDB_FILENAME_LEN); + TAOS_RETURN(0); +#endif } static int32_t tfsCheck(STfs *pTfs) { @@ -672,7 +684,7 @@ static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pTfsDir) { } pTfsDir->pDir = taosOpenDir(adir); if (pTfsDir->pDir != NULL) break; - fWarn("%s failed to open dir %s since %s", __func__, adir, tstrerror(TAOS_SYSTEM_ERROR(errno))); + fWarn("%s failed to open dir %s since %s", __func__, adir, tstrerror(TAOS_SYSTEM_ERROR(ERRNO))); } TAOS_RETURN(0); diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index 8f229f1f84..0fdff6530d 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -42,7 +42,7 @@ int32_t tfsInitTier(STfsTier *pTier, int32_t level) { (void)memset(pTier, 0, sizeof(STfsTier)); if (taosThreadSpinInit(&pTier->lock, 0) != 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } pTier->level = level; diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 44559f2db2..118ff98080 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -18,6 +18,14 @@ extern "C" { #endif +#ifdef TD_ASTRA_RPC +#include "tmsg.h" +#include "transLog.h" +#include "transportInt.h" +#include "trpc.h" +#include "tversion.h" + +#else #include #include "theap.h" #include "tmsg.h" @@ -25,6 +33,7 @@ extern "C" { #include "transportInt.h" #include "trpc.h" #include "ttrace.h" +#endif typedef bool (*FilteFunc)(void* arg); @@ -110,6 +119,11 @@ typedef SRpcCtxVal STransCtxVal; typedef SRpcInfo STrans; typedef SRpcConnInfo STransHandleInfo; +typedef struct { + tsem_t* sem; + STransMsg pMsg; +} STransReqWithSem; + // ref mgt handle typedef struct SExHandle { void* handle; @@ -137,7 +151,7 @@ typedef struct SCvtAddr { bool cvt; } SCvtAddr; -#pragma pack(push, 1) +PACK_PUSH_MIN typedef struct { int8_t inUse; @@ -182,7 +196,7 @@ typedef struct { uint8_t secret[TSDB_PASSWORD_LEN]; } STransUserMsg; -#pragma pack(pop) +PACK_POP int32_t transCreateReqEpsetFromUserEpset(const SEpSet* pEpset, SReqEpSet** pReqEpSet); int32_t transCreateUserEpsetFromReqEpset(const SReqEpSet* pReqEpSet, SEpSet* pEpSet); @@ -231,7 +245,24 @@ typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } Co #define transIsReq(type) (type & 1U) #define transLabel(trans) ((STrans*)trans)->label +#ifdef TD_ASTRA_RPC +typedef struct SConnBuffer { + char* buf; + int len; + int cap; + int left; + int total; + int invalid; +} SConnBuffer; +int32_t transInitBuffer(SConnBuffer* buf); +int32_t transClearBuffer(SConnBuffer* buf); +void transDestroyBuffer(SConnBuffer* buf); +// int32_t transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); +// bool transReadComplete(SConnBuffer* connBuf); +int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf); + +#else typedef struct SConnBuffer { char* buf; int len; @@ -303,9 +334,22 @@ int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf); int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf); int32_t transSetConnOption(uv_tcp_t* stream, int keepalive); +#endif -void transRefSrvHandle(void* handle); -void transUnrefSrvHandle(void* handle); +#ifdef TD_ASTRA_RPC +int32_t transSendRequest(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx); +int32_t transSendRecv(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp); +int32_t transSendRecvWithTimeout(void* pInit, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs); +int32_t transSendRequestWithId(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId); +int32_t transFreeConnById(void* pInit, int64_t transpointId); + +int32_t transSendResponse(STransMsg* msg); +int32_t transRegisterMsg(const STransMsg* msg); +int32_t transSetDefaultAddr(void* pInit, const char* ip, const char* fqdn); +int32_t transSetIpWhiteList(void* pInit, void* arg, FilteFunc* func); +void transRefSrvHandle(void* handle); +void transUnrefSrvHandle(void* handle); void transRefCliHandle(void* handle); int32_t transUnrefCliHandle(void* handle); @@ -314,6 +358,27 @@ int32_t transGetRefCount(void* handle); int32_t transReleaseCliHandle(void* handle, int32_t status); int32_t transReleaseSrvHandle(void* handle, int32_t status); +int32_t transSendRequest2(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx); +int32_t transSendRecv2(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp); +int32_t transSendRecvWithTimeout2(void* pInit, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs); +int32_t transSendRequestWithId2(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId); +int32_t transFreeConnById2(void* pInit, int64_t transpointId); +int32_t transSendResponse2(STransMsg* msg); +int32_t transRegisterMsg2(const STransMsg* msg); +int32_t transSetDefaultAddr2(void* pInit, const char* ip, const char* fqdn); +int32_t transSetIpWhiteList2(void* pInit, void* arg, FilteFunc* func); +void transRefSrvHandle2(void* handle); +void transUnrefSrvHandle2(void* handle); + +void transRefCliHandle2(void* handle); +void transUnrefCliHandle2(void* handle); +int32_t transGetRefCount(void* handle); + +int32_t transReleaseCliHandle2(void* handle, int32_t status); +int32_t transReleaseSrvHandle2(void* handle, int32_t status); + +#else int32_t transSendRequest(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx); int32_t transSendRecv(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp); int32_t transSendRecvWithTimeout(void* pInit, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, @@ -325,16 +390,32 @@ int32_t transSendResponse(const STransMsg* msg); int32_t transRegisterMsg(const STransMsg* msg); int32_t transSetDefaultAddr(void* pInit, const char* ip, const char* fqdn); int32_t transSetIpWhiteList(void* pInit, void* arg, FilteFunc* func); +void transRefSrvHandle(void* handle); +void transUnrefSrvHandle(void* handle); -void transSockInfo2Str(struct sockaddr* sockname, char* dst); +void transRefCliHandle(void* handle); +int32_t transUnrefCliHandle(void* handle); +int32_t transGetRefCount(void* handle); +int32_t transReleaseCliHandle(void* handle, int32_t status); +int32_t transReleaseSrvHandle(void* handle, int32_t status); + +#endif + +#ifndef TD_ASTRA_RPC +void transSockInfo2Str(struct sockaddr* sockname, char* dst); int32_t transAllocHandle(int64_t* refId); - -void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); -void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); - -void transCloseClient(void* arg); -void transCloseServer(void* arg); +void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); +void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); +void transCloseClient(void* arg); +void transCloseServer(void* arg); +#else +int32_t transAllocHandle(int64_t* refId); +void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); +void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); +void transCloseClient(void* arg); +void transCloseServer(void* arg); +#endif void transCtxInit(STransCtx* ctx); void transCtxCleanup(STransCtx* ctx); @@ -412,6 +493,7 @@ void transQueueClear(STransQueue* queue); */ void transQueueDestroy(STransQueue* queue); +#ifndef TD_ASTRA_RPC /* * delay queue based on uv loop and uv timer, and only used in retry */ @@ -437,6 +519,7 @@ int32_t transDQCreate(uv_loop_t* loop, SDelayQueue** queue); void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)); SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs); void transDQCancel(SDelayQueue* queue, SDelayTask* task); +#endif bool transReqEpsetIsEqual(SReqEpSet* a, SReqEpSet* b); @@ -486,6 +569,29 @@ int32_t subnetDebugInfoToBuf(SubnetUtils* pUtils, char* buf); int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf); int32_t transUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf); +#ifdef TD_ASTRA_RPC + +#define ASYNC_CHECK_HANDLE(idMgt, id, exh1) \ + do { \ + if (id > 0) { \ + SExHandle* exh2 = transAcquireExHandle(idMgt, id); \ + if (exh2 == NULL || exh1 != exh2 || (exh2 != NULL && exh2->refId != id)) { \ + tDebug("handle not match, exh1:%p, exh2:%p, refId:%" PRId64 "", exh1, exh2, id); \ + code = TSDB_CODE_INVALID_MSG; \ + goto _return1; \ + } \ + } else { \ + tDebug("invalid handle to release"); \ + goto _return2; \ + } \ + } while (0) +int32_t transUpdateCb(RPC_TYPE type, STrans* pTransport); + +int32_t transGetCb(RPC_TYPE type, STrans** ppTransport); +int32_t transSendReq(STrans* pTransport, SRpcMsg* pMsg, void* pEpSet); +int32_t transSendResp(const SRpcMsg* pMsg); +#endif + enum { REQ_STATUS_INIT = 0, REQ_STATUS_PROCESSING }; #if defined(WINDOWS) || defined(DARWIN) @@ -499,6 +605,7 @@ enum { REQ_STATUS_INIT = 0, REQ_STATUS_PROCESSING }; #define HEAP_MISS_HIT_LIMIT 100000 #define READ_TIMEOUT 100000 +#ifndef TD_ASTRA_RPC typedef struct { queue node; // queue for write queue q; // queue for reqs @@ -513,6 +620,8 @@ uv_write_t* allocWReqFromWQ(queue* wq, void* arg); void freeWReqToWQ(queue* wq, SWReqsWrapper* w); int32_t transSetReadOption(uv_handle_t* handle); +#endif + #ifdef __cplusplus } #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 39afb29342..5a66eff6f0 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -16,6 +16,11 @@ #ifndef _TD_TRANSPORT_INT_H_ #define _TD_TRANSPORT_INT_H_ +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef TD_ASTRA_RPC #include #include "lz4.h" #include "os.h" @@ -27,11 +32,20 @@ #include "tref.h" #include "trpc.h" #include "tutil.h" - -#ifdef __cplusplus -extern "C" { +#else +#include "lz4.h" +#include "os.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" +#include "tmsg.h" +#include "transLog.h" +#include "tref.h" +#include "trpc.h" +#include "tutil.h" #endif +#ifndef TD_ASTRA_RPC void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); @@ -82,6 +96,63 @@ typedef struct { int64_t readTimeout; TdThreadMutex mutex; } SRpcInfo; +#else +void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); +void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); + +void taosCloseServer(void* arg); +void taosCloseClient(void* arg); + +typedef struct { + int sessions; // number of sessions allowed + int numOfThreads; // number of threads to process incoming messages + int idleTime; // milliseconds; + uint16_t localPort; + int8_t connType; + char label[TSDB_LABEL_LEN]; + char user[TSDB_UNI_LEN]; // meter ID + int32_t compatibilityVer; + int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size + int8_t encryption; // encrypt or not + + int32_t retryMinInterval; // retry init interval + int32_t retryStepFactor; // retry interval factor + int32_t retryMaxInterval; // retry max interval + int32_t retryMaxTimeout; + + int32_t failFastThreshold; + int32_t failFastInterval; + + int8_t notWaitAvaliableConn; // 1: no delay, 0: delay + + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); + bool (*retry)(int32_t code, tmsg_t msgType); + bool (*startTimer)(int32_t code, tmsg_t msgType); + void (*destroyFp)(void* ahandle); + bool (*failFastFp)(tmsg_t msgType); + bool (*noDelayFp)(tmsg_t msgType); + + int32_t connLimitNum; + int8_t connLimitLock; // 0: no lock. 1. lock + int8_t supportBatch; // 0: no batch, 1: support batch + int32_t batchSize; + int32_t timeToGetConn; + int index; + void* parent; + void* tcphandle; // returned handle from TCP initialization + int64_t refId; + TdThreadMutex mutex; + int16_t type; + + TdThreadMutex sidMutx; + SHashObj* sidTable; + + TdThreadMutex seqMutex; + int64_t seq; + SHashObj* seqTable; + int8_t shareConnLimit; +} SRpcInfo; +#endif #ifdef __cplusplus } diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 811c044e47..f75bc8b0ec 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -13,11 +13,12 @@ * along with this program. If not, see . */ +#ifndef TD_ASTRA_RPC #define _DEFAULT_SOURCE // clang-format off -#include #include "zlib.h" #include "thttp.h" +#include #include "taoserror.h" #include "transComm.h" @@ -917,7 +918,7 @@ int64_t transInitHttpChanImpl() { err = taosThreadCreate(&http->thread, NULL, httpThread, (void*)http); if (err != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _ERROR; } @@ -1119,3 +1120,26 @@ _end: } return code; } +#else // TD_ASTRA_RPC + +#include "thttp.h" + +int32_t taosTelemetryMgtInit(STelemAddrMgmt* mgt, char* defaultAddr) { return 0; } +void taosTelemetryDestroy(STelemAddrMgmt* mgt) { return; } + +// not safe for multi-thread, should be called in the same thread +int32_t taosSendTelemReport(STelemAddrMgmt* mgt, const char* uri, uint16_t port, char* pCont, int32_t contLen, + EHttpCompFlag flag) { + return 0; +} + +int32_t taosSendRecvHttpReportWithQID(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, + EHttpCompFlag flag, const char* qid, int64_t recvBufId) { + return 0; +} + +int32_t taosSendHttpReportWithQID(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, + EHttpCompFlag flag, const char* qid) { + return 0; +} +#endif \ No newline at end of file diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 6507476713..0c2bba6395 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -15,6 +15,7 @@ #include "transComm.h" +#ifndef TD_ASTRA_RPC void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* pInit) = { transInitServer, transInitClient}; @@ -250,3 +251,309 @@ void rpcCleanup(void) { return; } +#else +#ifdef TD_ASTRA_RPC +void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = { + transInitServer, transInitClient}; +void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; +int (*transReleaseHandle[])(void* handle, int32_t status) = {transReleaseSrvHandle, transReleaseCliHandle}; +#else +void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = { + transInitServer, transInitClient}; +void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; +int (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle}; +#endif + +static int32_t transValidLocalFqdn(const char* localFqdn, uint32_t* ip) { return 0; } +typedef struct { + char* lablset; + RPC_TYPE type; +} SLableSet; +static SLableSet labelSet[] = { + {"TSC", TD_ASTRA_CLIENT | TD_ASTRA_DSVR}, + {"DNODE-CLI", TD_ASTRA_DSVR_CLIENT | TD_ASTRA_DSVR}, + {"DNODE-STA-CLI", TD_ASTRA_DSVR_STA_CLIENT | TD_ASTRA_DSVR}, + {"DNODE-SYNC-CLI", TD_ASTRA_DSVR_SYNC_CLIENT | TD_ASTRA_DSVR}, + {"DND-S", TD_ASTRA_DSVR}, +}; + +void* rpcOpen(const SRpcInit* pInit) { + int32_t code = rpcInit(); + if (code != 0) { + TAOS_CHECK_GOTO(code, NULL, _end); + } + + SRpcInfo* pRpc = taosMemoryCalloc(1, sizeof(SRpcInfo)); + if (pRpc == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _end); + } + if (pInit->label) { + int len = strlen(pInit->label) > sizeof(pRpc->label) ? sizeof(pRpc->label) : strlen(pInit->label); + memcpy(pRpc->label, pInit->label, len); + } + + pRpc->compressSize = pInit->compressSize; + if (pRpc->compressSize < 0) { + pRpc->compressSize = -1; + } + + pRpc->encryption = pInit->encryption; + pRpc->compatibilityVer = pInit->compatibilityVer; + + pRpc->retryMinInterval = pInit->retryMinInterval; // retry init interval + pRpc->retryStepFactor = pInit->retryStepFactor; + pRpc->retryMaxInterval = pInit->retryMaxInterval; + pRpc->retryMaxTimeout = pInit->retryMaxTimeout; + + pRpc->failFastThreshold = pInit->failFastThreshold; + pRpc->failFastInterval = pInit->failFastInterval; + + // register callback handle + pRpc->cfp = pInit->cfp; + pRpc->retry = pInit->rfp; + pRpc->startTimer = pInit->tfp; + pRpc->destroyFp = pInit->dfp; + pRpc->failFastFp = pInit->ffp; + pRpc->noDelayFp = pInit->noDelayFp; + pRpc->connLimitNum = pInit->connLimitNum; + if (pRpc->connLimitNum == 0) { + pRpc->connLimitNum = 20; + } + + pRpc->connLimitLock = pInit->connLimitLock; + pRpc->supportBatch = pInit->supportBatch; + pRpc->batchSize = pInit->batchSize; + + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + if (pRpc->numOfThreads <= 0) { + pRpc->numOfThreads = 1; + } + + uint32_t ip = 0; + if (pInit->connType == TAOS_CONN_SERVER) { + if ((code = transValidLocalFqdn(pInit->localFqdn, &ip)) != 0) { + tError("invalid fqdn:%s, errmsg:%s", pInit->localFqdn, tstrerror(code)); + TAOS_CHECK_GOTO(code, NULL, _end); + } + } + + pRpc->connType = pInit->connType; + pRpc->idleTime = pInit->idleTime; + pRpc->parent = pInit->parent; + if (pInit->user) { + tstrncpy(pRpc->user, pInit->user, sizeof(pRpc->user)); + } + pRpc->timeToGetConn = pInit->timeToGetConn; + if (pRpc->timeToGetConn == 0) { + pRpc->timeToGetConn = 10 * 1000; + } + pRpc->notWaitAvaliableConn = pInit->notWaitAvaliableConn; + + pRpc->tcphandle = + (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + + if (pRpc->tcphandle == NULL) { + // tError("failed to init rpc handle"); + // TAOS_CHECK_GOTO(terrno, NULL, _end); + } + for (int8_t i = 0; i < sizeof(labelSet) / sizeof(labelSet[0]); i++) { + if (strcmp(labelSet[i].lablset, pRpc->label) == 0) { + pRpc->type = labelSet[i].type; + break; + } + } + + taosThreadMutexInit(&pRpc->sidMutx, NULL); + pRpc->sidTable = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + taosThreadMutexInit(&pRpc->seqMutex, NULL); + pRpc->seqTable = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + pRpc->seq = 1; + transUpdateCb(pRpc->type, pRpc); + + int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); + void* tmp = transAcquireExHandle(transGetInstMgt(), refId); + pRpc->refId = refId; + return (void*)refId; +_end: + taosMemoryFree(pRpc); + terrno = code; + + return NULL; +} +void rpcClose(void* arg) { + tInfo("start to close rpc"); + return; +} +void rpcCloseImpl(void* arg) { + if (arg == NULL) return; + return; + // SRpcInfo* pRpc = (SRpcInfo*)arg; + // if (pRpc->tcphandle != NULL) { + // (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); + // } + // taosMemoryFree(pRpc); +} + +void* rpcMallocCont(int64_t contLen) { + int64_t size = contLen + TRANS_MSG_OVERHEAD; + char* start = taosMemoryCalloc(1, size); + if (start == NULL) { + tError("failed to malloc msg, size:%" PRId64, size); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } else { + tTrace("malloc mem:%p size:%" PRId64, start, size); + } + + return start + sizeof(STransMsgHead); +} + +void rpcFreeCont(void* cont) { transFreeMsg(cont); } + +void* rpcReallocCont(void* ptr, int64_t contLen) { + if (ptr == NULL) return rpcMallocCont(contLen); + + char* st = (char*)ptr - TRANS_MSG_OVERHEAD; + int64_t sz = contLen + TRANS_MSG_OVERHEAD; + char* nst = taosMemoryRealloc(st, sz); + if (nst == NULL) { + taosMemoryFree(st); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } else { + st = nst; + } + + return st + TRANS_MSG_OVERHEAD; +} + +int32_t rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { + return transSendRequest(shandle, pEpSet, pMsg, NULL); +#ifdef TD_ASTRA_RPC + // return transSendRequest2(shandle, pEpSet, pMsg, NULL); +#else + return transSendRequest(shandle, pEpSet, pMsg, NULL); +#endif +} +int32_t rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { + if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { + return transSendRequest(shandle, pEpSet, pMsg, pCtx); + } else { + return transSendRequestWithId(shandle, pEpSet, pMsg, pRid); + } +#ifdef TD_ASTRA_RPC + // if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { + // return transSendRequest2(shandle, pEpSet, pMsg, pCtx); + // } else { + // return transSendRequestWithId2(shandle, pEpSet, pMsg, pRid); + // } +#else + if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { + return transSendRequest(shandle, pEpSet, pMsg, pCtx); + } else { + return transSendRequestWithId(shandle, pEpSet, pMsg, pRid); + } +#endif +} + +int32_t rpcSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { + return transSendRequestWithId(shandle, pEpSet, pReq, transpointId); +#ifdef TD_ASTRA_RPC + // return transSendRequestWithId2(shandle, pEpSet, pReq, transpointId); +#else + return transSendRequestWithId(shandle, pEpSet, pReq, transpointId); +#endif +} + +int32_t rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { + return transSendRecv(shandle, pEpSet, pMsg, pRsp); +#ifdef TD_ASTRA_RPC + // return transSendRecv2(shandle, pEpSet, pMsg, pRsp); +#else + return transSendRecv(shandle, pEpSet, pMsg, pRsp); +#endif +} +int32_t rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { + return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); +#ifdef TD_ASTRA_RPC + // return transSendRecvWithTimeout2(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); +#else + return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); +#endif +} +int32_t rpcFreeConnById(void* shandle, int64_t connId) { + return transFreeConnById(shandle, connId); +#ifdef TD_ASTRA_RPC + // return transFreeConnById2(shandle, connId); +#else + return transFreeConnById(shandle, connId); +#endif +} + +int32_t rpcSendResponse(SRpcMsg* pMsg) { + return transSendResponse(pMsg); +#ifdef TD_ASTRA_RPC + // return transSendResponse2(pMsg); +#else + return transSendResponse(pMsg); +#endif +} + +int32_t rpcRegisterBrokenLinkArg(SRpcMsg* msg) { + return transRegisterMsg(msg); +#ifdef TD_ASTRA_RPC + // return transRegisterMsg2(msg); +#else + return transRegisterMsg(msg); +#endif +} +int32_t rpcReleaseHandle(void* handle, int8_t type, int32_t status) { + return (*transReleaseHandle[type])(handle, status); +} + +// client only +int32_t rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { + // later + return transSetDefaultAddr(thandle, ip, fqdn); +#ifdef TD_ASTRA_RPC + // return transSetDefaultAddr2(thandle, ip, fqdn); +#else + return transSetDefaultAddr(thandle, ip, fqdn); +#endif +} +// server only +int32_t rpcSetIpWhite(void* thandle, void* arg) { + return transSetIpWhiteList(thandle, arg, NULL); +#ifdef TD_ASTRA_RPC + // return transSetIpWhiteList2(thandle, arg, NULL); +#else + return transSetIpWhiteList(thandle, arg, NULL); +#endif +} + +int32_t rpcAllocHandle(int64_t* refId) { return transAllocHandle(refId); } + +int32_t rpcUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) { return transUtilSIpRangeToStr(pRange, buf); } +int32_t rpcUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf) { + return transUtilSWhiteListToStr(pWhiteList, ppBuf); +} + +int32_t rpcCvtErrCode(int32_t code) { + if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + return TSDB_CODE_RPC_NETWORK_ERROR; + } + return code; +} + +int32_t rpcInit() { return transInit(); } + +void rpcCleanup(void) { + transCleanup(); + // transHttpEnvDestroy(); + + return; +} + +#endif \ No newline at end of file diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index abbf1f9b47..bcbf498141 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -15,10 +15,12 @@ // clang-format off #include "taoserror.h" #include "transComm.h" +#include "tversion.h" #include "tmisce.h" #include "transLog.h" // clang-format on +#ifndef TD_ASTRA_RPC typedef struct { int32_t numOfConn; queue msgQ; @@ -2232,7 +2234,7 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, int err = taosThreadCreate(&pThrd->thread, NULL, cliWorkThread, (void*)(pThrd)); if (err != 0) { destroyThrdObj(pThrd); - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); TAOS_CHECK_GOTO(code, NULL, _err); } else { tDebug("success to create tranport-cli thread:%d", i); @@ -3295,7 +3297,7 @@ int32_t transCreateSyncMsg(STransMsg* pTransMsg, int64_t* refId) { } if (tsem2_init(sem, 0, 0) != 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _EXIT); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(ERRNO), NULL, _EXIT); } STransSyncMsg* pSyncMsg = taosMemoryCalloc(1, sizeof(STransSyncMsg)); @@ -3941,3 +3943,209 @@ int32_t transHeapBalance(SHeap* heap, SCliConn* p) { heapInsert(heap->heap, &p->node); return 0; } + +#else +void transRefCliHandle(void* handle) { return; } +int32_t transUnrefCliHandle(void* handle) { return 0; } +int32_t transReleaseCliHandle(void* handle, int32_t status) { + int32_t code = 0; + + STransMsg tmsg = {.msgType = TDMT_SCH_TASK_RELEASE, + .info.handle = handle, + .info.ahandle = (void*)0, + .info.qId = (int64_t)handle, + code = status}; + + SExHandle* exh = transAcquireExHandle(transGetRefMgt(), (int64_t)handle); + if (exh == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_MODULE_QUIT, NULL, _exception); + } + int64_t transId = (int64_t)handle; + + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)exh->handle); + taosThreadMutexLock(&pTransInst->sidMutx); + STransCtx* pCtx = taosHashGet(pTransInst->sidTable, &transId, sizeof(int64_t)); + transCtxCleanup(pCtx); + taosHashRemove(pTransInst->sidTable, &transId, sizeof(transId)); + + taosThreadMutexUnlock(&pTransInst->sidMutx); + transReleaseExHandle(transGetInstMgt(), (int64_t)exh->handle); + +_exception: + if (code != 0) { + tError("failed to release handle since %s", tstrerror(code)); + } + + transReleaseExHandle(transGetRefMgt(), (int64_t)handle); + return code; +} +int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { + int32_t code = 0; + int32_t cliVer = 0; + + code = taosVersionStrToInt(td_version, &cliVer); + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pTransInst == NULL) { + return TSDB_CODE_RPC_MODULE_QUIT; + } + + TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); + pReq->type = pTransInst->type; + pReq->info.connType = pReq->type; + pReq->info.cliVer = cliVer; + pReq->info.msgType = pReq->msgType; + // pReq->info.ahandle = pReq->ahandle; + if (pReq->info.handle != NULL) { + pReq->info.qId = (int64_t)pReq->info.handle; + } else { + pReq->info.handle = (void*)-1; + } + + if (pReq->info.qId != 0) { + taosThreadMutexLock(&pTransInst->sidMutx); + /******/ + if (ctx != NULL) { + STransCtx* pUserCtx = taosHashGet(pTransInst->sidTable, &pReq->info.qId, sizeof(pReq->info.qId)); + if (pUserCtx != NULL) { + transCtxMerge(pUserCtx, ctx); + } else { + taosHashPut(pTransInst->sidTable, &pReq->info.qId, sizeof(pReq->info.qId), ctx, sizeof(STransCtx)); + } + } + if (pReq->msgType == TDMT_SCH_DROP_TASK) { + (pTransInst->destroyFp)(pReq->info.ahandle); + } + taosThreadMutexUnlock(&pTransInst->sidMutx); + } + taosThreadMutexLock(&pTransInst->seqMutex); + pReq->info.seq = pTransInst->seq++; + pReq->parent = pTransInst; + taosHashPut(pTransInst->seqTable, &pReq->info.seq, sizeof(pReq->info.seq), &pReq->msgType, sizeof(pReq->msgType)); + taosThreadMutexUnlock(&pTransInst->seqMutex); + + sprintf(pReq->info.conn.user, "root"); + + code = transSendReq(pTransInst, pReq, NULL); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); + return code; +} + +int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { + int32_t code; + int8_t idInited = 0; + + STrans* pInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pInst == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _exception); + } + + TAOS_CHECK_GOTO(transAllocHandle(transpointId), NULL, _exception); + + SExHandle* exh = transAcquireExHandle(transGetRefMgt(), *transpointId); + if (exh == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_MODULE_QUIT, NULL, _exception); + } + idInited = 1; + + pReq->info.handle = (void*)(*transpointId); + pReq->info.qId = *transpointId; + exh->handle = shandle; + + code = transSendRequest(shandle, pEpSet, pReq, NULL); + TAOS_CHECK_GOTO(terrno, NULL, _exception); + +_exception: + if (code != 0) { + tError("failedt to send request with id since %s", tstrerror(code)); + } + + if (idInited) transReleaseExHandle(transGetRefMgt(), *transpointId); + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + return code; +} + +int32_t transCreateSyncMsg(STransMsg* pTransMsg, int64_t* refId) { return 0; } +int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { + int32_t code = 0; + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pTransInst == NULL) { + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + return TSDB_CODE_RPC_MODULE_QUIT; + } + + if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); + + STransReqWithSem* tmsg = taosMemoryCalloc(1, sizeof(STransReqWithSem)); + tmsg->sem = taosMemoryCalloc(1, sizeof(tsem_t)); + code = tsem_init(tmsg->sem, 0, 0); + if (code != 0) { + taosMemoryFree(tmsg->sem); + return code; + } + pReq->info.reqWithSem = tmsg; + transSendRequest(shandle, NULL, pReq, NULL); + TAOS_UNUSED(tsem_wait(tmsg->sem)); + tsem_destroy(tmsg->sem); + taosMemFree(tmsg->sem); + + memcpy(pRsp, &tmsg->pMsg, sizeof(STransMsg)); + taosMemoryFree(tmsg); + + taosReleaseRef(transGetInstMgt(), (int64_t)shandle); + return 0; +} +int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { + return transSendRecvWithTimeout(shandle, (SEpSet*)pEpSet, pReq, pRsp, NULL, 0); +} +int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { return 0; } +int32_t transAllocHandle(int64_t* refId) { + SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); + if (exh == NULL) { + return terrno; + } + + exh->refId = transAddExHandle(transGetRefMgt(), exh); + if (exh->refId < 0) { + taosMemoryFree(exh); + return TSDB_CODE_REF_INVALID_ID; + } + + SExHandle* self = transAcquireExHandle(transGetRefMgt(), exh->refId); + if (exh != self) { + taosMemoryFree(exh); + return TSDB_CODE_REF_INVALID_ID; + } + + QUEUE_INIT(&exh->q); + taosInitRWLatch(&exh->latch); + tDebug("trans alloc sid:%" PRId64 ", malloc:%p", exh->refId, exh); + *refId = exh->refId; + return 0; +} +int32_t transFreeConnById(void* pInstRef, int64_t transpointId) { + int32_t code = 0; + STrans* pInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + return TSDB_CODE_RPC_MODULE_QUIT; + } + if (transpointId == 0) { + tDebug("not free by refId:%" PRId64, transpointId); + TAOS_CHECK_GOTO(0, NULL, _exception); + } +_exception: + if (code != 0) { + tDebug("failed to free by refId:%" PRId64, transpointId); + } + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + + return code; +} + +void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { + return NULL; +} +void transCloseClient(void* arg) { return; } + +#endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index f41463c5db..87be3a88be 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -14,7 +14,9 @@ */ #include "transComm.h" +#include "tqueue.h" +#ifndef TD_ASTRA_RPC #define BUFFER_CAP 8 * 1024 static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; @@ -451,6 +453,7 @@ void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) { return ret; } +#if 0 int32_t transQueueInit(STransQueue* wq, void (*freeFunc)(void* arg)) { QUEUE_INIT(&wq->node); wq->freeFunc = (void (*)(void*))freeFunc; @@ -483,7 +486,8 @@ void* transQueueGet(STransQueue* q, int idx) { return NULL; } -void transQueueRemoveByFilter(STransQueue* q, bool (*filter)(void* e, void* arg), void* arg, void* dst, int32_t size) { +void transQueueRemoveByFilter(STransQueue* q, bool (*filter)(void* e, void* arg), void* arg, void* dst, int32_t size) +{ queue* d = dst; queue* node = QUEUE_NEXT(&q->node); while (node != &q->node) { @@ -539,6 +543,7 @@ void transQueueClear(STransQueue* q) { } } void transQueueDestroy(STransQueue* q) { transQueueClear(q); } +#endif static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) { SDelayTask* arg1 = container_of(a, SDelayTask, node); @@ -684,6 +689,7 @@ SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, return task; } +#if 0 void transPrintEpSet(SEpSet* pEpSet) { if (pEpSet == NULL) { tTrace("NULL epset"); @@ -729,6 +735,7 @@ bool transCompareReqAndUserEpset(SReqEpSet* a, SEpSet* b) { } return true; } +#endif static void transInitEnv() { refMgt = transOpenRefMgt(50000, transDestroyExHandle); @@ -748,7 +755,7 @@ int32_t transInit() { // init env int32_t code = taosThreadOnce(&transModuleInit, transInitEnv); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); } return code; } @@ -1019,4 +1026,728 @@ int32_t transValidReqEpset(SReqEpSet* pReqEpSet) { return TSDB_CODE_INVALID_PARA; } return TSDB_CODE_SUCCESS; +} + +#else +#define BUFFER_CAP 4096 + +typedef struct { + int32_t numOfThread; + STaosQueue** qhandle; + STaosQset** qset; + int64_t idx; + +} MultiThreadQhandle; +typedef struct TThread { + TdThread thread; + int idx; +} TThread; + +TdThreadMutex mutex[2]; +MultiThreadQhandle* multiQ[2] = {NULL, NULL}; +static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; + +static int32_t refMgt; +static int32_t svrRefMgt; +static int32_t instMgt; +static int32_t transSyncMsgMgt; +TdThreadMutex mutex[2]; + +TdThreadMutex tableMutex; +SHashObj* hashTable = NULL; + +void transDestroySyncMsg(void* msg); + +int32_t transCompressMsg(char* msg, int32_t len) { + int32_t ret = 0; + int compHdr = sizeof(STransCompMsg); + STransMsgHead* pHead = transHeadFromCont(msg); + + char* buf = taosMemoryMalloc(len + compHdr + 8); // 8 extra bytes + if (buf == NULL) { + tWarn("failed to allocate memory for rpc msg compression, contLen:%d", len); + ret = len; + return ret; + } + + int32_t clen = LZ4_compress_default(msg, buf, len, len + compHdr); + /* + * only the compressed size is less than the value of contLen - overhead, the compression is applied + * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message + */ + if (clen > 0 && clen < len - compHdr) { + STransCompMsg* pComp = (STransCompMsg*)msg; + pComp->reserved = 0; + pComp->contLen = htonl(len); + memcpy(msg + compHdr, buf, clen); + + tDebug("compress rpc msg, before:%d, after:%d", len, clen); + ret = clen + compHdr; + pHead->comp = 1; + } else { + ret = len; + pHead->comp = 0; + } + taosMemoryFree(buf); + return ret; +} +int32_t transDecompressMsg(char** msg, int32_t* len) { + return 0; +} + +void transFreeMsg(void* msg) { + if (msg == NULL) { + return; + } + tTrace("rpc free cont:%p", (char*)msg - TRANS_MSG_OVERHEAD); + taosMemoryFree((char*)msg - sizeof(STransMsgHead)); +} + +void transCtxInit(STransCtx* ctx) { + // init transCtx + ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK); +} +void transCtxCleanup(STransCtx* ctx) { + if (ctx == NULL || ctx->args == NULL) { + return; + } + + STransCtxVal* iter = taosHashIterate(ctx->args, NULL); + while (iter) { + ctx->freeFunc(iter->val); + iter = taosHashIterate(ctx->args, iter); + } + if (ctx->freeFunc) ctx->freeFunc(ctx->brokenVal.val); + taosHashCleanup(ctx->args); + ctx->args = NULL; +} + +void transCtxMerge(STransCtx* dst, STransCtx* src) { + if (src->args == NULL || src->freeFunc == NULL) { + return; + } + if (dst->args == NULL) { + dst->args = src->args; + dst->brokenVal = src->brokenVal; + dst->freeFunc = src->freeFunc; + src->args = NULL; + return; + } + void* key = NULL; + size_t klen = 0; + void* iter = taosHashIterate(src->args, NULL); + while (iter) { + STransCtxVal* sVal = (STransCtxVal*)iter; + key = taosHashGetKey(sVal, &klen); + + int32_t code = taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal)); + if (code != 0) { + tError("failed to put val to hash, reason:%s", tstrerror(code)); + } + iter = taosHashIterate(src->args, iter); + } + taosHashCleanup(src->args); +} +void* transCtxDumpVal(STransCtx* ctx, int32_t key) { + if (ctx->args == NULL) { + return NULL; + } + STransCtxVal* cVal = taosHashGet(ctx->args, (const void*)&key, sizeof(key)); + if (cVal == NULL) { + return NULL; + } + void* ret = NULL; + TAOS_UNUSED((*cVal->clone)(cVal->val, &ret)); + return ret; +} +void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) { + void* ret = NULL; + if (ctx->brokenVal.clone == NULL) { + return ret; + } + TAOS_UNUSED((*ctx->brokenVal.clone)(ctx->brokenVal.val, &ret)); + + *msgType = ctx->brokenVal.msgType; + + return ret; +} + +bool cliMayGetAhandle(STrans* pTrans, SRpcMsg* pMsg) { + int64_t seq = pMsg->info.seq; + int32_t* msgType = NULL; + + if (pMsg->msgType == TDMT_SCH_TASK_RELEASE || pMsg->msgType == TDMT_SCH_TASK_RELEASE + 1) { + STransCtx* ctx = taosHashGet(pTrans->sidTable, &pMsg->info.qId, sizeof(pMsg->info.qId)); + transCtxCleanup(ctx); + taosHashRemove(pTrans->sidTable, &pMsg->info.qId, sizeof(pMsg->info.qId)); + return true; + } + taosThreadMutexLock(&pTrans->seqMutex); + msgType = taosHashGet(pTrans->seqTable, &seq, sizeof(seq)); + taosThreadMutexUnlock(&pTrans->seqMutex); + if (msgType == NULL) { + STransCtx* ctx = taosHashGet(pTrans->sidTable, &pMsg->info.qId, sizeof(pMsg->info.qId)); + if (ctx == NULL) { + return false; + } + pMsg->info.ahandle = transCtxDumpVal(ctx, pMsg->msgType); + tError("failed to find msg type for seq:%" PRId64 ", gen ahandle for type %s", seq, TMSG_INFO(pMsg->msgType)); + } else { + taosThreadMutexLock(&pTrans->seqMutex); + taosHashRemove(pTrans->seqTable, &seq, sizeof(seq)); + msgType = taosHashGet(pTrans->seqTable, &seq, sizeof(seq)); + taosThreadMutexUnlock(&pTrans->seqMutex); + } + return true; +} + +void* processSvrMsg(void* arg) { + TThread* thread = (TThread*)arg; + + int32_t idx = thread->idx; + static int num = 0; + STaosQall* qall; + SRpcMsg * pRpcMsg, rpcMsg; + int type; + SQueueInfo qinfo = {0}; + + taosAllocateQall(&qall); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(multiQ[1]->qset[idx], qall, &qinfo); + if (numOfMsgs <= 0) break; + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; i++) { + taosGetQitem(qall, (void**)&pRpcMsg); + taosThreadMutexLock(&mutex[1]); + RpcCfp fp = NULL; + void* parent = NULL; + STraceId* trace = &pRpcMsg->info.traceId; + tGDebug("taos %s received from taosd", TMSG_INFO(pRpcMsg->msgType)); + STrans* pTrans = NULL; + transGetCb(pRpcMsg->type, &pTrans); + + taosThreadMutexUnlock(&mutex[1]); + + if (pTrans != NULL) { + if (cliMayGetAhandle(pTrans, pRpcMsg)) { + if (pRpcMsg->info.reqWithSem == NULL) { + (pTrans->cfp)(pTrans->parent, pRpcMsg, NULL); + } else { + STransReqWithSem* reqWithSem = pRpcMsg->info.reqWithSem; + memcpy(&reqWithSem->pMsg, pRpcMsg, sizeof(SRpcMsg)); + tsem_post(reqWithSem->sem); + } + } else { + tDebug("taosd %s received from taosd, ignore", TMSG_INFO(pRpcMsg->msgType)); + } + } + taosFreeQitem(pRpcMsg); + } + taosUpdateItemSize(qinfo.queue, numOfMsgs); + } + + taosFreeQall(qall); + return NULL; +} +void* procClientMsg(void* arg) { + TThread* thread = (TThread*)arg; + + int32_t idx = thread->idx; + static int num = 0; + STaosQall* qall; + SRpcMsg * pRpcMsg, rpcMsg; + int type; + SQueueInfo qinfo = {0}; + + taosAllocateQall(&qall); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(multiQ[0]->qset[idx], qall, &qinfo); + tDebug("%d msgs are received", numOfMsgs); + if (numOfMsgs <= 0) break; + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; i++) { + taosGetQitem(qall, (void**)&pRpcMsg); + + STraceId* trace = &pRpcMsg->info.traceId; + tDebug("taosc %s received from taosc", TMSG_INFO(pRpcMsg->msgType)); + RpcCfp fp = NULL; + // void* parent; + STrans* pTrans = NULL; + taosThreadMutexLock(&mutex[1]); + if ((pRpcMsg->type & TD_ASTRA_DSVR) != 0) { + transGetCb(TD_ASTRA_DSVR, &pTrans); + } + taosThreadMutexUnlock(&mutex[1]); + if (pTrans->cfp != NULL) { + (pTrans->cfp)(pTrans->parent, pRpcMsg, NULL); + } else { + tError("taosc failed to find callback for msg type:%s", TMSG_INFO(pRpcMsg->msgType)); + } + taosFreeQitem(pRpcMsg); + } + taosUpdateItemSize(qinfo.queue, numOfMsgs); + } + + taosFreeQall(qall); + return NULL; +} +static void transInitEnv() { + refMgt = transOpenRefMgt(50000, transDestroyExHandle); + svrRefMgt = transOpenRefMgt(50000, transDestroyExHandle); + instMgt = taosOpenRef(50, rpcCloseImpl); + transSyncMsgMgt = taosOpenRef(50, transDestroySyncMsg); + + taosThreadMutexInit(&tableMutex, NULL); + hashTable = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK); + + int32_t numOfAthread = 1; + + multiQ[0] = taosMemoryMalloc(sizeof(MultiThreadQhandle)); + multiQ[0]->numOfThread = numOfAthread; + multiQ[0]->qhandle = (STaosQueue**)taosMemoryMalloc(sizeof(STaosQueue*) * numOfAthread); + multiQ[0]->qset = (STaosQset**)taosMemoryMalloc(sizeof(STaosQset*) * numOfAthread); + + taosThreadMutexInit(&mutex[0], NULL); + + for (int i = 0; i < numOfAthread; i++) { + taosOpenQueue(&(multiQ[0]->qhandle[i])); + taosOpenQset(&multiQ[0]->qset[i]); + taosAddIntoQset(multiQ[0]->qset[i], multiQ[0]->qhandle[i], NULL); + } + { + TThread* threads = taosMemoryMalloc(sizeof(TThread) * numOfAthread); + for (int i = 0; i < numOfAthread; i++) { + threads[i].idx = i; + taosThreadCreate(&(threads[i].thread), NULL, procClientMsg, (void*)&threads[i]); + } + } + + multiQ[1] = taosMemoryMalloc(sizeof(MultiThreadQhandle)); + multiQ[1]->numOfThread = numOfAthread; + multiQ[1]->qhandle = (STaosQueue**)taosMemoryMalloc(sizeof(STaosQueue*) * numOfAthread); + multiQ[1]->qset = (STaosQset**)taosMemoryMalloc(sizeof(STaosQset*) * numOfAthread); + taosThreadMutexInit(&mutex[1], NULL); + + for (int i = 0; i < numOfAthread; i++) { + taosOpenQueue(&(multiQ[1]->qhandle[i])); + taosOpenQset(&multiQ[1]->qset[i]); + taosAddIntoQset(multiQ[1]->qset[i], multiQ[1]->qhandle[i], NULL); + } + { + TThread* threads = taosMemoryMalloc(sizeof(TThread) * numOfAthread); + for (int i = 0; i < numOfAthread; i++) { + threads[i].idx = i; + taosThreadCreate(&(threads[i].thread), NULL, processSvrMsg, (void*)&threads[i]); + } + } +} +static void transDestroyEnv() { + transCloseRefMgt(refMgt); + transCloseRefMgt(svrRefMgt); +} + +typedef struct { + void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); + RPC_TYPE type; + void* parant; + STrans* pTransport; +} FP_TYPE; +int32_t transUpdateCb(RPC_TYPE type, STrans* pTransport) { + taosThreadMutexLock(&tableMutex); + + FP_TYPE t = {.type = type, .pTransport = pTransport}; + int32_t code = taosHashPut(hashTable, &type, sizeof(type), &t, sizeof(t)); + taosThreadMutexUnlock(&tableMutex); + return 0; +} +int32_t transGetCb(RPC_TYPE type, STrans** ppTransport) { + taosThreadMutexLock(&tableMutex); + void* p = taosHashGet(hashTable, &type, sizeof(type)); + if (p == NULL) { + taosThreadMutexUnlock(&tableMutex); + return TSDB_CODE_INVALID_MSG; + } + FP_TYPE* t = p; + *ppTransport = t->pTransport; + // *fp = t->fp; + // *arg = t->parant; + taosThreadMutexUnlock(&tableMutex); + return 0; +} + +int32_t transSendReq(STrans* pTransport, SRpcMsg* pMsg, void* pEpSet) { + SRpcMsg* pTemp; + + taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0, (void**)&pTemp); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + int64_t cidx = multiQ[0]->idx++; + int32_t idx = cidx % (multiQ[0]->numOfThread); + tDebug("taos request is sent , type:%s, contLen:%d, item:%p", TMSG_INFO(pMsg->msgType), pMsg->contLen, pTemp); + taosWriteQitem(multiQ[0]->qhandle[idx], pTemp); + return 0; +} +int32_t transSendResp(const SRpcMsg* pMsg) { + SRpcMsg* pTemp; + + taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0, (void**)&pTemp); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + int64_t cidx = multiQ[1]->idx++; + int32_t idx = cidx % (multiQ[1]->numOfThread); + tDebug("taos resp is sent to, type:%s, contLen:%d, item:%p", TMSG_INFO(pMsg->msgType), pMsg->contLen, pTemp); + taosWriteQitem(multiQ[1]->qhandle[idx], pTemp); + return 0; +} + +int32_t transInit() { + // init env + int32_t code = taosThreadOnce(&transModuleInit, transInitEnv); + if (code != 0) { + code = TAOS_SYSTEM_ERROR(ERRNO); + } + return code; +} + +int32_t transGetRefMgt() { return refMgt; } +int32_t transGetSvrRefMgt() { return svrRefMgt; } +int32_t transGetInstMgt() { return instMgt; } +int32_t transGetSyncMsgMgt() { return transSyncMsgMgt; } + +void transCleanup() { + // clean env + transDestroyEnv(); + return; +} +int32_t transOpenRefMgt(int size, void (*func)(void*)) { + /// add later + return taosOpenRef(size, func); +} +void transCloseRefMgt(int32_t mgt) { + // close ref + taosCloseRef(mgt); + return; +} +int64_t transAddExHandle(int32_t refMgt, void* p) { + return taosAddRef(refMgt, p); + // acquire extern handle +} +void transRemoveExHandle(int32_t refMgt, int64_t refId) { + // acquire extern handle + int32_t code = taosRemoveRef(refMgt, refId); + return; +} + +void* transAcquireExHandle(int32_t refMgt, int64_t refId) { + // acquire extern handle + return (void*)taosAcquireRef(refMgt, refId); +} + +void transReleaseExHandle(int32_t refMgt, int64_t refId) { + // release extern handle + int32_t code = taosReleaseRef(refMgt, refId); + return; +} +void transDestroyExHandle(void* handle) { + if (handle == NULL) { + return; + } + SExHandle* eh = handle; + if (!QUEUE_IS_EMPTY(&eh->q)) { + tDebug("handle %p mem leak", handle); + } + tDebug("free exhandle %p", handle); + taosMemoryFree(handle); + return; +} + +void transDestroySyncMsg(void* msg) { + if (msg == NULL) return; + + STransSyncMsg* pSyncMsg = msg; + TAOS_UNUSED(tsem2_destroy(pSyncMsg->pSem)); + taosMemoryFree(pSyncMsg->pSem); + transFreeMsg(pSyncMsg->pRsp->pCont); + taosMemoryFree(pSyncMsg->pRsp); + taosMemoryFree(pSyncMsg); + return; +} + +uint32_t subnetIpRang2Int(SIpV4Range* pRange) { return 0; } +int32_t subnetInit(SubnetUtils* pUtils, SIpV4Range* pRange) { return 0; } +int32_t subnetDebugInfoToBuf(SubnetUtils* pUtils, char* buf) { return 0; } +int32_t subnetCheckIp(SubnetUtils* pUtils, uint32_t ip) { return 0; } + +int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) { return 0; } + +int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) { return 0; } + +int32_t transInitBuffer(SConnBuffer* buf) { + buf->buf = taosMemoryCalloc(1, BUFFER_CAP); + if (buf->buf == NULL) { + return terrno; + } + + buf->cap = BUFFER_CAP; + buf->left = -1; + buf->len = 0; + buf->total = 0; + buf->invalid = 0; + return 0; +} +void transDestroyBuffer(SConnBuffer* p) { + taosMemoryFree(p->buf); + p->buf = NULL; +} + +int32_t transClearBuffer(SConnBuffer* buf) { + SConnBuffer* p = buf; + if (p->cap > BUFFER_CAP) { + p->cap = BUFFER_CAP; + p->buf = taosMemoryRealloc(p->buf, BUFFER_CAP); + if (p->buf == NULL) { + return terrno; + } + } + p->left = -1; + p->len = 0; + p->total = 0; + p->invalid = 0; + return 0; +} + +int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf) { + static const int HEADSIZE = sizeof(STransMsgHead); + int32_t code = 0; + SConnBuffer* p = connBuf; + if (p->left != 0 || p->total <= 0) { + return TSDB_CODE_INVALID_MSG; + } + int total = p->total; + if (total >= HEADSIZE && !p->invalid) { + *buf = taosMemoryCalloc(1, total); + if (*buf == NULL) { + return terrno; + } + memcpy(*buf, p->buf, total); + if ((code = transResetBuffer(connBuf, resetBuf)) < 0) { + return code; + } + } else { + total = -1; + return TSDB_CODE_INVALID_MSG; + } + return total; +} + +int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) { + SConnBuffer* p = connBuf; + if (p->total < p->len) { + int left = p->len - p->total; + memmove(p->buf, p->buf + p->total, left); + p->left = -1; + p->total = 0; + p->len = left; + } else if (p->total == p->len) { + p->left = -1; + p->total = 0; + p->len = 0; + if (p->cap > BUFFER_CAP) { + if (resetBuf) { + p->cap = BUFFER_CAP; + p->buf = taosMemoryRealloc(p->buf, p->cap); + if (p->buf == NULL) { + return terrno; + } + } + } + } else { + tError("failed to reset buffer, total:%d, len:%d since %s", p->total, p->len, tstrerror(TSDB_CODE_INVALID_MSG)); + return TSDB_CODE_INVALID_MSG; + } + return 0; +} + +int32_t transCreateReqEpsetFromUserEpset(const SEpSet* pEpset, SReqEpSet** pReqEpSet) { + if (pEpset == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + if (pReqEpSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + int32_t size = sizeof(SReqEpSet) + sizeof(SEp) * pEpset->numOfEps; + SReqEpSet* pReq = (SReqEpSet*)taosMemoryCalloc(1, size); + if (pReq == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy((char*)pReq, (char*)pEpset, size); + // clear previous + taosMemoryFree(*pReqEpSet); + + if (transValidReqEpset(pReq) != TSDB_CODE_SUCCESS) { + taosMemoryFree(pReq); + return TSDB_CODE_INVALID_PARA; + } + + *pReqEpSet = pReq; + return TSDB_CODE_SUCCESS; +} + +int32_t transCreateUserEpsetFromReqEpset(const SReqEpSet* pReqEpSet, SEpSet* pEpSet) { + if (pReqEpSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } + memcpy((char*)pEpSet, (char*)pReqEpSet, sizeof(SReqEpSet) + sizeof(SEp) * pReqEpSet->numOfEps); + return TSDB_CODE_SUCCESS; +} + +int32_t transValidReqEpset(SReqEpSet* pReqEpSet) { + if (pReqEpSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (pReqEpSet->numOfEps == 0 || pReqEpSet->numOfEps > TSDB_MAX_EP_NUM || pReqEpSet->inUse >= TSDB_MAX_EP_NUM) { + return TSDB_CODE_INVALID_PARA; + } + return TSDB_CODE_SUCCESS; +} + +#endif // TD_ASTRA_RPC + +int32_t transQueueInit(STransQueue* wq, void (*freeFunc)(void* arg)) { + QUEUE_INIT(&wq->node); + wq->freeFunc = (void (*)(void*))freeFunc; + wq->size = 0; + wq->inited = 1; + return 0; +} +void transQueuePush(STransQueue* q, void* arg) { + queue* node = arg; + QUEUE_PUSH(&q->node, node); + q->size++; +} +void* transQueuePop(STransQueue* q) { + if (q->size == 0) return NULL; + + queue* head = QUEUE_HEAD(&q->node); + QUEUE_REMOVE(head); + q->size--; + return head; +} +int32_t transQueueSize(STransQueue* q) { return q->size; } + +void* transQueueGet(STransQueue* q, int idx) { + if (q->size == 0) return NULL; + + while (idx-- > 0) { + queue* node = QUEUE_NEXT(&q->node); + if (node == &q->node) return NULL; + } + return NULL; +} + +void transQueueRemoveByFilter(STransQueue* q, bool (*filter)(void* e, void* arg), void* arg, void* dst, int32_t size) { + queue* d = dst; + queue* node = QUEUE_NEXT(&q->node); + while (node != &q->node) { + queue* next = QUEUE_NEXT(node); + if (filter && filter(node, arg)) { + QUEUE_REMOVE(node); + q->size--; + QUEUE_PUSH(d, node); + if (--size == 0) { + break; + } + } + node = next; + } +} + +void* tranQueueHead(STransQueue* q) { + if (q->size == 0) return NULL; + + queue* head = QUEUE_HEAD(&q->node); + return head; +} + +void* transQueueRm(STransQueue* q, int i) { + // if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { + // return NULL; + // } + // if (i >= taosArrayGetSize(queue->q)) { + // return NULL; + // } + // void* ptr = taosArrayGetP(queue->q, i); + // taosArrayRemove(queue->q, i); + // return ptr; + return NULL; +} + +void transQueueRemove(STransQueue* q, void* e) { + if (q->size == 0) return; + queue* node = e; + QUEUE_REMOVE(node); + q->size--; +} + +bool transQueueEmpty(STransQueue* q) { return q->size == 0 ? true : false; } + +void transQueueClear(STransQueue* q) { + if (q->inited == 0) return; + while (!QUEUE_IS_EMPTY(&q->node)) { + queue* h = QUEUE_HEAD(&q->node); + QUEUE_REMOVE(h); + if (q->freeFunc != NULL) (q->freeFunc)(h); + q->size--; + } +} +void transQueueDestroy(STransQueue* q) { transQueueClear(q); } + +void transPrintEpSet(SEpSet* pEpSet) { + if (pEpSet == NULL) { + tTrace("NULL epset"); + return; + } + char buf[512] = {0}; + int len = tsnprintf(buf, sizeof(buf), "epset:{"); + for (int i = 0; i < pEpSet->numOfEps; i++) { + if (i == pEpSet->numOfEps - 1) { + len += tsnprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } else { + len += tsnprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } + } + len += tsnprintf(buf + len, sizeof(buf) - len, "}"); + tTrace("%s, inUse:%d", buf, pEpSet->inUse); +} +bool transReqEpsetIsEqual(SReqEpSet* a, SReqEpSet* b) { + if (a == NULL && b == NULL) { + return true; + } else if (a == NULL || b == NULL) { + return false; + } + + if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) { + return false; + } + for (int i = 0; i < a->numOfEps; i++) { + if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { + return false; + } + } + return true; +} +bool transCompareReqAndUserEpset(SReqEpSet* a, SEpSet* b) { + if (a->numOfEps != b->numOfEps) { + return false; + } + for (int i = 0; i < a->numOfEps; i++) { + if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { + return false; + } + } + return true; } \ No newline at end of file diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 35d74123f0..f28ad3e6b6 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -16,6 +16,7 @@ static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; +#ifndef TD_ASTRA_RPC static char* notify = "a"; typedef struct { @@ -1532,7 +1533,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, } if (false == taosValidIpAndPort(srv->ip, srv->port)) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); tError("invalid ip/port, %d:%d since %s", srv->ip, srv->port, terrstr()); goto End; } @@ -1684,7 +1685,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, if (code == 0) { tDebug("success to create accept-thread"); } else { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); tError("failed to create accept-thread since %s", tstrerror(code)); goto End; @@ -1903,8 +1904,8 @@ int32_t transReleaseSrvHandle(void* handle, int32_t status) { m->msg = tmsg; m->type = Normal; - tDebug("%s conn:%p, start to send %s, sid:%" PRId64 "", transLabel(pThrd->pInst), exh->handle, TMSG_INFO(tmsg.msgType), - qId); + tDebug("%s conn:%p, start to send %s, sid:%" PRId64 "", transLabel(pThrd->pInst), exh->handle, + TMSG_INFO(tmsg.msgType), qId); if ((code = transAsyncSend(pThrd->asyncPool, &m->q)) != 0) { destroySmsg(m); transReleaseExHandle(info->refIdMgt, refId); @@ -2071,3 +2072,39 @@ int32_t transSetIpWhiteList(void* thandle, void* arg, FilteFunc* func) { } return code; } +#else +int32_t transReleaseSrvHandle(void *handle, int32_t status) { + tDebug("rpc start to release svr handle"); + return 0; +} +void transRefSrvHandle(void *handle) { return; } + +void transUnrefSrvHandle(void *handle) { return; } +int32_t transSendResponse(STransMsg *msg) { + int32_t code = 0; + if (rpcIsReq(msg->info.msgType) && msg->info.msgType != 0) { + msg->msgType = msg->info.msgType + 1; + } + if (msg->info.noResp) { + rpcFreeCont(msg->pCont); + return 0; + } + int32_t svrVer = 0; + code = taosVersionStrToInt(td_version, &svrVer); + msg->info.cliVer = svrVer; + msg->type = msg->info.connType; + return transSendResp(msg); +} +int32_t transRegisterMsg(const STransMsg *msg) { + rpcFreeCont(msg->pCont); + return 0; +} +int32_t transSetIpWhiteList(void *thandle, void *arg, FilteFunc *func) { return 0; } + +void *transInitServer(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *pInit) { return NULL; } +void transCloseServer(void *arg) { + // impl later + return; +} + +#endif \ No newline at end of file diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 304cb52870..3d6708956e 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -114,11 +114,11 @@ static inline SWalFileInfo* walGetCurFileInfo(SWal* pWal) { } static inline void walBuildLogName(SWal* pWal, int64_t fileFirstVer, char* buf) { - TAOS_UNUSED(sprintf(buf, "%s/%020" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer)); + TAOS_UNUSED(sprintf(buf, "%s%s%020" PRId64 "." WAL_LOG_SUFFIX, pWal->path, TD_DIRSEP, fileFirstVer)); } static inline void walBuildIdxName(SWal* pWal, int64_t fileFirstVer, char* buf) { - TAOS_UNUSED(sprintf(buf, "%s/%020" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer)); + TAOS_UNUSED(sprintf(buf, "%s%s%020" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, TD_DIRSEP, fileFirstVer)); } static inline int walValidHeadCksum(SWalCkHead* pHead) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index b4eaa467af..fdf89e7414 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -38,12 +38,12 @@ int64_t FORCE_INLINE walGetCommittedVer(SWal* pWal) { return pWal->vers.commitVe int64_t FORCE_INLINE walGetAppliedVer(SWal* pWal) { return pWal->vers.appliedVer; } -static FORCE_INLINE int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { - return snprintf(buf, WAL_FILE_LEN, "%s/meta-ver%d", pWal->path, metaVer); +static FORCE_INLINE int walBuildMetaName(SWal* pWal, int64_t metaVer, char* buf) { + return snprintf(buf, WAL_FILE_LEN, "%s%smeta-ver%" PRIi64, pWal->path, TD_DIRSEP, metaVer); } static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { - return snprintf(buf, WAL_FILE_LEN, "%s/meta-ver.tmp", pWal->path); + return snprintf(buf, WAL_FILE_LEN, "%s%smeta-ver.tmp", pWal->path, TD_DIRSEP); } FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* lastVer) { @@ -99,12 +99,12 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); if (ret < 0) { - wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset); + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(ERRNO), offset); TAOS_CHECK_GOTO(terrno, &lino, _err); } if (readSize != taosReadFile(pFile, buf, readSize)) { - wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(ERRNO), readSize, fnameStr); TAOS_CHECK_GOTO(terrno, &lino, _err); } @@ -168,7 +168,7 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* } if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) { wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s", - pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr); + pWal->cfg.vgId, strerror(ERRNO), offset + readSize, extraSize, fnameStr); code = terrno; break; } @@ -190,7 +190,7 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* } // found one - retVer = logContent->head.version; + retVer = taosGetInt64Aligned(&logContent->head.version); lastEntryBeginOffset = offset + pos; lastEntryEndOffset = offset + pos + recordLen; @@ -217,7 +217,7 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* } if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pFile) < 0) { - wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); + wError("failed to fsync file due to %s. file:%s", strerror(ERRNO), fnameStr); TAOS_CHECK_GOTO(terrno, &lino, _err); } } @@ -226,9 +226,9 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* _err: if (code != 0) { - wError("vgId:%d, failed to scan log file due to %s, file:%s", pWal->cfg.vgId, tstrerror(terrno), fnameStr); + wError("vgId:%d, failed at line %d to scan log file due to %s, file:%s", pWal->cfg.vgId, lino, tstrerror(code), fnameStr); } - taosCloseFile(&pFile); + TAOS_UNUSED(taosCloseFile(&pFile)); taosMemoryFree(buf); *lastVer = retVer; @@ -316,7 +316,7 @@ static int32_t walRepairLogFileTs(SWal* pWal, bool* updateMeta) { walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); int64_t mtime = 0; if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) { - wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(ERRNO), fnameStr); TAOS_RETURN(terrno); } @@ -565,8 +565,8 @@ static int32_t walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); int64_t fileSize = 0; - if (taosStatFile(fnameStr, &fileSize, NULL, NULL) < 0 && errno != ENOENT) { - wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + if (taosStatFile(fnameStr, &fileSize, NULL, NULL) < 0 && ERRNO != ENOENT) { + wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(ERRNO), fnameStr); TAOS_RETURN(terrno); } @@ -586,7 +586,7 @@ static int32_t walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); if (pIdxFile == NULL) { - wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); TAOS_CHECK_GOTO(terrno, &lino, _err); } @@ -685,7 +685,7 @@ static int32_t walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pIdxFile) < 0) { wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(ERRNO), &lino, _err); } if (count > 0) { @@ -902,12 +902,17 @@ _err: return TSDB_CODE_FAILED; } -static int walFindCurMetaVer(SWal* pWal) { +static int walFindCurMetaVer(SWal* pWal, int64_t* pMetaVer) { const char* pattern = "^meta-ver[0-9]+$"; regex_t walMetaRegexPattern; - if (regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED) != 0) { - wError("failed to compile wal meta pattern, error %s", tstrerror(terrno)); - return terrno; + int ret = 0; + + if (pMetaVer) *pMetaVer = -1; + if ((ret = regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED)) != 0) { + char msgbuf[256] = {0}; + (void)regerror(ret, &walMetaRegexPattern, msgbuf, tListLen(msgbuf)); + wError("failed to compile wal meta pattern %s, error %s", pattern, msgbuf); + return TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR; } TdDirPtr pDir = taosOpenDir(pWal->path); @@ -920,13 +925,13 @@ static int walFindCurMetaVer(SWal* pWal) { TdDirEntryPtr pDirEntry; // find existing meta-ver[x].json - int metaVer = -1; + int64_t metaVer = -1; while ((pDirEntry = taosReadDir(pDir)) != NULL) { char* name = taosDirEntryBaseName(taosGetDirEntryName(pDirEntry)); int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0); if (code == 0) { - (void)sscanf(name, "meta-ver%d", &metaVer); - wDebug("vgId:%d, wal find current meta: %s is the meta file, ver %d", pWal->cfg.vgId, name, metaVer); + (void)sscanf(name, "meta-ver%" PRIi64, &metaVer); + wDebug("vgId:%d, wal find current meta: %s is the meta file, ver %" PRIi64, pWal->cfg.vgId, name, metaVer); break; } wDebug("vgId:%d, wal find current meta: %s is not meta file", pWal->cfg.vgId, name); @@ -937,7 +942,8 @@ static int walFindCurMetaVer(SWal* pWal) { return terrno; } regfree(&walMetaRegexPattern); - return metaVer; + if (pMetaVer) *pMetaVer = metaVer; + return 0; } static void walUpdateSyncedOffset(SWal* pWal) { @@ -948,22 +954,23 @@ static void walUpdateSyncedOffset(SWal* pWal) { int32_t walSaveMeta(SWal* pWal) { int code = 0, lino = 0; - int metaVer = walFindCurMetaVer(pWal); + int64_t metaVer = -1; char fnameStr[WAL_FILE_LEN]; char tmpFnameStr[WAL_FILE_LEN]; int n; + TAOS_CHECK_GOTO(walFindCurMetaVer(pWal, &metaVer), &lino, _err); // fsync the idx and log file at first to ensure validity of meta if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pWal->pIdxFile) < 0) { - wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno)); + wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(ERRNO)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pWal->pLogFile) < 0) { - wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno)); + wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(ERRNO)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } // update synced offset @@ -972,13 +979,13 @@ int32_t walSaveMeta(SWal* pWal) { // flush to a tmpfile n = walBuildTmpMetaName(pWal, tmpFnameStr); if (n >= sizeof(tmpFnameStr)) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pMetaFile == NULL) { - wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(ERRNO), tmpFnameStr); TAOS_RETURN(terrno); } @@ -987,21 +994,21 @@ int32_t walSaveMeta(SWal* pWal) { TAOS_CHECK_RETURN(walMetaSerialize(pWal, &serialized)); int len = strlen(serialized); if (pWal->cfg.level != TAOS_WAL_SKIP && len != taosWriteFile(pMetaFile, serialized, len)) { - wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(ERRNO), tmpFnameStr); TAOS_CHECK_GOTO(terrno, &lino, _err); } if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pMetaFile) < 0) { - wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(ERRNO), tmpFnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(ERRNO), &lino, _err); } if (taosCloseFile(&pMetaFile) < 0) { - wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(ERRNO), tmpFnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(ERRNO), &lino, _err); } wInfo("vgId:%d, save meta file: %s, firstVer:%" PRId64 ", lastVer:%" PRId64, pWal->cfg.vgId, tmpFnameStr, pWal->vers.firstVer, pWal->vers.lastVer); @@ -1013,20 +1020,20 @@ int32_t walSaveMeta(SWal* pWal) { } if (taosRenameFile(tmpFnameStr, fnameStr) < 0) { - wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr); + wError("failed to rename file due to %s. dest:%s", strerror(ERRNO), fnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(ERRNO), &lino, _err); } // delete old file if (metaVer > -1) { n = walBuildMetaName(pWal, metaVer, fnameStr); if (n >= sizeof(fnameStr)) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } code = taosRemoveFile(fnameStr); if (code) { - wError("vgId:%d, failed to remove file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + wError("vgId:%d, failed to remove file due to %s. file:%s", pWal->cfg.vgId, strerror(ERRNO), fnameStr); } else { wInfo("vgId:%d, remove old meta file: %s", pWal->cfg.vgId, fnameStr); } @@ -1048,18 +1055,15 @@ int32_t walLoadMeta(SWal* pWal) { int32_t lino = 0; char* buf = NULL; TdFilePtr pFile = NULL; + int64_t metaVer = -1; // find existing meta file - int metaVer = walFindCurMetaVer(pWal); - if (metaVer == -1) { - wDebug("vgId:%d, wal find meta ver %d", pWal->cfg.vgId, metaVer); + TAOS_CHECK_EXIT(walFindCurMetaVer(pWal, &metaVer)); - TAOS_RETURN(TSDB_CODE_FAILED); - } char fnameStr[WAL_FILE_LEN]; n = walBuildMetaName(pWal, metaVer, fnameStr); if (n >= sizeof(fnameStr)) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } // read metafile int64_t fileSize = 0; @@ -1067,11 +1071,11 @@ int32_t walLoadMeta(SWal* pWal) { if (fileSize == 0) { code = taosRemoveFile(fnameStr); if (code) { - wError("vgId:%d, failed to remove file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + wError("vgId:%d, failed to remove file due to %s. file:%s", pWal->cfg.vgId, strerror(ERRNO), fnameStr); } else { wInfo("vgId:%d, remove old meta file: %s", pWal->cfg.vgId, fnameStr); } - wDebug("vgId:%d, wal find empty meta ver %d", pWal->cfg.vgId, metaVer); + wDebug("vgId:%d, wal find empty meta ver %" PRIi64, pWal->cfg.vgId, metaVer); TAOS_RETURN(TSDB_CODE_FAILED); } @@ -1110,12 +1114,18 @@ _exit: } int32_t walRemoveMeta(SWal* pWal) { - int metaVer = walFindCurMetaVer(pWal); + int code = 0; + int64_t metaVer = -1; + + if ((code = walFindCurMetaVer(pWal, &metaVer))) { + wError("failed at line %d to find current meta file since %s", __LINE__, tstrerror(code)); + TAOS_RETURN(code); + } if (metaVer == -1) return 0; char fnameStr[WAL_FILE_LEN]; int n = walBuildMetaName(pWal, metaVer, fnameStr); if (n >= sizeof(fnameStr)) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } return taosRemoveFile(fnameStr); } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index e2c5e66158..e25b00d8c9 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -133,12 +133,12 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { int32_t code = 0; SWal *pWal = taosMemoryCalloc(1, sizeof(SWal)); if (pWal == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return NULL; } if (walInitLock(pWal) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); taosMemoryFree(pWal); return NULL; } @@ -177,7 +177,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeCur = -1; pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { - wError("vgId:%d, failed to init taosArray of fileInfoSet due to %s. path:%s", pWal->cfg.vgId, strerror(errno), + wError("vgId:%d, failed to init taosArray of fileInfoSet due to %s. path:%s", pWal->cfg.vgId, strerror(ERRNO), pWal->path); goto _err; } @@ -185,7 +185,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // init gc pWal->toDeleteFiles = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->toDeleteFiles == NULL) { - wError("vgId:%d, failed to init taosArray of toDeleteFiles due to %s. path:%s", pWal->cfg.vgId, strerror(errno), + wError("vgId:%d, failed to init taosArray of toDeleteFiles due to %s. path:%s", pWal->cfg.vgId, strerror(ERRNO), pWal->path); goto _err; } @@ -354,7 +354,7 @@ static bool walNeedFsync(SWal *pWal) { static void walUpdateSeq() { taosMsleep(WAL_REFRESH_MS); if (atomic_add_fetch_32((volatile int32_t *)&tsWal.seq, 1) < 0) { - wError("failed to update wal seq since %s", strerror(errno)); + wError("failed to update wal seq since %s", strerror(ERRNO)); } } @@ -367,7 +367,7 @@ static void walFsyncAll() { int32_t code = taosFsyncFile(pWal->pLogFile); if (code != 0) { wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), - strerror(errno)); + strerror(ERRNO)); } } pWal = taosIterateRef(tsWal.refSetId, pWal->refId); @@ -390,11 +390,13 @@ static int32_t walCreateThread() { TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL); +#endif if (taosThreadCreate(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) { - wError("failed to create wal thread since %s", strerror(errno)); + wError("failed to create wal thread since %s", strerror(ERRNO)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } (void)taosThreadAttrDestroy(&thAttr); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 4eaf0a9399..64d4c1a65f 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -284,11 +284,11 @@ int32_t walRollImpl(SWal *pWal) { int64_t newFileFirstVer = pWal->vers.lastVer + 1; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVer, fnameStr); - pIdxFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + pIdxFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_READ | TD_FILE_WRITE | TD_FILE_APPEND); TSDB_CHECK_NULL(pIdxFile, code, lino, _exit, terrno); walBuildLogName(pWal, newFileFirstVer, fnameStr); - pLogFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + pLogFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_READ | TD_FILE_WRITE | TD_FILE_APPEND); wDebug("vgId:%d, wal create new file for write:%s", pWal->cfg.vgId, fnameStr); TSDB_CHECK_NULL(pLogFile, code, lino, _exit, terrno); @@ -463,13 +463,13 @@ int32_t walEndSnapshot(SWal *pWal) { pInfo = taosArrayGet(pWal->toDeleteFiles, i); walBuildLogName(pWal, pInfo->firstVer, fnameStr); - if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) { - wError("vgId:%d, failed to remove log file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno)); + if (taosRemoveFile(fnameStr) < 0 && ERRNO != ENOENT) { + wError("vgId:%d, failed to remove log file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(ERRNO)); goto _exit; } walBuildIdxName(pWal, pInfo->firstVer, fnameStr); - if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) { - wError("vgId:%d, failed to remove idx file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno)); + if (taosRemoveFile(fnameStr) < 0 && ERRNO != ENOENT) { + wError("vgId:%d, failed to remove idx file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(ERRNO)); goto _exit; } } @@ -508,7 +508,7 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { int64_t size = taosWriteFile(pWal->pIdxFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { - wError("vgId:%d, failed to write idx entry due to %s. ver:%" PRId64, pWal->cfg.vgId, strerror(errno), ver); + wError("vgId:%d, failed to write idx entry due to %s. ver:%" PRId64, pWal->cfg.vgId, strerror(ERRNO), ver); walStopDnode(pWal); TAOS_RETURN(terrno); } @@ -516,8 +516,8 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { // check alignment of idx entries int64_t endOffset = taosLSeekFile(pWal->pIdxFile, 0, SEEK_END); if (endOffset < 0) { - wFatal("vgId:%d, failed to seek end of WAL idxfile due to %s. ver:%" PRId64 "", pWal->cfg.vgId, strerror(terrno), - ver); + wFatal("vgId:%d, failed to seek end of WAL idxfile due to %s. ver:%" PRId64 ", endOffset:%" PRId64, pWal->cfg.vgId, + tstrerror(terrno), ver, endOffset); taosMsleep(100); exit(EXIT_FAILURE); } @@ -554,7 +554,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { code = terrno; wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), - strerror(errno)); + strerror(ERRNO)); walStopDnode(pWal); TAOS_CHECK_GOTO(code, &lino, _exit); } @@ -576,7 +576,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy newBodyEncrypted = taosMemoryMalloc(cyptedBodyLen); if (newBodyEncrypted == NULL) { wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), - strerror(errno)); + strerror(ERRNO)); if (newBody != NULL) taosMemoryFreeClear(newBody); @@ -601,7 +601,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy if (pWal->cfg.level != TAOS_WAL_SKIP && taosWriteFile(pWal->pLogFile, (char *)buf, cyptedBodyLen) != cyptedBodyLen) { code = terrno; wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), - strerror(errno)); + strerror(ERRNO)); if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) { taosMemoryFreeClear(newBody); @@ -665,12 +665,12 @@ static int32_t walInitWriteFile(SWal *pWal) { char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); - pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_READ | TD_FILE_WRITE | TD_FILE_APPEND); if (pIdxTFile == NULL) { TAOS_RETURN(terrno); } walBuildLogName(pWal, fileFirstVer, fnameStr); - pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_READ |TD_FILE_WRITE | TD_FILE_APPEND); if (pLogTFile == NULL) { TAOS_UNUSED(taosCloseFile(&pIdxTFile)); TAOS_RETURN(terrno); @@ -729,7 +729,7 @@ int32_t walFsync(SWal *pWal, bool forceFsync) { wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (taosFsyncFile(pWal->pLogFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), - strerror(errno)); + strerror(ERRNO)); code = terrno; } } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 410f9da623..cf3b84d24e 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -81,7 +81,9 @@ typedef struct TdDir { #include #include #include +#ifndef TD_ASTRA #include +#endif typedef struct dirent dirent; typedef struct DIR TdDir; @@ -131,10 +133,10 @@ int32_t taosMkDir(const char *dirname) { int32_t code = mkdir(dirname, 0755); #endif if (-1 == code) { - if (errno == EEXIST) { + if (ERRNO == EEXIST) { return 0; } else { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); code = terrno; } } @@ -175,8 +177,8 @@ int32_t taosMulMkDir(const char *dirname) { #else code = mkdir(temp, 0755); #endif - if (code < 0 && errno != EEXIST) { - terrno = TAOS_SYSTEM_ERROR(errno); + if (code < 0 && ERRNO != EEXIST) { + terrno = TAOS_SYSTEM_ERROR(ERRNO); return code; } *pos = TD_DIRSEP[0]; @@ -191,8 +193,8 @@ int32_t taosMulMkDir(const char *dirname) { #else code = mkdir(temp, 0755); #endif - if (code < 0 && errno != EEXIST) { - terrno = TAOS_SYSTEM_ERROR(errno); + if (code < 0 && ERRNO != EEXIST) { + terrno = TAOS_SYSTEM_ERROR(ERRNO); return code; } } @@ -223,16 +225,19 @@ int32_t taosMulModeMkDir(const char *dirname, int32_t mode, bool checkAccess) { taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) { return 0; } - +#ifndef TD_ASTRA // TD_ASTRA_TODO IMPORTANT code = chmod(temp, mode); if (-1 == code) { struct stat statbuf = {0}; code = stat(temp, &statbuf); if (code != 0 || (statbuf.st_mode & mode) != mode) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } } +#else + return 0; +#endif } if (strncmp(temp, TD_DIRSEP, 1) == 0) { @@ -251,8 +256,8 @@ int32_t taosMulModeMkDir(const char *dirname, int32_t mode, bool checkAccess) { #else code = mkdir(temp, mode); #endif - if (code < 0 && errno != EEXIST) { - terrno = TAOS_SYSTEM_ERROR(errno); + if (code < 0 && ERRNO != EEXIST) { + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } *pos = TD_DIRSEP[0]; @@ -267,24 +272,27 @@ int32_t taosMulModeMkDir(const char *dirname, int32_t mode, bool checkAccess) { #else code = mkdir(temp, mode); #endif - if (code < 0 && errno != EEXIST) { - terrno = TAOS_SYSTEM_ERROR(errno); + if (code < 0 && ERRNO != EEXIST) { + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } } - if (code < 0 && errno == EEXIST) { + if (code < 0 && ERRNO == EEXIST) { if (checkAccess && taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) { return 0; } } - +#ifndef TD_ASTRA // TD_ASTRA_TODO IMPORTANT code = chmod(temp, mode); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } +#else + code = 0; +#endif return code; } @@ -338,6 +346,7 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { OS_PARAM_CHECK(outname); if (dirname[0] == 0) return 0; +#ifndef TD_ASTRA wordexp_t full_path = {0}; int32_t code = wordexp(dirname, &full_path, 0); switch (code) { @@ -355,12 +364,15 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { } wordfree(&full_path); +#else // TD_ASTRA_TODO + tstrncpy(outname, dirname, maxlen); +#endif return 0; } int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { OS_PARAM_CHECK(dirname); - +#ifndef TD_ASTRA char tmp[PATH_MAX] = {0}; #ifdef WINDOWS if (_fullpath(tmp, dirname, maxlen) != NULL) { @@ -379,9 +391,13 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { return terrno; } } else { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } +#else // TD_ASTRA_TODO + if (realPath) tstrncpy(realPath, dirname, maxlen); + return 0; +#endif } bool taosIsDir(const char *dirname) { @@ -477,7 +493,7 @@ TdDirPtr taosOpenDir(const char *dirname) { TdDirPtr dirPtr = (TdDirPtr)taosMemoryMalloc(sizeof(TdDir)); if (dirPtr == NULL) { (void)closedir(pDir); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return NULL; } dirPtr->dirEntryPtr = (TdDirEntryPtr) & (dirPtr->dirEntry1); @@ -486,7 +502,7 @@ TdDirPtr taosOpenDir(const char *dirname) { #else TdDirPtr ptr = (TdDirPtr)opendir(dirname); if (NULL == ptr) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } return ptr; #endif @@ -509,11 +525,11 @@ TdDirEntryPtr taosReadDir(TdDirPtr pDir) { return NULL; } #else - errno = 0; + SET_ERRNO(0); terrno = 0; TdDirEntryPtr p = (TdDirEntryPtr)readdir((DIR *)pDir); - if (NULL == p && errno) { - terrno = TAOS_SYSTEM_ERROR(errno); + if (NULL == p && ERRNO) { + terrno = TAOS_SYSTEM_ERROR(ERRNO); } return p; #endif @@ -525,6 +541,8 @@ bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry) { } #ifdef WINDOWS return (pDirEntry->findFileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; +#elif defined(TD_ASTRA) // TD_ASTRA_TODO + return ((dirent *)pDirEntry)->d_mode == 1; // DIRECTORY_ENTRY; #else return (((dirent *)pDirEntry)->d_type & DT_DIR) != 0; #endif @@ -558,7 +576,7 @@ int32_t taosCloseDir(TdDirPtr *ppDir) { #elif defined(DARWIN) code = closedir((*ppDir)->pDir); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } taosMemoryFree(*ppDir); @@ -568,7 +586,7 @@ int32_t taosCloseDir(TdDirPtr *ppDir) { code = closedir((DIR *)*ppDir); *ppDir = NULL; if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } return 0; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index f5b6d48877..6af9bcb472 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -24,6 +24,7 @@ #include #include #define F_OK 0 +#define X_OK 1 #define W_OK 2 #define R_OK 4 @@ -31,11 +32,13 @@ #else #include +#ifndef TD_ASTRA #include #if !defined(_TD_DARWIN_64) #include #endif +#endif #include #include #define LINUX_FILE_NO_TEXT_OPTION 0 @@ -96,7 +99,7 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha char rand[8] = {0}; taosRandStr(rand, tListLen(rand) - 1); - snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); + snprintf(dstPath, PATH_MAX, tmpPath, taosGetPId(), rand); #else @@ -115,7 +118,7 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha (void)snprintf(rand, sizeof(rand), "%" PRIu64, atomic_add_fetch_64(&seqId, 1)); - (void)snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); + (void)snprintf(dstPath, PATH_MAX, tmpPath, taosGetPId(), rand); #endif } @@ -225,7 +228,7 @@ int32_t taosRemoveFile(const char *path) { OS_PARAM_CHECK(path); int32_t code = remove(path); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } return code; @@ -263,9 +266,12 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { return finished ? 0 : terrno; #else +#ifdef TD_ASTRA // TD_ASTRA_TODO + if (taosCheckExistFile(newName)) taosRemoveFile(newName); +#endif int32_t code = rename(oldName, newName); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -283,7 +289,7 @@ int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *a int32_t code = stat(path, &fileStat); #endif if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -312,7 +318,7 @@ int32_t taosGetFileDiskID(const char *path, int64_t *diskid) { int32_t code = stat(path, &fileStat); #endif if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -353,7 +359,7 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -390,7 +396,7 @@ FILE *taosOpenFileForStream(const char *path, int32_t tdFileOptions) { } FILE *f = fopen(path, mode); if (NULL == f) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } return f; } @@ -492,8 +498,8 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { DWORD bytesWritten; if (!WriteFile(pFile->hFile, buf, count, &bytesWritten, NULL)) { - errno = GetLastError(); - terrno = TAOS_SYSTEM_WINAPI_ERROR(errno); + SET_ERRNO(GetLastError()); + terrno = TAOS_SYSTEM_WINAPI_ERROR(ERRNO); bytesWritten = -1; } @@ -526,8 +532,8 @@ int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t SetLastError(0); BOOL result = WriteFile(pFile->hFile, buf, count, &ret, &ol); if (!result) { - errno = GetLastError(); - terrno = TAOS_SYSTEM_WINAPI_ERROR(errno); + SET_ERRNO(GetLastError()); + terrno = TAOS_SYSTEM_WINAPI_ERROR(ERRNO); ret = -1; } @@ -549,15 +555,15 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { LARGE_INTEGER liOffset; liOffset.QuadPart = offset; if (!SetFilePointerEx(pFile->hFile, liOffset, NULL, whence)) { - errno = GetLastError(); - terrno = TAOS_SYSTEM_WINAPI_ERROR(errno); + SET_ERRNO(GetLastError()); + terrno = TAOS_SYSTEM_WINAPI_ERROR(ERRNO); return -1; } liOffset.QuadPart = 0; if (!SetFilePointerEx(pFile->hFile, liOffset, &liOffset, FILE_CURRENT)) { - errno = GetLastError(); - terrno = TAOS_SYSTEM_WINAPI_ERROR(errno); + SET_ERRNO(GetLastError()); + terrno = TAOS_SYSTEM_WINAPI_ERROR(ERRNO); return -1; } #if FILE_WITH_LOCK @@ -575,8 +581,8 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) { if (size != NULL) { LARGE_INTEGER fileSize; if (!GetFileSizeEx(pFile->hFile, &fileSize)) { - errno = GetLastError(); - terrno = TAOS_SYSTEM_WINAPI_ERROR(errno); + SET_ERRNO(GetLastError()); + terrno = TAOS_SYSTEM_WINAPI_ERROR(ERRNO); return terrno; // Error getting file size } *size = fileSize.QuadPart; @@ -585,8 +591,8 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) { if (mtime != NULL) { FILETIME creationTime, lastAccessTime, lastWriteTime; if (!GetFileTime(pFile->hFile, &creationTime, &lastAccessTime, &lastWriteTime)) { - errno = GetLastError(); - terrno = TAOS_SYSTEM_WINAPI_ERROR(errno); + SET_ERRNO(GetLastError()); + terrno = TAOS_SYSTEM_WINAPI_ERROR(ERRNO); return terrno; // Error getting file time } // Convert the FILETIME structure to a time_t value @@ -661,13 +667,13 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { int error = GetLastError(); switch (error) { case ERROR_INVALID_HANDLE: - errno = EBADF; + SET_ERRNO(EBADF); break; default: - errno = EIO; + SET_ERRNO(EIO); break; } - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -676,13 +682,13 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { printf("SetEndOfFile GetLastError is:%d", error); switch (error) { case ERROR_INVALID_HANDLE: - errno = EBADF; + SET_ERRNO(EBADF); break; default: - errno = EIO; + SET_ERRNO(EIO); break; } - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } return 0; @@ -787,7 +793,7 @@ int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); if (-1 == fd) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } return fd; } @@ -822,10 +828,10 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); #endif if (readbytes < 0) { - if (errno == EINTR) { + if (ERRNO == EINTR) { continue; } else { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); #if FILE_WITH_LOCK (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif @@ -875,10 +881,10 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { while (nleft > 0) { nwritten = write(pFile->fd, (void *)tbuf, (uint32_t)nleft); if (nwritten < 0) { - if (errno == EINTR) { + if (ERRNO == EINTR) { continue; } - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); #if FILE_WITH_LOCK (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif @@ -914,12 +920,31 @@ int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t return 0; } #endif - +#ifndef TD_ASTRA int64_t ret = pwrite(pFile->fd, buf, count, offset); if (-1 == ret) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); } - +#else // TD_ASTRA_TODO + int64_t ret = -1; + int64_t cur = lseek(pFile->fd, 0, SEEK_CUR); + if (cur < 0) { + code = TAOS_SYSTEM_ERROR(ERRNO); + goto _exit; + } + if (lseek(pFile->fd, offset, SEEK_SET) < 0) { + code = TAOS_SYSTEM_ERROR(ERRNO); + goto _exit; + } + if ((ret = write(pFile->fd, buf, count)) < 0) { + code = TAOS_SYSTEM_ERROR(ERRNO); + goto _exit; + } +_exit: + if (cur >= 0 && lseek(pFile->fd, cur, SEEK_SET) < 0) { + code = TAOS_SYSTEM_ERROR(ERRNO); + } +#endif #if FILE_WITH_LOCK (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif @@ -944,7 +969,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { int64_t ret = lseek(pFile->fd, offset, whence); if (-1 == ret) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); } #if FILE_WITH_LOCK @@ -973,7 +998,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime) { struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -993,11 +1018,24 @@ int32_t taosLockFile(TdFilePtr pFile) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } +#ifndef TD_ASTRA int32_t code = (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } +#else // TD_ASTRA_TODO + struct flock lock; + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + int32_t code = fcntl(pFile->fd, F_SETLK, &lock); + if (-1 == code) { + // terrno = TAOS_SYSTEM_ERROR(ERRNO); // TD_ASTRA_TODO + // return terrno; // TD_ASTRA_TODO + } +#endif return 0; } @@ -1006,11 +1044,24 @@ int32_t taosUnLockFile(TdFilePtr pFile) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } +#ifndef TD_ASTRA int32_t code = (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } +#else // TD_ASTRA_TODO + struct flock lock; + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + int32_t code = fcntl(pFile->fd, F_SETLK, &lock); + if (-1 == code) { + // terrno = TAOS_SYSTEM_ERROR(ERRNO);// TD_ASTRA_TODO + // return terrno;// TD_ASTRA_TODO + } +#endif return 0; } @@ -1022,7 +1073,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { int32_t code = ftruncate(pFile->fd, l_size); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } return 0; @@ -1038,9 +1089,9 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in return -1; } -#ifdef _TD_DARWIN_64 +#if defined(_TD_DARWIN_64) || defined(TD_ASTRA) // TD_ASTRA_TODO if (lseek(pFileIn->fd, (int32_t)(*offset), 0) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return -1; } int64_t writeLen = 0; @@ -1083,10 +1134,10 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in sentbytes = sendfile(pFileOut->fd, pFileIn->fd, offset, leftbytes); #endif if (sentbytes == -1) { - if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { + if (ERRNO == EINTR || ERRNO == EAGAIN || ERRNO == EWOULDBLOCK) { continue; } else { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return -1; } } else if (sentbytes == 0) { @@ -1158,7 +1209,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { // will be automatically remove by OS. if (tdFileOptions & TD_FILE_AUTO_DEL) { if (-1 == unlink(path)) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); (void)close(fd); taosMemoryFree(pFile); return NULL; @@ -1196,7 +1247,7 @@ int32_t taosCloseFile(TdFilePtr *ppFile) { /*fsync((*ppFile)->fd);*/ code = close((*ppFile)->fd); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } (*ppFile)->fd = -1; #endif @@ -1257,10 +1308,31 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) terrno = TSDB_CODE_INVALID_PARA; return -1; } +#ifndef TD_ASTRA int64_t ret = pread(pFile->fd, buf, count, offset); if (-1 == ret) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); } +#else // TD_ASTRA_TODO + int64_t ret = -1; + int64_t cur = lseek(pFile->fd, 0, SEEK_CUR); + if (cur < 0) { + code = TAOS_SYSTEM_ERROR(ERRNO); + goto _exit; + } + if (lseek(pFile->fd, offset, SEEK_SET) < 0) { + code = TAOS_SYSTEM_ERROR(ERRNO); + goto _exit; + } + if ((ret = read(pFile->fd, buf, count)) < 0) { + code = TAOS_SYSTEM_ERROR(ERRNO); + goto _exit; + } +_exit: + if (cur >= 0 && lseek(pFile->fd, cur, SEEK_SET) < 0) { + code = TAOS_SYSTEM_ERROR(ERRNO); + } +#endif #endif #if FILE_WITH_LOCK (void)taosThreadRwlockUnlock(&(pFile->rwlock)); @@ -1285,7 +1357,7 @@ int32_t taosFsyncFile(TdFilePtr pFile) { if (pFile->fp != NULL) { code = fflush(pFile->fp); if (0 != code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -1308,7 +1380,11 @@ int32_t taosFsyncFile(TdFilePtr pFile) { if (pFile->fd >= 0) { code = fsync(pFile->fd); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); +#ifndef TD_ASTRA + terrno = TAOS_SYSTEM_ERROR(ERRNO); +#else + terrno = 0; // TD_ASTRA_TODO +#endif return terrno; } } @@ -1405,11 +1481,41 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { (*ptrBuf)[totalBytesRead] = '\0'; ret = (totalBytesRead > 0 ? totalBytesRead : -1); // -1 means EOF +#elif defined(TD_ASTRA) + size_t bufsize = 128; + if (*ptrBuf == NULL) { + *ptrBuf = (char *)taosMemoryMalloc(bufsize); + if (*ptrBuf == NULL) { + goto END; + } + } + size_t pos = 0; + int c; + while ((c = fgetc(pFile->fp)) != EOF) { + if (pos + 1 >= bufsize) { + size_t new_size = bufsize << 1; + char *new_line = (char *)taosMemoryRealloc(*ptrBuf, new_size); + if (new_line == NULL) { + goto END; + } + *ptrBuf = new_line; + bufsize = new_size; + } + (*ptrBuf)[pos++] = (char)c; + if (c == '\n') { + break; + } + } + if (pos == 0 && c == EOF) { + goto END; + } + (*ptrBuf)[pos] = '\0'; + ret = pos; #else size_t len = 0; ret = getline(ptrBuf, &len, pFile->fp); if (-1 == ret) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } #endif @@ -1476,6 +1582,10 @@ bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) { if (tdFileAccessOptions & TD_FILE_ACCESS_WRITE_OK) { flags |= W_OK; } + + if (tdFileAccessOptions & TD_FILE_ACCESS_EXEC_OK) { + flags |= X_OK; + } #ifdef WINDOWS return _access(pathname, flags) == 0; #else @@ -1513,7 +1623,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { #endif int fd = open(destFileName, access, pmode); if (-1 == fd) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); ret = terrno; goto cmp_end; } @@ -1521,13 +1631,13 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { // Both gzclose() and fclose() will close the associated fd, so they need to have different fds. FileFd gzFd = dup(fd); if (-1 == gzFd) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); ret = terrno; goto cmp_end; } dstFp = gzdopen(gzFd, "wb6f"); if (dstFp == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); ret = terrno; (void)close(gzFd); goto cmp_end; @@ -1537,7 +1647,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { len = (int32_t)fread(data, 1, compressSize, pSrcFile->fp); if (len > 0) { if (gzwrite(dstFp, data, len) == 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); ret = terrno; goto cmp_end; } @@ -1574,7 +1684,7 @@ int32_t taosSetFileHandlesLimit() { int32_t taosLinkFile(char *src, char *dst) { #ifndef WINDOWS if (-1 == link(src, dst)) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } #endif @@ -1589,7 +1699,7 @@ FILE *taosOpenCFile(const char *filename, const char *mode) { STUB_RAND_IO_ERR(NULL) FILE *f = fopen(filename, mode); if (NULL == f) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } return f; } @@ -1604,7 +1714,7 @@ int taosSeekCFile(FILE *file, int64_t offset, int whence) { #else int code = fseeko(file, offset, whence); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); code = terrno; } return code; @@ -1640,7 +1750,7 @@ int taosSetAutoDelFile(char *path) { } #else if (-1 == unlink(path)) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } return 0; diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 484436fafe..1157040dce 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -16,6 +16,7 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "osLocale.h" +#include "tutil.h" #ifdef WINDOWS #if (_WIN64) @@ -37,11 +38,13 @@ #include #else #include +#ifndef TD_ASTRA #include #include #include #include #include +#endif #include #include #endif @@ -61,7 +64,7 @@ char *taosCharsetReplace(char *charsetstr) { }; for (int32_t i = 0; i < tListLen(charsetRep); ++i) { - if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { + if (taosStrcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { return taosStrdup(charsetRep[i].newCharset); } } @@ -122,7 +125,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { locale = setlocale(LC_CTYPE, ""); if (locale == NULL) { - // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", ERRNO, strerror(ERRNO)); tstrncpy(outLocale, "en_US.UTF-8", TD_LOCALE_LEN); } else { tstrncpy(outLocale, locale, TD_LOCALE_LEN); diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 51454653ca..cbf86ff511 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -235,10 +235,10 @@ void taosPrintBackTrace() { return; } #endif int32_t taosMemoryDbgInit() { -#if defined(LINUX) && !defined(_ALPINE) +#if defined(LINUX) && !defined(_ALPINE) && !defined(TD_ASTRA) int ret = mallopt(M_MMAP_THRESHOLD, 0); if (0 == ret) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } return 0; @@ -248,10 +248,10 @@ int32_t taosMemoryDbgInit() { } int32_t taosMemoryDbgInitRestore() { -#if defined(LINUX) && !defined(_ALPINE) +#if defined(LINUX) && !defined(_ALPINE) && !defined(TD_ASTRA) int ret = mallopt(M_MMAP_THRESHOLD, 128 * 1024); if (0 == ret) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } return 0; @@ -281,6 +281,9 @@ void *taosMemMalloc(int64_t size) { return NULL; } } +#endif +#ifdef TD_ASTRA + if (size == 0) size = 1; #endif void *p = malloc(size); if (NULL == p) { @@ -316,7 +319,9 @@ void *taosMemCalloc(int64_t num, int64_t size) { } } #endif - +#ifdef TD_ASTRA + if (0 == num || 0 == size) num = size = 1; +#endif void *p = calloc(num, size); if (NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -358,7 +363,9 @@ void *taosMemRealloc(void *ptr, int64_t size) { } } #endif - +#ifdef TD_ASTRA + if (size == 0) size = 1; +#endif void *p = realloc(ptr, size); if (size > 0 && NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -429,6 +436,8 @@ int64_t taosMemSize(void *ptr) { return _msize(ptr); #elif defined(_TD_DARWIN_64) return malloc_size(ptr); +#elif defined(TD_ASTRA) + return 0; // TD_ASTRA_TODO N/A #else return malloc_usable_size(ptr); #endif @@ -436,14 +445,14 @@ int64_t taosMemSize(void *ptr) { } int32_t taosMemTrim(int32_t size, bool* trimed) { -#if defined(WINDOWS) || defined(DARWIN) || defined(_ALPINE) +#if defined(WINDOWS) || defined(DARWIN) || defined(_ALPINE) || defined(TD_ASTRA) // do nothing return TSDB_CODE_SUCCESS; #else if (trimed) { *trimed = malloc_trim(size); } else { - malloc_trim(size); + TAOS_UNUSED(malloc_trim(size)); } return TSDB_CODE_SUCCESS; @@ -467,10 +476,10 @@ void *taosMemMallocAlign(uint32_t alignment, int64_t size) { void *p = memalign(alignment, size); if (NULL == p) { - if (ENOMEM == errno) { + if (ENOMEM == ERRNO) { terrno = TSDB_CODE_OUT_OF_MEMORY; } else { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } } return p; diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index d13342ca28..60e14bc219 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -19,7 +19,9 @@ #include "wincrypt.h" #include "windows.h" #else +#ifndef TD_ASTRA #include +#endif // !TD_ASTRA #include #endif @@ -57,6 +59,8 @@ uint32_t taosSafeRand(void) { } if (hCryptProv != NULL) CryptReleaseContext(hCryptProv, 0); return seed; +#elif defined(TD_ASTRA) + return (uint32_t)taosGetTimestampSec(); #else TdFilePtr pFile; int seed; diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 9327b33f79..e71ba82e88 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -131,7 +131,7 @@ int32_t tsem_destroy(tsem_t* sem) { int32_t tsem_init(tsem_t *psem, int flags, unsigned int count) { OS_PARAM_CHECK(psem); *psem = dispatch_semaphore_create(count); - if (*psem == NULL) return TAOS_SYSTEM_ERROR(errno); + if (*psem == NULL) return TAOS_SYSTEM_ERROR(ERRNO); return 0; } @@ -202,16 +202,21 @@ int32_t taosGetPIdByName(const char* name, int32_t* pPId) {return -1;} /* * linux implementation */ - +#ifndef TD_ASTRA #include +#endif #include bool taosCheckPthreadValid(TdThread thread) { return thread != 0; } int64_t taosGetSelfPthreadId() { - static __thread int id = 0; + static __thread int64_t id = 0; if (id != 0) return id; +#ifndef TD_ASTRA id = syscall(SYS_gettid); +#else + id = (int64_t) taosThreadSelf(); +#endif return id; } @@ -226,17 +231,22 @@ bool taosComparePthread(TdThread first, TdThread second) { return first == se int32_t taosGetPId() { static int32_t pid; if (pid != 0) return pid; +#ifndef TD_ASTRA pid = getpid(); +#else + pid = (int32_t)taosThreadSelf(); // TD_ASTRA_TODO +#endif return pid; } int32_t taosGetAppName(char* name, int32_t* len) { +#ifndef TD_ASTRA OS_PARAM_CHECK(name); const char* self = "/proc/self/exe"; char path[PATH_MAX] = {0}; if (-1 == readlink(self, path, PATH_MAX)) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -249,7 +259,9 @@ int32_t taosGetAppName(char* name, int32_t* len) { } tstrncpy(name, end, TSDB_APP_NAME_LEN); - +#else + tstrncpy(name, "tdastra", TSDB_APP_NAME_LEN); // TD_ASTRA_TODO +#endif if (len != NULL) { *len = strlen(name); } @@ -258,6 +270,7 @@ int32_t taosGetAppName(char* name, int32_t* len) { } int32_t taosGetPIdByName(const char* name, int32_t* pPId) { +#ifndef TD_ASTRA OS_PARAM_CHECK(name); OS_PARAM_CHECK(pPId); DIR* dir = NULL; @@ -270,7 +283,7 @@ int32_t taosGetPIdByName(const char* name, int32_t* pPId) { *pPId = -1; dir = opendir("/proc"); if (dir == NULL) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } while ((ptr = readdir(dir)) != NULL) { @@ -310,13 +323,16 @@ int32_t taosGetPIdByName(const char* name, int32_t* pPId) { } else { return TSDB_CODE_SUCCESS; } +#else + return TSDB_CODE_APP_ERROR; +#endif } int32_t tsem_init(tsem_t* psem, int flags, unsigned int count) { if (sem_init(psem, flags, count) == 0) { return 0; } else { - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(ERRNO); } } @@ -327,7 +343,7 @@ int32_t tsem_timewait(tsem_t* sem, int64_t ms) { struct timespec ts = {0}; if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -336,12 +352,12 @@ int32_t tsem_timewait(tsem_t* sem, int64_t ms) { ts.tv_nsec %= 1000000000; while ((ret = sem_timedwait(sem, &ts)) == -1) { - if (errno == EINTR) { + if (ERRNO == EINTR) { continue; - } else if (errno == ETIMEDOUT) { + } else if (ERRNO == ETIMEDOUT) { return TSDB_CODE_TIMEOUT_ERROR; } else { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } } @@ -354,10 +370,10 @@ int32_t tsem_wait(tsem_t* sem) { int ret = 0; do { ret = sem_wait(sem); - } while (-1 == ret && errno == EINTR); + } while (-1 == ret && ERRNO == EINTR); if (-1 == ret) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -399,7 +415,7 @@ int32_t tsem_post(tsem_t* psem) { if (sem_post(psem) == 0) { return 0; } else { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } } @@ -408,7 +424,7 @@ int32_t tsem_destroy(tsem_t* sem) { if (sem_destroy(sem) == 0) { return 0; } else { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } } @@ -480,7 +496,7 @@ int32_t tsem2_timewait(tsem2_t* sem, int64_t ms) { if (sem->count <= 0) { struct timespec ts = {0}; if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); (void)taosThreadMutexUnlock(&sem->mutex); terrno = code; return code; diff --git a/source/os/src/osSignal.c b/source/os/src/osSignal.c index 238093e418..2c7af40148 100644 --- a/source/os/src/osSignal.c +++ b/source/os/src/osSignal.c @@ -37,7 +37,7 @@ int32_t taosSetSignal(int32_t signum, FSignalHandler sigfp) { } } else { if(signal(signum, (FWinSignalHandler)sigfp) == SIG_ERR) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } } @@ -47,7 +47,7 @@ int32_t taosSetSignal(int32_t signum, FSignalHandler sigfp) { int32_t taosIgnSignal(int32_t signum) { if (signum == SIGUSR1 || signum == SIGHUP) return 0; if(signal(signum, SIG_IGN) == SIG_ERR) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -57,7 +57,7 @@ int32_t taosIgnSignal(int32_t signum) { int32_t taosDflSignal(int32_t signum) { if (signum == SIGUSR1 || signum == SIGHUP) return 0; if(signal(signum, SIG_DFL) == SIG_ERR) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -87,7 +87,7 @@ int32_t taosSetSignal(int32_t signum, FSignalHandler sigfp) { #endif int32_t code = sigaction(signum, &act, NULL); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -97,7 +97,7 @@ int32_t taosSetSignal(int32_t signum, FSignalHandler sigfp) { int32_t taosIgnSignal(int32_t signum) { sighandler_t h = signal(signum, SIG_IGN); if (SIG_ERR == h) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -107,7 +107,7 @@ int32_t taosIgnSignal(int32_t signum) { int32_t taosDflSignal(int32_t signum) { sighandler_t h = signal(signum, SIG_DFL); if (SIG_ERR == h) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -116,10 +116,10 @@ int32_t taosDflSignal(int32_t signum) { #if 0 int32_t taosKillChildOnParentStopped() { -#ifndef _TD_DARWIN_64 +#if !defined(_TD_DARWIN_64) && !defined(TD_ASTRA) int32_t code = prctl(PR_SET_PDEATHSIG, SIGKILL); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 3051005789..dce3e2885d 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -40,11 +40,25 @@ #if defined(DARWIN) #include #include "osEok.h" -#else +#elif !defined(TD_ASTRA) #include #endif #endif +#ifdef TD_ASTRA +#ifndef __BYTE_ORDER +#define __BYTE_ORDER _BYTE_ORDER +#endif + +#ifndef __BIG_ENDIAN +#define __BIG_ENDIAN _BIG_ENDIAN +#endif + +#ifndef __LITTLE_ENDIAN +#define __LITTLE_ENDIAN _LITTLE_ENDIAN +#endif +#endif // TD_ASTRA + #ifndef INVALID_SOCKET #define INVALID_SOCKET -1 #endif @@ -84,7 +98,7 @@ int32_t taosCloseSocketNoCheck1(SocketFd fd) { #else int32_t code = close(fd); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } return code; @@ -143,7 +157,7 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void #else int32_t code = setsockopt(pSocket->fd, level, optname, optval, (int)optlen); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } return 0; @@ -153,7 +167,7 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void const char *taosInetNtop(struct in_addr ipInt, char *dstStr, int32_t len) { const char *r = inet_ntop(AF_INET, &ipInt, dstStr, len); if (NULL == r) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } return r; @@ -184,7 +198,7 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); if (-1 == fd) { // exception - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return false; } @@ -205,7 +219,7 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { /* bind socket to server address */ if (-1 == bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd))) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); return false; } @@ -264,11 +278,11 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { inRetry = true; continue; } else if (EAI_SYSTEM == ret) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _err; } - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); goto _err; } @@ -299,7 +313,7 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { } else { #ifdef EAI_SYSTEM if (ret == EAI_SYSTEM) { - // printf("failed to get the ip address, fqdn:%s, errno:%d, since:%s", fqdn, errno, strerror(errno)); + // printf("failed to get the ip address, fqdn:%s, errno:%d, since:%s", fqdn, ERRNO, strerror(ERRNO)); } else { // printf("failed to get the ip address, fqdn:%s, ret:%d, since:%s", fqdn, ret, gai_strerror(ret)); } @@ -350,8 +364,9 @@ int32_t taosGetFqdn(char *fqdn) { #else // linux #endif // linux - -#if defined(LINUX) +#if defined(TD_ASTRA) + tstrncpy(fqdn, hostname, TD_FQDN_LEN); +#elif defined(LINUX) struct addrinfo hints = {0}; struct addrinfo *result = NULL; @@ -363,7 +378,7 @@ int32_t taosGetFqdn(char *fqdn) { if (EAI_AGAIN == ret) { continue; } else if (EAI_SYSTEM == ret) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -414,10 +429,9 @@ uint32_t taosInetAddr(const char *ipstr){ int32_t taosIgnSIGPIPE() { sighandler_t h = signal(SIGPIPE, SIG_IGN); if (SIG_ERR == h) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } - return 0; } @@ -433,7 +447,7 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) { #endif if ((fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) == INVALID_SOCKET) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -449,10 +463,18 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) { // taosCloseSocketNoCheck1(fd); // return -1; //} +#elif defined(TD_ASTRA) // TD_ASTRA_TODO + uint32_t conn_timeout_ms = timeout; + if (0 != setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms)) || + 0 != setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { + terrno = TAOS_SYSTEM_ERROR(ERRNO); + TAOS_SKIP_ERROR(taosCloseSocketNoCheck1(fd)); + return terrno; + } #else // Linux like systems uint32_t conn_timeout_ms = timeout; if (-1 == setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); TAOS_SKIP_ERROR(taosCloseSocketNoCheck1(fd)); return terrno; } @@ -508,12 +530,12 @@ uint64_t taosNtoh64(uint64_t val) { } int32_t taosSetSockOpt2(int32_t fd) { -#if defined(WINDOWS) || defined(DARWIN) +#if defined(WINDOWS) || defined(DARWIN) || defined(TD_ASTRA) return 0; #else int32_t ret = setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, (int[]){1}, sizeof(int)); if (ret < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } else { return 0; diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 24fd8ca034..6890d93a3e 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -103,6 +103,46 @@ char *stpncpy(char *dest, const char *src, int n) { if (size == n) return dest; return memset(dest, '\0', n - size); } +#elif defined(TD_ASTRA) +/* Copy no more than N characters of SRC to DEST, returning the address of + the terminating '\0' in DEST, if any, or else DEST + N. */ +char *stpncpy(char *dest, const char *src, int n) { + if (dest == NULL || src == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + if (n == 0) { + return dest; + } + char *orig_dest = dest; + const char *end = (const char *)memchr(src, '\0', n); + size_t len = (end != NULL) ? (size_t)(end - src) : n; + memcpy(dest, src, len); + if (len < n) { + memset(dest + len, '\0', n - len); + } + return orig_dest + len; +} +char *taosStrndupi(const char *s, int64_t size) { + if (s == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + + const char *end = (const char *)memchr(s, '\0', size); + size_t actual_len = (end != NULL) ? (size_t)(end - s) : (size_t)size; + + char *p = (char *)malloc(actual_len + 1); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + memcpy(p, s, actual_len); + p[actual_len] = '\0'; + + return p; +} #else char *taosStrndupi(const char *s, int64_t size) { if (s == NULL) { @@ -117,28 +157,27 @@ char *taosStrndupi(const char *s, int64_t size) { #endif char *tstrndup(const char *str, int64_t size) { -#ifdef WINDOWS +#if defined(WINDOWS) || defined(TD_ASTRA) return taosStrndupi(str, size); #else - char* p = strndup(str, size); + char *p = strndup(str, size); if (str != NULL && NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; } return p; - + #endif } - int32_t taosStr2int64(const char *str, int64_t *val) { if (str == NULL || val == NULL) { return TSDB_CODE_INVALID_PARA; } - errno = 0; + SET_ERRNO(0); char *endptr = NULL; int64_t ret = strtoll(str, &endptr, 10); - if (errno != 0) { - return TAOS_SYSTEM_ERROR(errno); + if (ERRNO != 0) { + return TAOS_SYSTEM_ERROR(ERRNO); } else { if (endptr == str) { return TSDB_CODE_INVALID_PARA; @@ -197,11 +236,11 @@ int32_t taosStr2Uint64(const char *str, uint64_t *val) { return TSDB_CODE_INVALID_PARA; } char *endptr = NULL; - errno = 0; + SET_ERRNO(0); uint64_t ret = strtoull(str, &endptr, 10); - if (errno != 0) { - return TAOS_SYSTEM_ERROR(errno); + if (ERRNO != 0) { + return TAOS_SYSTEM_ERROR(ERRNO); } else { if (endptr == str) { return TSDB_CODE_INVALID_PARA; @@ -256,7 +295,7 @@ int32_t taosStr2Uint8(const char *str, uint8_t *val) { } } -int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { +int32_t taosUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { if ((f1_ucs4 == NULL || f2_ucs4 == NULL)) { return TSDB_CODE_INVALID_PARA; } @@ -299,6 +338,7 @@ int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) if (target_ucs4 == NULL || source_ucs4 == NULL || len_ucs4 <= 0) { return TSDB_CODE_INVALID_PARA; } +#ifndef TD_ASTRA if (taosMemorySize(target_ucs4) < len_ucs4 * sizeof(TdUcs4)) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -307,10 +347,14 @@ int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) (void)memcpy(target_ucs4, source_ucs4, len_ucs4 * sizeof(TdUcs4)); return TSDB_CODE_SUCCESS; +#else + return TSDB_CODE_APP_ERROR; +#endif } #endif iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt) { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV if(idx == NULL) { terrno = TSDB_CODE_INVALID_PARA; return (iconv_t)NULL; @@ -324,13 +368,13 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt) { if (type == M2C) { iconv_t c = iconv_open(DEFAULT_UNICODE_ENCODEC, "UTF-8"); if ((iconv_t)-1 == c) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } return c; } else { iconv_t c = iconv_open("UTF-8", DEFAULT_UNICODE_ENCODEC); if ((iconv_t)-1 == c) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } return c; } @@ -366,21 +410,27 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt) { } else { return info->gConv[type][startId].conv; } +#else + terrno = TSDB_CODE_APP_ERROR; + return (iconv_t)-1; +#endif } -void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt) { +void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void *charsetCxt) { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV if (idx < 0) { (void)iconv_close(conv); return; } - if (charsetCxt == NULL){ + if (charsetCxt == NULL) { charsetCxt = tsCharsetCxt; } SConvInfo *info = (SConvInfo *)charsetCxt; atomic_store_8(&info->gConv[type][idx].inUse, 0); (void)atomic_sub_fetch_32(&info->convUsed[type], 1); +#endif } bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt) { @@ -407,7 +457,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 size_t ucs4_input_len = mbsLength; size_t outLeft = ucs4_max_len; if (iconv(conv, (char **)&mbs, &ucs4_input_len, (char **)&ucs4, &outLeft) == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); taosReleaseConv(idx, conv, M2C, charsetCxt); return false; } @@ -452,7 +502,7 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* chars size_t ucs4_input_len = ucs4_max_len; size_t outLen = ucs4_max_len; if (iconv(conv, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { - code = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(ERRNO); taosReleaseConv(idx, conv, C2M, charsetCxt); terrno = code; return code; @@ -483,7 +533,7 @@ int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t c size_t ucs4_input_len = ucs4_max_len; size_t outLen = ucs4_max_len; if (iconv(conv, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -503,7 +553,7 @@ bool taosValidateEncodec(const char *encodec) { #else iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); if (cd == (iconv_t)(-1)) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return false; } @@ -561,6 +611,56 @@ int32_t taosHexDecode(const char *src, char *dst, int32_t len) { return 0; } +#ifdef TD_ASTRA +#include +#include +#include + +typedef struct { + uint32_t start; + uint32_t end; +} SUnicodeRange; + +static const SUnicodeRange __eaw_ranges[] = { + {0x1100, 0x115F}, {0x2329, 0x232A}, {0x2E80, 0x303E}, {0x3040, 0xA4CF}, + {0xAC00, 0xD7AF}, {0xF900, 0xFAFF}, {0xFE10, 0xFE19}, {0x20000, 0x2FFFD}, + {0x30000, 0x3FFFD}, {0xFF00, 0xFFEF}, {0xA000, 0xA48F}, {0x13A0, 0x13FF} +}; + +static const int __n_eaw_ranges = sizeof(__eaw_ranges) / sizeof(__eaw_ranges[0]); + +static bool isEawWideChar(wchar_t code_point) { + int low = 0; + int high = __n_eaw_ranges - 1; + + while (low <= high) { + int mid = (low + high) / 2; + if (code_point < __eaw_ranges[mid].start) { + high = mid - 1; + } else if (code_point > __eaw_ranges[mid].end) { + low = mid + 1; + } else { + return true; + } + } + return false; +} + +int wcwidth(wchar_t c) { + if (c < 0x20 || (c >= 0x7F && c <= 0x9F)) { + return 0; + } + if (isEawWideChar(c)) { + return 2; + } + if (c >= 0x0300 && c <= 0x036F) { + return 0; + } + return 1; +} + +#endif + int32_t taosWcharWidth(TdWchar wchar) { return wcwidth(wchar); } int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size) { @@ -568,7 +668,15 @@ int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } +#ifndef TD_ASTRA return wcswidth(pWchar, size); +#else + int32_t width = 0; + for (int32_t i = 0; i < size; ++i) { + width += wcwidth(pWchar[i]); + } + return width; +#endif } int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size) { @@ -576,9 +684,38 @@ int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } +#ifndef TD_ASTRA return mbtowc(pWchar, pStr, size); +#else + return mbrtowc(pWchar, &pStr, size, NULL); +#endif } +#ifdef TD_ASTRA +size_t mbstowcs(wchar_t *dest, const char *src, size_t n) { + if (src == NULL) { + return 0; + } + + size_t count = 0; + int result; + while (*src && count < n) { + result = mbrtowc(dest, src, MB_CUR_MAX, NULL); + if (result == -1 || result == 0) { + return -1; + } + src += result; + dest++; + count++; + } + + if (count < n) { + *dest = L'\0'; + } + return count; +} +#endif + int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { if (pWchars == NULL || pStrs == NULL || size <= 0) { terrno = TSDB_CODE_INVALID_PARA; @@ -589,7 +726,11 @@ int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { int32_t taosWcharToMb(char *pStr, TdWchar wchar) { OS_PARAM_CHECK(pStr); +#ifndef TD_ASTRA return wctomb(pStr, wchar); +#else + return wcrtomb(pStr, wchar, NULL); +#endif } char *taosStrCaseStr(const char *str, const char *pattern) { @@ -618,7 +759,7 @@ int64_t taosStr2Int64(const char *str, char **pEnd, int32_t radix) { } int64_t tmp = strtoll(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) - if (errno == EINVAL) errno = 0; + if (ERRNO == EINVAL) SET_ERRNO(0); #endif return tmp; } @@ -630,7 +771,7 @@ uint64_t taosStr2UInt64(const char *str, char **pEnd, int32_t radix) { } uint64_t tmp = strtoull(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) - if (errno == EINVAL) errno = 0; + if (ERRNO == EINVAL) SET_ERRNO(0); #endif return tmp; } @@ -642,7 +783,7 @@ int32_t taosStr2Int32(const char *str, char **pEnd, int32_t radix) { } int32_t tmp = strtol(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) - if (errno == EINVAL) errno = 0; + if (ERRNO == EINVAL) SET_ERRNO(0); #endif return tmp; } @@ -654,7 +795,7 @@ uint32_t taosStr2UInt32(const char *str, char **pEnd, int32_t radix) { } uint32_t tmp = strtol(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) - if (errno == EINVAL) errno = 0; + if (ERRNO == EINVAL) SET_ERRNO(0); #endif return tmp; } @@ -666,7 +807,7 @@ int16_t taosStr2Int16(const char *str, char **pEnd, int32_t radix) { } int32_t tmp = strtol(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) - if (errno == EINVAL) errno = 0; + if (ERRNO == EINVAL) SET_ERRNO(0); #endif return (int16_t)tmp; } @@ -678,7 +819,7 @@ uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) { } uint32_t tmp = strtoul(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) - if (errno == EINVAL) errno = 0; + if (ERRNO == EINVAL) SET_ERRNO(0); #endif return (uint16_t)tmp; } @@ -699,7 +840,7 @@ uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) { } uint32_t tmp = strtoul(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) - if (errno == EINVAL) errno = 0; + if (ERRNO == EINVAL) SET_ERRNO(0); #endif return tmp; } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index dccce2d987..51f352e321 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -117,11 +117,13 @@ LONG WINAPI exceptionHandler(LPEXCEPTION_POINTERS exception); #else #include +#ifndef TD_ASTRA #include #include #include #include #include +#endif #include #include @@ -138,11 +140,13 @@ static void taosGetProcIOnfos() { tsPageSizeKB = sysconf(_SC_PAGESIZE) / 1024; tsOpenMax = sysconf(_SC_OPEN_MAX); tsStreamMax = TMAX(sysconf(_SC_STREAM_MAX), 0); +#ifndef TD_ASTRA tsProcId = (pid_t)syscall(SYS_gettid); (void)snprintf(tsProcMemFile, sizeof(tsProcMemFile), "/proc/%d/status", tsProcId); (void)snprintf(tsProcCpuFile, sizeof(tsProcCpuFile), "/proc/%d/stat", tsProcId); (void)snprintf(tsProcIOFile, sizeof(tsProcIOFile), "/proc/%d/io", tsProcId); +#endif } #endif @@ -162,7 +166,7 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { cpuInfo->user = CompareFileTime(&pre_userTime, &userTime); cpuInfo->nice = 0; } -#elif defined(DARWIN) +#elif defined(DARWIN) || defined(TD_ASTRA) cpuInfo->idle = 0; cpuInfo->system = 0; cpuInfo->user = 0; @@ -187,7 +191,7 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle, &cpuInfo->wa, &cpuInfo->hi, &cpuInfo->si, &cpuInfo->st, &cpuInfo->guest, &cpuInfo->guest_nice); if (EOF == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); TAOS_SKIP_ERROR(taosCloseFile(&pFile)); return terrno; } @@ -212,7 +216,7 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { cpuInfo->cutime = 0; cpuInfo->cstime = 0; } -#elif defined(DARWIN) +#elif defined(DARWIN) || defined(TD_ASTRA) cpuInfo->stime = 0; cpuInfo->utime = 0; cpuInfo->cutime = 0; @@ -236,7 +240,7 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { code = sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, &cpuInfo->cutime, &cpuInfo->cstime); if (EOF == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -270,11 +274,16 @@ void taosGetSystemInfo() { tsTotalMemoryKB = physical_pages * page_size / 1024; tsPageSizeKB = page_size / 1024; tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); -#else +#elif defined(TD_ASTRA) taosGetProcIOnfos(); TAOS_SKIP_ERROR(taosGetCpuCores(&tsNumOfCores, false)); TAOS_SKIP_ERROR(taosGetTotalMemory(&tsTotalMemoryKB)); TAOS_SKIP_ERROR(taosGetCpuUsage(NULL, NULL)); +#else + taosGetProcIOnfos(); + TAOS_SKIP_ERROR(taosGetCpuCores(&tsNumOfCores, false)); + TAOS_SKIP_ERROR(taosGetTotalMemory(&tsTotalMemoryKB)); + TAOS_SKIP_ERROR(taosGetCpuUsage(NULL, NULL)); TAOS_SKIP_ERROR(taosGetCpuInstructions(&tsSSE42Supported, &tsAVXSupported, &tsAVX2Supported, &tsFMASupported, &tsAVX512Supported)); #endif } @@ -376,12 +385,12 @@ int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t if(sName) snprintf(sName, maxLen, "macOS"); if (sysctl(osversion_name, 2, osversion, &osversion_len, NULL, 0) == -1) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } uint32_t major, minor; if (sscanf(osversion, "%u.%u", &major, &minor) == EOF) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } if (major >= 20) { major -= 9; // macOS 11 and newer @@ -391,6 +400,10 @@ int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t snprintf(releaseName, maxLen, "macOS 10.%d.%d", major, minor); } + return 0; +#elif defined(TD_ASTRA) // TD_ASTRA_TODO + if(sName) snprintf(sName, maxLen, "Astra"); + snprintf(releaseName, maxLen, "Astra"); return 0; #else char line[1024]; @@ -477,6 +490,10 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { taosCloseCmd(&pCmd); return code; +#elif defined(TD_ASTRA) // TD_ASTRA_TODO + tstrncpy(cpuModel, "ft_2000_4", maxLen); + TAOS_SKIP_ERROR(taosGetCpuCores(numOfCores, false)); + return 0; #else char line[1024] = {0}; size_t size = 0; @@ -541,7 +558,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { static int32_t taosCntrGetCpuCores(float *numOfCores) { #ifdef WINDOWS return TSDB_CODE_UNSUPPORT_OS; -#elif defined(_TD_DARWIN_64) +#elif defined(_TD_DARWIN_64) || defined(TD_ASTRA) return TSDB_CODE_UNSUPPORT_OS; #else TdFilePtr pFile = NULL; @@ -581,14 +598,14 @@ static int32_t taosCntrGetCpuCores(float *numOfCores) { *numOfCores = sysCores; } if(*numOfCores <= 0) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } goto _end; _sys: *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); if(*numOfCores <= 0) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } _end: @@ -607,14 +624,17 @@ int32_t taosGetCpuCores(float *numOfCores, bool physical) { #elif defined(_TD_DARWIN_64) *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); if(*numOfCores <= 0) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } return 0; +#elif defined(TD_ASTRA) // TD_ASTRA_TODO + *numOfCores = 4; + return 0; #else if (physical) { *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); if(*numOfCores <= 0) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } } else { int code= taosCntrGetCpuCores(numOfCores); @@ -713,10 +733,13 @@ int32_t taosGetTotalMemory(int64_t *totalKB) { return 0; #elif defined(_TD_DARWIN_64) return 0; +#elif defined(TD_ASTRA) // TD_ASTRA_TODO + *totalKB = (int64_t)256 * 1024; + return 0; #else *totalKB = (int64_t)(sysconf(_SC_PHYS_PAGES) * tsPageSizeKB); if(*totalKB <= 0) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } return 0; #endif @@ -738,7 +761,7 @@ int32_t taosGetProcMemory(int64_t *usedKB) { *usedKB = bytes_used / 1024; return 0; -#elif defined(_TD_DARWIN_64) +#elif defined(_TD_DARWIN_64) || defined(TD_ASTRA) *usedKB = 0; return 0; #else @@ -782,7 +805,7 @@ int32_t taosGetSysAvailMemory(int64_t *availSize) { *availSize = nMemTotal - nMemFree; return 0; -#elif defined(_TD_DARWIN_64) +#elif defined(_TD_DARWIN_64) || defined(TD_ASTRA) *availSize = 0; return 0; #else @@ -836,13 +859,13 @@ int32_t taosGetSysMemory(int64_t *usedKB) { *usedKB = nMemTotal - nMemFree; return 0; -#elif defined(_TD_DARWIN_64) +#elif defined(_TD_DARWIN_64) || defined(TD_ASTRA) // TD_ASTRA_TODO *usedKB = 0; return 0; #else *usedKB = sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB; if(*usedKB <= 0) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } return 0; #endif @@ -864,15 +887,15 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { diskSize->used = (int64_t)(i64TotalBytes - i64FreeBytes); return 0; } else { - // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(ERRNO)); terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); return terrno; } #elif defined(_TD_DARWIN_64) struct statvfs info; if (statvfs(dataDir, &info)) { - // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(ERRNO)); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } else { diskSize->total = info.f_blocks * info.f_frsize; @@ -880,10 +903,23 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize; return 0; } +#elif defined(TD_ASTRA) // TD_ASTRA_TODO + // if (-1 == ioctl(dataDir, FIOFSTATVFSGETBYNAME, &info)) { // TODO:try to check whether the API is available + // terrno = TAOS_SYSTEM_ERROR(ERRNO); + // return terrno; + diskSize->total = 100LL * 1024 * 1024 * 1024; + diskSize->avail = 50LL * 1024 * 1024 * 1024; + diskSize->used = 50LL * 1024 * 1024 * 1024; + // } else { + // diskSize->total = info.f_blocks * info.f_frsize; + // diskSize->avail = info.f_bavail * info.f_frsize; + // diskSize->used = diskSize->total - diskSize->avail; + // } + return 0; #else struct statvfs info; if (-1 == statvfs(dataDir, &info)) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } else { diskSize->total = info.f_blocks * info.f_frsize; @@ -910,7 +946,7 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int return 0; } return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); -#elif defined(_TD_DARWIN_64) +#elif defined(_TD_DARWIN_64) || defined(TD_ASTRA) *rchars = 0; *wchars = 0; *read_bytes = 0; @@ -1010,7 +1046,7 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { #ifdef WINDOWS return 0; -#elif defined(_TD_DARWIN_64) +#elif defined(_TD_DARWIN_64) || defined(TD_ASTRA) return 0; #else TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ | TD_FILE_STREAM); @@ -1094,7 +1130,7 @@ void taosKillSystem() { #ifdef WINDOWS printf("function taosKillSystem, exit!"); exit(0); -#elif defined(_TD_DARWIN_64) +#elif defined(_TD_DARWIN_64) || defined(TD_ASTRA) printf("function taosKillSystem, exit!"); exit(0); #else @@ -1127,6 +1163,25 @@ int32_t taosGetSystemUUIDLimit36(char *uid, int32_t uidlen) { // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null uuid_unparse_lower(uuid, buf); (void)snprintf(uid, uidlen, "%.*s", (int)sizeof(buf), buf); + return 0; +#elif defined(TD_ASTRA) + const char *template = "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx"; + const char *hex_chars = "0123456789abcdef"; + int32_t len = uidlen > 36 ? 36 : uidlen; + + for (int32_t i = 0; i < len; i++) { + if (template[i] == 'x') { + uid[i] = hex_chars[taosRand() & 15]; + } else if (template[i] == 'y') { + uid[i] = hex_chars[(taosRand() & 3) + 8]; // 8, 9, a, or b + } else { + uid[i] = template[i]; + } + } + if (len >= 0) { + uid[len] = 0; + } + return 0; #else int64_t len = 0; @@ -1173,14 +1228,16 @@ char *taosGetCmdlineByPID(int pid) { return ""; #elif defined(_TD_DARWIN_64) static char cmdline[1024]; - errno = 0; + SET_ERRNO(0); if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) { - fprintf(stderr, "PID is %d, %s", pid, strerror(errno)); - return strerror(errno); + fprintf(stderr, "PID is %d, %s", pid, strerror(ERRNO)); + return strerror(ERRNO); } return cmdline; +#elif defined(TD_ASTRA) + return ""; #else static char cmdline[1024]; (void)snprintf(cmdline, sizeof(cmdline), "/proc/%d/cmdline", pid); @@ -1206,11 +1263,11 @@ char *taosGetCmdlineByPID(int pid) { int64_t taosGetOsUptime() { #ifdef WINDOWS -#elif defined(_TD_DARWIN_64) +#elif defined(_TD_DARWIN_64) || defined(TD_ASTRA) #else struct sysinfo info; if (-1 == sysinfo(&info)) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -1225,7 +1282,7 @@ void taosSetCoreDump(bool enable) { #ifdef WINDOWS SetUnhandledExceptionFilter(exceptionHandler); SetUnhandledExceptionFilter(&FlCrashDump); -#elif defined(_TD_DARWIN_64) +#elif defined(_TD_DARWIN_64) || defined(TD_ASTRA) #else // 1. set ulimit -c unlimited struct rlimit rlim; @@ -1239,7 +1296,7 @@ void taosSetCoreDump(bool enable) { rlim_new.rlim_cur = RLIM_INFINITY; rlim_new.rlim_max = RLIM_INFINITY; if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { - // printf("set unlimited fail, error: %s", strerror(errno)); + // printf("set unlimited fail, error: %s", strerror(ERRNO)); rlim_new.rlim_cur = rlim.rlim_max; rlim_new.rlim_max = rlim.rlim_max; (void)setrlimit(RLIMIT_CORE, &rlim_new); @@ -1277,7 +1334,7 @@ void taosSetCoreDump(bool enable) { #ifndef __loongarch64 if (syscall(SYS__sysctl, &args) == -1) { - // printf("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); + // printf("_sysctl(kern_core_uses_pid) set fail: %s", strerror(ERRNO)); } #endif @@ -1295,7 +1352,7 @@ void taosSetCoreDump(bool enable) { #ifndef __loongarch64 if (syscall(SYS__sysctl, &args) == -1) { - // printf("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); + // printf("_sysctl(kern_core_uses_pid) get fail: %s", strerror(ERRNO)); } #endif @@ -1348,7 +1405,7 @@ SysNameInfo taosGetSysNameInfo() { tstrncpy(info.version, uts.version, sizeof(info.version)); tstrncpy(info.machine, uts.machine, sizeof(info.machine)); } else { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } return info; @@ -1387,7 +1444,7 @@ int32_t taosGetMaclocalhostnameByCommand(char *hostname, size_t maxLen) { } taosCloseCmd(&pCmd); } - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } int32_t getMacLocalHostNameBySCD(char *hostname, size_t maxLen) { @@ -1416,7 +1473,7 @@ int32_t taosGetlocalhostname(char *hostname, size_t maxLen) { #else int r = gethostname(hostname, maxLen); if (-1 == r) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } return r; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index fefada2142..38114a8deb 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -75,14 +75,14 @@ void stratWindowsService(MainWindows mainWindows) { StartServiceCtrlDispatcher(ServiceTable); } -#elif defined(_TD_DARWIN_64) +#elif defined(_TD_DARWIN_64) || defined(TD_ASTRA) #else #include #include #include #endif -#if !defined(WINDOWS) +#if !defined(WINDOWS) && !defined(TD_ASTRA) struct termios oldtio; #endif @@ -142,6 +142,8 @@ int32_t taosSetConsoleEcho(bool on) { return terrno; } + return 0; +#elif defined(TD_ASTRA) // TD_ASTRA_TODO return 0; #else #define ECHOFLAGS (ECHO | ECHOE | ECHOK | ECHONL) @@ -149,7 +151,7 @@ int32_t taosSetConsoleEcho(bool on) { struct termios term; if (tcgetattr(STDIN_FILENO, &term) == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -160,7 +162,7 @@ int32_t taosSetConsoleEcho(bool on) { err = tcsetattr(STDIN_FILENO, TCSAFLUSH, &term); if (err == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -169,7 +171,7 @@ int32_t taosSetConsoleEcho(bool on) { } int32_t taosSetTerminalMode() { -#if defined(WINDOWS) +#if defined(WINDOWS) || defined(TD_ASTRA) // TD_ASTRA_TODO return 0; #else struct termios newtio; @@ -195,7 +197,7 @@ int32_t taosSetTerminalMode() { newtio.c_cc[VTIME] = 0; if (-1 == tcsetattr(0, TCSANOW, &newtio)) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); (void)fprintf(stderr, "Fail to set terminal properties!\n"); return terrno; } @@ -205,17 +207,18 @@ int32_t taosSetTerminalMode() { } int32_t taosGetOldTerminalMode() { -#if defined(WINDOWS) +#if defined(WINDOWS) || defined(TD_ASTRA) // TD_ASTRA_TODO + return 0; #else /* Make sure stdin is a terminal. */ if (!isatty(STDIN_FILENO)) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } // Get the parameter of current terminal if (-1 == tcgetattr(0, &oldtio)) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -224,10 +227,11 @@ int32_t taosGetOldTerminalMode() { } int32_t taosResetTerminalMode() { -#if defined(WINDOWS) +#if defined(WINDOWS) || defined(TD_ASTRA) // TD_ASTRA_TODO + return 0; #else if (-1 == tcsetattr(0, TCSANOW, &oldtio)) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); (void)fprintf(stderr, "Fail to reset the terminal properties!\n"); return terrno; } @@ -243,10 +247,12 @@ TdCmdPtr taosOpenCmd(const char* cmd) { #ifdef WINDOWS return (TdCmdPtr)_popen(cmd, "r"); +#elif defined(TD_ASTRA) + return NULL; #else TdCmdPtr p = (TdCmdPtr)popen(cmd, "r"); if (NULL == p) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } return p; #endif @@ -287,11 +293,41 @@ int64_t taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf) { } (*ptrBuf)[1023] = 0; return strlen(*ptrBuf); +#elif defined(TD_ASTRA) + size_t bufsize = 128; + size_t pos = 0; + int c; + if (*ptrBuf == NULL) { + *ptrBuf = (char*)taosMemoryMalloc(bufsize); + if (*ptrBuf == NULL) { + return terrno; + } + } + while ((c = fgetc((FILE*)pCmd)) != EOF) { + if (pos + 1 >= bufsize) { + size_t new_size = bufsize << 1; + char* new_line = (char*)taosMemoryRealloc(*ptrBuf, new_size); + if (new_line == NULL) { + return terrno; + } + *ptrBuf = new_line; + bufsize = new_size; + } + (*ptrBuf)[pos++] = (char)c; + if (c == '\n') { + break; + } + } + if (pos == 0 && c == EOF) { + return TSDB_CODE_INVALID_PARA; + } + (*ptrBuf)[pos] = '\0'; + return (ssize_t)pos; #else ssize_t len = 0; len = getline(ptrBuf, (size_t*)&len, (FILE*)pCmd); if (-1 == len) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } return len; @@ -311,6 +347,7 @@ void taosCloseCmd(TdCmdPtr* ppCmd) { } #ifdef WINDOWS _pclose((FILE*)(*ppCmd)); +#elif defined(TD_ASTRA) // TD_ASTRA_TODO #else (void)pclose((FILE*)(*ppCmd)); #endif diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 603f5da3f7..458c89d9a5 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -20,7 +20,25 @@ int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void *(*start)(void *), void *arg) { OS_PARAM_CHECK(tid); OS_PARAM_CHECK(start); +#ifdef TD_ASTRA + int32_t code = 0; + if (!attr) { + pthread_attr_t threadAttr; + pthread_attr_init(&threadAttr); + pthread_attr_setstacksize(&threadAttr, STACK_SIZE_DEFAULT); + code = pthread_create(tid, &threadAttr, start, arg); + pthread_attr_destroy(&threadAttr); + } else { + int32_t stackSize = 0; + pthread_attr_getstacksize(attr, &stackSize); + if (stackSize == 0) { + pthread_attr_setstacksize(attr, STACK_SIZE_DEFAULT); + } + code = pthread_create(tid, attr, start, arg); + } +#else int32_t code = pthread_create(tid, attr, start, arg); +#endif if (code) { taosThreadClear(tid); return (terrno = TAOS_SYSTEM_ERROR(code)); diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 3bab3e7e25..18d71abee8 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -26,7 +26,7 @@ #include "os.h" -#ifdef WINDOWS +#if defined(WINDOWS) || defined(TD_ASTRA) #include #include @@ -82,7 +82,7 @@ static const char *am_pm[2] = {"AM", "PM"}; char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { if (!buf || !fmt || !tm) return NULL; -#ifdef WINDOWS +#if defined(WINDOWS) || defined(TD_ASTRA) char c; const char *bp; size_t len = 0; @@ -370,7 +370,7 @@ int32_t taosGetTimeOfDay(struct timeval *tv) { return 0; #else code = gettimeofday(tv, NULL); - return (-1 == code) ? (terrno = TAOS_SYSTEM_ERROR(errno)) : 0; + return (-1 == code) ? (terrno = TAOS_SYSTEM_ERROR(ERRNO)) : 0; #endif } @@ -380,7 +380,7 @@ int32_t taosTime(time_t *t) { } time_t r = time(t); if (r == (time_t)-1) { - return TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(ERRNO); } return 0; } @@ -439,10 +439,16 @@ time_t taosMktime(struct tm *timep, timezone_t tz) { #endif return user_mktime64(timep->tm_year + 1900, timep->tm_mon + 1, timep->tm_mday, timep->tm_hour, timep->tm_min, timep->tm_sec, tzw); +#elif defined(TD_ASTRA) + time_t r = mktime(timep); + if (r == (time_t)-1) { + terrno = TAOS_SYSTEM_ERROR(ERRNO); + } + return r; #else time_t r = (tz != NULL ? mktime_z(tz, timep) : mktime(timep)); if (r == (time_t)-1) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); } timezone = -timep->tm_gmtoff; return r; @@ -467,6 +473,13 @@ time_t taosTimeGm(struct tm *tmp) { } #ifdef WINDOWS return _mkgmtime(tmp); +#elif defined(TD_ASTRA) + time_t local = mktime(tmp); + struct tm local_tm = *localtime(&local); + struct tm utc_tm = *gmtime(&local); + time_t offset = (local_tm.tm_hour - utc_tm.tm_hour) * 3600 + (local_tm.tm_min - utc_tm.tm_min) * 60 + + (local_tm.tm_sec - utc_tm.tm_sec); + return local - offset; #else return timegm(tmp); #endif @@ -529,6 +542,12 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int3 } } return result; +#elif defined(TD_ASTRA) + res = localtime_r(timep, result); + if (res == NULL && buf != NULL) { + (void)sprintf(buf, "NaN"); + } + return res; #else res = (tz != NULL ? localtime_rz(tz, timep, result) : localtime_r(timep, result)); if (res == NULL && buf != NULL) { @@ -558,6 +577,6 @@ int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { return (0); #else code = clock_gettime(clock_id, pTS); - return (-1 == code) ? (terrno = TAOS_SYSTEM_ERROR(errno)) : 0; + return (-1 == code) ? (terrno = TAOS_SYSTEM_ERROR(ERRNO)) : 0; #endif } diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index 12a43925cd..f3fd478a6b 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -75,7 +75,9 @@ void taos_block_sigalrm(void) { } #else +#ifndef TD_ASTRA #include +#endif #include static void taosDeleteTimer(void *tharg) { @@ -104,6 +106,8 @@ static void *taosProcessAlarmSignal(void *tharg) { #ifdef _ALPINE sevent.sigev_notify = SIGEV_THREAD_ID; sevent.sigev_notify_thread_id = syscall(__NR_gettid); +#elif defined(TD_ASTRA) // TD_ASTRA_TODO + sevent.sigev_notify = SIGEV_THREAD; #else sevent.sigev_notify = SIGEV_THREAD_ID; sevent._sigev_un._tid = syscall(__NR_gettid); @@ -112,7 +116,7 @@ static void *taosProcessAlarmSignal(void *tharg) { sevent.sigev_signo = SIGALRM; if (timer_create(CLOCK_REALTIME, &sevent, &timerId) == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return NULL; } @@ -126,7 +130,7 @@ static void *taosProcessAlarmSignal(void *tharg) { ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK; if (-1 == timer_settime(timerId, 0, &ts, NULL)) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); break; } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 4b06930118..5e90f70c70 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -733,11 +733,13 @@ char *tz_win[W_TZ_CITY_NUM][2] = {{"Asia/Shanghai", "China Standard Time"}, #include #else #include +#ifndef TD_ASTRA #include #include #include #include #include +#endif #include #include #endif @@ -773,7 +775,7 @@ int32_t taosSetGlobalTimezone(const char *tz) { #else code = setenv("TZ", tz, 1); if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); return terrno; } @@ -788,11 +790,13 @@ int32_t taosGetLocalTimezoneOffset() { time_t tx1 = taosGetTimestampSec(); struct tm tm1; if (taosLocalTime(&tx1, &tm1, NULL, 0, NULL) == NULL) { - uError("%s failed to get local time: code:%d", __FUNCTION__, errno); + uError("%s failed to get local time: code:%d", __FUNCTION__, ERRNO); return TSDB_CODE_TIME_ERROR; } #ifdef WINDOWS return -_timezone; +#elif defined(TD_ASTRA) + return -(int32_t)timezone; #else return (int32_t)(tm1.tm_gmtoff); #endif @@ -801,7 +805,7 @@ int32_t taosGetLocalTimezoneOffset() { int32_t taosFormatTimezoneStr(time_t t, const char *tz, timezone_t sp, char *outTimezoneStr) { struct tm tm1; if (taosLocalTime(&t, &tm1, NULL, 0, sp) == NULL) { - uError("%s failed to get local time: code:%d", __FUNCTION__, errno); + uError("%s failed to get local time: code:%d", __FUNCTION__, ERRNO); return TSDB_CODE_TIME_ERROR; } @@ -830,12 +834,14 @@ int32_t taosFormatTimezoneStr(time_t t, const char *tz, timezone_t sp, char *out return 0; } -#ifndef WINDOWS void getTimezoneStr(char *tz) { +#ifdef TD_ASTRA // TD_ASTRA_TODO + memcpy(tz, "Asia/Shanghai", sizeof("Asia/Shanghai")); +#elif !defined(WINDOWS) do { int n = readlink("/etc/localtime", tz, TD_TIMEZONE_LEN - 1); if (n < 0) { - uWarn("[tz] failed to readlink /etc/localtime, reason:%s", strerror(errno)); + uWarn("[tz] failed to readlink /etc/localtime, reason:%s", strerror(ERRNO)); break; } @@ -851,7 +857,7 @@ void getTimezoneStr(char *tz) { TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); if (pFile == NULL) { - uWarn("[tz] failed to open /etc/timezone, reason:%s", strerror(errno)); + uWarn("[tz] failed to open /etc/timezone, reason:%s", strerror(ERRNO)); goto END; } int len = taosReadFile(pFile, tz, TD_TIMEZONE_LEN - 1); @@ -869,8 +875,8 @@ END: memcpy(tz, TZ_UNKNOWN, sizeof(TZ_UNKNOWN)); } uDebug("[tz] system timezone:%s", tz); -} #endif +} void truncateTimezoneString(char *tz) { char *spacePos = strchr(tz, ' '); diff --git a/source/os/test/osStringTests.cpp b/source/os/test/osStringTests.cpp index 4e71b4b884..dbd3cd64cb 100644 --- a/source/os/test/osStringTests.cpp +++ b/source/os/test/osStringTests.cpp @@ -87,19 +87,19 @@ TEST(osStringTests, osUcs4Tests1) { TdUcs4 f1_ucs4[] = {0x0048, 0x0065, 0x006C, 0x006C, 0x006F, 0x0000}; TdUcs4 f2_ucs4[] = {0x0048, 0x0065, 0x006C, 0x006C, 0x006F, 0x0000}; - EXPECT_EQ(tasoUcs4Compare(f1_ucs4, f2_ucs4, sizeof(f1_ucs4)), 0); + EXPECT_EQ(taosUcs4Compare(f1_ucs4, f2_ucs4, sizeof(f1_ucs4)), 0); TdUcs4 f3_ucs4[] = {0x0048, 0x0065, 0x006C, 0x006C, 0x006F, 0x0020, 0x0077, 0x006F, 0x0072, 0x006C, 0x0064, 0x0021, 0x0000}; TdUcs4 f4_ucs4[] = {0x0048, 0x0065, 0x006C, 0x006C, 0x006F, 0x0000}; - EXPECT_GT(tasoUcs4Compare(f3_ucs4, f4_ucs4, sizeof(f3_ucs4)), 0); + EXPECT_GT(taosUcs4Compare(f3_ucs4, f4_ucs4, sizeof(f3_ucs4)), 0); TdUcs4 f5_ucs4[] = {0x0048, 0x0065, 0x006C, 0x006C, 0x006F, 0x0000}; TdUcs4 f6_ucs4[] = {0x0048, 0x0065, 0x006C, 0x006C, 0x006F, 0x0020, 0x0077, 0x006F, 0x0072, 0x006C, 0x0064, 0x0021, 0x0000}; - EXPECT_LT(tasoUcs4Compare(f5_ucs4, f6_ucs4, sizeof(f5_ucs4)), 0); + EXPECT_LT(taosUcs4Compare(f5_ucs4, f6_ucs4, sizeof(f5_ucs4)), 0); } TEST(osStringTests, osUcs4lenTests2) { @@ -650,9 +650,9 @@ TEST(osStringTests, strint2) { TdUcs4 p1, p2; int32_t val32 = 0; - ret32 = tasoUcs4Compare(&p1, NULL, val32); + ret32 = taosUcs4Compare(&p1, NULL, val32); EXPECT_NE(ret32, 0); - ret32 = tasoUcs4Compare(NULL, &p2, val32); + ret32 = taosUcs4Compare(NULL, &p2, val32); EXPECT_NE(ret32, 0); void* retptr = taosAcquireConv(NULL, M2C, NULL); diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index d5001f0264..38c313a435 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -155,7 +155,7 @@ static void *taosCacheTimedRefresh(void *handle); static void doInitRefreshThread(void) { pCacheArrayList = taosArrayInit(4, POINTER_BYTES); if (pCacheArrayList == NULL) { - uError("failed to allocate memory, reason:%s", strerror(errno)); + uError("failed to allocate memory, reason:%s", strerror(ERRNO)); return; } @@ -174,7 +174,7 @@ TdThread doRegisterCacheObj(SCacheObj *pCacheObj) { (void)taosThreadMutexLock(&guard); if (taosArrayPush(pCacheArrayList, &pCacheObj) == NULL) { - uError("failed to add cache object into array, reason:%s", strerror(errno)); + uError("failed to add cache object into array, reason:%s", strerror(ERRNO)); (void)taosThreadMutexUnlock(&guard); return cacheRefreshWorker; } @@ -366,7 +366,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLi SCacheObj *pCacheObj = (SCacheObj *)taosMemoryCalloc(1, sizeof(SCacheObj)); if (pCacheObj == NULL) { - uError("failed to allocate memory, reason:%s", strerror(errno)); + uError("failed to allocate memory, reason:%s", strerror(ERRNO)); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -376,7 +376,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLi pCacheObj->pEntryList = taosMemoryCalloc(pCacheObj->capacity, sizeof(SCacheEntry)); if (pCacheObj->pEntryList == NULL) { taosMemoryFree(pCacheObj); - uError("failed to allocate memory, reason:%s", strerror(errno)); + uError("failed to allocate memory, reason:%s", strerror(ERRNO)); return NULL; } @@ -400,7 +400,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLi taosMemoryFreeClear(pCacheObj->name); taosMemoryFree(pCacheObj); - uError("failed to init lock, reason:%s", strerror(errno)); + uError("failed to init lock, reason:%s", strerror(ERRNO)); return NULL; } @@ -725,7 +725,7 @@ SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pDat SCacheNode *pNewNode = taosMemoryCalloc(1, sizeInBytes); if (pNewNode == NULL) { - uError("failed to allocate memory, reason:%s", strerror(errno)); + uError("failed to allocate memory, reason:%s", strerror(ERRNO)); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index a253c58415..18229f2e81 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -210,7 +210,7 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { int32_t len1 = varDataLen(pLeft); int32_t len2 = varDataLen(pRight); - int32_t ret = tasoUcs4Compare((TdUcs4 *)varDataVal(pLeft), (TdUcs4 *)varDataVal(pRight), len1>len2 ? len2:len1); + int32_t ret = taosUcs4Compare((TdUcs4 *)varDataVal(pLeft), (TdUcs4 *)varDataVal(pRight), len1>len2 ? len2:len1); if (ret == 0) { if (len1 > len2) return 1; @@ -1224,11 +1224,12 @@ typedef struct UsingRegex { typedef UsingRegex* HashRegexPtr; typedef struct RegexCache { - SHashObj *regexHash; - void *regexCacheTmr; - void *timer; - SRWLatch mutex; - bool exit; + SHashObj *regexHash; + void *regexCacheTmr; + void *timer; + SRWLatch mutex; + bool exit; + int8_t inited; } RegexCache; static RegexCache sRegexCache; #define MAX_REGEX_CACHE_SIZE 20 @@ -1276,6 +1277,7 @@ int32_t InitRegexCache() { #ifdef WINDOWS return 0; #endif + if (atomic_val_compare_exchange_8(&sRegexCache.inited, 0, 1) != 0) return TSDB_CODE_SUCCESS; sRegexCache.regexHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (sRegexCache.regexHash == NULL) { uError("failed to create RegexCache"); @@ -1312,7 +1314,9 @@ void DestroyRegexCache(){ taosWLockLatch(&sRegexCache.mutex); sRegexCache.exit = true; taosHashCleanup(sRegexCache.regexHash); + sRegexCache.regexHash = NULL; taosTmrCleanUp(sRegexCache.regexCacheTmr); + sRegexCache.regexCacheTmr = NULL; taosWUnLockLatch(&sRegexCache.mutex); } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 32ffa04beb..3dfe8040fb 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -355,7 +355,7 @@ static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType cfgStypeStr(stype), value); TAOS_RETURN(TSDB_CODE_INVALID_CFG); } - +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV if (!taosValidateEncodec(value)) { uError("invalid charset:%s", value); TAOS_RETURN(terrno); @@ -366,7 +366,7 @@ static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType } (void)memcpy(tsCharset, value, strlen(value) + 1); TAOS_CHECK_RETURN(doSetConf(pItem, value, stype)); - +#endif TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -582,14 +582,14 @@ SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) { int32_t size = taosArrayGetSize(pCfg->localArray); for (int32_t i = 0; i < size; ++i) { SConfigItem *pItem = taosArrayGet(pCfg->localArray, i); - if (strcasecmp(pItem->name, pName) == 0) { + if (taosStrcasecmp(pItem->name, pName) == 0) { return pItem; } } size = taosArrayGetSize(pCfg->globalArray); for (int32_t i = 0; i < size; ++i) { SConfigItem *pItem = taosArrayGet(pCfg->globalArray, i); - if (strcasecmp(pItem->name, pName) == 0) { + if (taosStrcasecmp(pItem->name, pName) == 0) { return pItem; } } @@ -964,7 +964,7 @@ int32_t cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t } if (len < 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } if (len > bufSize) { @@ -990,7 +990,7 @@ int32_t cfgDumpItemCategory(SConfigItem *pItem, char *buf, int32_t bufSize, int3 } if (len < 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(ERRNO)); } if (len > bufSize) { @@ -1363,7 +1363,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { if (pFile == NULL) { // success when the file does not exist code = terrno; - if (errno == ENOENT) { + if (ERRNO == ENOENT) { uInfo("failed to load from cfg file %s since %s, use default parameters", filepath, tstrerror(code)); TAOS_RETURN(TSDB_CODE_SUCCESS); } else { @@ -1440,7 +1440,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { size_t len = strlen(name); const char *debugFlagStr = "debugFlag"; const size_t debugFlagLen = strlen(debugFlagStr); - if (len >= debugFlagLen && strcasecmp(name + len - debugFlagLen, debugFlagStr) == 0) { + if (len >= debugFlagLen && taosStrcasecmp(name + len - debugFlagLen, debugFlagStr) == 0) { code = cfgUpdateDebugFlagItem(pConfig, name, len == debugFlagLen); if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; } diff --git a/source/util/src/tconv.c b/source/util/src/tconv.c index d9932489f8..2bc0e2559c 100644 --- a/source/util/src/tconv.c +++ b/source/util/src/tconv.c @@ -26,6 +26,7 @@ SHashObj *gConvInfo = NULL; // C2M: Ucs4--> Mbs static void taosConvDestroyInner(void *arg) { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV SConvInfo *info = (SConvInfo *)arg; if (info == NULL) { return; @@ -41,9 +42,11 @@ static void taosConvDestroyInner(void *arg) { info->gConvMaxNum[M2C] = -1; info->gConvMaxNum[C2M] = -1; +#endif } void* taosConvInit(const char* charset) { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV if (charset == NULL){ terrno = TSDB_CODE_INVALID_PARA; return NULL; @@ -91,14 +94,14 @@ void* taosConvInit(const char* charset) { for (int32_t i = 0; i < info.gConvMaxNum[M2C]; ++i) { info.gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, charset); if ((iconv_t)-1 == info.gConv[M2C][i].conv) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); goto FAILED; } } for (int32_t i = 0; i < info.gConvMaxNum[C2M]; ++i) { info.gConv[C2M][i].conv = iconv_open(charset, DEFAULT_UNICODE_ENCODEC); if ((iconv_t)-1 == info.gConv[C2M][i].conv) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(ERRNO); goto FAILED; } } @@ -116,8 +119,14 @@ FAILED: END: atomic_store_32(&lock_c, 0); return conv; +#else + return NULL; +#endif } void taosConvDestroy() { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV taosHashCleanup(gConvInfo); + gConvInfo = NULL; +#endif } diff --git a/source/util/src/tcrc32c.c b/source/util/src/tcrc32c.c index 8b1bcc659f..99e6c666b2 100644 --- a/source/util/src/tcrc32c.c +++ b/source/util/src/tcrc32c.c @@ -19,7 +19,7 @@ */ #define _DEFAULT_SOURCE -#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) && !defined(_TD_LOONGARCH_) +#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) && !defined(_TD_LOONGARCH_) && !defined(TD_ASTRA) #include #endif @@ -856,7 +856,7 @@ static uint32_t shift_crc(uint32_t shift_table[][256], uint32_t crc) { version. Otherwise, use the software version. */ uint32_t (*crc32c)(uint32_t crci, crc_stream bytes, size_t len) = crc32c_sf; -#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) && !defined(_TD_LOONGARCH_) +#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) && !defined(_TD_LOONGARCH_) && !defined(TD_ASTRA) /* Compute CRC-32C using the Intel hardware instruction. */ uint32_t crc32c_hw(uint32_t crc, crc_stream buf, size_t len) { crc_stream next = buf; @@ -1011,7 +1011,7 @@ uint32_t crc32c_hw(uint32_t crc, crc_stream buf, size_t len) { #endif // #ifndef _TD_ARM_ void taosResolveCRC() { -#if defined _TD_ARM_ || defined _TD_MIPS_ || defined WINDOWS || defined _TD_LOONGARCH_ +#if defined _TD_ARM_ || defined _TD_MIPS_ || defined WINDOWS || defined _TD_LOONGARCH_ || defined TD_ASTRA crc32c = crc32c_sf; #else int32_t sse42; diff --git a/source/util/src/tgeosctx.c b/source/util/src/tgeosctx.c index 8de8d26fad..49ae22a654 100644 --- a/source/util/src/tgeosctx.c +++ b/source/util/src/tgeosctx.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +#ifdef USE_GEOS #include "tgeosctx.h" #include "tlog.h" #include "tutil.h" @@ -86,7 +86,7 @@ int32_t getThreadLocalGeosCtx(SGeosContext **ppCtx) { if (old == 0) { if ((taosThreadKeyCreate(&tlGeosCtxKey, destroyThreadLocalGeosCtx)) != 0) { atomic_store_8(&tlGeosCtxKeyInited, 0); - TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(errno)); + TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(ERRNO)); } atomic_store_8(&tlGeosCtxKeyInited, 1); } @@ -97,7 +97,7 @@ int32_t getThreadLocalGeosCtx(SGeosContext **ppCtx) { } if ((taosThreadSetSpecific(tlGeosCtxKey, (const void *)tlGeosCtxObj)) != 0) { taosMemoryFreeClear(tlGeosCtxObj); - TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(errno)); + TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(ERRNO)); } *ppCtx = tlGeosCtx = tlGeosCtxObj; @@ -113,3 +113,4 @@ _exit: const char *getGeosErrMsg(int32_t code) { return (tlGeosCtx && tlGeosCtx->errMsg[0] != 0) ? tlGeosCtx->errMsg : (code ? tstrerror(code) : ""); } +#endif \ No newline at end of file diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 562fef2027..52cc05087a 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -46,7 +46,10 @@ struct SHashNode { uint32_t keyLen; // length of the key uint16_t refCount; // reference count int8_t removed; // flag to indicate removed - char data[]; +#ifdef TD_ASTRA_32 + uint32_t paddingAligned; +#endif + char data[]; }; typedef struct SHashEntry { @@ -56,9 +59,9 @@ typedef struct SHashEntry { } SHashEntry; struct SHashObj { - SHashEntry **hashList; - size_t capacity; // number of slots int64_t size; // number of elements in hash table + size_t capacity; // number of slots + SHashEntry **hashList; _hash_fn_t hashFp; // hash function _equal_fn_t equalFp; // equal function _hash_free_fn_t freeFp; // hash node free callback function diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index b466e1b351..aab0fd454b 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "tcompare.h" +#include "tutil.h" #include "thash.h" #include "types.h" #include "xxhash.h" @@ -106,8 +107,12 @@ uint64_t MurmurHash3_64(const char *key, uint32_t len) { const uint8_t *end = data + (len - (len & 7)); while (data != end) { +#ifndef NO_UNALIGNED_ACCESS uint64_t k = *((uint64_t *)data); - +#else + uint64_t k = 0; + memcpy(&k, data, sizeof(uint64_t)); +#endif k *= m; k ^= k >> r; k *= m; @@ -176,7 +181,7 @@ uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) { } } uint32_t taosIntHash_64(const char *key, uint32_t UNUSED_PARAM(len)) { - uint64_t val = *(uint64_t *)key; + uint64_t val = taosGetUInt64Aligned((uint64_t *)key); uint64_t hash = val >> 16U; hash += (val & 0xFFFFU); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 2460c7c650..647feecd5f 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -159,6 +159,9 @@ static void taosWriteSlowLog(SLogBuff *pLogBuf); static int32_t taosStartLog() { TdThreadAttr threadAttr; (void)taosThreadAttrInit(&threadAttr); +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&threadAttr, STACK_SIZE_SMALL); +#endif if (taosThreadCreate(&(tsLogObj.logHandle->asyncThread), &threadAttr, taosAsyncOutputLog, tsLogObj.logHandle) != 0) { return terrno; } @@ -219,9 +222,9 @@ int32_t taosInitSlowLog() { if (tsLogObj.slowHandle == NULL) return terrno; TAOS_UNUSED(taosUmaskFile(0)); - tsLogObj.slowHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + tsLogObj.slowHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_READ | TD_FILE_WRITE | TD_FILE_APPEND); if (tsLogObj.slowHandle->pFile == NULL) { - (void)printf("\nfailed to open slow log file:%s, reason:%s\n", name, strerror(errno)); + (void)printf("\nfailed to open slow log file:%s, reason:%s\n", name, strerror(ERRNO)); return terrno; } @@ -476,7 +479,9 @@ static int32_t taosOpenNewLogFile() { TdThreadAttr attr; (void)taosThreadAttrInit(&attr); (void)taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED); - +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&attr, STACK_SIZE_SMALL); +#endif OldFileKeeper *oldFileKeeper = taosOpenNewFile(); if (!oldFileKeeper) { tsLogObj.openInProgress = 0; @@ -527,7 +532,7 @@ void taosOpenNewSlowLogFile() { (void)snprintf(name, PATH_MAX + TD_TIME_STR_LEN, "%s.%s", tsLogObj.slowLogName, day); pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pFile == NULL) { - uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); + uError("open new log file fail! reason:%s, reuse lastlog", strerror(ERRNO)); (void)taosThreadMutexUnlock(&tsLogObj.logMutex); return; } @@ -561,7 +566,7 @@ static bool taosCheckFileIsOpen(char *logFileName) { if (lastErrorIsFileNotExist()) { return false; } else { - printf("\nfailed to open log file:%s, reason:%s\n", logFileName, strerror(errno)); + printf("\n%s:%d failed to open log file:%s, reason:%s\n", __func__, __LINE__, logFileName, strerror(ERRNO)); return true; } } @@ -661,9 +666,9 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE); if (tsLogObj.logHandle == NULL) return terrno; - tsLogObj.logHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE); + tsLogObj.logHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_READ | TD_FILE_WRITE); if (tsLogObj.logHandle->pFile == NULL) { - (void)printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno)); + (void)printf("\n%s:%d failed to open log file:%s, reason:%s\n", __func__, __LINE__, name, strerror(ERRNO)); return terrno; } TAOS_UNUSED(taosLockLogFile(tsLogObj.logHandle->pFile)); @@ -722,7 +727,7 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { time_t curTime = timeSecs.tv_sec; ptm = taosLocalTime(&curTime, &Tm, NULL, 0, NULL); if (ptm == NULL) { - uError("%s failed to get local time, code:%d", __FUNCTION__, errno); + uError("%s failed to get local time, code:%d", __FUNCTION__, ERRNO); return 0; } return snprintf(buffer, LOG_MAX_STACK_LINE_BUFFER_SIZE, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " %s %s", @@ -749,7 +754,7 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b int fd = 0; if (tsLogObj.outputType == LOG_OUTPUT_FILE) { - if (dflag & DEBUG_SCREEN) fd = 1; + if ((dflag & DEBUG_SCREEN) && tsLogEmbedded) fd = 1; } else if (tsLogObj.outputType == LOG_OUTPUT_STDOUT) { fd = 1; } else if (tsLogObj.outputType == LOG_OUTPUT_STDERR) { @@ -759,9 +764,13 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b if (fd) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" +#ifndef TD_ASTRA if (write(fd, buffer, (uint32_t)len) < 0) { - TAOS_UNUSED(printf("failed to write log to screen, reason:%s\n", strerror(errno))); + TAOS_UNUSED(printf("failed to write log to screen, reason:%s\n", strerror(ERRNO))); } +#else + TAOS_UNUSED(fprintf(fd == 1 ? stdout : stderr, "%s", buffer)); +#endif #pragma GCC diagnostic pop } } @@ -1180,6 +1189,9 @@ static void *taosAsyncOutputLog(void *param) { TdThreadAttr attr; (void)taosThreadAttrInit(&attr); (void)taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED); +#ifdef TD_COMPACT_OS + (void)taosThreadAttrSetStackSize(&attr, STACK_SIZE_SMALL); +#endif if (taosThreadCreate(&thread, &attr, taosLogRotateFunc, tsLogObj.logHandle) == 0) { uInfo("process log rotation"); lastCheckSec = curSec; @@ -1214,8 +1226,9 @@ bool taosAssertDebug(bool condition, const char *file, int32_t line, bool core, taosPrintLogImp(1, 255, buffer, len); taosPrintLog(flags, level, dflag, "tAssert at file %s:%d exit:%d", file, line, tsAssert); +#ifndef TD_ASTRA taosPrintTrace(flags, level, dflag, -1); - +#endif if (tsAssert || core) { taosCloseLog(); taosMsleep(300); @@ -1229,7 +1242,7 @@ bool taosAssertDebug(bool condition, const char *file, int32_t line, bool core, return true; } - +#ifdef USE_REPORT void taosLogCrashInfo(char *nodeType, char *pMsg, int64_t msgLen, int signum, void *sigInfo) { const char *flags = "UTL FATAL "; ELogLevel level = DEBUG_FATAL; @@ -1357,7 +1370,7 @@ static void checkWriteCrashLogToFileInNewThead() { } taosLogCrashInfo(gCrashBasicInfo.nodeType, pMsg, msgLen, gCrashBasicInfo.signum, gCrashBasicInfo.sigInfo); setCrashWriterStatus(CRASH_LOG_WRITER_INIT); - tsem_post(&gCrashBasicInfo.sem); + TAOS_UNUSED(tsem_post(&gCrashBasicInfo.sem)); } } @@ -1386,7 +1399,7 @@ void writeCrashLogToFile(int signum, void *sigInfo, char *nodeType, int64_t clus gCrashBasicInfo.nodeType = nodeType; gCrashBasicInfo.signum = signum; gCrashBasicInfo.sigInfo = sigInfo; - tsem_wait(&gCrashBasicInfo.sem); + TAOS_UNUSED(tsem_wait(&gCrashBasicInfo.sem)); } } @@ -1415,7 +1428,7 @@ void taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_WRITE); if (pFile == NULL) { - if (ENOENT == errno) { + if (ENOENT == ERRNO) { return; } @@ -1481,6 +1494,7 @@ void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile) { TAOS_UNUSED(taosUnLockFile(pFile)); TAOS_UNUSED(taosCloseFile(&pFile)); } +#endif // USE_REPORT #ifdef NDEBUG bool taosAssertRelease(bool condition) { diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index 06ed9b7690..8756903763 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -42,7 +42,7 @@ int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, SMultiwayMergeTreeInfo* pTreeInfo = (SMultiwayMergeTreeInfo*)taosMemoryCalloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); if (pTreeInfo == NULL) { - uError("allocate memory for loser-tree failed. reason:%s", strerror(errno)); + uError("allocate memory for loser-tree failed. reason:%s", strerror(ERRNO)); return terrno; } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 3752ff8207..6f71d83214 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -652,7 +652,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { if (needRemoveFile) { int32_t ret = taosRemoveFile(pBuf->path); if (ret != 0) { // print the error and discard this error info - uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(errno)); + uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(ERRNO)); } } diff --git a/source/util/src/tpcre2.c b/source/util/src/tpcre2.c index ba9bd51510..bfd73c1964 100644 --- a/source/util/src/tpcre2.c +++ b/source/util/src/tpcre2.c @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifdef USE_PRCE2 #include "tpcre2.h" int32_t doRegComp(pcre2_code** ppRegex, pcre2_match_data** ppMatchData, const char* pattern) { @@ -43,3 +59,4 @@ void destroyRegexes(pcre2_code* pWktRegex, pcre2_match_data* pWktMatchData) { pcre2_code_free(pWktRegex); pcre2_match_data_free(pWktMatchData); } +#endif // USE_PRCE2 diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index de6f8bd83b..1aab8cd43a 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -32,12 +32,12 @@ struct STaosQueue { void *ahandle; // for queue set FItem itemFp; FItems itemsFp; - TdThreadMutex mutex; - int64_t memOfItems; int32_t numOfItems; + int64_t memOfItems; int64_t threadId; int64_t memLimit; int64_t itemLimit; + TdThreadMutex mutex; }; struct STaosQset { diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 9f0392c418..8fa9f3a6a2 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -68,7 +68,7 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab pSched->emptySlot = 0; if (taosThreadMutexInit(&pSched->queueMutex, NULL) < 0) { - uError("init %s:queueMutex failed(%s)", label, strerror(errno)); + uError("init %s:queueMutex failed(%s)", label, strerror(ERRNO)); taosCleanUpScheduler(pSched); if (schedMalloced) { taosMemoryFree(pSched); @@ -77,7 +77,7 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab } if (tsem_init(&pSched->emptySem, 0, (uint32_t)pSched->queueSize) != 0) { - uError("init %s:empty semaphore failed(%s)", label, strerror(errno)); + uError("init %s:empty semaphore failed(%s)", label, strerror(ERRNO)); taosCleanUpScheduler(pSched); if (schedMalloced) { taosMemoryFree(pSched); @@ -86,7 +86,7 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab } if (tsem_init(&pSched->fullSem, 0, 0) != 0) { - uError("init %s:full semaphore failed(%s)", label, strerror(errno)); + uError("init %s:full semaphore failed(%s)", label, strerror(ERRNO)); taosCleanUpScheduler(pSched); if (schedMalloced) { taosMemoryFree(pSched); @@ -159,7 +159,7 @@ void *taosProcessSchedQueue(void *scheduler) { } if ((ret = taosThreadMutexLock(&pSched->queueMutex)) != 0) { - uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(ERRNO)); } msg = pSched->queue[pSched->fullSlot]; @@ -167,7 +167,7 @@ void *taosProcessSchedQueue(void *scheduler) { pSched->fullSlot = (pSched->fullSlot + 1) % pSched->queueSize; if ((ret = taosThreadMutexUnlock(&pSched->queueMutex)) != 0) { - uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(ERRNO)); } if ((ret = tsem_post(&pSched->emptySem)) != 0) { @@ -202,14 +202,14 @@ int taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { } if ((ret = taosThreadMutexLock(&pSched->queueMutex)) != 0) { - uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(ERRNO)); } pSched->queue[pSched->emptySlot] = *pMsg; pSched->emptySlot = (pSched->emptySlot + 1) % pSched->queueSize; if ((ret = taosThreadMutexUnlock(&pSched->queueMutex)) != 0) { - uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(ERRNO)); } if ((ret = tsem_post(&pSched->fullSem)) != 0) { diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index b0b124acc1..833aee3643 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -553,7 +553,7 @@ static int32_t taosTmrModuleInit(void) { for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; if (taosThreadMutexInit(&wheel->mutex, NULL) != 0) { - tmrError("failed to create the mutex for wheel, reason:%s", strerror(errno)); + tmrError("failed to create the mutex for wheel, reason:%s", strerror(ERRNO)); return terrno; } wheel->nextScanAt = now + wheel->resolution; diff --git a/source/util/src/tunit.c b/source/util/src/tunit.c index 357e942faa..8b8c57d26d 100644 --- a/source/util/src/tunit.c +++ b/source/util/src/tunit.c @@ -27,14 +27,14 @@ static int32_t parseCfgIntWithUnit(const char* str, int64_t* res) { double val = 0, temp = (double)INT64_MAX; char* endPtr; bool useDouble = false; - errno = 0; + SET_ERRNO(0); int64_t int64Val = taosStr2Int64(str, &endPtr, 0); - if (*endPtr == '.' || errno == ERANGE) { - errno = 0; + if (*endPtr == '.' || ERRNO == ERANGE) { + SET_ERRNO(0); val = taosStr2Double(str, &endPtr); useDouble = true; } - if (endPtr == str || errno == ERANGE || isnan(val)) { + if (endPtr == str || ERRNO == ERANGE || isnan(val)) { return terrno = TSDB_CODE_INVALID_CFG_VALUE; } while (isspace((unsigned char)*endPtr)) endPtr++; diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index cdcef4b807..4ce04b1201 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -490,9 +490,9 @@ size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rs int32_t parseCfgReal(const char *str, float *out) { float val; char *endPtr; - errno = 0; + SET_ERRNO(0); val = taosStr2Float(str, &endPtr); - if (str == endPtr || errno == ERANGE || isnan(val)) { + if (str == endPtr || ERRNO == ERANGE || isnan(val)) { return terrno = TSDB_CODE_INVALID_CFG_VALUE; } while (isspace((unsigned char)*endPtr)) endPtr++; diff --git a/source/util/test/log.cpp b/source/util/test/log.cpp index ae1be94e40..0080ca7831 100644 --- a/source/util/test/log.cpp +++ b/source/util/test/log.cpp @@ -93,6 +93,7 @@ TEST(log, misc) { tsAssert = true; // test taosLogCrashInfo, taosReadCrashInfo and taosReleaseCrashLogFile +#ifdef USE_REPORT char nodeType[16] = "nodeType"; char *pCrashMsg = (char *)taosMemoryCalloc(1, 16); EXPECT_NE(pCrashMsg, nullptr); @@ -133,7 +134,7 @@ TEST(log, misc) { pFile = taosOpenFile(crashInfo, TD_FILE_WRITE); EXPECT_NE(pFile, nullptr); taosReleaseCrashLogFile(pFile, true); - +#endif // clean up taosRemoveDir(path); diff --git a/tests/army/tools/benchmark/basic/bugs.py b/tests/army/tools/benchmark/basic/bugs.py index 2af1e1d0f3..f7b10c6cc7 100644 --- a/tests/army/tools/benchmark/basic/bugs.py +++ b/tests/army/tools/benchmark/basic/bugs.py @@ -127,7 +127,7 @@ class TDTestCase(TBase): def bugsTD(self, benchmark): self.testBenchmarkJson(benchmark, "./tools/benchmark/basic/json/TD-31490.json", checkStep = False) self.testBenchmarkJson(benchmark, "./tools/benchmark/basic/json/TD-31575.json") - self.testBenchmarkJson(benchmark, "./tools/benchmark/basic/json/TD-32846.json") + # self.testBenchmarkJson(benchmark, "./tools/benchmark/basic/json/TD-32846.json") # no drop db = "td32913db" diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index f79c4c291b..ca0002a8f6 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -216,6 +216,7 @@ # # system test # +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/ts-5617.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_multi_agg.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/scalar_function.py diff --git a/tests/script/api/stmt2-performance.c b/tests/script/api/stmt2-performance.c index 7ab20d873f..70bd0568ce 100644 --- a/tests/script/api/stmt2-performance.c +++ b/tests/script/api/stmt2-performance.c @@ -5,9 +5,9 @@ #include #include "taos.h" -int CTB_NUMS = 1; -int ROW_NUMS = 1; -int CYC_NUMS = 2; +int CTB_NUMS = 10000; +int ROW_NUMS = 10; +int CYC_NUMS = 1; void do_query(TAOS* taos, const char* sql) { TAOS_RES* result = taos_query(taos, sql); @@ -44,10 +44,10 @@ void initEnv(TAOS* taos) { do_query(taos, "use db"); } -void do_stmt(TAOS* taos, const char* sql) { +void do_stmt(TAOS* taos, const char* sql, bool hasTag) { initEnv(taos); - TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL}; + TAOS_STMT2_OPTION option = {0, false, true, NULL, NULL}; TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); int code = taos_stmt2_prepare(stmt, sql, 0); @@ -74,8 +74,11 @@ void do_stmt(TAOS* taos, const char* sql) { for (int i = 0; i < CTB_NUMS; i++) { tbs[i] = (char*)malloc(sizeof(char) * 20); sprintf(tbs[i], "ctb_%d", i); - // createCtb(taos, tbs[i]); + createCtb(taos, tbs[i]); } + + double bind_time = 0; + double exec_time = 0; for (int r = 0; r < CYC_NUMS; r++) { // col params int64_t** ts = (int64_t**)malloc(CTB_NUMS * sizeof(int64_t*)); @@ -100,9 +103,6 @@ void do_stmt(TAOS* taos, const char* sql) { int t2len = 3; // TAOS_STMT2_BIND* tagv[2] = {&tags[0][0], &tags[1][0]}; - clock_t start, end; - double cpu_time_used; - // bind params TAOS_STMT2_BIND** paramv = (TAOS_STMT2_BIND**)malloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); TAOS_STMT2_BIND** tags = (TAOS_STMT2_BIND**)malloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); @@ -118,26 +118,32 @@ void do_stmt(TAOS* taos, const char* sql) { paramv[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS}; } // bind - start = clock(); - TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, tags, paramv}; + struct timespec start, end; + clock_gettime(CLOCK_MONOTONIC, &start); // 获取开始时间 + TAOS_STMT2_BINDV bindv; + if (hasTag) { + bindv = (TAOS_STMT2_BINDV){CTB_NUMS, tbs, tags, paramv}; + } else { + bindv = (TAOS_STMT2_BINDV){CTB_NUMS, tbs, NULL, paramv}; + } if (taos_stmt2_bind_param(stmt, &bindv, -1)) { printf("failed to execute taos_stmt2_bind_param statement.error:%s\n", taos_stmt2_error(stmt)); taos_stmt2_close(stmt); return; } - end = clock(); - cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; - printf("stmt2-bind [%s] insert Time used: %f seconds\n", sql, cpu_time_used); - start = clock(); + + clock_gettime(CLOCK_MONOTONIC, &end); // 获取开始时间 TAOS_STMT2_BINDV bindv; + bind_time += ((double)(end.tv_sec - start.tv_sec) + (end.tv_nsec - start.tv_nsec) / 1e9); + + clock_gettime(CLOCK_MONOTONIC, &start); // 获取开始时间 // exec if (taos_stmt2_exec(stmt, NULL)) { printf("failed to execute taos_stmt2_exec statement.error:%s\n", taos_stmt2_error(stmt)); taos_stmt2_close(stmt); return; } - end = clock(); - cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; - printf("stmt2-exec [%s] insert Time used: %f seconds\n", sql, cpu_time_used); + clock_gettime(CLOCK_MONOTONIC, &end); // 获取开始时间 + exec_time += ((double)(end.tv_sec - start.tv_sec) + (end.tv_nsec - start.tv_nsec) / 1e9); for (int i = 0; i < CTB_NUMS; i++) { free(tags[i]); @@ -152,6 +158,10 @@ void do_stmt(TAOS* taos, const char* sql) { free(paramv); free(tags); } + + printf("stmt2-bind [%s] insert Time used: %f seconds\n", sql, bind_time); + printf("stmt2-exec [%s] insert Time used: %f seconds\n", sql, exec_time); + for (int i = 0; i < CTB_NUMS; i++) { free(tbs[i]); } @@ -217,8 +227,12 @@ int main() { exit(1); } - do_stmt(taos, "insert into `db`.`stb` (tbname,ts,b,t1,t2) values(?,?,?,?,?)"); - // do_stmt(taos, "insert into db.? using db.stb tags(?,?)values(?,?)"); + // do_stmt(taos, "insert into `db`.`stb` (tbname,ts,b,t1,t2) values(?,?,?,?,?)", true); + // printf("no interlace\n"); + do_stmt(taos, "insert into db.? using db.stb tags(?,?)values(?,?)", true); + + do_stmt(taos, "insert into db.? values(?,?)", false); + // do_taosc(taos); taos_close(taos); taos_cleanup(); diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index 0acb97f1b6..a5750e20ec 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -230,7 +230,7 @@ endi sql_error show create stable t0; sql show variables; -if $rows != 88 then +if $rows != 89 then return -1 endi diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index 8ac43ebaf3..b81cf80548 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -120,7 +120,7 @@ if $rows != 3 then endi sql show variables; -if $rows != 88 then +if $rows != 89 then return -1 endi diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 63f0a236c3..01e6123f3e 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -222,7 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(320, 321)) + tdSql.checkEqual(len(tdSql.queryResult), 321) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(61, len(tdSql.queryResult)) diff --git a/tests/system-test/8-stream/ts-5617.py b/tests/system-test/8-stream/ts-5617.py new file mode 100755 index 0000000000..2112c46c0f --- /dev/null +++ b/tests/system-test/8-stream/ts-5617.py @@ -0,0 +1,187 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +insertJson = '''{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "localhost", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 10, + "thread_count": 10, + "create_table_thread_count": 10, + "result_file": "./insert-2-2-1.txt", + "confirm_parameter_prompt": "no", + "num_of_records_per_req": 3600, + "prepared_rand": 3600, + "chinese": "no", + "escape_character": "yes", + "continue_if_fail": "no", + "databases": [ + { + "dbinfo": { + "name": "ts5617", + "drop": "yes", + "vgroups": 10, + "precision": "ms", + "buffer": 512, + "cachemodel":"'both'", + "stt_trigger": 1 + }, + "super_tables": [ + { + "name": "stb_2_2_1", + "child_table_exists": "no", + "childtable_count": 10000, + "childtable_prefix": "d_", + "auto_create_table": "yes", + "batch_create_tbl_num": 10, + "data_source": "csv", + "insert_mode": "stmt", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 10000, + "childtable_limit": 0, + "childtable_offset": 0, + "interlace_rows": 0, + "insert_interval": 0, + "partial_col_num": 0, + "timestamp_step": 1000, + "start_timestamp": "2024-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./td_double10000_juchi.csv", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + {"type": "DOUBLE", "name": "val"}, + { "type": "INT", "name": "quality"} + ], + "tags": [ + {"type": "INT", "name": "id", "max": 100, "min": 1} + ] + } + ] + } + ] +}''' + +class TDTestCase: + updatecfgDict = {'debugFlag': 135, 'asynclog': 0, 'streamFailedTimeout': 10000} + clientCfgDict = {'debugFlag': 135, 'asynclog': 0, 'streamRunHistoryAsync': 1} + updatecfgDict["clientCfg"] = clientCfgDict + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + + with open('ts-5617.json', 'w') as file: + file.write(insertJson) + + tdLog.info("start to insert data: taosBenchmark -f ts-5617.json") + if os.system("taosBenchmark -f ts-5617.json") != 0: + tdLog.exit("taosBenchmark -f ts-5617.json") + + tdLog.info("test creating stream with history in normal ......") + start_time = time.time() + tdSql.execute(f'create stream s21 fill_history 1 into ts5617.st21 tags(tname varchar(20)) subtable(tname) as select last(val), last(quality) from ts5617.stb_2_2_1 partition by tbname tname interval(1800s);') + end_time = time.time() + if end_time - start_time > 1: + tdLog.exit("create history stream sync too long") + + tdSql.query("show streams") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "init") + + while 1: + tdSql.query("show streams") + tdLog.info(f"streams is creating ...") + if tdSql.getData(0, 1) == "ready": + break + else: + time.sleep(5) + + tdSql.execute(f'drop stream s21') + tdSql.execute(f'drop table if exists ts5617.st21') + + tdLog.info("test creating stream with history in taosd error ......") + tdSql.execute(f'create stream s211 fill_history 1 into ts5617.st211 tags(tname varchar(20)) subtable(tname) as select last(val), last(quality) from ts5617.stb_2_2_1 partition by tbname tname interval(1800s);') + tdSql.execute(f'create stable ts5617.st211(ts timestamp, i int) tags(tname varchar(20))') + + tdSql.query("show streams") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "init") + + while 1: + tdSql.query("show streams") + tdLog.info(f"streams is creating ...") + tdLog.info(tdSql.queryResult) + + if tdSql.getData(0, 1) == "failed" and tdSql.getData(0, 2) == "STable already exists": + break + else: + time.sleep(5) + time.sleep(10) + tdSql.execute(f'drop stream s211') + tdSql.execute(f'drop table if exists ts5617.st211') + + tdLog.info("test creating stream with history in taosd error ......") + tdSql.execute(f'create stream s21 fill_history 1 into ts5617.st21 as select last(val), last(quality) from ts5617.d_0 interval(1800s);') + tdSql.execute(f'create stream s211 fill_history 1 into ts5617.st211 as select last(val), last(quality) from ts5617.d_0 interval(1800s);') + + while 1: + tdSql.query("show streams") + tdLog.info(tdSql.queryResult) + + tdLog.info(f"streams is creating ...") + if "failed" in [tdSql.getData(0, 1), tdSql.getData(1, 1)] and "Conflict transaction not completed" in [tdSql.getData(0, 2), tdSql.getData(1, 2)]: + break + else: + time.sleep(5) + + tdSql.execute(f'drop stream s21') + tdSql.execute(f'drop stream s211') + tdSql.execute(f'drop table if exists ts5617.st21') + tdSql.execute(f'drop table if exists ts5617.st211') + + tdLog.info("test creating stream with history in taosd restart ......") + tdSql.execute(f'create stream s21 fill_history 1 into ts5617.st21 tags(tname varchar(20)) subtable(tname) as select last(val), last(quality) from ts5617.stb_2_2_1 partition by tbname tname interval(1800s);') + tdSql.query("show streams") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "init") + + tdLog.debug("restart taosd") + tdDnodes.forcestop(1) + time.sleep(20) + tdDnodes.start(1) + + while 1: + tdSql.query("show streams") + tdLog.info(f"streams is creating ...") + tdLog.info(tdSql.queryResult) + if tdSql.getData(0, 1) == "failed" and tdSql.getData(0, 2) == "timeout": + break + else: + time.sleep(5) + + return + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index 9e14cd32f0..6dbc5db94f 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -129,7 +129,7 @@ int32_t shellParseArgs(int32_t argc, char* argv[]); int32_t shellReadCommand(char* command); // shellEngine.c -int32_t shellExecute(); +int32_t shellExecute(int argc, char *argv[]); int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision); void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields); void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision); diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 93381b4f51..0cebd3a047 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -96,7 +96,7 @@ void shellPrintHelp() { #endif } -#ifdef LINUX +#if defined(LINUX) && !defined(TD_ASTRA) #include #ifdef _ALPINE #include @@ -270,7 +270,7 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { } return 0; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) || defined(_TD_DARWIN_64) +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) || defined(_TD_DARWIN_64) || defined(TD_ASTRA) int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { SShellArgs *pArgs = &shell.args; @@ -477,6 +477,11 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { shell.info.osname = "Darwin"; snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getpwuid(getuid())->pw_dir, SHELL_HISTORY_FILE); if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1; +#elif defined(TD_ASTRA) + shell.info.osname = "Astra"; + snprintf(shell.history.file, TSDB_FILENAME_LEN, "C:%sTDengine%s%s", TD_DIRSEP, TD_DIRSEP, + SHELL_HISTORY_FILE); // TD_ASTRA_TODO getenv("HOME") + if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1; #else shell.info.osname = "Linux"; snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), SHELL_HISTORY_FILE); diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index a160d2780c..bf2f9fd0db 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -345,6 +345,9 @@ void shellGetScreenSize(int32_t *ws_col, int32_t *ws_row) { GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi); if (ws_col != NULL) *ws_col = csbi.srWindow.Right - csbi.srWindow.Left + 1; if (ws_row != NULL) *ws_row = csbi.srWindow.Bottom - csbi.srWindow.Top + 1; +#elif defined(TD_ASTRA) + if (ws_col != NULL) *ws_col = 120; + if (ws_row != NULL) *ws_row = 30; #else struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { @@ -556,7 +559,11 @@ int32_t shellReadCommand(char *command) { case 12: // Ctrl + L; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" +#ifndef TD_ASTRA system("clear"); +#else + printf("\033[2J\033[H"); +#endif #pragma GCC diagnostic pop shellShowOnScreen(&cmd); break; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 99b1c204a2..93a478dda8 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -89,7 +89,11 @@ int32_t shellRunSingleCommand(char *command) { if (shellRegexMatch(command, "^[\t ]*clear[ \t;]*$", REG_EXTENDED | REG_ICASE)) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" +#ifndef TD_ASTRA system("clear"); +#else + printf("\033[2J\033[H"); +#endif #pragma GCC diagnostic pop return 0; } @@ -662,27 +666,27 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t printf("%*u", width, *((uint32_t *)val)); break; case TSDB_DATA_TYPE_BIGINT: - printf("%*" PRId64, width, *((int64_t *)val)); + printf("%*" PRId64, width, taosGetInt64Aligned((int64_t *)val)); break; case TSDB_DATA_TYPE_UBIGINT: - printf("%*" PRIu64, width, *((uint64_t *)val)); + printf("%*" PRIu64, width, taosGetUInt64Aligned((uint64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: width = width >= LENGTH ? LENGTH - 1 : width; if (tsEnableScience) { - printf("%*.7e", width, GET_FLOAT_VAL(val)); + printf("%*.7e", width, taosGetFloatAligned((float *)val)); } else { - snprintf(buf, LENGTH, "%*.*g", width, FLT_DIG, GET_FLOAT_VAL(val)); + snprintf(buf, LENGTH, "%*.*g", width, FLT_DIG, taosGetFloatAligned((float *)val)); printf("%s", buf); } break; case TSDB_DATA_TYPE_DOUBLE: width = width >= LENGTH ? LENGTH - 1 : width; if (tsEnableScience) { - snprintf(buf, LENGTH, "%*.15e", width, GET_DOUBLE_VAL(val)); + snprintf(buf, LENGTH, "%*.15e", width, taosGetDoubleAligned((double *)val)); printf("%s", buf); } else { - snprintf(buf, LENGTH, "%*.*g", width, DBL_DIG, GET_DOUBLE_VAL(val)); + snprintf(buf, LENGTH, "%*.*g", width, DBL_DIG, taosGetDoubleAligned((double *)val)); printf("%*s", width, buf); } break; @@ -705,7 +709,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t shellPrintGeometry(val, length, width); break; case TSDB_DATA_TYPE_TIMESTAMP: - shellFormatTimestamp(buf, sizeof(buf), *(int64_t *)val, precision); + shellFormatTimestamp(buf, sizeof(buf), taosGetInt64Aligned((int64_t *)val), precision); printf("%s", buf); break; default: @@ -993,7 +997,7 @@ void shellDumpResultCallback(void *param, TAOS_RES *tres, int num_of_rows) { } } else { if (num_of_rows < 0) { - printf("\033[31masync retrieve failed, code: %d\033[0m\n", num_of_rows); + printf("\033[31masync retrieve failed, code: %d\033, %s[0m\n", num_of_rows, tstrerror(num_of_rows)); } tsem_post(&dump_info->sem); } @@ -1162,6 +1166,7 @@ int32_t shellGetGrantInfo(char *buf) { tstrncpy(sinfo, taos_get_server_info(shell.conn), sizeof(sinfo)); strtok(sinfo, "\r\n"); +#ifndef TD_ASTRA char sql[] = "show grants"; TAOS_RES *tres = taos_query(shell.conn, sql); @@ -1214,6 +1219,10 @@ int32_t shellGetGrantInfo(char *buf) { } fprintf(stdout, "\r\n"); +#else + verType = TSDB_VERSION_ENTERPRISE; + sprintf(buf, "Server is %s, %s and will never expire.\r\n", TD_PRODUCT_NAME, sinfo); +#endif return verType; } @@ -1296,7 +1305,8 @@ void *shellThreadLoop(void *arg) { } #pragma GCC diagnostic pop -int32_t shellExecute() { +int32_t shellExecute(int argc, char *argv[]) { + int32_t code = 0; printf(shell.info.clientVersion, shell.info.cusName, taos_get_client_info(), shell.info.cusName); fflush(stdout); @@ -1363,9 +1373,9 @@ int32_t shellExecute() { return 0; } - if (tsem_init(&shell.cancelSem, 0, 0) != 0) { - printf("failed to create cancel semaphore\r\n"); - return -1; + if ((code = tsem_init(&shell.cancelSem, 0, 0)) != 0) { + printf("failed to create cancel semaphore since %s\r\n", tstrerror(code)); + return code; } TdThread spid = {0}; @@ -1419,5 +1429,5 @@ int32_t shellExecute() { taos_kill_query(shell.conn); taos_close(shell.conn); - return 0; + TAOS_RETURN(code); } diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 1f6f8fe3df..19277de1dd 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -31,9 +31,9 @@ void shellCrashHandler(int signum, void *sigInfo, void *context) { taosIgnSignal(SIGABRT); taosIgnSignal(SIGFPE); taosIgnSignal(SIGSEGV); - +#ifdef USE_REPORT tscWriteCrashInfo(signum, sigInfo, context); - +#endif #ifdef _TD_DARWIN_64 exit(signum); #elif defined(WINDOWS) @@ -88,6 +88,7 @@ int main(int argc, char *argv[]) { // kill heart-beat thread when quit taos_set_hb_quit(1); +#ifndef TD_ASTRA if (shell.args.is_dump_config) { shellDumpConfig(); taos_cleanup(); @@ -105,10 +106,10 @@ int main(int argc, char *argv[]) { taos_cleanup(); return 0; } - +#endif // support port feature shellAutoInit(); - int32_t ret = shellExecute(); + int32_t ret = shellExecute(argc, argv); shellAutoExit(); return ret; } diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index 1ed9af00f2..c5b6f748fb 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -185,7 +185,7 @@ void initLogFile() { char filename[256]; char tmpString[128]; - pid_t process_id = getpid(); + pid_t process_id = taosGetPId(); if (0 != strlen(g_stConfInfo.topic)) { sprintf(filename, "/tmp/tmqlog-%d-%s.txt", process_id, getCurrentTimeString(tmpString));