diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index b45064ce08..47a56d3175 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -49,6 +49,7 @@ table_option: { 7. Escape character "\`" can be used to avoid the conflict between table names and reserved keywords, above rules will be bypassed when using escape character on table names, but the upper limit for the name length is still valid. The table names specified using escape character are case sensitive. For example \`aBc\` and \`abc\` are different table names but `abc` and `aBc` are same table names because they are both converted to `abc` internally. Only ASCII visible characters can be used with escape character. +8. For the details of using `ENCODE` and `COMPRESS`, please refer to [Encode and Compress for Column](../compress). **Parameter description** diff --git a/docs/en/12-taos-sql/04-stable.md b/docs/en/12-taos-sql/04-stable.md index ad179a5fe3..e7101eaba6 100644 --- a/docs/en/12-taos-sql/04-stable.md +++ b/docs/en/12-taos-sql/04-stable.md @@ -13,17 +13,29 @@ create_definition: col_name column_definition column_definition: - type_name + type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type'] + +table_options: + table_option ... + +table_option: { + COMMENT 'string_value' + | SMA(col_name [, col_name] ...) + | TTL value +} + ``` **More explanations** -- Each supertable can have a maximum of 4096 columns, including tags. The minimum number of columns is 3: a timestamp column used as the key, one tag column, and one data column. -- The TAGS keyword defines the tag columns for the supertable. The following restrictions apply to tag columns: +1. Each supertable can have a maximum of 4096 columns, including tags. The minimum number of columns is 3: a timestamp column used as the key, one tag column, and one data column. +2. Since version 3.3.0.0, besides the timestamp, you can specify another column as primary key using `PRIMARY KEY` keyword, the column specified using `primary key` must be type of integer or varchar. +2. The TAGS keyword defines the tag columns for the supertable. The following restrictions apply to tag columns: - A tag column can use the TIMESTAMP data type, but the values in the column must be fixed numbers. Timestamps including formulae, such as "now + 10s", cannot be stored in a tag column. - The name of a tag column cannot be the same as the name of any other column. - The name of a tag column cannot be a reserved keyword. - Each supertable must contain between 1 and 128 tags. The total length of the TAGS keyword cannot exceed 16 KB. -- For more information about table parameters, see Create a Table. +3. Regarding how to use `ENCODE` and `COMPRESS`, please refer to [Encode and Compress for Column](../compress). +3. For more information about table parameters, see [Create a Table](../table). ## View a Supertable diff --git a/docs/en/21-tdinternal/08-compress.md b/docs/en/12-taos-sql/31-compress.md similarity index 100% rename from docs/en/21-tdinternal/08-compress.md rename to docs/en/12-taos-sql/31-compress.md diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index f50551b5de..5020296422 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -729,6 +729,57 @@ The charset that takes effect is UTF-8. | Value Range | -1: none message is compressed; 0: all messages are compressed; N (N>0): messages exceeding N bytes are compressed | | Default | -1 | +### fPrecision + +| Attribute | Description | +| -------- | -------------------------------- | +| Application | Server Only | +| Meaning | Compression precision for float data type | +| Value Range | 0.1 ~ 0.00000001 | +| Default | 0.00000001 | +| Note | The floating value below this setting will be cut off | + +### dPrecision + +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Server Only | +| Meaning | Compression precision for double data type | +| Value Range | 0.1 ~ 0.0000000000000001 | +| Default | 0.0000000000000001 | +| Note | The floating value below this setting will be cut off | + +### lossyColumn + +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Server Only | +| Meaning | Enable TSZ lossy compression for float and/or double | +| Value Range | float, double | +| Default | none: disable TSZ lossy compression | + +**补充说明** +1. It's only available since 3.2.0.0 version, and can't downgrade to previous version once upgrading to 3.2.0.0 and enabling this parameter +2. TSZ compression algorithm compresses data based on data prediction technique, so it's more suitable for data with specific pattern +3. TSZ compression algorithm may take longer time but it has better compression ratio, so it's suitable when you have enough CPU resources and are more sensitive to disk occupation +4. Example: enable TSZ for both float and double +```shell +lossyColumns float|double +``` +5. After configuring, taosd service needs to restarted. After restarting, if you see the following output in taosd logfile, it means the function has been enabled +```sql + 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double +``` + +### ifAdtFse + +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Server Only | +| Meaning | Replace HUFFMAN with FSE in TSZ, FSE is faster when compressing but slower when uncompressing | +| Value Range | 0: Use HUFFMAN, 1: Use FSE | +| Default | 0: Use HUFFMAN | + ## Other Parameters diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index f5ff1b83ef..f665aa2277 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -49,6 +49,7 @@ table_option: { 6. 使用数据类型 BINARY/NCHAR/GEOMETRY,需指定其最长的字节数,如 BINARY(20),表示 20 字节。 7. 为了兼容支持更多形式的表名,TDengine 引入新的转义符 "\`",可以让表名与关键词不冲突,同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后,不再对转义字符中的内容进行大小写统一, 例如:\`aBc\` 和 \`abc\` 是不同的表名,但是 abc 和 aBc 是相同的表名。 +8. 关于 `ENCODE` 和 `COMPRESS` 的使用,请参考[按列压缩](../compress) **参数说明** diff --git a/docs/zh/12-taos-sql/04-stable.md b/docs/zh/12-taos-sql/04-stable.md index 9c6de9e220..5ad2f4a19d 100644 --- a/docs/zh/12-taos-sql/04-stable.md +++ b/docs/zh/12-taos-sql/04-stable.md @@ -13,17 +13,28 @@ create_definition: col_name column_definition column_definition: - type_name + type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type'] + +table_options: + table_option ... + +table_option: { + COMMENT 'string_value' + | SMA(col_name [, col_name] ...) + | TTL value +} ``` **使用说明** -- 超级表中列的最大个数为 4096,需要注意,这里的 4096 是包含 TAG 列在内的,最小个数为 3,包含一个时间戳主键、一个 TAG 列和一个数据列。 -- TAGS语法指定超级表的标签列,标签列需要遵循以下约定: +1. 超级表中列的最大个数为 4096,需要注意,这里的 4096 是包含 TAG 列在内的,最小个数为 3,包含一个时间戳主键、一个 TAG 列和一个数据列。 +2. 除时间戳主键列之外,还可以通过 PRIMARY KEY 关键字指定第二列为额外的主键列。被指定为主键列的第二列必须为整型或字符串类型(varchar) +3. TAGS语法指定超级表的标签列,标签列需要遵循以下约定: - TAGS 中的 TIMESTAMP 列写入数据时需要提供给定值,而暂不支持四则运算,例如 NOW + 10s 这类表达式。 - TAGS 列名不能与其他列名相同。 - TAGS 列名不能为预留关键字。 - TAGS 最多允许 128 个,至少 1 个,总长度不超过 16 KB。 -- 关于表参数的详细说明,参见 CREATE TABLE 中的介绍。 +4. 关于 `ENCODE` 和 `COMPRESS` 的使用,请参考 [按列压缩](../compress) +5. 关于 table_option 中的参数说明,请参考 [建表 SQL 说明](../table) ## 查看超级表 diff --git a/docs/zh/21-tdinternal/08-compress.md b/docs/zh/12-taos-sql/31-compress.md similarity index 98% rename from docs/zh/21-tdinternal/08-compress.md rename to docs/zh/12-taos-sql/31-compress.md index dfec63c212..0f2b260832 100644 --- a/docs/zh/21-tdinternal/08-compress.md +++ b/docs/zh/12-taos-sql/31-compress.md @@ -1,5 +1,6 @@ --- title: 可配置压缩算法 +sidebar_label: 可配置压缩 description: 可配置压缩算法 --- diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 814b543859..c3d23d624d 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -784,6 +784,57 @@ charset 的有效值是 UTF-8。 | 取值范围 | -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩 | | 缺省值 | -1 | +### fPrecision +FLOAT 类型压缩精度控制: + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 设置 float 类型浮点数压缩精度 | +| 取值范围 | 0.1 ~ 0.00000001 | +| 缺省值 | 0.00000001 | +| 补充说明 | 小于此值的浮点数尾数部分将被截取 | + +### dPrecision +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 设置 double 类型浮点数压缩精度 | +| 取值范围 | 0.1 ~ 0.0000000000000001 | +| 缺省值 | 0.0000000000000001 | +| 补充说明 | 小于此值的浮点数尾数部分将被截取 | + +### lossyColumn + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 对 float 和/或 double 类型启用 TSZ 有损压缩 | +| 取值范围 | float, double | +| 缺省值 | none:表示关闭无损压缩 | + +**补充说明** +1. 在 3.2.0.0 及以后版本生效,启用该参数后不能回退到升级前的版本 +2. TSZ 压缩算法是通过数据预测技术完成的压缩,所以更适合有规律变化的数据 +3. TSZ 压缩时间会更长一些,如果您的服务器 CPU 空闲多,存储空间小的情况下适合选用 +4. 示例:对 float 和 double 类型都启用有损压缩 +```shell +lossyColumns float|double +``` +5. 配置需重启服务生效,重启如果在 taosd 日志中看到以下内容,表明配置已生效: +```sql + 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double +``` + +### ifAdtFse + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 在启用 TSZ 有损压缩时,使用 FSE 算法替换 HUFFMAN 算法, FSE 算法压缩速度更快,但解压稍慢,追求压缩速度可选用此算法 | +| 取值范围 | 0:关闭 1:打开 | +| 缺省值 | 0:关闭 | + ## 3.0 中有效的配置参数列表 | # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | @@ -922,8 +973,5 @@ charset 的有效值是 UTF-8。 | 73 | probeSeconds | 是 | 否 | 3.0 行为未知 | | 74 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | | 75 | probeInterval | 是 | 否 | 3.0 行为未知 | -| 76 | lossyColumns | 是 | 否 | 3.0 行为未知 | -| 77 | fPrecision | 是 | 否 | 3.0 行为未知 | -| 78 | dPrecision | 是 | 否 | 3.0 行为未知 | | 79 | maxRange | 是 | 否 | 3.0 行为未知 | | 80 | range | 是 | 否 | 3.0 行为未知 | diff --git a/docs/zh/21-tdinternal/07-tsz.md b/docs/zh/21-tdinternal/07-tsz.md deleted file mode 100644 index db1a340ab8..0000000000 --- a/docs/zh/21-tdinternal/07-tsz.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: TSZ 压缩算法 -description: TDengine 对浮点数进行高效压缩的算法 ---- - -TSZ 压缩算法是 TDengine 为浮点数据类型提供的可选压缩算法,可以实现浮点数有损至无损全状态压缩,相比默认压缩算法, TSZ 压缩算法压缩率更高,即使切至无损状态,压缩率也会比默认压缩高一倍。 - -## 适合场景 - -- TSZ 压缩算法是通过数据预测技术完成的压缩,所以更适合有规律变化的数据 -- TSZ 压缩时间会更长一些,如果您的服务器 CPU 空闲多,存储空间小的情况下适合选用 - -## 使用步骤 -- TDengine 支持版本为 3.2.0.0 或以上 -- 开启选项 - 在 taos.cfg 配置中增加以下内容,即可开启 TSZ 压缩算法,功能打开后,会替换默认算法。 - 以下表示字段类型是 float 及 double 类型都使用此压缩算法,也可以单独只配置一个 - -```sql - lossyColumns float|double -``` - -- 配置需重启服务生效 -- Taosd 日志输出以下内容,表明功能已生效: - -```sql - 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double -``` - -## 配置参数 - -### fPrecision -FLOAT 类型精度控制: - -| 属性 | 说明 | -| -------- | -------------------------------- | -| 适用范围 | 服务器端 | -| 含义 | 设置 float 类型浮点数压缩精度 | -| 取值范围 | 0.1 ~ 0.00000001 | -| 缺省值 | 0.00000001 | -| 补充说明 | 小于此值的浮点数尾数部分将被截取 | - - - -### dPrecision -DOUBLE 类型精度控制: - -| 属性 | 说明 | -| -------- | -------------------------------- | -| 适用范围 | 服务器端 | -| 含义 | 设置 double 类型浮点数压缩精度 | -| 取值范围 | 0.1 ~ 0.0000000000000001 | -| 缺省值 | 0.0000000000000001 | -| 补充说明 | 小于此值的浮点数尾数部分将被截取 | - - -### ifAdtFse -TSZ 压缩中可选择的算法 FSE,默认为 HUFFMAN: - -| 属性 | 说明 | -| -------- | -------------------------------- | -| 适用范围 | 服务器端 | -| 含义 | 使用 FSE 算法替换 HUFFMAN 算法, FSE 算法压缩速度更快,但解压稍慢,追求压缩速度可选用此算法 | -| 取值范围 | 0:关闭 1:打开 | -| 缺省值 | 0:关闭 | - - -## 注意事项 -- 打开 TSZ 后生成的存储数据格式,回退至 3.2.0.0 之前的版本,数据将不能被识别 diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 7aec39817a..04e13fbdb3 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -136,7 +136,7 @@ int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag); int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag); void tRowGetPrimaryKey(SRow *pRow, SRowKey *key); int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2); -int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc); +void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc); // SRowIter ================================ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 03bf8da707..0c61aa5a51 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -17,13 +17,13 @@ #define _TD_COMMON_MSG_CB_H_ #include "os.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { #endif typedef struct SRpcMsg SRpcMsg; -typedef struct SEpSet SEpSet; typedef struct SMgmtWrapper SMgmtWrapper; typedef struct SRpcHandleInfo SRpcHandleInfo; @@ -46,7 +46,7 @@ typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); typedef void (*SendRspFp)(SRpcMsg* pMsg); -typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg); +typedef void (*RegisterBrokenLinkArgFp)(struct SRpcMsg* pMsg); typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type); typedef void (*ReportStartup)(const char* name, const char* desc); diff --git a/include/common/tname.h b/include/common/tname.h index 6a89d2a6be..f0a68f28c4 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -81,7 +81,7 @@ typedef struct { // uint64_t uid; // child table uid, may be useful } RandTableName; -void buildChildTableName(RandTableName* rName); +int32_t buildChildTableName(RandTableName* rName); #ifdef __cplusplus } diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 45f9f73fb1..0a33798f0c 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -105,7 +105,7 @@ typedef struct SMTbCursor { } SMTbCursor; typedef struct SMCtbCursor { - SMeta* pMeta; + struct SMeta* pMeta; void* pCur; tb_uid_t suid; void* pKey; @@ -134,7 +134,7 @@ typedef struct SMetaTableInfo { } SMetaTableInfo; typedef struct SSnapContext { - SMeta* pMeta; + struct SMeta* pMeta; int64_t snapVersion; void* pCur; int64_t suid; @@ -178,7 +178,7 @@ typedef struct TsdReader { int32_t (*tsdNextDataBlock)(); int32_t (*tsdReaderRetrieveBlockSMAInfo)(); - SSDataBlock *(*tsdReaderRetrieveDataBlock)(); + int32_t (*tsdReaderRetrieveDataBlock)(); void (*tsdReaderReleaseDataBlock)(); @@ -381,7 +381,7 @@ typedef struct SStateStore { void** ppVal, int32_t* pVLen); int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); - SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); + int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); @@ -389,12 +389,12 @@ typedef struct SStateStore { void (*updateInfoDestroy)(SUpdateInfo* pInfo); void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count); - void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); + int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); - SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); + int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); - int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); + int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen); int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo); SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 3ac357055e..4ad24fe91d 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -149,10 +149,10 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void qDestroyBoundColInfo(void* pInfo); -SQuery* smlInitHandle(); +int32_t smlInitHandle(SQuery** query); int32_t smlBuildRow(STableDataCxt* pTableCxt); int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index); -STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); +int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt); void clearColValArraySml(SArray* pCols); int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 5ba0ce454c..fd9b5667d7 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -237,6 +237,8 @@ typedef struct { int64_t stbUid; char stbFullName[TSDB_TABLE_FNAME_LEN]; SSchemaWrapper* pSchemaWrapper; + SSchemaWrapper* pTagSchema; + bool autoCreateCtb; void* vnode; // not available to encoder and decoder FTbSink* tbSinkFunc; STSchema* pTSchema; diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 632a82743f..e1b33de429 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -15,29 +15,29 @@ #ifndef _TSTREAMUPDATE_H_ #define _TSTREAMUPDATE_H_ +#include "storageapi.h" #include "taosdef.h" #include "tarray.h" #include "tcommon.h" #include "tmsg.h" -#include "storageapi.h" #ifdef __cplusplus extern "C" { #endif -SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); -SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); -TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); -bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); -bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid); -void updateInfoDestroy(SUpdateInfo *pInfo); -void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); -void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); -int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); -int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); -void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count); -void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); -bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); +int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); +int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); +TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); +bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); +bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid); +void updateInfoDestroy(SUpdateInfo* pInfo); +void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo); +void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo); +int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen); +int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo); +void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count); +int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count); +bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); #ifdef __cplusplus } diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 0be03b82f3..1fb077e3ca 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -70,7 +70,6 @@ typedef int64_t SyncIndex; typedef int64_t SyncTerm; typedef struct SSyncNode SSyncNode; -typedef struct SWal SWal; typedef struct SSyncRaftEntry SSyncRaftEntry; typedef enum { @@ -238,7 +237,7 @@ typedef struct SSyncInfo { int32_t batchSize; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; - SWal* pWal; + struct SWal* pWal; SSyncFSM* pFsm; SMsgCb* msgcb; int32_t pingMs; diff --git a/include/os/osMath.h b/include/os/osMath.h index 3c05d15397..edbfa935ce 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -32,9 +32,9 @@ extern "C" { #define TSWAP(a, b) \ do { \ char *__tmp = (char*)alloca(sizeof(a)); \ - memcpy(__tmp, &(a), sizeof(a)); \ - memcpy(&(a), &(b), sizeof(a)); \ - memcpy(&(b), __tmp, sizeof(a)); \ + (void)memcpy(__tmp, &(a), sizeof(a)); \ + (void)memcpy(&(a), &(b), sizeof(a)); \ + (void)memcpy(&(b), __tmp, sizeof(a)); \ } while (0) #ifdef WINDOWS diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 6166f1dc07..c6a5ce27c4 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -54,6 +54,7 @@ void taosMemoryTrim(int32_t size); void *taosMemoryMallocAlign(uint32_t alignment, int64_t size); #define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n)) +#define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n)) #define taosMemoryFreeClear(ptr) \ do { \ diff --git a/include/os/osString.h b/include/os/osString.h index 5fa5a25c6d..c834eb8e65 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -55,7 +55,7 @@ typedef enum { M2C = 0, C2M } ConvType; #define tstrncpy(dst, src, size) \ do { \ - strncpy((dst), (src), (size)); \ + (void)strncpy((dst), (src), (size)); \ (dst)[(size)-1] = 0; \ } while (0) diff --git a/include/util/tbloomfilter.h b/include/util/tbloomfilter.h index 23bf5aaafb..f0450cb862 100644 --- a/include/util/tbloomfilter.h +++ b/include/util/tbloomfilter.h @@ -19,6 +19,8 @@ #include "os.h" #include "tencode.h" #include "thash.h" +#include "tlog.h" +#include "tutil.h" #ifdef __cplusplus extern "C" { @@ -35,19 +37,19 @@ typedef struct SBloomFilter { uint64_t size; _hash_fn_t hashFn1; _hash_fn_t hashFn2; - void *buffer; + void* buffer; double errorRate; } SBloomFilter; -SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate); -int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2); -int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len); -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2); -void tBloomFilterDestroy(SBloomFilter *pBF); -void tBloomFilterDump(const SBloomFilter *pBF); -bool tBloomFilterIsFull(const SBloomFilter *pBF); -int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder); -SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder); +int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF); +int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2); +int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len); +int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t h1, uint64_t h2); +void tBloomFilterDestroy(SBloomFilter* pBF); +void tBloomFilterDump(const SBloomFilter* pBF); +bool tBloomFilterIsFull(const SBloomFilter* pBF); +int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder); +int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF); #ifdef __cplusplus } diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index d3ce2eb23b..f82493aca2 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -23,22 +23,22 @@ extern "C" { #endif typedef struct SScalableBf { - SArray *bfArray; // array of bloom filters - uint32_t growth; - uint64_t numBits; - uint32_t maxBloomFilters; - int8_t status; + SArray* bfArray; // array of bloom filters + uint32_t growth; + uint64_t numBits; + uint32_t maxBloomFilters; + int8_t status; _hash_fn_t hashFn1; _hash_fn_t hashFn2; } SScalableBf; -SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate); -int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); -int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); -int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); -void tScalableBfDestroy(SScalableBf *pSBf); -int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder); -SScalableBf *tScalableBfDecode(SDecoder *pDecoder); +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf); +int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len); +int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len); +int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len); +void tScalableBfDestroy(SScalableBf* pSBf); +int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder); +int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf); #ifdef __cplusplus } diff --git a/include/util/tutil.h b/include/util/tutil.h index d1fc11d0fe..31ce34f667 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -21,6 +21,7 @@ #include "tdef.h" #include "thash.h" #include "tmd5.h" +#include "tutil.h" #ifdef __cplusplus extern "C" { diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 122914fd34..e901f86b41 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -82,7 +82,7 @@ extern "C" { #define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH)) -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) +#define MOVE_FORWARD_ONE(sql, len) ((void)memmove((void *)((sql)-1), (sql), len)) #define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ for (int i = 1; i < keyLen; ++i) { \ @@ -211,19 +211,19 @@ extern int64_t smlFactorS[]; typedef int32_t (*_equal_fn_sml)(const void *, const void *); -SSmlHandle *smlBuildSmlInfo(TAOS *taos); +int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle); void smlDestroyInfo(SSmlHandle *info); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); -int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); -bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); +void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); +int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); -SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); -SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); +int32_t smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen, SSmlTableInfo** tInfo); +int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta** sMeta); int32_t smlSetCTableName(SSmlTableInfo *oneTable); -void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo); -STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen); +int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo); +int32_t smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen, STableMeta **pTableMeta); int32_t is_same_child_table_telnet(const void *a, const void *b); int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); int32_t smlClearForRerun(SSmlHandle *info); @@ -236,7 +236,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); -SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement); +int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta); bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv); int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index ce404092b0..fe12850e14 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1572,9 +1572,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat } // uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid); - pQuery = smlInitHandle(); - if (pQuery == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1656,9 +1655,9 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha if (code != TSDB_CODE_SUCCESS) { goto end; } - pQuery = smlInitHandle(); - if (pQuery == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1755,12 +1754,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - pQuery = smlInitHandle(); - if (pQuery == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { SET_ERROR_MSG("init sml handle failed"); goto end; } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); @@ -1903,12 +1902,12 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - pQuery = smlInitHandle(); - if (pQuery == NULL) { + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { SET_ERROR_MSG("init sml handle failed"); - code = TSDB_CODE_OUT_OF_MEMORY; goto end; } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 7e63dcd36a..00445b4d12 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -110,11 +110,13 @@ int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL}; static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char *pTabName, AUTH_TYPE type) { SUserAuthInfo pAuth = {0}; - snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user); + (void)snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user); if (NULL == pTabName) { - tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb)); + if (tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb)) != 0){ + return TSDB_CODE_SML_INVALID_DATA; + } } else { - toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); + (void)toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); //ignore } pAuth.type = type; @@ -129,17 +131,19 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char : code; } -int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { - if (pBuf->buf) { - memset(pBuf->buf, 0, pBuf->len); - if (msg1) strncat(pBuf->buf, msg1, pBuf->len); - int32_t left = pBuf->len - strlen(pBuf->buf); - if (left > 2 && msg2) { - strncat(pBuf->buf, ":", left - 1); - strncat(pBuf->buf, msg2, left - 2); - } +void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { + if (pBuf->buf == NULL) { + return; + } + (void)memset(pBuf->buf, 0, pBuf->len); + if (msg1) { + (void)strncat(pBuf->buf, msg1, pBuf->len); + } + int32_t left = pBuf->len - strlen(pBuf->buf); + if (left > 2 && msg2) { + (void)strncat(pBuf->buf, ":", left - 1); + (void)strncat(pBuf->buf, msg2, left - 2); } - return TSDB_CODE_SML_INVALID_DATA; } int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) { @@ -161,10 +165,10 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) { +int32_t smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen, SSmlTableInfo** tInfo) { SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); if (!tag) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } tag->sTableName = measure; @@ -173,14 +177,12 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measu tag->cols = taosArrayInit(numRows, POINTER_BYTES); if (tag->cols == NULL) { uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; + taosMemoryFree(tag); + return TSDB_CODE_OUT_OF_MEMORY; } - return tag; - -cleanup: - taosMemoryFree(tag); - return NULL; + *tInfo = tag; + return TSDB_CODE_SUCCESS; } void smlBuildTsKv(SSmlKv *kv, int64_t ts) { @@ -191,34 +193,50 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts) { kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; } -SSmlSTableMeta *smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement) { - SSmlSTableMeta *sMeta = NULL; +static void smlDestroySTableMeta(void *para) { + SSmlSTableMeta *meta = *(SSmlSTableMeta **)para; + taosHashCleanup(meta->tagHash); + taosHashCleanup(meta->colHash); + (void)taosArrayDestroy(meta->tags); + (void)taosArrayDestroy(meta->cols); + taosMemoryFreeClear(meta->tableMeta); + taosMemoryFree(meta); +} + +int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta) { + int32_t code = TSDB_CODE_SUCCESS; char *measure = currElement->measure; int measureLen = currElement->measureLen; if (currElement->measureEscaped) { measure = (char *)taosMemoryMalloc(measureLen); - memcpy(measure, currElement->measure, measureLen); + if (measure == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(measure, currElement->measure, measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); smlStrReplace(measure, measureLen); } - STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + STableMeta *pTableMeta = NULL; + code = smlGetMeta(info, measure, measureLen, &pTableMeta); if (currElement->measureEscaped) { taosMemoryFree(measure); } - if (pTableMeta == NULL) { + if (code != TSDB_CODE_SUCCESS) { info->dataFormat = false; info->reRun = true; - terrno = TSDB_CODE_SUCCESS; - return sMeta; + return code; } - sMeta = smlBuildSTableMeta(info->dataFormat); - if (sMeta == NULL) { + code = smlBuildSTableMeta(info->dataFormat, sMeta); + if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pTableMeta); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return sMeta; + return code; + } + (*sMeta)->tableMeta = pTableMeta; + code = taosHashPut(info->superTables, currElement->measure, currElement->measureLen, sMeta, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS){ + smlDestroySTableMeta(*sMeta); + return code; } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { SSchema *col = pTableMeta->schema + i; SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type}; @@ -232,40 +250,38 @@ SSmlSTableMeta *smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currEleme } if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(sMeta->cols, &kv); + if(taosArrayPush((*sMeta)->cols, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } else { - taosArrayPush(sMeta->tags, &kv); + if(taosArrayPush((*sMeta)->tags, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } } - return sMeta; + return TSDB_CODE_SUCCESS; } bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { // cnt begin 0, add ts so + 2 if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } // bind data int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1); if (unlikely(ret != TSDB_CODE_SUCCESS)) { uDebug("smlBuildCol error, retry"); - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } if (cnt >= taosArrayGetSize(info->maxColKVs)) { - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt); - + if (maxKV == NULL){ + goto END; + } if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } if (unlikely(IS_VAR_DATA_TYPE(kv->type) && kv->length > maxKV->length)) { @@ -273,6 +289,11 @@ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { info->needModifySchema = true; } return true; + +END: + info->dataFormat = false; + info->reRun = true; + return false; } bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { @@ -285,6 +306,9 @@ bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); + if (maxKV == NULL) { + goto END; + } if (unlikely(!IS_SAME_KEY)) { goto END; } @@ -306,8 +330,8 @@ int32_t smlJoinMeasureTag(SSmlLineInfo *elements) { if (elements->measureTag == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(elements->measureTag, elements->measure, elements->measureLen); - memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); + (void)memcpy(elements->measureTag, elements->measure, elements->measureLen); + (void)memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); elements->measureTagsLen = elements->measureLen + elements->tagsLen; return TSDB_CODE_SUCCESS; } @@ -325,14 +349,14 @@ static bool smlIsPKTable(STableMeta *pTableMeta){ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { bool isSameMeasure = IS_SAME_SUPER_TABLE; if (isSameMeasure) { - return 0; + return TSDB_CODE_SUCCESS; } SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); SSmlSTableMeta *sMeta = NULL; if (unlikely(tmp == NULL)) { - sMeta = smlBuildSuperTableInfo(info, elements); - if (sMeta == NULL) return -1; + int32_t code = smlBuildSuperTableInfo(info, elements, &sMeta); + if (code != 0) return code; } else { sMeta = *tmp; } @@ -342,39 +366,59 @@ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { info->maxColKVs = sMeta->cols; if(smlIsPKTable(sMeta->tableMeta)){ - terrno = TSDB_CODE_SML_NOT_SUPPORT_PK; - return -1; + return TSDB_CODE_SML_NOT_SUPPORT_PK; } return 0; } int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { + int32_t code = TSDB_CODE_SUCCESS; SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); SSmlTableInfo *tinfo = NULL; if (unlikely(oneTable == NULL)) { - tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); - if (unlikely(!tinfo)) { - return TSDB_CODE_OUT_OF_MEMORY; + code = smlBuildTableInfo(1, elements->measure, elements->measureLen, &tinfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); + if(code != 0){ + smlDestroyTableInfo(&tinfo); + return code; } - taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); tinfo->tags = taosArrayDup(info->preLineTagKV, NULL); + if(tinfo->tags == NULL){ + smlDestroyTableInfo(&tinfo); + return TSDB_CODE_OUT_OF_MEMORY; + } for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i); + if(kv == NULL){ + smlDestroyTableInfo(&tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } if (kv->keyEscaped) kv->key = NULL; if (kv->valueEscaped) kv->value = NULL; } - smlSetCTableName(tinfo); - getTableUid(info, elements, tinfo); + code = smlSetCTableName(tinfo); + if (code != TSDB_CODE_SUCCESS){ + smlDestroyTableInfo(&tinfo); + return code; + } + code = getTableUid(info, elements, tinfo); + if (code != TSDB_CODE_SUCCESS){ + smlDestroyTableInfo(&tinfo); + return code; + } if (info->dataFormat) { info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { + code = smlInitTableDataCtx(info->pQuery, info->currSTableMeta, &tinfo->tableDataCtx); + if (code != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; + return code; } } } else { @@ -404,9 +448,16 @@ int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv * uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); if (elements->colArray == NULL) { elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + if (elements->colArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (taosArrayPush(elements->colArray, kvTs) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + if (taosArrayPush(elements->colArray, kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(elements->colArray, kvTs); - taosArrayPush(elements->colArray, kv); } info->preLine = *elements; @@ -442,6 +493,9 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + if(tag == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } totalNameLen += tag->length; } if (totalNameLen < TSDB_TABLE_NAME_LEN) { @@ -449,12 +503,15 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { } } if (autoChildName) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - strncat(childTableName, tag->value, tag->length); + if(tag == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } + (void)strncat(childTableName, tag->value, tag->length); if (i != taosArrayGetSize(tags) - 1) { - strcat(childTableName, tsSmlAutoChildTableNameDelimiter); + (void)strcat(childTableName, tsSmlAutoChildTableNameDelimiter); } } if (tsSmlDot2Underline) { @@ -466,10 +523,13 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + if(tag == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + (void)strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); } @@ -483,60 +543,60 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { } int32_t smlSetCTableName(SSmlTableInfo *oneTable) { - smlParseTableName(oneTable->tags, oneTable->childTableName); + int32_t code = smlParseTableName(oneTable->tags, oneTable->childTableName); + if(code != TSDB_CODE_SUCCESS){ + return code; + } if (strlen(oneTable->childTableName) == 0) { SArray *dst = taosArrayDup(oneTable->tags, NULL); + if (dst == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } ASSERT(oneTable->sTableNameLen < TSDB_TABLE_NAME_LEN); char superName[TSDB_TABLE_NAME_LEN] = {0}; RandTableName rName = {dst, NULL, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; if (tsSmlDot2Underline) { - memcpy(superName, oneTable->sTableName, oneTable->sTableNameLen); + (void)memcpy(superName, oneTable->sTableName, oneTable->sTableNameLen); smlStrReplace(superName, oneTable->sTableNameLen); rName.stbFullName = superName; } else { rName.stbFullName = oneTable->sTableName; } - buildChildTableName(&rName); - taosArrayDestroy(dst); + code = buildChildTableName(&rName); + if (code != TSDB_CODE_SUCCESS){ + return code; + } + (void)taosArrayDestroy(dst); } return TSDB_CODE_SUCCESS; } -void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { +int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { char key[TSDB_TABLE_NAME_LEN * 2 + 1] = {0}; size_t nLen = strlen(tinfo->childTableName); - memcpy(key, currElement->measure, currElement->measureLen); + (void)memcpy(key, currElement->measure, currElement->measureLen); if (tsSmlDot2Underline) { smlStrReplace(key, currElement->measureLen); } - memcpy(key + currElement->measureLen + 1, tinfo->childTableName, nLen); + (void)memcpy(key + currElement->measureLen + 1, tinfo->childTableName, nLen); void *uid = taosHashGet(info->tableUids, key, currElement->measureLen + 1 + nLen); // use \0 as separator for stable name and child table name if (uid == NULL) { tinfo->uid = info->uid++; - taosHashPut(info->tableUids, key, currElement->measureLen + 1 + nLen, &tinfo->uid, sizeof(uint64_t)); + return taosHashPut(info->tableUids, key, currElement->measureLen + 1 + nLen, &tinfo->uid, sizeof(uint64_t)); } else { tinfo->uid = *(uint64_t *)uid; } + return TSDB_CODE_SUCCESS; } -static void smlDestroySTableMeta(void *para) { - SSmlSTableMeta *meta = *(SSmlSTableMeta **)para; - taosHashCleanup(meta->tagHash); - taosHashCleanup(meta->colHash); - taosArrayDestroy(meta->tags); - taosArrayDestroy(meta->cols); - taosMemoryFreeClear(meta->tableMeta); - taosMemoryFree(meta); -} - -SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { +int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); if (!meta) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } if (unlikely(!isDataFormat)) { @@ -564,14 +624,15 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { uError("SML:smlBuildSTableMeta failed to allocate memory"); goto cleanup; } - return meta; + *sMeta = meta; + return TSDB_CODE_SUCCESS; cleanup: smlDestroySTableMeta(meta); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } -bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { +int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; @@ -743,8 +804,8 @@ bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { return true; } -STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen) { - STableMeta *pTableMeta = NULL; +int32_t smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen, STableMeta **pTableMeta) { + *pTableMeta = NULL; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); @@ -754,14 +815,14 @@ STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen conn.requestId = info->pRequest->requestId; conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, measure, measureLen); + (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + (void)memcpy(pName.tname, measure, measureLen); - int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, pTableMeta); if (code != TSDB_CODE_SUCCESS) { - return NULL; + return code; } - return pTableMeta; + return TSDB_CODE_SUCCESS; } static int64_t smlGenId() { @@ -803,7 +864,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm *action = SCHEMA_ACTION_ADD_COLUMN; } } - return 0; + return TSDB_CODE_SUCCESS; } #define BOUNDARY 1024 @@ -838,6 +899,9 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH for (int j = 0; j < taosArrayGetSize(cols); ++j) { if (j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); if (code != TSDB_CODE_SUCCESS) { return code; @@ -846,6 +910,9 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } if(taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL){ return TSDB_CODE_PAR_DUPLICATED_COLUMN; } @@ -854,10 +921,18 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH } static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { + int32_t code = TSDB_CODE_SUCCESS; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (hashTmp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } int32_t i = 0; for (; i < length; i++) { - taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); + code = taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); + if (code != 0){ + goto END; + } } if (isTag) { @@ -867,13 +942,19 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool } for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (kv == NULL){ + code = TSDB_CODE_SML_INVALID_DATA; + goto END; + } if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { - taosHashCleanup(hashTmp); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto END; } } + +END: taosHashCleanup(hashTmp); - return 0; + return code; } static int32_t getBytes(uint8_t type, int32_t length) { @@ -889,6 +970,9 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO SArray *results, int32_t numOfCols, bool isTag) { for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } ESchemaAction action = SCHEMA_ACTION_NULL; int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); if (code != 0) { @@ -898,8 +982,10 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO SField field = {0}; field.type = kv->type; field.bytes = getBytes(kv->type, kv->length); - memcpy(field.name, kv->key, kv->keyLen); - taosArrayPush(results, &field); + (void)memcpy(field.name, kv->key, kv->keyLen); + if (taosArrayPush(results, &field) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); if (index == NULL) { @@ -909,6 +995,9 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO uint16_t newIndex = *index; if (isTag) newIndex -= numOfCols; SField *field = (SField *)taosArrayGet(results, newIndex); + if (field == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } field->bytes = getBytes(kv->type, kv->length); } } @@ -917,6 +1006,9 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO int32_t len = 0; for (int j = 0; j < taosArrayGetSize(results); ++j) { SField *field = taosArrayGet(results, j); + if (field == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } len += field->bytes; } if (len > maxLen) { @@ -936,18 +1028,29 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, // put front for free pReq.numOfColumns = taosArrayGetSize(pColumns); + pReq.pTags = pTags; + pReq.numOfTags = taosArrayGetSize(pTags); + pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions)); + if (pReq.pColumns == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (int32_t i = 0; i < pReq.numOfColumns; ++i) { - SField *pField = taosArrayGet(pColumns, i); + SField *pField = taosArrayGet(pColumns, i); + if (pField == NULL){ + code = TSDB_CODE_SML_INVALID_DATA; + goto end; + } SFieldWithOptions fieldWithOption = {0}; setFieldWithOptions(&fieldWithOption, pField); setDefaultOptionsForField(&fieldWithOption); - taosArrayPush(pReq.pColumns, &fieldWithOption); + if (taosArrayPush(pReq.pColumns, &fieldWithOption) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } - pReq.numOfTags = taosArrayGetSize(pTags); - pReq.pTags = pTags; - if (action == SCHEMA_ACTION_CREATE_STABLE) { pReq.colVer = 1; pReq.tagVer = 1; @@ -968,6 +1071,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; } else { uError("SML:0x%" PRIx64 " invalid action:%d", info->id, action); + code = TSDB_CODE_SML_INVALID_DATA; goto end; } @@ -987,38 +1091,52 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SField field = {0}; field.type = TSDB_DATA_TYPE_NCHAR; field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - strcpy(field.name, tsSmlTagName); - taosArrayPush(pReq.pTags, &field); + tstrncpy(field.name, tsSmlTagName, sizeof(field.name)); + if (taosArrayPush(pReq.pTags, &field) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } pReq.commentLen = -1; pReq.igExists = true; - tNameExtractFullName(pName, pReq.name); + (void)tNameExtractFullName(pName, pReq.name); pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); + if (pCmdMsg.msgLen < 0){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); if (NULL == pCmdMsg.pMsg) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery; - memset(&pQuery, 0, sizeof(pQuery)); + if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) < 0){ + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pCmdMsg.pMsg); + goto end; + } + + SQuery pQuery = {0}; + (void)memset(&pQuery, 0, sizeof(pQuery)); pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; pQuery.stableQuery = true; - launchQueryImpl(pRequest, &pQuery, true, NULL); + (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // no need to check return value if (pRequest->code == TSDB_CODE_SUCCESS) { - catalogRemoveTableMeta(info->pCatalog, pName); + code = catalogRemoveTableMeta(info->pCatalog, pName); + if (code != TSDB_CODE_SUCCESS){ + goto end; + } } code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); end: destroyRequest(pRequest); @@ -1053,11 +1171,15 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { size_t superTableLen = 0; void *superTable = taosHashGetKey(tmp, &superTableLen); char *measure = taosMemoryMalloc(superTableLen); - memcpy(measure, superTable, superTableLen); + if (measure == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + (void)memcpy(measure, superTable, superTableLen); PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); smlStrReplace(measure, superTableLen); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, measure, superTableLen); + (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + (void)memcpy(pName.tname, measure, superTableLen); taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); @@ -1069,23 +1191,32 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); + if (pColumns == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); + if (pTags == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + (void)taosArrayDestroy(pColumns); + goto end; + } code = smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList tag1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList col1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); - taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pColumns); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; @@ -1107,9 +1238,16 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (hashTmp == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (uint16_t i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + if (code != 0){ + goto end; + } } ESchemaAction action = SCHEMA_ACTION_NULL; @@ -1126,46 +1264,64 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { action); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + if (pColumns == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - + if (pTags == NULL){ + (void)taosArrayDestroy(pColumns); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { SField field = {0}; field.type = pTableMeta->schema[i].type; field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); + tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); + if (taosArrayPush(pColumns, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } else { - taosArrayPush(pTags, &field); + if (taosArrayPush(pTags, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } } code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, pTableMeta->tableInfo.numOfColumns, true); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } if (taosArrayGetSize(pTags) + pTableMeta->tableInfo.numOfColumns > TSDB_MAX_COLUMNS) { uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } if (taosArrayGetSize(pTags) > TSDB_MAX_TAGS) { uError("SML:0x%" PRIx64 " too many tags than 128", info->id); code = TSDB_CODE_PAR_INVALID_TAGS_NUM; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pColumns); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; @@ -1185,7 +1341,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { taosHashClear(hashTmp); for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } } action = SCHEMA_ACTION_NULL; code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false); @@ -1201,18 +1360,36 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { action); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + if (pColumns == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - + if (pTags == NULL){ + (void)taosArrayDestroy(pColumns); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { SField field = {0}; field.type = pTableMeta->schema[i].type; field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); + tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); + if (taosArrayPush(pColumns, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } else { - taosArrayPush(pTags, &field); + if (taosArrayPush(pTags, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } } @@ -1220,21 +1397,21 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { pTableMeta->tableInfo.numOfColumns, false); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } if (taosArrayGetSize(pColumns) + pTableMeta->tableInfo.numOfTags > TSDB_MAX_COLUMNS) { uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pColumns); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; @@ -1284,13 +1461,13 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema); - return 0; + return TSDB_CODE_SUCCESS; end: taosHashCancelIterate(info->superTables, tmp); taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); - catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); + (void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, tstrerror(code), info->dataFormat, info->needModifySchema); @@ -1301,9 +1478,14 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols terrno = 0; for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); if (ret == 0) { - taosArrayPush(metaArray, kv); + if (taosArrayPush(metaArray, kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } @@ -1317,10 +1499,16 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg, SHashObj* checkDuplicate) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); if (index) { SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); + if (value == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } + if (isTag) { if (kv->length > value->length) { value->length = kv->length; @@ -1345,7 +1533,9 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols int16_t size = tmp; int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); if (ret == 0) { - taosArrayPush(metaArray, kv); + if(taosArrayPush(metaArray, kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } @@ -1365,7 +1555,7 @@ void smlDestroyTableInfo(void *para) { taosHashCleanup(kvHash); } - taosArrayDestroy(tag->cols); + (void)taosArrayDestroy(tag->cols); taosArrayDestroyEx(tag->tags, freeSSmlKv); taosMemoryFree(tag); } @@ -1391,13 +1581,13 @@ void smlDestroyInfo(SSmlHandle *info) { cJSON *tags = (cJSON *)taosArrayGetP(info->tagJsonArray, i); cJSON_Delete(tags); } - taosArrayDestroy(info->tagJsonArray); + (void)taosArrayDestroy(info->tagJsonArray); for (int i = 0; i < taosArrayGetSize(info->valueJsonArray); i++) { cJSON *value = (cJSON *)taosArrayGetP(info->valueJsonArray, i); cJSON_Delete(value); } - taosArrayDestroy(info->valueJsonArray); + (void)taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); @@ -1418,21 +1608,22 @@ void smlDestroyInfo(SSmlHandle *info) { taosMemoryFreeClear(info); } -SSmlHandle *smlBuildSmlInfo(TAOS *taos) { +int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { int32_t code = TSDB_CODE_SUCCESS; SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); if (NULL == info) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } if (taos != NULL) { info->taos = acquireTscObj(*(int64_t *)taos); if (info->taos == NULL) { - goto cleanup; + code = TSDB_CODE_TSC_DISCONNECTED; + goto FAILED; } code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto cleanup; + goto FAILED; } } @@ -1440,27 +1631,37 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (info->pVgHash == NULL || info->childTables == NULL || info->superTables == NULL || info->tableUids == NULL) { + uError("create SSmlHandle hash obj failed"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto FAILED; + } taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); info->id = smlGenId(); - info->pQuery = smlInitHandle(); + code = smlInitHandle(&info->pQuery); + if (code != TSDB_CODE_SUCCESS){ + goto FAILED; + } info->dataFormat = true; info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); - if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables || NULL == info->tableUids) { - uError("create SSmlHandle failed"); - goto cleanup; + if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) { + uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); + code = TSDB_CODE_OUT_OF_MEMORY; + goto FAILED; } - return info; + *handle = info; + return code; -cleanup: +FAILED: smlDestroyInfo(info); - return NULL; + return code; } static int32_t smlPushCols(SArray *colsArray, SArray *cols) { @@ -1471,15 +1672,26 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (kv == NULL){ + taosHashCleanup(kvHash); + return TSDB_CODE_SML_INVALID_DATA; + } terrno = 0; - taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + int32_t code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { taosHashCleanup(kvHash); return TSDB_CODE_PAR_DUPLICATED_COLUMN; } + if (code != TSDB_CODE_SUCCESS){ + taosHashCleanup(kvHash); + return code; + } } - taosArrayPush(colsArray, &kvHash); + if (taosArrayPush(colsArray, &kvHash) == NULL) { + taosHashCleanup(kvHash); + return TSDB_CODE_OUT_OF_MEMORY; + } return TSDB_CODE_SUCCESS; } @@ -1542,9 +1754,10 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { } else { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); - SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); - if (meta == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + SSmlSTableMeta *meta = NULL; + ret = smlBuildSTableMeta(info->dataFormat, &meta); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } ret = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); if (ret != TSDB_CODE_SUCCESS) { @@ -1572,11 +1785,17 @@ static int32_t smlInsertData(SSmlHandle *info) { if (info->pRequest->dbList == NULL) { info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); + if (info->pRequest->dbList == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } char *data = (char *)taosArrayReserve(info->pRequest->dbList, 1); + if (data == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - tNameGetFullDbName(&pName, data); + (void)tNameGetFullDbName(&pName, data); //ignore SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); while (oneTable) { @@ -1584,18 +1803,26 @@ static int32_t smlInsertData(SSmlHandle *info) { int measureLen = tableData->sTableNameLen; char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); - memcpy(measure, tableData->sTableName, tableData->sTableNameLen); + if (measure == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(measure, tableData->sTableName, tableData->sTableNameLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); smlStrReplace(measure, measureLen); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, measure, measureLen); + (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + (void)memcpy(pName.tname, measure, measureLen); if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); + if (info->pRequest->tableList == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (taosArrayPush(info->pRequest->tableList, &pName) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(info->pRequest->tableList, &pName); - strcpy(pName.tname, tableData->childTableName); + tstrncpy(pName.tname, tableData->childTableName, sizeof(pName.tname)); SRequestConnInfo conn = {0}; conn.pTrans = info->taos->pAppInfo->pTransporter; @@ -1618,7 +1845,13 @@ static int32_t smlInsertData(SSmlHandle *info) { taosHashCancelIterate(info->childTables, oneTable); return code; } - taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); + code = taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); + if (code != TSDB_CODE_SUCCESS){ + uError("SML:0x%" PRIx64 " taosHashPut failed. table name: %s", info->id, tableData->childTableName); + taosMemoryFree(measure); + taosHashCancelIterate(info->childTables, oneTable); + return code; + } SSmlSTableMeta **pMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); @@ -1655,9 +1888,10 @@ static int32_t smlInsertData(SSmlHandle *info) { info->cost.insertRpcTime = taosGetTimestampUs(); SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; - atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); + (void)atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); // no need to check return code + + (void)launchQueryImpl(info->pRequest, info->pQuery, true, NULL); // no need to check return code - launchQueryImpl(info->pRequest, info->pQuery, true, NULL); uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code, tstrerror(info->pRequest->code)); @@ -1691,7 +1925,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); } - memset(&info->preLine, 0, sizeof(SSmlLineInfo)); + (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; @@ -1721,7 +1955,7 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi if (*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)) { char *print = taosMemoryCalloc(*len + 1, 1); - memcpy(print, *tmp, *len); + (void)memcpy(print, *tmp, *len); uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines, info->protocol, *len, print); taosMemoryFree(print); @@ -1777,7 +2011,11 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char if (code != TSDB_CODE_SUCCESS) { if (rawLine != NULL) { char *print = taosMemoryCalloc(len + 1, 1); - memcpy(print, tmp, len); + if (print == NULL) { + uError("SML:0x%" PRIx64 " smlParseLine failed. out of memory", info->id); + return code; + } + (void)memcpy(print, tmp, len); uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print); taosMemoryFree(print); } else { @@ -1880,7 +2118,7 @@ void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawL uError("malloc %d for sml sql failed", rlen + 1); return; } - memcpy(sql, p, rlen); + (void)memcpy(sql, p, rlen); sql[rlen] = 0; request->sqlstr = sql; @@ -1892,7 +2130,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, int protocol, int precision, int32_t ttl, int64_t reqid) { int32_t code = TSDB_CODE_SUCCESS; if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; + uError("SML:taos_schemaless_insert error taos is null"); return NULL; } SRequestObj *request = NULL; @@ -1905,11 +2143,13 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, return NULL; } - info = smlBuildSmlInfo(taos); - if (info == NULL) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error SSmlHandle is null"); - return (TAOS_RES *)request; + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + code = smlBuildSmlInfo(taos, &info); + if (code != TSDB_CODE_SUCCESS) { + request->code = code; + smlBuildInvalidDataMsg(&msg, "init SSmlHandle failed", NULL); + uError("SML:taos_schemaless_insert error SSmlHandle is null, err msg:%s", tstrerror(code)); + goto end; } info->pRequest = request; info->pRequest->pQuery = info->pQuery; @@ -1922,7 +2162,6 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, smlSetReqSQL(request, lines, rawLine, rawLineEnd); - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); @@ -1961,7 +2200,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, break; } taosMsleep(100); - refreshMeta(request->pTscObj, request); + (void)refreshMeta(request->pTscObj, request); //ignore return code,try again uInfo("SML:%" PRIx64 " retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code, tstrerror(code)); smlDestroyInfo(info); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 845884d5ac..14d6a95e22 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -29,29 +29,29 @@ (start)++; \ } -static char *smlJsonGetObj(char *payload) { +static int32_t smlJsonGetObj(char **payload) { int leftBracketCnt = 0; bool isInQuote = false; - while (*payload) { - if (*payload == '"' && *(payload - 1) != '\\') { + while (**payload) { + if (**payload == '"' && *((*payload) - 1) != '\\') { isInQuote = !isInQuote; - } else if (!isInQuote && unlikely(*payload == '{')) { + } else if (!isInQuote && unlikely(**payload == '{')) { leftBracketCnt++; - payload++; + (*payload)++; continue; - } else if (!isInQuote && unlikely(*payload == '}')) { + } else if (!isInQuote && unlikely(**payload == '}')) { leftBracketCnt--; - payload++; + (*payload)++; if (leftBracketCnt == 0) { - return payload; + return 0; } else if (leftBracketCnt < 0) { - return NULL; + return -1; } continue; } - payload++; + (*payload)++; } - return NULL; + return -1; } int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { @@ -99,8 +99,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { offset[index++] = *start - sTmp; element->timestamp = (*start); if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->timestampLen = tmp - (*start); *start = tmp; } @@ -127,8 +128,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { offset[index++] = *start - sTmp; element->cols = (*start); if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->colsLen = tmp - (*start); *start = tmp; } @@ -153,8 +155,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { JUMP_JSON_SPACE((*start)) offset[index++] = *start - sTmp; element->tags = (*start); - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->tagsLen = tmp - (*start); *start = tmp; } @@ -209,8 +212,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { (*start) += offset[index++]; element->timestamp = *start; if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->timestampLen = tmp - (*start); *start = tmp; } @@ -227,8 +231,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { (*start) += offset[index++]; element->cols = *start; if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->colsLen = tmp - (*start); *start = tmp; } @@ -244,8 +249,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { } else if ((*start)[1] == 't' && (*start)[2] == 'a') { (*start) += offset[index++]; element->tags = (*start); - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->tagsLen = tmp - (*start); *start = tmp; } @@ -261,7 +267,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { uError("elements != %d", OTD_JSON_FIELDS_NUM); return TSDB_CODE_TSC_INVALID_JSON; } - return 0; + return TSDB_CODE_SUCCESS; } static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { @@ -466,7 +472,11 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { break; } case cJSON_String: { - smlConvertJSONString(kv, "binary", root); + int32_t ret = smlConvertJSONString(kv, "binary", root); + if (ret != TSDB_CODE_SUCCESS) { + uError("OTD:Failed to parse binary value from JSON Obj"); + return ret; + } break; } case cJSON_Object: { @@ -492,20 +502,17 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ int32_t tagNum = cJSON_GetArraySize(tags); if (unlikely(tagNum == 0)) { uError("SML:Tag should not be empty"); - terrno = TSDB_CODE_TSC_INVALID_JSON; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } for (int32_t i = 0; i < tagNum; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); if (unlikely(tag == NULL)) { - terrno = TSDB_CODE_TSC_INVALID_JSON; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } size_t keyLen = strlen(tag->string); if (unlikely(IS_INVALID_COL_LEN(keyLen))) { uError("OTD:Tag key length is 0 or too large than 64"); - terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - return -1; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } // add kv to SSmlKv @@ -516,19 +523,19 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ // value int32_t ret = smlParseValueFromJSON(tag, &kv); if (unlikely(ret != TSDB_CODE_SUCCESS)) { - terrno = ret; - return -1; + return ret; + } + if (taosArrayPush(preLineKV, &kv) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(preLineKV, &kv); if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - terrno = TSDB_CODE_SUCCESS; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } cnt++; } - return 0; + return TSDB_CODE_SUCCESS; } static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { @@ -536,12 +543,18 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo if(info->dataFormat){ ret = smlProcessSuperTable(info, elements); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } } ret = smlProcessTagJson(info, tags); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } ret = smlJoinMeasureTag(elements); if(ret != 0){ @@ -703,6 +716,9 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo // Parse tags bool needFree = info->dataFormat; elements->tags = cJSON_PrintUnformatted(tagsJson); + if (elements->tags == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } elements->tagsLen = strlen(elements->tags); if (is_same_child_table_telnet(elements, &info->preLine) != 0) { ret = smlParseTagsFromJSON(info, tagsJson, elements); @@ -729,7 +745,8 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo // notice!!! put ts back to tag to ensure get meta->precision int64_t ts = smlParseTSFromJSON(info, tsJson); if (unlikely(ts < 0)) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + char* tmp = cJSON_PrintUnformatted(tsJson); + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); return TSDB_CODE_INVALID_TIMESTAMP; } SSmlKv kvTs = {0}; @@ -823,7 +840,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * } if (ret != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_VALUE; + return ret; } if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; @@ -844,18 +861,23 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * cJSON *valueJson = cJSON_Parse(elements->cols); if (unlikely(valueJson == NULL)) { uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols); + elements->cols[elements->colsLen] = tmp; return TSDB_CODE_TSC_INVALID_JSON; } - taosArrayPush(info->tagJsonArray, &valueJson); + if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){ + cJSON_Delete(valueJson); + elements->cols[elements->colsLen] = tmp; + return TSDB_CODE_OUT_OF_MEMORY; + } ret = smlParseValueFromJSONObj(valueJson, &kv); if (ret != TSDB_CODE_SUCCESS) { - uError("SML:Failed to parse value from JSON Obj:%s", elements->cols); + uError("SML:0x%" PRIx64 " Failed to parse value from JSON Obj:%s", info->id, elements->cols); elements->cols[elements->colsLen] = tmp; return TSDB_CODE_TSC_INVALID_VALUE; } elements->cols[elements->colsLen] = tmp; } else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { - uError("SML:cols invalidate:%s", elements->cols); + uError("SML:0x%" PRIx64 " cols invalidate:%s", info->id, elements->cols); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -870,7 +892,11 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * return TSDB_CODE_TSC_INVALID_JSON; } - taosArrayPush(info->tagJsonArray, &tagsJson); + if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){ + cJSON_Delete(tagsJson); + uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id); + return TSDB_CODE_OUT_OF_MEMORY; + } ret = smlParseTagsFromJSON(info, tagsJson, elements); if (unlikely(ret)) { uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); @@ -937,7 +963,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { return ret; } info->lines = (SSmlLineInfo *)tmp; - memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); + (void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); } ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); if ((info->lines + cnt)->measure == NULL) break; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 7535cbfd0c..3456b7df17 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -103,7 +103,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return code; } char* tmp = taosMemoryCalloc(pVal->length, 1); - memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length); taosMemoryFree(tmp); if (code != TSDB_CODE_SUCCESS) { @@ -149,7 +152,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { if(data == NULL){ return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); + (void)memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); pVal->value = data; } @@ -212,8 +215,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(IS_EQUAL(*sql,escapeChar))) { keyLen = *sql - key; @@ -230,8 +232,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); - terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - return -1; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } // parse value @@ -245,8 +246,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ break; } else if (unlikely(IS_EQUAL(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { @@ -261,24 +261,28 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ if (unlikely(valueLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - return -1; + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } if (keyEscaped) { char *tmp = (char *)taosMemoryMalloc(keyLen); - memcpy(tmp, key, keyLen); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, key, keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); key = tmp; } if (valueEscaped) { char *tmp = (char *)taosMemoryMalloc(valueLen); - memcpy(tmp, value, valueLen); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, value, valueLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen); value = tmp; } @@ -289,11 +293,12 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; - taosArrayPush(preLineKV, &kv); + if(taosArrayPush(preLineKV, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - terrno = TSDB_CODE_SUCCESS; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } cnt++; @@ -302,7 +307,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ } (*sql)++; } - return 0; + return TSDB_CODE_SUCCESS; } static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { @@ -315,13 +320,19 @@ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if(info->dataFormat){ ret = smlProcessSuperTable(info, elements); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } } ret = smlProcessTagLine(info, sql, sqlEnd); if(ret != 0){ - return terrno; + if (info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } return smlProcessChildTable(info, elements); @@ -410,7 +421,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (keyEscaped) { char *tmp = (char *)taosMemoryMalloc(kv.keyLen); - memcpy(tmp, key, kv.keyLen); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, key, kv.keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen); kv.key = tmp; kv.keyEscaped = keyEscaped; @@ -418,7 +432,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (valueEscaped) { char *tmp = (char *)taosMemoryMalloc(kv.length); - memcpy(tmp, kv.value, kv.length); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, kv.value, kv.length); PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY); if(kv.type == TSDB_DATA_TYPE_VARBINARY){ @@ -437,8 +454,13 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } else { if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); + if (currElement->colArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(currElement->colArray, &kv); // reserve for timestamp } cnt++; diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index bc0e560178..a25ef4ec95 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -84,8 +84,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ while (sql < sqlEnd) { if (unlikely(*sql == SPACE)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(*sql == EQUAL)) { keyLen = sql - key; @@ -97,8 +96,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ if (unlikely(IS_INVALID_COL_LEN(keyLen))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); - terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - return -1; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } // parse value @@ -111,8 +109,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ } if (unlikely(*sql == EQUAL)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } sql++; } @@ -120,13 +117,11 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ if (unlikely(valueLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); - terrno = TSDB_CODE_TSC_INVALID_VALUE; - return -1; + return TSDB_CODE_TSC_INVALID_VALUE; } if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - return -1; + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } SSmlKv kv = {.key = key, @@ -136,14 +131,15 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ .length = valueLen, .keyEscaped = false, .valueEscaped = false}; - taosArrayPush(preLineKV, &kv); + if (taosArrayPush(preLineKV, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - terrno = TSDB_CODE_SUCCESS; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } cnt++; } - return 0; + return TSDB_CODE_SUCCESS; } static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) { @@ -156,13 +152,19 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS if(info->dataFormat){ ret = smlProcessSuperTable(info, elements); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } } ret = smlProcessTagTelnet(info, data, sqlEnd); if(ret != 0){ - return terrno; + if (info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } ret = smlJoinMeasureTag(elements); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 054698fff0..77079ed863 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -40,13 +40,16 @@ TEST(testCase, smlParseInfluxString_Test) { msgBuf.len = 256; SSmlLineInfo elements = {0}; - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, 0); info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; // case 1 char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *sql = (char *)taosMemoryCalloc(256, 1); - memcpy(sql, tmp, strlen(tmp) + 1); + ASSERT_NE(sql, nullptr); + (void)memcpy(sql, tmp, strlen(tmp) + 1); int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); @@ -62,22 +65,22 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 2 false tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 4 tag is null tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); @@ -92,13 +95,13 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 5 tag is null tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql + 1); @@ -112,36 +115,36 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 6 tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; - smlClearForRerun(info); + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; + ASSERT_EQ(smlClearForRerun(info), 0); // case 7 tmp = " st , "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 8 false tmp = ", st , "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; taosMemoryFree(sql); smlDestroyInfo(info); @@ -228,7 +231,9 @@ TEST(testCase, smlParseCols_Error_Test) { "escape_test,tag1=\"tag1_value\",tag2=\"tag2_value\" col0=\"co\"l\"0_value\",col1=\"col1_value\" 1680918783010000000" }; - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) { @@ -238,13 +243,14 @@ TEST(testCase, smlParseCols_Error_Test) { msgBuf.len = 256; int32_t len = strlen(data[i]); char *sql = (char *)taosMemoryCalloc(256, 1); - memcpy(sql, data[i], len + 1); + ASSERT_NE(sql, nullptr); + (void)memcpy(sql, data[i], len + 1); SSmlLineInfo elements = {0}; int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); // printf("i:%d\n", i); ASSERT_NE(ret, TSDB_CODE_SUCCESS); taosMemoryFree(sql); - taosArrayDestroy(elements.colArray); + (void)taosArrayDestroy(elements.colArray); } smlDestroyInfo(info); } @@ -254,7 +260,9 @@ TEST(testCase, smlParseCols_Test) { SSmlMsgBuf msgBuf; msgBuf.buf = msg; msgBuf.len = 256; - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; SSmlLineInfo elements = {0}; @@ -267,7 +275,8 @@ TEST(testCase, smlParseCols_Test) { "cboolf=f,cnch_=l\"iuwq\" 1626006833639000000"; int32_t len = strlen(data); char *sql = (char *)taosMemoryCalloc(1024, 1); - memcpy(sql, data, len + 1); + ASSERT_NE(sql, nullptr); + (void)memcpy(sql, data, len + 1); int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); int32_t size = taosArrayGetSize(elements.colArray); @@ -275,6 +284,7 @@ TEST(testCase, smlParseCols_Test) { // binary SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); @@ -285,6 +295,7 @@ TEST(testCase, smlParseCols_Test) { // nchar kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); @@ -293,6 +304,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -301,6 +313,7 @@ TEST(testCase, smlParseCols_Test) { // double kv = (SSmlKv *)taosArrayGet(elements.colArray, 4); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); @@ -310,6 +323,7 @@ TEST(testCase, smlParseCols_Test) { // float kv = (SSmlKv *)taosArrayGet(elements.colArray, 5); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); @@ -319,6 +333,7 @@ TEST(testCase, smlParseCols_Test) { // float kv = (SSmlKv *)taosArrayGet(elements.colArray, 6); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); @@ -328,6 +343,7 @@ TEST(testCase, smlParseCols_Test) { // tiny int kv = (SSmlKv *)taosArrayGet(elements.colArray, 7); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); @@ -336,6 +352,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned tiny int kv = (SSmlKv *)taosArrayGet(elements.colArray, 8); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); @@ -344,6 +361,7 @@ TEST(testCase, smlParseCols_Test) { // small int kv = (SSmlKv *)taosArrayGet(elements.colArray, 9); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); @@ -352,6 +370,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned smallint kv = (SSmlKv *)taosArrayGet(elements.colArray, 10); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); @@ -360,6 +379,7 @@ TEST(testCase, smlParseCols_Test) { // int kv = (SSmlKv *)taosArrayGet(elements.colArray, 11); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); @@ -368,6 +388,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned int kv = (SSmlKv *)taosArrayGet(elements.colArray, 12); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); @@ -376,6 +397,7 @@ TEST(testCase, smlParseCols_Test) { // bigint kv = (SSmlKv *)taosArrayGet(elements.colArray, 13); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); @@ -384,6 +406,7 @@ TEST(testCase, smlParseCols_Test) { // bigint kv = (SSmlKv *)taosArrayGet(elements.colArray, 14); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); ASSERT_EQ(kv->keyLen, 2); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); @@ -392,6 +415,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned bigint kv = (SSmlKv *)taosArrayGet(elements.colArray, 15); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); @@ -400,6 +424,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 16); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); ASSERT_EQ(kv->keyLen, 9); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -408,6 +433,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 17); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -416,6 +442,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 18); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -424,13 +451,14 @@ TEST(testCase, smlParseCols_Test) { // nchar kv = (SSmlKv *)taosArrayGet(elements.colArray, 19); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 4); ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); - taosArrayDestroy(elements.colArray); + (void)taosArrayDestroy(elements.colArray); taosMemoryFree(sql); smlDestroyInfo(info); } @@ -471,7 +499,9 @@ TEST(testCase, smlParseNumber_Test) { } TEST(testCase, smlParseTelnetLine_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->dataFormat = false; info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); @@ -509,7 +539,9 @@ TEST(testCase, smlParseTelnetLine_error_Test) { } TEST(testCase, smlParseTelnetLine_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->dataFormat = false; info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); @@ -528,7 +560,9 @@ TEST(testCase, smlParseTelnetLine_Test) { } //TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { -// SSmlHandle *info = smlBuildSmlInfo(NULL); +// SSmlHandle *info = nullptr; +// int32_t code = smlBuildSmlInfo(nullptr, &info); +// ASSERT_EQ(code, TSDB_CODE_SUCCESS); // info->protocol = TSDB_SML_JSON_PROTOCOL; // ASSERT_NE(info, nullptr); // @@ -568,7 +602,7 @@ TEST(testCase, smlParseNumber_performance_Test) { for (int j = 0; j < 10000000; ++j) { kv.value = str[i]; kv.length = strlen(str[i]); - smlParseNumber(&kv, &msgBuf); + (void)smlParseNumber(&kv, &msgBuf); //ignore result } printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1); printf("\n"); @@ -576,7 +610,7 @@ TEST(testCase, smlParseNumber_performance_Test) { for (int j = 0; j < 10000000; ++j) { kv.value = str[i]; kv.length = strlen(str[i]); - smlParseNumberOld(&kv, &msgBuf); + (void)smlParseNumberOld(&kv, &msgBuf); //ignore result } printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); printf("\n\n"); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 7710dbdb3f..965246326a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -615,6 +615,10 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex; SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, index); + if (pColInfoData == NULL) { + return 0; + } + if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) { return 0; } @@ -1514,7 +1518,7 @@ void blockDataFreeRes(SSDataBlock* pBlock) { taosArrayDestroy(pBlock->pDataBlock); pBlock->pDataBlock = NULL; - + taosMemoryFreeClear(pBlock->pBlockAgg); memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); } @@ -2488,7 +2492,10 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha RandTableName rname = { .tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname}; - buildChildTableName(&rname); + int32_t code = buildChildTableName(&rname); + if(code != TSDB_CODE_SUCCESS){ + return code; + } taosArrayDestroy(tags); if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 61af2f6aae..27f865353e 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -305,11 +305,11 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo *(int32_t *)(fixed + schema->columns[i].offset) = varlen - fixed - sinfo->tupleFixedSize; varlen += tPutU32v(varlen, colValArray[colValIndex].value.nData); if (colValArray[colValIndex].value.nData) { - memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); + (void)memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); varlen += colValArray[colValIndex].value.nData; } } else { - memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val, + (void)memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val, tDataTypes[schema->columns[i].type].bytes); } } else if (COL_VAL_IS_NULL(&colValArray[colValIndex])) { // NULL @@ -384,12 +384,12 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); payloadSize += tPutU32v(payload + payloadSize, colValArray[colValIndex].value.nData); if (colValArray[colValIndex].value.nData > 0) { - memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); + (void)memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); } payloadSize += colValArray[colValIndex].value.nData; } else { payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); - memcpy(payload + payloadSize, &colValArray[colValIndex].value.val, + (void)memcpy(payload + payloadSize, &colValArray[colValIndex].value.val, tDataTypes[schema->columns[i].type].bytes); payloadSize += tDataTypes[schema->columns[i].type].bytes; } @@ -475,7 +475,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, value.nData = infos[iInfo].bind->length[iRow]; value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow; } else { - memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow, + (void)memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow, infos[iInfo].bind->buffer_length); } colVal = COL_VAL_VALUE(infos[iInfo].columnId, value); @@ -509,7 +509,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->cid = pTColumn->colId; pColVal->value.type = pTColumn->type; pColVal->flag = CV_FLAG_VALUE; - memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); + (void)memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); return 0; } @@ -573,7 +573,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->value.pData = NULL; } } else { - memcpy(&pColVal->value.val, pData, pTColumn->bytes); + (void)memcpy(&pColVal->value.val, pData, pTColumn->bytes); } } return 0; @@ -624,7 +624,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->value.pData = varlen + *(int32_t *)(fixed + pTColumn->offset); pColVal->value.pData += tGetU32v(pColVal->value.pData, &pColVal->value.nData); } else { - memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]); + (void)memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } } @@ -861,7 +861,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.cid = pTColumn->colId; pIter->cv.value.type = pTColumn->type; pIter->cv.flag = CV_FLAG_VALUE; - memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); + (void)memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); goto _exit; } @@ -906,7 +906,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.value.pData = NULL; } } else { - memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); + (void)memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); } } @@ -965,7 +965,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.value.pData = NULL; } } else { - memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); + (void)memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } goto _exit; } @@ -1288,7 +1288,7 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) { key->pks[i].pData = tdata; key->pks[i].pData += tGetU32v(key->pks[i].pData, &key->pks[i].nData); } else { - memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes); + (void)memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes); } } } @@ -1378,7 +1378,7 @@ FORCE_INLINE int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2) { return 0; } -int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { +void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { pDst->ts = pSrc->ts; pDst->numOfPKs = pSrc->numOfPKs; @@ -1392,12 +1392,10 @@ int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { } else { pVal->nData = pSrc->pks[i].nData; ASSERT(pSrc->pks[i].pData != NULL); - memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData); + (void)memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData); } } } - - return TSDB_CODE_SUCCESS; } // STag ======================================== @@ -1528,7 +1526,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { } else { p = p ? p + n : p; n += tDataTypes[pTagVal->type].bytes; - if (p) memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes); + if (p) (void)memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes); } return n; @@ -1550,7 +1548,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (IS_VAR_DATA_TYPE(pTagVal->type)) { n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); } else { - memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); + (void)memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); n += tDataTypes[pTagVal->type].bytes; } @@ -1661,7 +1659,7 @@ char *tTagValToData(const STagVal *value, bool isJson) { } varDataLen(data + typeBytes) = value->nData; - memcpy(varDataVal(data + typeBytes), value->pData, value->nData); + (void)memcpy(varDataVal(data + typeBytes), value->pData, value->nData); } else { data = ((char *)&(value->i64)) - typeBytes; // json with type } @@ -1713,7 +1711,7 @@ bool tTagGet(const STag *pTag, STagVal *pTagVal) { } else if (c > 0) { lidx = midx + 1; } else { - memcpy(pTagVal, &tv, sizeof(tv)); + (void)memcpy(pTagVal, &tv, sizeof(tv)); return true; } } @@ -1892,7 +1890,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData, if (nData) { code = tRealloc(&pColData->pData, pColData->nData + nData); if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, pData, nData); + (void)memcpy(pColData->pData + pColData->nData, pData, nData); pColData->nData += nData; } } else { @@ -1900,7 +1898,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData, code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); if (code) goto _exit; if (pData) { - memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); + (void)memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); } else { memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]); } @@ -2594,7 +2592,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo } value.pData = pColData->pData + pColData->aOffset[iVal]; } else { - memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); + (void)memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); } *pColVal = COL_VAL_VALUE(pColData->cid, value); } @@ -2692,7 +2690,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal)); + (void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal)); break; case (HAS_VALUE | HAS_NULL | HAS_NONE): pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal)); @@ -2700,7 +2698,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal)); + (void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal)); break; default: pColData->pBitMap = NULL; @@ -2714,7 +2712,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2); + (void)memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2); } else { pColData->aOffset = NULL; } @@ -2727,7 +2725,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall goto _exit; } - memcpy(pColData->pData, pColDataFrom->pData, pColData->nData); + (void)memcpy(pColData->pData, pColDataFrom->pData, pColData->nData); } else { pColData->pData = NULL; } @@ -3119,10 +3117,10 @@ static int32_t tColDataCopyRowCell(SColData *pFromColData, int32_t iFromRow, SCo pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData; } - memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow], + (void)memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow], nData); } else { - memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow], + (void)memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow], &pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]); } return code; @@ -3361,7 +3359,7 @@ static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd / ASSERT(0); } else { if (iv != iStart) { - memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], + (void)memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], &pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]); } memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)], @@ -3579,11 +3577,11 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) { case (HAS_NULL | HAS_NONE): case (HAS_VALUE | HAS_NONE): case (HAS_VALUE | HAS_NULL): - if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); + if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); n += BIT1_SIZE(pColData->nVal); break; case (HAS_VALUE | HAS_NULL | HAS_NONE): - if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); + if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); n += BIT2_SIZE(pColData->nVal); break; default: @@ -3593,14 +3591,14 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) { // value if (pColData->flag & HAS_VALUE) { if (IS_VAR_DATA_TYPE(pColData->type)) { - if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); + if (pBuf) (void)memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); n += (pColData->nVal << 2); n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData); - if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData); n += pColData->nData; } else { - if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData); n += pColData->nData; } } @@ -4348,7 +4346,7 @@ int32_t tCompressData(void *input, // input ASSERT(outputSize >= extraSizeNeeded); if (info->cmprAlg == NO_COMPRESSION) { - memcpy(output, input, info->originalSize); + (void)memcpy(output, input, info->originalSize); info->compressedSize = info->originalSize; } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) { SBuffer local; @@ -4385,7 +4383,7 @@ int32_t tCompressData(void *input, // input } else { DEFINE_VAR(info->cmprAlg) if ((l1 == L1_UNKNOWN && l2 == L2_UNKNOWN) || (l1 == L1_DISABLED && l2 == L2_DISABLED)) { - memcpy(output, input, info->originalSize); + (void)memcpy(output, input, info->originalSize); info->compressedSize = info->originalSize; return 0; } @@ -4431,7 +4429,7 @@ int32_t tDecompressData(void *input, // input if (info->cmprAlg == NO_COMPRESSION) { ASSERT(info->compressedSize == info->originalSize); - memcpy(output, input, info->compressedSize); + (void)memcpy(output, input, info->compressedSize); } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) { SBuffer local; @@ -4468,7 +4466,7 @@ int32_t tDecompressData(void *input, // input } else { DEFINE_VAR(info->cmprAlg); if (l1 == L1_DISABLED && l2 == L2_DISABLED) { - memcpy(output, input, info->compressedSize); + (void)memcpy(output, input, info->compressedSize); return 0; } SBuffer local; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 119438fbd6..7538b504e3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -504,7 +504,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 4fe2beb6aa..e4604c6bab 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -293,17 +293,19 @@ static int compareKv(const void* p1, const void* p2) { /* * use stable name and tags to grearate child table name */ -void buildChildTableName(RandTableName* rName) { +int32_t buildChildTableName(RandTableName* rName) { SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen); if (sb.buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return; + return TSDB_CODE_OUT_OF_MEMORY; } taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { taosStringBuilderAppendChar(&sb, ','); SSmlKv* tagKv = taosArrayGet(rName->tags, j); + if(tagKv == NULL) { + return TSDB_CODE_SML_INVALID_DATA; + } taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendChar(&sb, '='); if (IS_VAR_DATA_TYPE(tagKv->type)) { @@ -323,8 +325,9 @@ void buildChildTableName(RandTableName* rName) { rName->ctbShortName[0] = 't'; rName->ctbShortName[1] = '_'; for (int i = 0; i < 16; i++) { - sprintf(temp, "%02x", context.digest[i]); - strcat(rName->ctbShortName, temp); + (void)sprintf(temp, "%02x", context.digest[i]); + (void)strcat(rName->ctbShortName, temp); } taosStringBuilderDestroy(&sb); + return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e35c152e9b..990d03f940 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -167,7 +167,7 @@ void tsdbReaderClose2(STsdbReader *pReader); int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext); int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); void tsdbReleaseDataBlock2(STsdbReader *pReader); -SSDataBlock *tsdbRetrieveDataBlock2(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); +int32_t tsdbRetrieveDataBlock2(STsdbReader *pReader, SSDataBlock **pBlock, SArray *pIdList); int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 84e1996f2c..943ba099f6 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -909,9 +909,9 @@ void tMergeTreeUnpinSttBlock(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); -SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols); -void * destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); -void * destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); +int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo); +void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); +void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); // tsdbCache ============================================================================================== typedef enum { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 4a47e08730..70aa55915a 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -54,7 +54,6 @@ extern "C" { #endif typedef struct SVnodeInfo SVnodeInfo; -typedef struct SMeta SMeta; typedef struct SSma SSma; typedef struct STsdb STsdb; typedef struct STQ STQ; @@ -153,7 +152,6 @@ int32_t vnodeBufPoolRegisterQuery(SVBufPool* pPool, SQueryNode* pQNode); void vnodeBufPoolDeregisterQuery(SVBufPool* pPool, SQueryNode* pQNode, bool proactive); // meta -typedef struct SMCtbCursor SMCtbCursor; typedef struct SMStbCursor SMStbCursor; typedef struct STbUidStore STbUidStore; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 9b1e8075da..3efc653f64 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -725,24 +725,33 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat if (metaGetTableEntryByName(&mr, dstTableName) < 0) { metaReaderClear(&mr); - tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName); + if (pTask->outputInfo.tbSink.autoCreateCtb) { + tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName); - SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal)); + SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal)); - pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; - pTableData->pCreateTbReq = - buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray, - pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); - taosArrayDestroy(pTagArray); + pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + pTableData->pCreateTbReq = + buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray, + pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); + taosArrayDestroy(pTagArray); - if (pTableData->pCreateTbReq == NULL) { - tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno)); - taosMemoryFree(pTableSinkInfo); - return terrno; + if (pTableData->pCreateTbReq == NULL) { + tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName, + tstrerror(terrno)); + taosMemoryFree(pTableSinkInfo); + return terrno; + } + + pTableSinkInfo->uid = 0; + doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); + } else { + metaReaderClear(&mr); + + tqError("s-task:%s vgId:%d dst-table:%s not auto-created, and not create in tsdb, discard data", id, + vgId, dstTableName); + return TSDB_CODE_TDB_TABLE_NOT_EXIST; } - - pTableSinkInfo->uid = 0; - doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); } else { bool isValid = isValidDstChildTable(&mr, vgId, dstTableName, suid); if (!isValid) { @@ -788,16 +797,34 @@ int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_ } void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { - const SArray* pBlocks = (const SArray*)data; - SVnode* pVnode = (SVnode*)vnode; - int64_t suid = pTask->outputInfo.tbSink.stbUid; - char* stbFullName = pTask->outputInfo.tbSink.stbFullName; - STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; - int32_t vgId = TD_VID(pVnode); - int32_t numOfBlocks = taosArrayGetSize(pBlocks); - int32_t code = TSDB_CODE_SUCCESS; - const char* id = pTask->id.idStr; - int64_t earlyTs = tsdbGetEarliestTs(pVnode->pTsdb); + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->outputInfo.tbSink.stbUid; + char* stbFullName = pTask->outputInfo.tbSink.stbFullName; + STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; + int32_t vgId = TD_VID(pVnode); + int32_t numOfBlocks = taosArrayGetSize(pBlocks); + int32_t code = TSDB_CODE_SUCCESS; + const char* id = pTask->id.idStr; + int64_t earlyTs = tsdbGetEarliestTs(pVnode->pTsdb); + STaskOutputInfo* pOutputInfo = &pTask->outputInfo; + + if (pTask->outputInfo.tbSink.pTagSchema == NULL) { + SMetaReader mer1 = {0}; + metaReaderDoInit(&mer1, pVnode->pMeta, META_READER_LOCK); + + code = metaReaderGetTableEntryByUid(&mer1, pOutputInfo->tbSink.stbUid); + pOutputInfo->tbSink.pTagSchema = tCloneSSchemaWrapper(&mer1.me.stbEntry.schemaTag); + metaReaderClear(&mer1); + + SSchemaWrapper* pTagSchema = pOutputInfo->tbSink.pTagSchema; + SSchema* pCol1 = &pTagSchema->pSchema[0]; + if (pTagSchema->nCols == 1 && pCol1->type == TSDB_DATA_TYPE_UBIGINT && strcmp(pCol1->name, "group_id") == 0) { + pOutputInfo->tbSink.autoCreateCtb = true; + } else { + pOutputInfo->tbSink.autoCreateCtb = false; + } + } bool onlySubmitData = hasOnlySubmitData(pBlocks, numOfBlocks); if (!onlySubmitData) { @@ -829,6 +856,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP}; code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d s-task:%s dst-table not exist, stb:%s discard stream results", vgId, id, stbFullName); continue; } @@ -882,6 +910,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { if (index == NULL) { // no data yet, append it code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d dst-table gid:%" PRId64 " not exist, discard stream results", vgId, groupId); continue; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 38dd945b28..ed09ca821b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -2579,7 +2579,7 @@ typedef struct { static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) { int32_t code = 0; - pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, NULL); + destroySttBlockReader(pr->pLDataIterArray, NULL); pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); SMergeTreeConf conf = { @@ -3212,7 +3212,10 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->pMemDelData = NULL; - loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer); + code = loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 817468c6b6..46c3ba4785 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -203,7 +203,7 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf pReader->pTableList = pTableIdList; pReader->numOfTables = numOfTables; pReader->lastTs = INT64_MIN; - pReader->pLDataIterArray = destroySttBlockReader(pReader->pLDataIterArray, NULL); + destroySttBlockReader(pReader->pLDataIterArray, NULL); pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); return TSDB_CODE_SUCCESS; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 57b8a99fb1..2c03603d73 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -23,11 +23,12 @@ static void tLDataIterClose2(SLDataIter *pIter); // SLDataIter ================================================= -SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols) { +int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo) { + *pInfo = NULL; + SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo)); if (pLoadInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pLoadInfo->blockData[0].sttBlockIndex = -1; @@ -37,26 +38,29 @@ SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t code = tBlockDataCreate(&pLoadInfo->blockData[0].data); if (code) { - terrno = code; + taosMemoryFreeClear(pLoadInfo); + return code; } code = tBlockDataCreate(&pLoadInfo->blockData[1].data); if (code) { - terrno = code; + taosMemoryFreeClear(pLoadInfo); + return code; } pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); if (pLoadInfo->aSttBlk == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFreeClear(pLoadInfo); - return NULL; + return code; } pLoadInfo->pSchema = pSchema; pLoadInfo->colIds = colList; pLoadInfo->numOfCols = numOfCols; - return pLoadInfo; + *pInfo = pLoadInfo; + return code; } static void freeItem(void* pValue) { @@ -66,9 +70,9 @@ static void freeItem(void* pValue) { } } -void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { +void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { if (pLoadInfo == NULL) { - return NULL; + return; } pLoadInfo->currentLoadBlockIndex = 1; @@ -94,7 +98,6 @@ void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { taosArrayDestroy(pLoadInfo->aSttBlk); taosMemoryFree(pLoadInfo); - return NULL; } void destroyLDataIter(SLDataIter *pIter) { @@ -103,9 +106,9 @@ void destroyLDataIter(SLDataIter *pIter) { taosMemoryFree(pIter); } -void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) { +void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) { if (pLDataIterArray == NULL) { - return NULL; + return; } int32_t numOfLevel = taosArrayGetSize(pLDataIterArray); @@ -132,7 +135,6 @@ void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoa } taosArrayDestroy(pLDataIterArray); - return NULL; } // choose the unpinned slot to load next data block @@ -914,9 +916,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF } if (pLoadInfo == NULL) { - pLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); - if (pLoadInfo == NULL) { - code = terrno; + code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pLoadInfo); + if (code != TSDB_CODE_SUCCESS) { goto _end; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 272c0762d8..cf2d23cfc8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -45,7 +45,7 @@ typedef struct { bool moreThanCapcity; } SDataBlockToLoadInfo; -static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); +static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo); static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader); static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); @@ -123,7 +123,7 @@ static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_ pKey->pks[0].val = cv.value.val; } else { pKey->pks[0].nData = cv.value.nData; - memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); + (void)(void)memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); } } @@ -155,9 +155,9 @@ static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) { if (IS_VAR_DATA_TYPE(indices[i].type)) { tdata += tGetU32v(tdata, &pKey->pks[i].nData); - memcpy(pKey->pks[i].pData, tdata, pKey->pks[i].nData); + (void)memcpy(pKey->pks[i].pData, tdata, pKey->pks[i].nData); } else { - memcpy(&pKey->pks[i].val, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes); + (void)memcpy(&pKey->pks[i].val, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes); } } } @@ -185,6 +185,10 @@ static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pC if (IS_VAR_DATA_TYPE(pCols[i].type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); + if (pSupInfo->buildBuf[i] == NULL) { + tsdbError("failed to prepare memory for set columnId slot list, size:%d, code:out of memory", pCols[i].bytes); + return TSDB_CODE_OUT_OF_MEMORY; + } } else { pSupInfo->buildBuf[i] = NULL; } @@ -242,6 +246,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetA SBlockLoadSuppInfo* pInfo = &pReader->suppInfo; size_t numOfFileset = TARRAY2_SIZE(pFileSetArray); bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int32_t code = TSDB_CODE_SUCCESS; pIter->index = asc ? -1 : numOfFileset; pIter->order = pReader->info.order; @@ -251,7 +256,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetA if (pIter->pSttBlockReader == NULL) { pIter->pSttBlockReader = taosMemoryCalloc(1, sizeof(struct SSttBlockReader)); if (pIter->pSttBlockReader == NULL) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; tsdbError("failed to prepare the last block iterator, since:%s %s", tstrerror(code), pReader->idStr); return code; } @@ -265,10 +270,14 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetA pSttReader->uid = 0; tMergeTreeClose(&pSttReader->mergeTree); - initRowKey(&pSttReader->currentKey, INT64_MIN, pInfo->numOfPks, pInfo->pk.type, pInfo->pk.bytes, asc); + code = initRowKey(&pSttReader->currentKey, INT64_MIN, pInfo->numOfPks, pInfo->pk.type, pInfo->pk.bytes, asc); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed init row key, %s", pReader->idStr); + } else { + tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); + } - tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); - return TSDB_CODE_SUCCESS; + return code; } static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bool* hasNext) { @@ -286,7 +295,8 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo pIter->pSttBlockReader->uid = 0; tMergeTreeClose(&pIter->pSttBlockReader->mergeTree); - pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); + destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); + pReader->status.pLDataIterArray = NULL; pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); // check file the time range of coverage @@ -294,7 +304,7 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo while (1) { if (pReader->pFileReader != NULL) { - tsdbDataFileReaderClose(&pReader->pFileReader); + (void) tsdbDataFileReaderClose(&pReader->pFileReader); } pReader->status.pCurrentFileset = pIter->pFilesetList->data[pIter->index]; @@ -367,15 +377,20 @@ _err: bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp) { return (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type); } -void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree) { +int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree) { pIter->order = order; pIter->index = -1; pIter->numOfBlocks = 0; if (pIter->blockList == NULL) { pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); + if (pIter->blockList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { clearDataBlockIterator(pIter, needFree); } + + return TSDB_CODE_SUCCESS; } static void initReaderStatus(SReaderStatus* pStatus) { @@ -383,75 +398,85 @@ static void initReaderStatus(SReaderStatus* pStatus) { pStatus->loadFromFile = true; } -static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) { - SSDataBlock* pResBlock = createDataBlock(); - if (pResBlock == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; +static int32_t createResBlock(SQueryTableDataCond* pCond, int32_t capacity, SSDataBlock** pResBlock) { + *pResBlock = createDataBlock(); + if (*pResBlock == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < pCond->numOfCols; ++i) { SColumnInfoData colInfo = {0}; colInfo.info = pCond->colList[i]; - blockDataAppendColInfo(pResBlock, &colInfo); + int32_t code = blockDataAppendColInfo(*pResBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(*pResBlock); + *pResBlock = NULL; + return code; + } } - int32_t code = blockDataEnsureCapacity(pResBlock, capacity); + int32_t code = blockDataEnsureCapacity(*pResBlock, capacity); if (code != TSDB_CODE_SUCCESS) { - terrno = code; - taosMemoryFree(pResBlock); - return NULL; + taosMemoryFree(*pResBlock); + *pResBlock = NULL; } - return pResBlock; + + return code; } static int32_t tsdbInitReaderLock(STsdbReader* pReader) { int32_t code = taosThreadMutexInit(&pReader->readerMutex, NULL); - qTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + tsdbTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; } static int32_t tsdbUninitReaderLock(STsdbReader* pReader) { - int32_t code = -1; - qTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + int32_t code = TSDB_CODE_SUCCESS; + tsdbTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); code = taosThreadMutexDestroy(&pReader->readerMutex); - qTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + tsdbTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; } static int32_t tsdbAcquireReader(STsdbReader* pReader) { int32_t code = -1; - qTrace("tsdb/read: %p, pre-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + tsdbTrace("tsdb/read: %p, pre-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); code = taosThreadMutexLock(&pReader->readerMutex); - qTrace("tsdb/read: %p, post-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + tsdbTrace("tsdb/read: %p, post-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; } static int32_t tsdbTryAcquireReader(STsdbReader* pReader) { int32_t code = taosThreadMutexTryLock(&pReader->readerMutex); - qTrace("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); - + if (code != TSDB_CODE_SUCCESS) { + tsdbError("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + } else { + tsdbTrace("tsdb/read: %p, post-trytask read mutex: %p", pReader, &pReader->readerMutex); + } return code; } static int32_t tsdbReleaseReader(STsdbReader* pReader) { int32_t code = taosThreadMutexUnlock(&pReader->readerMutex); - qTrace("tsdb/read: %p, post-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); - + if (code != TSDB_CODE_SUCCESS) { + tsdbError("tsdb/read: %p post-untake read mutex:%p failed, code:%d", pReader, &pReader->readerMutex, code); + } else { + tsdbTrace("tsdb/read: %p, post-untake read mutex: %p", pReader, &pReader->readerMutex); + } return code; } void tsdbReleaseDataBlock2(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; if (!pStatus->composedDataBlock) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); } } @@ -459,11 +484,16 @@ static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacit SQueryTableDataCond* pCond, SBlockLoadSuppInfo* pSup) { pResBlockInfo->capacity = capacity; pResBlockInfo->pResBlock = pResBlock; - terrno = 0; + int32_t code = 0; if (pResBlockInfo->pResBlock == NULL) { pResBlockInfo->freeBlock = true; - pResBlockInfo->pResBlock = createResBlock(pCond, pResBlockInfo->capacity); + pResBlockInfo->pResBlock = NULL; + + code = createResBlock(pCond, pResBlockInfo->capacity, &pResBlockInfo->pResBlock); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (pSup->numOfPks > 0) { SSDataBlock* p = pResBlockInfo->pResBlock; @@ -490,7 +520,7 @@ static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacit pResBlockInfo->freeBlock = false; } - return terrno; + return code; } static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void** ppReader, int32_t capacity, @@ -530,7 +560,10 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void // allocate buffer in order to load data blocks from file SBlockLoadSuppInfo* pSup = &pReader->suppInfo; pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; - setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + code = setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } code = initResBlockInfo(&pReader->resBlockInfo, capacity, pResBlock, pCond, pSup); if (code != TSDB_CODE_SUCCESS) { @@ -558,8 +591,15 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void goto _end; } - tsdbInitReaderLock(pReader); - tsem_init(&pReader->resumeAfterSuspend, 0, 0); + code = tsdbInitReaderLock(pReader); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + + code = tsem_init(&pReader->resumeAfterSuspend, 0, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } *ppReader = pReader; return code; @@ -630,7 +670,11 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead break; } - taosArrayPush(pIndexList, pBrinBlk); + void* p1 = taosArrayPush(pIndexList, pBrinBlk); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + i += 1; } @@ -661,7 +705,12 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S cleanupInfoForNextFileset(pReader->status.pTableMap); initBrinRecordIter(&iter, pReader->pFileReader, pIndexList); - while (((pRecord = getNextBrinRecord(&iter)) != NULL)) { + while (1) { + int32_t code = getNextBrinRecord(&iter, &pRecord); + if (code != TSDB_CODE_SUCCESS) { + break; + } + if (pRecord->suid > pReader->info.suid) { break; } @@ -689,7 +738,11 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S ASSERT(pRecord->suid == pReader->info.suid && uid == pRecord->uid); - STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + STableBlockScanInfo* pScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // here we should find the first timestamp that is greater than the lastProcKey // the window is an open interval NOW. @@ -761,13 +814,17 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S pBlockNum->numOfBlocks += 1; if (taosArrayGetSize(pTableScanInfoList) == 0) { - taosArrayPush(pTableScanInfoList, &pScanInfo); + p1 = taosArrayPush(pTableScanInfoList, &pScanInfo); } else { STableBlockScanInfo** p = taosArrayGetLast(pTableScanInfoList); if ((*p)->uid != uid) { - taosArrayPush(pTableScanInfoList, &pScanInfo); + p1 = taosArrayPush(pTableScanInfoList, &pScanInfo); } } + + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } clearBrinBlockIter(&iter); @@ -805,12 +862,14 @@ static void updateLastKeyInfo(SRowKey* pKey, SFileDataBlockInfo* pBlockInfo, SDa } else { uint8_t* p = asc ? pBlockInfo->lastPk.pData : pBlockInfo->firstPk.pData; pKey->pks[0].nData = asc ? varDataLen(pBlockInfo->lastPk.pData) : varDataLen(pBlockInfo->firstPk.pData); - memcpy(pKey->pks[0].pData, p, pKey->pks[0].nData); + (void)memcpy(pKey->pks[0].pData, p, pKey->pks[0].nData); } } static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, SBlockLoadSuppInfo* pSup) { + int32_t code = 0; + if (IS_VAR_DATA_TYPE(pColVal->value.type)) { if (!COL_VAL_IS_VALUE(pColVal)) { colDataSetNULL(pColInfoData, rowIndex); @@ -823,31 +882,33 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int } if (pColVal->value.nData > 0) { // pData may be null, if nData is 0 - memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); + (void)memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); } - colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); + code = colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); } } else { - colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal)); + code = colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal)); } - return TSDB_CODE_SUCCESS; + return code; } -static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { +static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo) { + *pInfo = NULL; + if (pBlockIter->blockList == NULL) { - return NULL; + return TSDB_CODE_FAILED; } size_t num = TARRAY_SIZE(pBlockIter->blockList); if (num == 0) { ASSERT(pBlockIter->numOfBlocks == num); - return NULL; + return TSDB_CODE_FAILED; } - SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); - return pBlockInfo; + *pInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); + return (*pInfo) != NULL? TSDB_CODE_SUCCESS:TSDB_CODE_FAILED; } static int doBinarySearchKey(const TSKEY* keyList, int num, int pos, TSKEY key, int order) { @@ -961,10 +1022,10 @@ static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, int32_t dumpedRows, bool asc) { if (asc) { - memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); + (void)memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); } else { int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; - memcpy(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t)); + (void)memcpy(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t)); // todo: opt perf by extract the loop // reverse the array list @@ -995,7 +1056,7 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo // ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); // 1. copy data in a batch model - memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); + (void)memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); // 2. reverse the array list in case of descending order scan data block if (!asc) { @@ -1110,7 +1171,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pStatus->fileBlockData; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pBlockInfo = NULL; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; int32_t numOfOutputCols = pSupInfo->numOfCols; int32_t code = TSDB_CODE_SUCCESS; @@ -1118,6 +1179,11 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t step = asc ? 1 : -1; + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + SColVal cv = {0}; SBrinRecord tmp; blockInfoToRecord(&tmp, pBlockInfo, pSupInfo); @@ -1301,9 +1367,11 @@ static FORCE_INLINE STSchema* getTableSchemaImpl(STsdbReader* pReader, uint64_t static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData, uint64_t uid) { - int32_t code = 0; - STSchema* pSchema = pReader->info.pSchema; - int64_t st = taosGetTimestampUs(); + int32_t code = 0; + STSchema* pSchema = pReader->info.pSchema; + int64_t st = taosGetTimestampUs(); + SFileDataBlockInfo* pBlockInfo = NULL; + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; tBlockDataReset(pBlockData); @@ -1315,8 +1383,11 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI } } - SBlockLoadSuppInfo* pSup = &pReader->suppInfo; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBrinRecord tmp; @@ -1379,8 +1450,9 @@ static bool getNeighborBlockOfTable(SDataBlockIter* pBlockIter, SFileDataBlockIn static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t index, int32_t step) { + int32_t code = TSDB_CODE_SUCCESS; if (index < 0 || index >= pBlockIter->numOfBlocks) { - return -1; + return TSDB_CODE_FAILED; } SFileDataBlockInfo fblock = *(SFileDataBlockInfo*)taosArrayGet(pBlockIter->blockList, index); @@ -1391,8 +1463,12 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte for (int32_t i = index - 1; i >= pBlockIter->index; --i) { SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i); - STableBlockScanInfo* pBlockScanInfo = - getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); pTableDataBlockIdx->globalIndex = i + 1; @@ -1402,8 +1478,12 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte for (int32_t i = index + 1; i <= pBlockIter->index; ++i) { SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i); - STableBlockScanInfo* pBlockScanInfo = - getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); pTableDataBlockIdx->globalIndex = i - 1; @@ -1412,7 +1492,12 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte } taosArraySet(pBlockIter->blockList, pBlockIter->index, &fblock); - STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, fblock.uid, pReader->idStr); + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, fblock.uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, fblock.tbBlockIdx); pTableDataBlockIdx->globalIndex = pBlockIter->index; } @@ -1759,7 +1844,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (ps == NULL) { return terrno; } - tsdbRowMergerInit(pMerger, ps); + int32_t code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } SRowKey minKey = k; @@ -1792,7 +1880,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pkCompEx(&minKey, pSttKey) == 0) { @@ -1801,7 +1892,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pkCompEx(&minKey, &k) == 0) { @@ -1843,7 +1937,10 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* if (ps == NULL) { return terrno; } - tsdbRowMergerInit(pMerger, ps); + code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } bool dataInDataFile = hasDataInFileBlock(pBlockData, pDumpInfo); @@ -1890,7 +1987,10 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); code = tsdbRowMergerAdd(pMerger, pRow1, NULL); @@ -1899,7 +1999,10 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* } // pSttKey will be changed when sttBlockReader iterates to the next row, so use pKey instead. - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { @@ -1965,7 +2068,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (ps == NULL) { return terrno; } - tsdbRowMergerInit(pMerger, ps); + + code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } SRowKey minKey = k; @@ -2005,7 +2112,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pkCompEx(&minKey, pSttKey) == 0) { @@ -2015,7 +2125,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pkCompEx(&minKey, &ik) == 0) { @@ -2106,10 +2219,9 @@ static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanIn } // handle the open interval issue. Find the first row key that is greater than the given one. -static int32_t forwardDataIter(SRowKey* pKey, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { +static void forwardDataIter(SRowKey* pKey, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { doForwardDataIter(pKey, &pBlockScanInfo->iter, pBlockScanInfo, pReader); doForwardDataIter(pKey, &pBlockScanInfo->iiter, pBlockScanInfo, pReader); - return TSDB_CODE_SUCCESS; } static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { @@ -2142,7 +2254,10 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea return code; } - loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer); + code = loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (forward) { forwardDataIter(&startKey.key, pBlockScanInfo, pReader); @@ -2255,8 +2370,15 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan return false; } - initMemDataIterator(pScanInfo, pReader); - initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); + code = initMemDataIterator(pScanInfo, pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + code = initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (conf.rspRows) { pScanInfo->cleanSttBlocks = isCleanSttBlock(info.pKeyRangeList, &pReader->info.window, pScanInfo, order); @@ -2329,7 +2451,11 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc if (ps == NULL) { return terrno; } - tsdbRowMergerInit(pMerger, ps); + + code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } tRowKeyAssign(&pBlockScanInfo->lastProcKey, pKey); @@ -2345,7 +2471,11 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2392,8 +2522,16 @@ int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanIn } TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, pRow1, NULL); - doMergeRowsInSttBlock(pSttBlockReader, pScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInSttBlock(pSttBlockReader, pScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2473,7 +2611,10 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock // 2. remove it from the scan block list int32_t neighborIndex = tableDataBlockIdx->globalIndex; - setFileBlockActiveInBlockIter(pReader, pBlockIter, neighborIndex, step); + code = setFileBlockActiveInBlockIter(pReader, pBlockIter, neighborIndex, step); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // 3. load the neighbor block, and set it to be the currently accessed file data block code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pBlockInfo->uid); @@ -2496,8 +2637,8 @@ void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInf pResBlock->info.dataLoad = 1; pResBlock->info.version = pReader->info.verRange.maxVer; - blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); - blockDataUpdatePkRange(pResBlock, pReader->suppInfo.pkDstSlot, ASCENDING_TRAVERSE(pReader->info.order)); + int32_t code = blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); + code = blockDataUpdatePkRange(pResBlock, pReader->suppInfo.pkDstSlot, ASCENDING_TRAVERSE(pReader->info.order)); setComposedBlockFlag(pReader, true); pReader->cost.composedBlocks += 1; @@ -2511,11 +2652,12 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { int32_t step = asc ? 1 : -1; double el = 0; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + SFileDataBlockInfo* pBlockInfo = NULL; SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - if (pBlockInfo == NULL) { + code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { return 0; } @@ -2524,9 +2666,9 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { return code; } - STableBlockScanInfo* pBlockScanInfo = - getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); - if (pBlockScanInfo == NULL) { + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -2544,7 +2686,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } SBlockData* pBlockData = &pReader->status.fileBlockData; - initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader); + (void) initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader); while (1) { bool hasBlockData = false; @@ -2559,7 +2701,10 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pDumpInfo->rowIndex += step; if (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0) { - pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); // NOTE: get the new block info + code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo); // NOTE: get the new block info + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } // continue check for the next file block if the last ts in the current block // is overlapped with the next neighbor block @@ -2638,7 +2783,10 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t orde if (pSource == NULL) { pSource = pBlockScanInfo->pMemDelData; } else { - taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData); + void* p1 = taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } code = tsdbBuildDeleteSkyline(pSource, 0, taosArrayGetSize(pSource) - 1, pBlockScanInfo->delSkyline); @@ -2736,6 +2884,9 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBrinBlk)); + if (pIndexList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } while (1) { // only check here, since the iterate data in memory is very fast. @@ -2784,6 +2935,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr return loadDataFileTombDataForAll(pReader); } +// pTableIter can be NULL, no need to handle the return value static void resetTableListIndex(SReaderStatus* pStatus) { STableUidList* pList = &pStatus->uidList; @@ -2825,7 +2977,7 @@ static bool moveToNextTableForPreFileSetMem(SReaderStatus* pStatus) { return (pStatus->pProcMemTableIter != NULL); } -static void buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo) { +static int32_t buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo) { SReaderStatus* pStatus = &pReader->status; SSttBlockReader* pSttBlockReader = pStatus->fileIter.pSttBlockReader; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; @@ -2833,7 +2985,10 @@ static void buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInf bool asc = ASCENDING_TRAVERSE(pReader->info.order); SDataBlockInfo* pInfo = &pResBlock->info; - blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt); + int32_t code = blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pInfo->rows = pScanInfo->numOfRowsInStt; pInfo->id.uid = pScanInfo->uid; @@ -2858,6 +3013,7 @@ static void buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInf tsdbDebug("%p uid:%" PRId64 " return clean stt block as one, brange:%" PRId64 "-%" PRId64 " rows:%" PRId64 " %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, pReader->idStr); + return code; } static void buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, @@ -2878,8 +3034,8 @@ static void buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanIn pInfo->pks[0].val = pBlockInfo->firstPk.val; pInfo->pks[1].val = pBlockInfo->lastPk.val; } else { - memcpy(pInfo->pks[0].pData, varDataVal(pBlockInfo->firstPk.pData), varDataLen(pBlockInfo->firstPk.pData)); - memcpy(pInfo->pks[1].pData, varDataVal(pBlockInfo->lastPk.pData), varDataLen(pBlockInfo->lastPk.pData)); + (void)memcpy(pInfo->pks[0].pData, varDataVal(pBlockInfo->firstPk.pData), varDataLen(pBlockInfo->firstPk.pData)); + (void)memcpy(pInfo->pks[1].pData, varDataVal(pBlockInfo->lastPk.pData), varDataLen(pBlockInfo->lastPk.pData)); pInfo->pks[0].nData = varDataLen(pBlockInfo->firstPk.pData); pInfo->pks[1].nData = varDataLen(pBlockInfo->lastPk.pData); @@ -2950,8 +3106,8 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { // if only require the total rows, no need to load data from stt file if it is clean stt blocks if (pReader->info.execMode == READER_EXEC_ROWS && pScanInfo->cleanSttBlocks) { - buildCleanBlockFromSttFiles(pReader, pScanInfo); - return TSDB_CODE_SUCCESS; + code = buildCleanBlockFromSttFiles(pReader, pScanInfo); + return code; } int64_t st = taosGetTimestampUs(); @@ -3006,11 +3162,16 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; STableBlockScanInfo* pScanInfo = NULL; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pBlockInfo = NULL; SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t code = TSDB_CODE_SUCCESS; + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); return code; @@ -3020,13 +3181,13 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return pReader->code; } - pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); - if (pScanInfo == NULL) { - return terrno; + code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; } if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) { - initSttBlockReader(pSttBlockReader, pScanInfo, pReader); + (void) initSttBlockReader(pSttBlockReader, pScanInfo, pReader); } TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); @@ -3067,8 +3228,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // return the stt file block ASSERT(pReader->info.execMode == READER_EXEC_ROWS && pSttBlockReader->mergeTree.pIter == NULL); - buildCleanBlockFromSttFiles(pReader, pScanInfo); - return TSDB_CODE_SUCCESS; + code = buildCleanBlockFromSttFiles(pReader, pScanInfo); + return code; } } else { SBlockData* pBData = &pReader->status.fileBlockData; @@ -3080,7 +3241,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { int64_t st = taosGetTimestampUs(); // let's load data from stt files, make sure clear the cleanStt block flag before load the data from stt files - initSttBlockReader(pSttBlockReader, pScanInfo, pReader); + (void) initSttBlockReader(pSttBlockReader, pScanInfo, pReader); // no data in stt block, no need to proceed. while (hasDataInSttBlock(pScanInfo)) { @@ -3141,10 +3302,17 @@ static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_ continue; } - initMemDataIterator(*pBlockScanInfo, pReader); - initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + int32_t code = initMemDataIterator(*pBlockScanInfo, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); + code = initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3183,10 +3351,17 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en continue; } - initMemDataIterator(*pBlockScanInfo, pReader); - initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + int32_t code = initMemDataIterator(*pBlockScanInfo, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); + code = initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3205,12 +3380,12 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en // set the correct start position in case of the first/last file block, according to the query time window static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { - int64_t lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? INT64_MIN : INT64_MAX; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pBlockInfo = NULL; SReaderStatus* pStatus = &pReader->status; SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; - if (pBlockInfo) { + int32_t code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code == TSDB_CODE_SUCCESS) { pDumpInfo->totalRows = pBlockInfo->numRow; pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1; } else { @@ -3243,7 +3418,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks, pTableList); } else { // no block data, only last block exists tBlockDataReset(&pReader->status.fileBlockData); - resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); resetTableListIndex(&pReader->status); } @@ -3353,7 +3528,10 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } tBlockDataReset(pBlockData); - resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } resetTableListIndex(&pReader->status); ERetrieveType type = doReadDataFromSttFiles(pReader); @@ -3622,6 +3800,7 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, STsdbReader* pReader) { SRowMerger* pMerger = &pReader->status.merger; + int32_t code = 0; while (1) { pIter->hasVal = tsdbTbDataIterNext(pIter->iter); @@ -3656,10 +3835,13 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArra } } - tsdbRowMergerAdd(pMerger, pRow, pTSchema); + code = tsdbRowMergerAdd(pMerger, pRow, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } - return TSDB_CODE_SUCCESS; + return code; } static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowIndex, SRowKey* pKey, SRowMerger* pMerger, @@ -3677,7 +3859,7 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowInd } TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - tsdbRowMergerAdd(pMerger, &fRow, NULL); + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, NULL); rowIndex += step; } @@ -3719,6 +3901,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t step = asc ? 1 : -1; SVersionRange* pRange = &pReader->info.verRange; + int32_t code = 0; pDumpInfo->rowIndex += step; if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) { @@ -3730,25 +3913,31 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc while (1) { CHECK_FILEBLOCK_STATE st; - SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + SFileDataBlockInfo* pFileBlockInfo = NULL; + code = getCurrentBlockInfo(&pReader->status.blockIter, &pFileBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pFileBlockInfo == NULL) { st = CHECK_FILEBLOCK_QUIT; break; } - checkForNeighborFileBlock(pReader, pScanInfo, pFileBlockInfo, pMerger, pKey, &st); - if (st == CHECK_FILEBLOCK_QUIT) { + code = checkForNeighborFileBlock(pReader, pScanInfo, pFileBlockInfo, pMerger, pKey, &st); + if (st == CHECK_FILEBLOCK_QUIT || code != TSDB_CODE_SUCCESS) { break; } } } - return TSDB_CODE_SUCCESS; + return code; } int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, SRowMerger* pMerger, int32_t pkSrcSlot, SVersionRange* pVerRange, const char* idStr) { SRowKey* pRowKey = &pScanInfo->lastProcKey; + int32_t code = TSDB_CODE_SUCCESS; while (nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pkSrcSlot, pVerRange)) { SRowKey* pNextKey = getCurrentKeyInSttBlock(pSttBlockReader); @@ -3756,7 +3945,10 @@ int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanI int32_t ret = pkCompEx(pRowKey, pNextKey); if (ret == 0) { TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, pRow1, NULL); + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + break; + } } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), @@ -3765,7 +3957,7 @@ int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanI } } - return TSDB_CODE_SUCCESS; + return code; } int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIterInfo* pIter, SArray* pDelList, @@ -4024,7 +4216,11 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT if (colId == pSchema->columns[j].colId) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - tRowGet(pTSRow, pSchema, j, &colVal); + code = tRowGet(pTSRow, pSchema, j, &colVal); + if (code) { + return code; + } + code = doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); if (code) { return code; @@ -4109,7 +4305,11 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e do { TSDBROW row = {.type = -1}; bool freeTSRow = false; - tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); + code = tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (row.type == -1) { break; } @@ -4155,6 +4355,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e // TODO refactor: with createDataBlockScanInfo int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t num) { + int32_t code = TSDB_CODE_SUCCESS; int32_t size = tSimpleHashGetSize(pReader->status.pTableMap); STableBlockScanInfo** p = NULL; @@ -4165,7 +4366,7 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t } if (size < num) { - int32_t code = ensureBlockScanInfoBuf(&pReader->blockInfoBuf, num); + code = ensureBlockScanInfoBuf(&pReader->blockInfoBuf, num); if (code) { return code; } @@ -4186,42 +4387,43 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t for (int32_t i = 0; i < num; ++i) { pUidList->tableUidList[i] = pList[i].uid; - STableBlockScanInfo* pInfo = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i); - initTableBlockScanInfo(pInfo, pList[i].uid, pReader->status.pTableMap, pReader); + STableBlockScanInfo* pInfo = NULL; + code = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = initTableBlockScanInfo(pInfo, pList[i].uid, pReader->status.pTableMap, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } return TDB_CODE_SUCCESS; } -void* tsdbGetIdx2(SMeta* pMeta) { - if (pMeta == NULL) { - return NULL; - } - return metaGetIdx(pMeta); -} - -void* tsdbGetIvtIdx2(SMeta* pMeta) { - if (pMeta == NULL) { - return NULL; - } - return metaGetIvtIdx(pMeta); -} - uint64_t tsdbGetReaderMaxVersion2(STsdbReader* pReader) { return pReader->info.verRange.maxVer; } static int32_t doOpenReaderImpl(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; + int32_t code = TSDB_CODE_SUCCESS; if (pReader->bFilesetDelimited) { getMemTableTimeRange(pReader, &pReader->status.memTableMaxKey, &pReader->status.memTableMinKey); pReader->status.bProcMemFirstFileset = true; } - initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); - resetDataBlockIterator(&pStatus->blockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + code = initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = resetDataBlockIterator(&pStatus->blockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - int32_t code = TSDB_CODE_SUCCESS; if (pStatus->fileIter.numOfFiles == 0) { pStatus->loadFromFile = false; } else { @@ -4248,7 +4450,7 @@ static void clearSharedPtr(STsdbReader* p) { p->pSchemaMap = NULL; } -static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { +static int32_t setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { pDst->status.pTableMap = pSrc->status.pTableMap; pDst->status.uidList = pSrc->status.uidList; pDst->info.pSchema = pSrc->info.pSchema; @@ -4257,8 +4459,10 @@ static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { pDst->pReadSnap->pfSetArray = pSrc->pReadSnap->pfSetArray; if (pDst->info.pSchema) { - tsdbRowMergerInit(&pDst->status.merger, pDst->info.pSchema); + return tsdbRowMergerInit(&pDst->status.merger, pDst->info.pSchema); } + + return TSDB_CODE_SUCCESS; } // ====================================== EXPOSED APIs ====================================== @@ -4266,13 +4470,17 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables) { STimeWindow window = pCond->twindows; SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); + int32_t code = 0; int32_t capacity = pConf->tsdbCfg.maxRows; if (pResBlock != NULL) { - blockDataEnsureCapacity(pResBlock, capacity); + code = blockDataEnsureCapacity(pResBlock, capacity); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } } - int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); + code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -4335,7 +4543,10 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi } if (pReader->info.pSchema != NULL) { - tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); + code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); @@ -4354,11 +4565,10 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi } STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader; - pReader->status.pTableMap = - createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables); - if (pReader->status.pTableMap == NULL) { + + code = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables, &pReader->status.pTableMap); + if (code != TSDB_CODE_SUCCESS) { *ppReader = NULL; - code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -4391,7 +4601,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { return; } - tsdbAcquireReader(pReader); + (void)tsdbAcquireReader(pReader); { if (pReader->innerReader[0] != NULL || pReader->innerReader[1] != NULL) { @@ -4430,7 +4640,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { clearBlockScanInfoBuf(&pReader->blockInfoBuf); if (pReader->pFileReader != NULL) { - tsdbDataFileReaderClose(&pReader->pFileReader); + (void) tsdbDataFileReaderClose(&pReader->pFileReader); } SReadCostSummary* pCost = &pReader->cost; @@ -4446,15 +4656,15 @@ void tsdbReaderClose2(STsdbReader* pReader) { destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); taosMemoryFreeClear(pReader->status.uidList.tableUidList); - qTrace("tsdb/reader-close: %p, untake snapshot", pReader); + tsdbTrace("tsdb/reader-close: %p, untake snapshot", pReader); void* p = pReader->pReadSnap; if ((p == atomic_val_compare_exchange_ptr((void**)&pReader->pReadSnap, p, NULL)) && (p != NULL)) { tsdbUntakeReadSnap2(pReader, p, true); } - tsem_destroy(&pReader->resumeAfterSuspend); - tsdbReleaseReader(pReader); - tsdbUninitReaderLock(pReader); + (void) tsem_destroy(&pReader->resumeAfterSuspend); + (void) tsdbReleaseReader(pReader); + (void) tsdbUninitReaderLock(pReader); tsdbDebug( "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 @@ -4483,11 +4693,14 @@ static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) { SReaderStatus* pStatus = &pCurrentReader->status; if (pStatus->loadFromFile) { - tsdbDataFileReaderClose(&pCurrentReader->pFileReader); + (void) tsdbDataFileReaderClose(&pCurrentReader->pFileReader); SReadCostSummary* pCost = &pCurrentReader->cost; - pStatus->pLDataIterArray = destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost); + destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost); pStatus->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + if (pStatus->pLDataIterArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } // resetDataBlockScanInfo excluding lastKey @@ -4515,14 +4728,14 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { - doSuspendCurrentReader(pReader->innerReader[0]); + code = doSuspendCurrentReader(pReader->innerReader[0]); } else if (pReader->step == EXTERNAL_ROWS_MAIN) { - doSuspendCurrentReader(pReader); + code = doSuspendCurrentReader(pReader); } else { - doSuspendCurrentReader(pReader->innerReader[1]); + code = doSuspendCurrentReader(pReader->innerReader[1]); } } else { - doSuspendCurrentReader(pReader); + code = doSuspendCurrentReader(pReader); } // make sure only release once @@ -4557,13 +4770,12 @@ static int32_t tsdbSetQueryReseek(void* pQHandle) { code = tsdbTryAcquireReader(pReader); if (code == 0) { if (pReader->flag == READER_STATUS_SUSPEND) { - tsdbReleaseReader(pReader); + code = tsdbReleaseReader(pReader); return code; } - tsdbReaderSuspend2(pReader); - tsdbReleaseReader(pReader); - + code = tsdbReaderSuspend2(pReader); + (void) tsdbReleaseReader(pReader); return code; } else if (code == EBUSY) { return TSDB_CODE_VND_QUERY_BUSY; @@ -4580,7 +4792,7 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { // restore reader's state, task snapshot int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (numOfTables > 0) { - qTrace("tsdb/reader: %p, take snapshot", pReader); + tsdbTrace("tsdb/reader: %p, take snapshot", pReader); code = tsdbTakeReadSnap2(pReader, tsdbSetQueryReseek, &pReader->pReadSnap); if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -4601,10 +4813,16 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { // we need only one row pPrevReader->resBlockInfo.capacity = 1; - setSharedPtr(pPrevReader, pReader); + code = setSharedPtr(pPrevReader, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pNextReader->resBlockInfo.capacity = 1; - setSharedPtr(pNextReader, pReader); + code = setSharedPtr(pNextReader, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (pReader->step == 0 || pReader->step == EXTERNAL_ROWS_PREV) { code = doOpenReaderImpl(pPrevReader); @@ -4773,12 +4991,17 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { #endif code = tsdbAcquireReader(pReader); - qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + // release reader failure should be suppressed here, to avoid over-write the original error code + (void) tsdbReleaseReader(pReader); return code; } } @@ -4786,7 +5009,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (pReader->innerReader[0] != NULL && pReader->step == 0) { code = doTsdbNextDataBlock2(pReader->innerReader[0], hasNext); if (code) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } @@ -4794,8 +5017,8 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (*hasNext) { pStatus = &pReader->innerReader[0]->status; if (pStatus->composedDataBlock) { - qTrace("tsdb/read: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + code = tsdbReleaseReader(pReader); } return code; @@ -4817,14 +5040,14 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { code = doTsdbNextDataBlock2(pReader, hasNext); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } if (*hasNext) { if (pStatus->composedDataBlock) { - qTrace("tsdb/read: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + code = tsdbReleaseReader(pReader); } return code; } @@ -4840,7 +5063,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { code = doTsdbNextDataBlock2(pReader->innerReader[1], hasNext); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } @@ -4848,17 +5071,16 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (*hasNext) { pStatus = &pReader->innerReader[1]->status; if (pStatus->composedDataBlock) { - qTrace("tsdb/read: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + code = tsdbReleaseReader(pReader); } return code; } } - qTrace("tsdb/read: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); - + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + code = tsdbReleaseReader(pReader); return code; } @@ -4910,9 +5132,9 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ } int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) { - SColumnDataAgg** pBlockSMA = &pDataBlock->pBlockAgg; - - int32_t code = 0; + SColumnDataAgg** pBlockSMA = &pDataBlock->pBlockAgg; + SFileDataBlockInfo* pBlockInfo = NULL; + int32_t code = 0; *allHave = false; *pBlockSMA = NULL; @@ -4925,7 +5147,11 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, return TSDB_CODE_SUCCESS; } - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; @@ -5005,38 +5231,45 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, return code; } -static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { +static int32_t doRetrieveDataBlock(STsdbReader* pReader, SSDataBlock** pBlock) { SReaderStatus* pStatus = &pReader->status; int32_t code = TSDB_CODE_SUCCESS; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); + SFileDataBlockInfo* pBlockInfo = NULL; + *pBlock = NULL; - if (pReader->code != TSDB_CODE_SUCCESS) { - return NULL; + code = getCurrentBlockInfo(&pStatus->blockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; } - STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); - if (pBlockScanInfo == NULL) { - return NULL; + if (pReader->code != TSDB_CODE_SUCCESS) { + return pReader->code; + } + + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; } code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { tBlockDataReset(&pStatus->fileBlockData); - terrno = code; - return NULL; + return code; } code = copyBlockDataToSDataBlock(pReader, &pBlockScanInfo->lastProcKey); if (code != TSDB_CODE_SUCCESS) { tBlockDataReset(&pStatus->fileBlockData); - terrno = code; - return NULL; } - return pReader->resBlockInfo.pResBlock; + *pBlock = pReader->resBlockInfo.pResBlock; + return code; } -SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { +int32_t tsdbRetrieveDataBlock2(STsdbReader* pReader, SSDataBlock** pBlock, SArray* pIdList) { + *pBlock = NULL; + STsdbReader* pTReader = pReader; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { @@ -5050,39 +5283,40 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { if (pStatus->composedDataBlock || pReader->info.execMode == READER_EXEC_ROWS) { // tsdbReaderSuspend2(pReader); // tsdbReaderResume2(pReader); - - return pTReader->resBlockInfo.pResBlock; + *pBlock = pTReader->resBlockInfo.pResBlock; + return TSDB_CODE_SUCCESS; } - SSDataBlock* ret = doRetrieveDataBlock(pTReader); + int32_t code = doRetrieveDataBlock(pTReader, pBlock); - qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); + tsdbTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); + (void) tsdbReleaseReader(pReader); // tsdbReaderSuspend2(pReader); // tsdbReaderResume2(pReader); - - return ret; + return code; } int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { int32_t code = TSDB_CODE_SUCCESS; - qTrace("tsdb/reader-reset: %p, take read mutex", pReader); - tsdbAcquireReader(pReader); + tsdbTrace("tsdb/reader-reset: %p, take read mutex", pReader); + code = tsdbAcquireReader(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } } if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->pReadSnap == NULL) { tsdbDebug("tsdb reader reset return %p, %s", pReader->pReadSnap, pReader->idStr); - tsdbReleaseReader(pReader); - return TSDB_CODE_SUCCESS; + return tsdbReleaseReader(pReader); } SReaderStatus* pStatus = &pReader->status; @@ -5098,12 +5332,21 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg)); pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; - tsdbDataFileReaderClose(&pReader->pFileReader); + (void) tsdbDataFileReaderClose(&pReader->pFileReader); int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap); - initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); - resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + code = initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + if (code != TSDB_CODE_SUCCESS) { + (void) tsdbReleaseReader(pReader); + return code; + } + resetTableListIndex(&pReader->status); bool asc = ASCENDING_TRAVERSE(pReader->info.order); @@ -5126,7 +5369,7 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { tsdbError("%p reset reader failed, numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", pReader, numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } } @@ -5136,8 +5379,7 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { pReader, pReader->info.suid, numOfTables, pCond->twindows.skey, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); - tsdbReleaseReader(pReader); - + code = tsdbReleaseReader(pReader); return code; } @@ -5161,12 +5403,15 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT pTableBlockInfo->numOfVgroups = 1; // find the start data block in file - tsdbAcquireReader(pReader); + code = tsdbAcquireReader(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); - return code; + return tsdbReleaseReader(pReader); } } @@ -5206,8 +5451,13 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT while (true) { if (hasNext) { - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); - int32_t numOfRows = pBlockInfo->numRow; + SFileDataBlockInfo* pBlockInfo = NULL; + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + break; + } + + int32_t numOfRows = pBlockInfo->numRow; pTableBlockInfo->totalRows += numOfRows; @@ -5243,7 +5493,7 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT } // record the data in stt files - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } @@ -5300,11 +5550,15 @@ int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { int64_t rows = 0; SReaderStatus* pStatus = &pReader->status; - tsdbAcquireReader(pReader); + code = tsdbAcquireReader(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } } @@ -5335,7 +5589,7 @@ int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, pStatus->pTableIter, &iter); } - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return rows; } @@ -5382,12 +5636,16 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs SVersionRange* pRange = &pReader->info.verRange; // lock - taosThreadMutexLock(&pTsdb->mutex); + code = taosThreadMutexLock(&pTsdb->mutex); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed to lock tsdb, code:%s", tstrerror(code)); + return code; + } // alloc STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap)); if (pSnap == NULL) { - taosThreadMutexUnlock(&pTsdb->mutex); + (void) taosThreadMutexUnlock(&pTsdb->mutex); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -5397,7 +5655,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pMem = pTsdb->mem; pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode)); if (pSnap->pNode == NULL) { - taosThreadMutexUnlock(&pTsdb->mutex); + (void) taosThreadMutexUnlock(&pTsdb->mutex); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -5405,14 +5663,14 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pNode->pQHandle = pReader; pSnap->pNode->reseek = reseek; - tsdbRefMemTable(pTsdb->mem, pSnap->pNode); + (void)tsdbRefMemTable(pTsdb->mem, pSnap->pNode); } if (pTsdb->imem && (pRange->minVer <= pTsdb->imem->maxVer && pRange->maxVer >= pTsdb->imem->minVer)) { pSnap->pIMem = pTsdb->imem; pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode)); if (pSnap->pINode == NULL) { - taosThreadMutexUnlock(&pTsdb->mutex); + (void) taosThreadMutexUnlock(&pTsdb->mutex); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -5420,14 +5678,14 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pINode->pQHandle = pReader; pSnap->pINode->reseek = reseek; - tsdbRefMemTable(pTsdb->imem, pSnap->pINode); + (void)tsdbRefMemTable(pTsdb->imem, pSnap->pINode); } // fs code = tsdbFSCreateRefSnapshotWithoutLock(pTsdb->pFS, &pSnap->pfSetArray); // unlock - taosThreadMutexUnlock(&pTsdb->mutex); + (void) taosThreadMutexUnlock(&pTsdb->mutex); if (code == TSDB_CODE_SUCCESS) { tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); @@ -5455,17 +5713,17 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact if (pSnap) { if (pSnap->pMem) { - tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive); + (void) tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive); } if (pSnap->pIMem) { - tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); + (void) tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); } if (pSnap->pNode) taosMemoryFree(pSnap->pNode); if (pSnap->pINode) taosMemoryFree(pSnap->pINode); - tsdbFSDestroyRefSnapshot(&pSnap->pfSetArray); + (void) tsdbFSDestroyRefSnapshot(&pSnap->pfSetArray); taosMemoryFree(pSnap); } @@ -5476,6 +5734,9 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { taosMemoryFreeClear(pReader->idStr); pReader->idStr = taosStrdup(idstr); + if (pReader->idStr == NULL) { + // no need to do anything + } pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 4bd9bee050..84f235ab25 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -112,22 +112,28 @@ void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) { taosArrayDestroy(pBuf->pData); } -void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) { +int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pInfo) { + *pInfo = NULL; + int32_t bucketIndex = index / pBuf->numPerBucket; char** pBucket = taosArrayGet(pBuf->pData, bucketIndex); - return (*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo); -} - -STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id) { - STableBlockScanInfo** p = tSimpleHashGet(pTableMap, &uid, sizeof(uid)); - if (p == NULL || *p == NULL) { - terrno = TSDB_CODE_INVALID_PARA; - int32_t size = tSimpleHashGetSize(pTableMap); - tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); - return NULL; + if (pBucket == NULL) { + return TSDB_CODE_FAILED; } - return *p; + *pInfo = (STableBlockScanInfo*)((*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo)); + return TSDB_CODE_SUCCESS; +} + +int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) { + *pInfo = *(STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid)); + if (pInfo == NULL) { + int32_t size = tSimpleHashGetSize(pTableMap); + tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); + return TSDB_CODE_INVALID_PARA; + } + + return TSDB_CODE_SUCCESS; } int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) { @@ -146,17 +152,17 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in } case TSDB_DATA_TYPE_INT:{ int32_t min = INT32_MIN; - memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); + (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); break; } case TSDB_DATA_TYPE_SMALLINT:{ int16_t min = INT16_MIN; - memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); + (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); break; } case TSDB_DATA_TYPE_TINYINT:{ int8_t min = INT8_MIN; - memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); + (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); break; } case TSDB_DATA_TYPE_UTINYINT: @@ -245,20 +251,27 @@ int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSH initLastProcKey(pScanInfo, pReader); pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; - tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); + int32_t code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid, pScanInfo->lastProcKey.ts, pReader->idStr); - return TSDB_CODE_SUCCESS; + return code; } // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model -SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, - STableUidList* pUidList, int32_t numOfTables) { +int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj) { + int32_t code = 0; + *pHashObj = NULL; + // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); if (pTableMap == NULL) { - return NULL; + return terrno; } int64_t st = taosGetTimestampUs(); @@ -267,7 +280,7 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); if (pUidList->tableUidList == NULL) { tSimpleHashCleanup(pTableMap); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pUidList->currentIndex = 0; @@ -275,8 +288,16 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf for (int32_t j = 0; j < numOfTables; ++j) { pUidList->tableUidList[j] = idList[j].uid; - STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(pBuf, j); - initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader); + STableBlockScanInfo* pScanInfo = NULL; + code = getPosInBlockInfoBuf(pBuf, j, &pScanInfo); + if (code != TSDB_CODE_SUCCESS) { + break; + } + + code = initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader); + if (code != TSDB_CODE_SUCCESS) { + break; + } } taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc); @@ -286,7 +307,8 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList, pTsdbReader->idStr); - return pTableMap; + *pHashObj = pTableMap; + return code; } void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { @@ -391,11 +413,13 @@ void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray pIter->pBrinBlockList = pList; } -SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { +int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) { + *pRecord = NULL; + if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) { pIter->blockIndex += 1; if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) { - return NULL; + return TSDB_CODE_FAILED; } pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); @@ -404,7 +428,7 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code)); - return NULL; + return TSDB_CODE_FAILED; } pIter->recordIndex = -1; @@ -412,7 +436,9 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { pIter->recordIndex += 1; tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record); - return &pIter->record; + *pRecord = &pIter->record; + + return TSDB_CODE_SUCCESS; } void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); } @@ -670,7 +696,13 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ STombRecord record = {0}; uint64_t uid = pReader->status.uidList.tableUidList[*j]; - STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + STableBlockScanInfo* pScanInfo = NULL; + + code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pScanInfo->pFileDelData == NULL) { pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); } @@ -702,7 +734,11 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ } uid = pReader->status.uidList.tableUidList[*j]; - pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pScanInfo->pFileDelData == NULL) { pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); } @@ -806,9 +842,12 @@ int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileRead return doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false); } -void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) { +int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) { if (*ppMemDelData == NULL) { *ppMemDelData = taosArrayInit(4, sizeof(SDelData)); + if (*ppMemDelData == NULL) { + return TSDB_CODE_SUCCESS; + } } SArray* pMemDelData = *ppMemDelData; @@ -836,6 +875,8 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT p = p->pNext; } } + + return TSDB_CODE_SUCCESS; } int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo, @@ -960,6 +1001,7 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf, const char* pstr) { int32_t numOfRows = 0; + int32_t code = 0; // no data exists, go to end int32_t numOfLevels = pFileSet->lvlArr->size; @@ -984,7 +1026,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra conf.file[0] = *pSttLevel->fobjArr->data[i]->f; const char* pName = pSttLevel->fobjArr->data[i]->fname; - int32_t code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader); + code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader); if (code != TSDB_CODE_SUCCESS) { tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr); continue; @@ -992,8 +1034,8 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra } if (pIter->pBlockLoadInfo == NULL) { - pIter->pBlockLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); - if (pIter->pBlockLoadInfo == NULL) { + code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pIter->pBlockLoadInfo); + if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to create block load info, code: out of memory, %s", pstr); continue; } @@ -1001,7 +1043,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file TStatisBlkArray* pStatisBlkArray = NULL; - int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray); + code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray); if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr); continue; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 6c76f8c372..45a2384b65 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -315,32 +315,31 @@ typedef struct SBrinRecordIter { } SBrinRecordIter; int32_t uidComparFunc(const void* p1, const void* p2); +int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id); -STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id); - -SSHashObj* createDataBlockScanInfo(STsdbReader* pReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, - STableUidList* pUidList, int32_t numOfTables); -int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap, - STsdbReader* pReader); -void clearBlockScanInfo(STableBlockScanInfo* p); -void destroyAllBlockScanInfo(SSHashObj* pTableMap); -void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step); -void cleanupInfoForNextFileset(SSHashObj* pTableMap); -int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables); -void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf); -void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index); +int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj); +int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap, + STsdbReader* pReader); +void clearBlockScanInfo(STableBlockScanInfo* p); +void destroyAllBlockScanInfo(SSHashObj* pTableMap); +void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step); +void cleanupInfoForNextFileset(SSHashObj* pTableMap); +int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables); +void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf); +int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pRes); // brin records iterator -void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList); -SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter); -void clearBrinBlockIter(SBrinRecordIter* pIter); +void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList); +int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord); +void clearBrinBlockIter(SBrinRecordIter* pIter); // initialize block iterator API int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList); bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr); // load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time) -void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver); +int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver); int32_t loadDataFileTombDataForAll(STsdbReader* pReader); int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo); int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo, @@ -358,10 +357,10 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2); int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc); void clearRowKey(SRowKey* pKey); -bool shouldFreePkBuf(SBlockLoadSuppInfo *pSupp); -void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk); -void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree); -void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk); +bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp); +int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk); +void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree); +void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk); typedef struct { SArray* pTombData; diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 9547fa12f3..6db3c780e2 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -929,7 +929,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, i SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore, SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr, SStorageAPI* pApi, int32_t tsIndex); -void initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, +int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic); void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins); void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); diff --git a/source/libs/executor/inc/streamexecutorInt.h b/source/libs/executor/inc/streamexecutorInt.h index 8b61b93601..c6053cc96e 100644 --- a/source/libs/executor/inc/streamexecutorInt.h +++ b/source/libs/executor/inc/streamexecutorInt.h @@ -20,6 +20,7 @@ extern "C" { #endif #include "executorInt.h" +#include "tutil.h" void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type); bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index f0fddc28fc..c7236a88e8 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -17,6 +17,7 @@ #include "function.h" #include "os.h" #include "tname.h" +#include "tutil.h" #include "tdatablock.h" #include "tmsg.h" @@ -1357,11 +1358,13 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosMemoryFreeClear(param); } -void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { +int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI; if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - return; + return code; } SStreamScanInfo* pScanInfo = downstream->info; @@ -1369,8 +1372,9 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup pScanInfo->pPartScalarSup = pExpr; pScanInfo->pPartTbnameSup = pTbnameExpr; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); + code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); } + return code; } SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) { @@ -1421,6 +1425,7 @@ void freePartItem(void* ptr) { SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1511,14 +1516,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); - initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); + code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); + TSDB_CHECK_CODE(code, lino, _error); + return pOperator; _error: pTaskInfo->code = code; destroyStreamPartitionOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ce4915ca4d..678d8d569e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -388,9 +388,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SSDataBlock* p = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); - if (p == NULL) { - return terrno; + SSDataBlock* p = NULL; + int32_t code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, &p, NULL); + if (p == NULL || code != TSDB_CODE_SUCCESS) { + return code; } ASSERT(p == pBlock); @@ -1351,7 +1352,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } if (hasNext) { - /*SSDataBlock* p = */ pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, NULL); + SSDataBlock* p = NULL; + code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, &p, NULL); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } @@ -2477,27 +2483,54 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl } } -int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { - int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); - len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup); - *pBuff = taosMemoryCalloc(1, len); - void* buf = *pBuff; - encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup); - pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo); - return len; +int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t len = 0; + code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len); + TSDB_CHECK_CODE(code, lino, _end); + + len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup); + *pBuff = taosMemoryCalloc(1, len); + if (!(*pBuff)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + void* buf = *pBuff; + encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup); + int32_t tmp = 0; + code = pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo, &tmp); + TSDB_CHECK_CODE(code, lino, _end); + + *pLen = len; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (!pInfo->pState) { return; } if (needSaveStreamOperatorInfo(&pInfo->basic)) { void* pBuf = NULL; - int32_t len = streamScanOperatorEncode(pInfo, &pBuf); + int32_t len = 0; + code = streamScanOperatorEncode(pInfo, &pBuf, &len); + TSDB_CHECK_CODE(code, lino, _end); pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len); taosMemoryFree(pBuf); saveStreamOperatorStateComplete(&pInfo->basic); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } // other properties are recovered from the execution plan @@ -2929,8 +2962,9 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } - SSDataBlock* pBlock = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, NULL); - if (pBlock == NULL) { + SSDataBlock* pBlock = NULL; + code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, &pBlock, NULL); + if (pBlock == NULL || code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, terrno); } @@ -3094,17 +3128,35 @@ static void destroyStreamScanOperatorInfo(void* param) { } void streamScanReleaseState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamScanInfo* pInfo = pOperator->info; + void* pBuff = NULL; if (!pInfo->pState) { return; } if (!pInfo->pUpdateInfo) { return; } - int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); - void* pBuff = taosMemoryCalloc(1, len); - pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo); + int32_t len = 0; + code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len); + TSDB_CHECK_CODE(code, lino, _end); + + pBuff = taosMemoryCalloc(1, len); + if (!pBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; +TSDB_CHECK_CODE(code, lino, _end); + } + + int32_t tmp = 0; + code = pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo, &tmp); + TSDB_CHECK_CODE(code, lino, _end); + pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), pBuff, len); +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } taosMemoryFree(pBuff); } diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 857d048457..14f2dfbada 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -68,9 +68,7 @@ void destroyStreamCountAggOperatorInfo(void* param) { taosMemoryFreeClear(param); } -bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { - return pAggSup->windowCount != pAggSup->windowSliding; -} +bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; } void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin, SBuffInfo* pBuffInfo) { @@ -89,13 +87,14 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, ASSERT(pBuffInfo->pCur); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); + (void**)&pCurWin->winInfo.pStatePos, &size); if (code == TSDB_CODE_FAILED) { pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, (void**)&pCurWin->winInfo.pStatePos, &size); } } else { - pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount); + pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, + pAggSup->windowCount); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, (void**)&pCurWin->winInfo.pStatePos, &size); if (code == TSDB_CODE_FAILED) { @@ -107,15 +106,16 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pBuffInfo->rebuildWindow = true; } } else { - code = pAggSup->stateStore.streamStateCountWinAddIfNotExist( - pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size); + code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, + pAggSup->windowCount, + (void**)&pCurWin->winInfo.pStatePos, &size); } if (code == TSDB_CODE_SUCCESS) { pCurWin->winInfo.isOutput = true; } - pCurWin->pWindowCount= - (COUNT_TYPE*) ((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE))); + pCurWin->pWindowCount = + (COUNT_TYPE*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE))); if (*pCurWin->pWindowCount == pAggSup->windowCount) { pBuffInfo->rebuildWindow = true; @@ -133,7 +133,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; - int32_t num = 0; + int32_t num = 0; for (int32_t i = start; i < rows; i++) { if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) { num++; @@ -181,7 +181,7 @@ void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, ESt pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, pKey); } SSessionKey tmpKey = {.groupId = pKey->groupId, .win.ekey = INT64_MIN, .win.skey = INT64_MIN}; - int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); if (code != TSDB_CODE_SUCCESS) { pAggSup->stateStore.streamStateFreeCur(pCur); return; @@ -239,7 +239,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2); SStreamStateCur* pCur = NULL; - COUNT_TYPE slidingRows = 0; + COUNT_TYPE slidingRows = 0; for (int32_t i = 0; i < rows;) { if (pInfo->ignoreExpiredData && @@ -269,7 +269,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } getCountWinRange(pAggSup, &curWin.winInfo.sessionWin, STREAM_DELETE_DATA, &range); range.win.skey = TMIN(startTsCols[i], range.win.skey); - range.win.ekey = TMAX(startTsCols[rows-1], range.win.ekey); + range.win.ekey = TMAX(startTsCols[rows - 1], range.win.ekey); uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL); break; @@ -289,8 +289,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(curWin.winInfo, pStUpdated); if (code != TSDB_CODE_SUCCESS) { - qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), - tstrerror(code)); + qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); break; } } @@ -397,13 +396,14 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato int32_t mapSize = 0; buf = taosDecodeFixedI32(buf, &mapSize); for (int32_t i = 0; i < mapSize; i++) { - SSessionKey key = {0}; + SSessionKey key = {0}; SCountWindowInfo curWin = {0}; buf = decodeSSessionKey(buf, &key); SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL}; setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, + sizeof(SResultWindowInfo)); } // 2.twAggSup @@ -441,9 +441,9 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; SRowBuffPos* pPos = NULL; - int32_t size = 0; + int32_t size = 0; for (int32_t i = 0; i < pBlock->info.rows; i++) { - SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]}; + SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]}; SStreamStateCur* pCur = NULL; if (isSlidingCountWindow(pAggSup)) { pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &key, pAggSup->windowCount); @@ -452,7 +452,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { } while (1) { SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN}; - int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void**)&pPos, &size); if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) { pAggSup->stateStore.streamStateFreeCur(pCur); break; @@ -482,7 +482,7 @@ void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SAr SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]}; while (1) { SSessionKey curWin = {0}; - int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); + int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); if (code == TSDB_CODE_FAILED) { break; } @@ -511,11 +511,11 @@ void deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHa } static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { - SExprSupp* pSup = &pOperator->exprSupp; - SStreamCountAggOperatorInfo* pInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExprSupp* pSup = &pOperator->exprSupp; + SStreamCountAggOperatorInfo* pInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; qDebug("stask:%s %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -635,7 +635,7 @@ void streamCountReloadState(SOperatorInfo* pOperator) { int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME, strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size); - TSKEY ts = *(TSKEY*)pBuf; + TSKEY ts = *(TSKEY*)pBuf; pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); taosMemoryFree(pBuf); @@ -650,11 +650,13 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) { SCountWinodwPhysiNode* pCountNode = (SCountWinodwPhysiNode*)pPhyNode; int32_t numOfCols = 0; - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamCountAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamCountAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pOperator->pTaskInfo = pTaskInfo; @@ -664,18 +666,14 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); } SExprSupp* pExpSup = &pOperator->exprSupp; SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); pInfo->twAggSup = (STimeWindowAggSupp){ .waterMark = pCountNode->window.watermark, @@ -686,12 +684,11 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys }; pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId; - code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, - pTaskInfo->streamInfo.pState, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, - &pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, + sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, + GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); + TSDB_CHECK_CODE(code, lino, _error); + pInfo->streamAggSup.windowCount = pCountNode->windowCount; pInfo->streamAggSup.windowSliding = pCountNode->windowSliding; @@ -709,7 +706,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->dataVersion = 0; pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); if (!pInfo->historyWins) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); @@ -735,8 +733,12 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState); if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); + TSDB_CHECK_CODE(code, lino, _error); } return pOperator; @@ -747,6 +749,6 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } - diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 1809d0cc67..e6ff4fe242 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -28,9 +28,9 @@ #include "tlog.h" #include "ttime.h" -#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT) -#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState" -#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint" +#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT) +#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState" +#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint" typedef struct SEventWinfowFlag { bool startFlag; @@ -86,10 +86,11 @@ void destroyStreamEventOperatorInfo(void* param) { void setEventWindowFlag(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo) { char* pFlagInfo = (char*)pWinInfo->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize); - pWinInfo->pWinFlag = (SEventWinfowFlag*) pFlagInfo; + pWinInfo->pWinFlag = (SEventWinfowFlag*)pFlagInfo; } -void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, SEventWindowInfo* pWinInfo) { +void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, + SEventWindowInfo* pWinInfo) { pWinInfo->winInfo.sessionWin = *pKey; pWinInfo->winInfo.pStatePos = pPos; setEventWindowFlag(pAggSup, pWinInfo); @@ -122,22 +123,23 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI pCurWin->winInfo.sessionWin.groupId = groupId; pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.ekey = ts; - SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin); - SSessionKey leftWinKey = {.groupId = groupId}; - void* pVal = NULL; - int32_t len = 0; + SStreamStateCur* pCur = + pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin); + SSessionKey leftWinKey = {.groupId = groupId}; + void* pVal = NULL; + int32_t len = 0; code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len); - if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win) ) { + if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) { bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0); setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin); - if(inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag) ) { + if (inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag)) { pCurWin->winInfo.isOutput = !isWindowIncomplete(pCurWin); goto _end; } } pAggSup->stateStore.streamStateFreeCur(pCur); pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); - SSessionKey rightWinKey = {.groupId = groupId}; + SSessionKey rightWinKey = {.groupId = groupId}; code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len); bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0); if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) { @@ -149,7 +151,7 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI } } - SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; + SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len); setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin); pCurWin->pWinFlag->startFlag = start; @@ -164,7 +166,7 @@ _end: if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pNextWinKey); } - if(pCurWin->winInfo.pStatePos->needFree) { + if (pCurWin->winInfo.pStatePos->needFree) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin); } pAggSup->stateStore.streamStateFreeCur(pCur); @@ -173,14 +175,14 @@ _end: } int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey, - TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, SSHashObj* pResultRows, - SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { + TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, + SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { *pRebuild = false; - if (!pWinInfo->pWinFlag->startFlag && !(starts[start]) ) { + if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) { return 1; } - TSKEY maxTs = INT64_MAX; + TSKEY maxTs = INT64_MAX; STimeWindow* pWin = &pWinInfo->winInfo.sessionWin.win; if (pWinInfo->pWinFlag->endFlag) { maxTs = pWin->ekey + 1; @@ -219,7 +221,7 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); if (ends[i]) { - if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey ) { + if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey) { *pRebuild = true; } return i + 1 - start; @@ -230,26 +232,28 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - int32_t winNum = 0; - SStreamEventAggOperatorInfo* pInfo = pOperator->info; - SResultRow* pCurResult = NULL; - int32_t numOfOutput = pOperator->exprSupp.numOfExprs; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + int32_t winNum = 0; + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + SResultRow* pCurResult = NULL; + int32_t numOfOutput = pOperator->exprSupp.numOfExprs; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; while (1) { if (!pCurWin->pWinFlag->startFlag || pCurWin->pWinFlag->endFlag) { break; } SEventWindowInfo nextWinInfo = {0}; getNextSessionWinInfo(pAggSup, pStUpdated, &pCurWin->winInfo, &nextWinInfo.winInfo); - if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) { + if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || + !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) { releaseOutputBuf(pAggSup->pState, nextWinInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); break; } setEventWindowFlag(pAggSup, &nextWinInfo); - compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, pStDeleted, false); + compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, + pStDeleted, false); pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag; winNum++; } @@ -275,7 +279,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t winRows = 0; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SColumnInfoData* pColStart = NULL; - SColumnInfoData* pColEnd = NULL; + SColumnInfoData* pColEnd = NULL; pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; @@ -290,7 +294,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl return; } - SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; + SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), + .pDataBlock = pSDataBlock->pDataBlock}; code = filterSetDataFromSlotId(pInfo->pStartCondInfo, ¶mStart); if (code != TSDB_CODE_SUCCESS) { qError("set data from start slotId error."); @@ -299,7 +304,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t statusStart = 0; filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart); - SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; + SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), + .pDataBlock = pSDataBlock->pDataBlock}; code = filterSetDataFromSlotId(pInfo->pEndCondInfo, ¶mEnd); if (code != TSDB_CODE_SUCCESS) { qError("set data from end slotId error."); @@ -320,20 +326,23 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t winIndex = 0; bool allEqual = true; SEventWindowInfo curWin = {0}; - SSessionKey nextWinKey = {0}; - setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, &nextWinKey); + SSessionKey nextWinKey = {0}; + setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, + &nextWinKey); setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); bool rebuild = false; - winRows = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, rows, i, - pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); + winRows = + updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, + rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); ASSERT(winRows >= 1); if (rebuild) { uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, - &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); + &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin); - if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && !isWindowIncomplete(&curWin)) { + if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && + !isWindowIncomplete(&curWin)) { saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); } releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); @@ -603,7 +612,8 @@ void streamEventReleaseState(SOperatorInfo* pOperator) { char* pBuff = taosMemoryCalloc(1, resSize); memcpy(pBuff, pInfo->historyWins->pData, winSize); memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); - qDebug("===stream=== event window operator relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins)); + qDebug("===stream=== event window operator relase state. save result count:%d", + (int32_t)taosArrayGetSize(pInfo->historyWins)); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_STATE_NAME, strlen(STREAM_EVENT_OP_STATE_NAME), pBuff, resSize); pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); @@ -647,7 +657,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, i); getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo); - //event window has been deleted + // event window has been deleted if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) { continue; } @@ -659,7 +669,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, - curInfo.winInfo.sessionWin.groupId); + curInfo.winInfo.sessionWin.groupId); if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { saveSessionOutputBuf(pAggSup, &curInfo.winInfo); } @@ -693,6 +703,7 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys SStreamEventWinodwPhysiNode* pEventNode = (SStreamEventWinodwPhysiNode*)pPhyNode; int32_t tsSlotId = ((SColumnNode*)pEventNode->window.pTspk)->slotId; int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamEventAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamEventAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -783,20 +794,18 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState); - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); code = filterInitFromNode((SNode*)pEventNode->pStartCond, &pInfo->pStartCondInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); + code = filterInitFromNode((SNode*)pEventNode->pEndCond, &pInfo->pEndCondInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); return pOperator; @@ -804,5 +813,6 @@ _error: destroyStreamEventOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index d3f90c3134..96c6712866 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -28,14 +28,18 @@ #include "tlog.h" #include "ttime.h" -#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) -#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) -#define IS_NORMAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) +#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) +#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) +#define IS_NORMAL_INTERVAL_OP(op) \ + ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || \ + (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) -#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) -#define IS_NORMAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) +#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) +#define IS_NORMAL_SESSION_OP(op) \ + ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \ + (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) -#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) +#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) #define DEAULT_DELETE_MARK INT64_MAX #define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState" @@ -263,14 +267,16 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa if (chIds) { int32_t childId = getChildIndex(pBlock); if (pInvalidWins) { - qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, winRes.groupId, childId); + qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, + winRes.groupId, childId); taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0); } SArray* chArray = *(void**)chIds; int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ); if (index != -1) { - qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey, winGpId, childId); + qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey, + winGpId, childId); getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); continue; } @@ -390,7 +396,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin } void destroyFlusedPos(void* pRes) { - SRowBuffPos* pPos = (SRowBuffPos*) pRes; + SRowBuffPos* pPos = (SRowBuffPos*)pRes; if (!pPos->needFree && !pPos->pRowBuff) { taosMemoryFreeClear(pPos->pKey); taosMemoryFree(pPos); @@ -488,8 +494,11 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = - pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); + int32_t code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, + pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at %d since %s", __func__, __LINE__, tstrerror(code)); + } } pScanInfo->interval = pInfo->interval; @@ -624,17 +633,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB blockDataUpdateTsWindow(pBlock, 0); } -static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, SArray* pPullWins, - int32_t numOfCh, SOperatorInfo* pOperator) { +static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, + SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); - TSKEY* tsData = (TSKEY*)pStartCol->pData; - SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - TSKEY* tsEndData = (TSKEY*)pEndCol->pData; - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; - int32_t chId = getChildIndex(pBlock); - bool res = false; + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + TSKEY* tsData = (TSKEY*)pStartCol->pData; + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + TSKEY* tsEndData = (TSKEY*)pEndCol->pData; + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; + int32_t chId = getChildIndex(pBlock); + bool res = false; for (int32_t i = 0; i < pBlock->info.rows; i++) { TSKEY winTs = tsData[i]; while (winTs <= tsEndData[i]) { @@ -650,7 +659,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina // pull data is over taosArrayDestroy(chArray); taosHashRemove(pMap, &winRes, sizeof(SWinKey)); - res =true; + res = true; qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts); void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey)); @@ -663,7 +672,8 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request - qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d", winRes.ts, winRes.groupId, numOfCh); + qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d", + winRes.ts, winRes.groupId, numOfCh); if (IS_MID_INTERVAL_OP(pOperator)) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; taosArrayPush(pInfo->pMidPullDatas, &winRes); @@ -671,7 +681,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina taosArrayPush(pInfo->pDelWins, &winRes); addPullWindow(pMap, &winRes, numOfCh); if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); } qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh); } @@ -690,7 +700,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i for (int32_t i = 0; i < size; i++) { SWinKey* winKey = taosArrayGet(wins, i); STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval); - void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); + void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); if (!chIds) { SPullWindowInfo pull = { .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; @@ -698,7 +708,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild); if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap,winKey, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0); } qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild); } @@ -751,7 +761,8 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat if (pBlock->info.id.groupId == 0) { pBlock->info.id.groupId = groupId; void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, false) < 0) { + if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, + false) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); @@ -829,9 +840,7 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN return startPos; } -bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { - return pInfo->primaryPkIndex != -1; -} +bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; } static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId, SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { @@ -852,23 +861,22 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; - void* pPkVal = NULL; - int32_t pkLen = 0; + void* pPkVal = NULL; + int32_t pkLen = 0; SColumnInfoData* pPkColDataInfo = NULL; if (hasSrcPrimaryKeyCol(&pInfo->basic)) { pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); } if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) { - qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 - ",maxKey %" PRId64, - pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); + qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64, + pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex); // timestamp of the data is incorrect if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64, - pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); + pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); } } @@ -916,7 +924,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild); if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); } } } else { @@ -962,7 +970,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat }; if (pInfo->destHasPrimaryKey && code == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) { - tSimpleHashPut(pDeletedMap,&key, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { @@ -1365,7 +1373,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); + SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL; doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); if (IS_FINAL_INTERVAL_OP(pOperator)) { @@ -1399,7 +1407,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE) { - if(!IS_FINAL_INTERVAL_OP(pOperator)) { + if (!IS_FINAL_INTERVAL_OP(pOperator)) { pInfo->recvRetrive = true; copyDataBlock(pInfo->pMidRetriveRes, pBlock); pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE; @@ -1525,8 +1533,8 @@ void streamIntervalReloadState(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL && pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) { - int32_t size = 0; - void* pBuf = NULL; + int32_t size = 0; + void* pBuf = NULL; int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); if (code == 0) { @@ -1600,7 +1608,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pInfo->pState, &pTaskInfo->storageAPI.functionStore); + pInfo->pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1755,27 +1763,37 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num } } -void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, - STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) { +int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, + STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) { SStreamPartitionOperatorInfo* pScanInfo = downstream->info; pScanInfo->tsColIndex = tsColIndex; } if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic); - return; + code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic); + return code; } SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pState = pAggSup->pState; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, - pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); + code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, + pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, + &pScanInfo->pUpdateInfo); + TSDB_CHECK_CODE(code, lino, _end); } pScanInfo->twAggSup = *pTwSup; pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo; pBasic->primaryPkIndex = pScanInfo->primaryKeyIndex; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static TSKEY sesionTs(void* pKey) { @@ -2148,8 +2166,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData TSKEY* endTsCols = (int64_t*)pEndTsCol->pData; - void* pPkVal = NULL; - int32_t pkLen = 0; + void* pPkVal = NULL; + int32_t pkLen = 0; SColumnInfoData* pPkColDataInfo = NULL; if (hasSrcPrimaryKeyCol(&pInfo->basic)) { pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); @@ -2454,8 +2472,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa pBlock->info.id.groupId = pKey->groupId; void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, - &tbname, false) < 0) { + if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, + false) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); @@ -2935,7 +2953,7 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -2968,7 +2986,7 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -3018,6 +3036,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; int32_t numOfCols = 0; int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t lino = 0; SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -3112,8 +3131,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); + TSDB_CHECK_CODE(code, lino, _error); } return pOperator; @@ -3124,6 +3147,7 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } @@ -3574,7 +3598,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (!allEqual) { uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, - &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); + &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); @@ -3681,9 +3705,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); - pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, - pAggSup->stateKeySize, compareStateKey, - (void**)&winfo.pStatePos, &pAggSup->resultRowSize); + pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateKeySize, + compareStateKey, (void**)&winfo.pStatePos, + &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } @@ -3877,7 +3901,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, - strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); qDebug("===stream=== reload state. get result count:%d", num); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; @@ -3941,12 +3965,13 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey); int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _error; + TSDB_CHECK_CODE(code, lino, _error); } pInfo->stateCol = extractColumnFromColumnNode(pColNode); @@ -3955,9 +3980,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); } pInfo->twAggSup = (STimeWindowAggSupp){ @@ -3984,9 +4007,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, type, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pSeDeleted = tSimpleHashInit(64, hashFn); pInfo->pDelIterator = NULL; @@ -3999,7 +4021,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->dataVersion = 0; pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); if (!pInfo->historyWins) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } if (pHandle) { pInfo->isHistoryOp = pHandle->fillHistory; @@ -4026,17 +4049,20 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState); - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); + return pOperator; _error: destroyStreamStateOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } @@ -4203,11 +4229,12 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, }; - pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark, - .calTrigger = pIntervalPhyNode->window.triggerType, - .maxTs = INT64_MIN, - .minTs = INT64_MAX, - .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)}; + pInfo->twAggSup = + (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark, + .calTrigger = pIntervalPhyNode->window.triggerType, + .maxTs = INT64_MIN, + .minTs = INT64_MAX, + .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)}; ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); @@ -4335,7 +4362,7 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS .ts = nextWin.skey, .groupId = groupId, }; - void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey)); + void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey)); int32_t index = -1; SArray* chArray = NULL; int32_t chId = 0; @@ -4407,8 +4434,8 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) { int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { - SWinKey* winKey = taosArrayGet(wins, i); - void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey)); + SWinKey* winKey = taosArrayGet(wins, i); + void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey)); if (!chIds) { addPullWindow(pMidPullMap, winKey, numOfChild); qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild); @@ -4611,5 +4638,6 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { void setStreamOperatorCompleted(SOperatorInfo* pOperator) { setOperatorCompleted(pOperator); - qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); + qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), + getStreamOpName(pOperator->operatorType), pOperator->status); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 23bd2b87a1..41d398888e 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1139,7 +1139,6 @@ void destroyIntervalOperatorInfo(void* param) { taosArrayDestroy(pInfo->pInterpCols); pInfo->pInterpCols = NULL; taosArrayDestroyEx(pInfo->pPrevValues, freeItem); - pInfo->pPrevValues = NULL; cleanupGroupResInfo(&pInfo->groupResInfo); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index db2d34b844..89eaf14f75 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -112,7 +112,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem for (int i = 0; i < tags->numOfBound; ++i) { SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; SSmlKv* kv = taosArrayGet(cols, i); - + if (kv == NULL){ + code = TSDB_CODE_SML_INVALID_DATA; + uError("SML smlBuildTagRow error kv is null"); + goto end; + } if (kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type) { code = TSDB_CODE_SML_INVALID_DATA; @@ -120,7 +124,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem goto end; } - taosArrayPush(*tagName, pTagSchema->name); + if (taosArrayPush(*tagName, pTagSchema->name) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML smlBuildTagRow error push tag name"); + goto end; + } STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; // strcpy(val.colName, pTagSchema->name); if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY || @@ -141,7 +149,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem goto end; } char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + (void)snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); taosMemoryFree(p); code = buildSyntaxErrMsg(msg, buf, kv->value); goto end; @@ -149,9 +157,13 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem val.pData = p; val.nData = output; } else { - memcpy(&val.i64, &(kv->value), kv->length); + (void)memcpy(&val.i64, &(kv->value), kv->length); + } + if (taosArrayPush(pTagArray, &val) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML smlBuildTagRow error push tag array"); + goto end; } - taosArrayPush(pTagArray, &val); } code = tTagNew(pTagArray, 1, false, ppTag); @@ -166,20 +178,19 @@ end: return code; } -STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { - STableDataCxt* pTableCxt = NULL; +int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt) { SVCreateTbReq* pCreateTbReq = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, cxt, false, false); if (ret != TSDB_CODE_SUCCESS) { - return NULL; + return ret; } - ret = initTableColSubmitData(pTableCxt); + ret = initTableColSubmitData(*cxt); if (ret != TSDB_CODE_SUCCESS) { - return NULL; + return ret; } - return pTableCxt; + return TSDB_CODE_SUCCESS; } void clearColValArraySml(SArray* pCols) { @@ -197,6 +208,9 @@ void clearColValArraySml(SArray* pCols) { int32_t smlBuildRow(STableDataCxt* pTableCxt) { SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + if (pRow == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); if (TSDB_CODE_SUCCESS != ret) { return ret; @@ -211,18 +225,22 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 int ret = TSDB_CODE_SUCCESS; SSchema* pColSchema = schema + index; SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); + if (pVal == NULL) { + return TSDB_CODE_SUCCESS; + } SSmlKv* kv = (SSmlKv*)data; if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type) { ret = TSDB_CODE_SML_INVALID_DATA; char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1); if (tmp) { - memcpy(tmp, kv->key, kv->keyLen); + (void)memcpy(tmp, kv->key, kv->keyLen); uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name, pColSchema->name, tDataTypes[pColSchema->type].name); taosMemoryFree(tmp); } else { uError("SML smlBuildCol out of memory"); + ret = TSDB_CODE_OUT_OF_MEMORY; } goto end; } @@ -256,9 +274,9 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { pVal->value.nData = kv->length; pVal->value.pData = taosMemoryMalloc(kv->length); - memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); + (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); } else { - memcpy(&pVal->value.val, &(kv->value), kv->length); + (void)memcpy(&pVal->value.val, &(kv->value), kv->length); } pVal->flag = CV_FLAG_VALUE; @@ -276,8 +294,13 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc SVCreateTbReq* pCreateTblReq = NULL; SArray* tagName = NULL; - insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); - int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); + int ret = insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); + if (ret != TSDB_CODE_SUCCESS) { + buildInvalidOperationMsg(&pBuf, "init bound cols error"); + goto end; + } + + ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound tags error"); goto end; @@ -299,7 +322,11 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc ttl); pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); - memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); + if (pCreateTblReq->ctb.stbName == NULL){ + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + (void)memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); if (dataFormat) { STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, @@ -345,11 +372,18 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc for (int32_t r = 0; r < rowNum; ++r) { void* rowData = taosArrayGetP(cols, r); - + if (rowData == NULL) { + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } // 1. set the parsed value from sql string for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); + if (pVal == NULL) { + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); if (p == NULL) { continue; @@ -388,14 +422,22 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { pVal->value.nData = kv->length; pVal->value.pData = taosMemoryMalloc(kv->length); - memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); + if (NULL == pVal->value.pData) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); } else { - memcpy(&pVal->value.val, &(kv->value), kv->length); + (void)memcpy(&pVal->value.val, &(kv->value), kv->length); } pVal->flag = CV_FLAG_VALUE; } SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + if (NULL == pRow) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); if (TSDB_CODE_SUCCESS != ret) { buildInvalidOperationMsg(&pBuf, "tRowBuild error"); @@ -415,11 +457,12 @@ end: return ret; } -SQuery* smlInitHandle() { +int32_t smlInitHandle(SQuery** query) { + *query = NULL; SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); if (NULL == pQuery) { uError("create pQuery error"); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->haveResultSet = false; @@ -428,14 +471,22 @@ SQuery* smlInitHandle() { if (NULL == stmt) { uError("create SVnodeModifyOpStmt error"); qDestroyQuery(pQuery); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (stmt->pTableBlockHashObj == NULL){ + uError("create pTableBlockHashObj error"); + qDestroyQuery(pQuery); + nodesDestroyNode((SNode*)stmt); + return TSDB_CODE_OUT_OF_MEMORY; + } stmt->freeHashFunc = insDestroyTableDataCxtHashMap; stmt->freeArrayFunc = insDestroyVgroupDataCxtList; pQuery->pRoot = (SNode*)stmt; - return pQuery; + *query = pQuery; + + return TSDB_CODE_SUCCESS; } int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9b1c8096aa..104ffcc945 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -12815,6 +12815,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) int32_t numOfTags = getNumOfTags(pParFileCxt->pStbMeta); uint8_t precision = getTableInfo(pParFileCxt->pStbMeta).precision; SSchema* pSchemas = getTableTagSchema(pParFileCxt->pStbMeta); + bool isJson = false; for (int i = 0; i < sz; i++) { const char* pSql = pParFileCxt->pSql; @@ -12832,6 +12833,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) // parse tag const SSchema* pTagSchema = &pSchemas[index]; + isJson = (pTagSchema->type == TSDB_DATA_TYPE_JSON); code = checkAndTrimValue(&token, pParFileCxt->tmpTokenBuf, pMsgBuf, pTagSchema->type); if (TSDB_CODE_SUCCESS == code && TK_NK_VARIABLE == token.type) { code = buildInvalidOperationMsg(pMsgBuf, "not expected row value"); @@ -12859,7 +12861,9 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) if (TSDB_CODE_SUCCESS == code) { pParFileCxt->tagNameFilled = true; - code = tTagNew(pParFileCxt->aTagVals, 1, false, &pParFileCxt->pTag); + if (!isJson) { + code = tTagNew(pParFileCxt->aTagVals, 1, false, &pParFileCxt->pTag); + } } return code; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 17476072bb..5ea79c6ae9 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -336,7 +336,7 @@ extern SSchedulerMgmt schMgmt; ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && \ (!SCH_IS_DATA_BIND_QRY_TASK(_task))) -#define SCH_UPDATE_REDIRECT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) +#define SCH_UPDATE_REDIRECT_CODE(job, _code) (void)atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) #define SCH_GET_REDIRECT_CODE(job, _code) \ (((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode) @@ -413,7 +413,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOG_TASK_START_TS(_task) \ do { \ int64_t us = taosGetTimestampUs(); \ - taosArrayPush((_task)->profile.execTime, &us); \ + (void)taosArrayPush((_task)->profile.execTime, &us); \ if (0 == (_task)->execId) { \ (_task)->profile.startTs = us; \ } \ @@ -422,7 +422,10 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOG_TASK_WAIT_TS(_task) \ do { \ int64_t us = taosGetTimestampUs(); \ - (_task)->profile.waitTime += us - *(int64_t *)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ + int64_t* startus = (int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ + if (NULL != startus) { \ + (_task)->profile.waitTime += us - *startus; \ + } \ } while (0) #define SCH_LOG_TASK_END_TS(_task) \ @@ -430,7 +433,9 @@ extern SSchedulerMgmt schMgmt; int64_t us = taosGetTimestampUs(); \ int32_t idx = (_task)->execId % (_task)->maxExecTimes; \ int64_t *startts = taosArrayGet((_task)->profile.execTime, (_task)->execId); \ - *startts = us - *startts; \ + if (NULL != startts) { \ + *startts = us - *startts; \ + } \ (_task)->profile.endTs = us; \ } while (0) @@ -538,7 +543,7 @@ void schCleanClusterHb(void *pTrans); int32_t schLaunchTask(SSchJob *job, SSchTask *task); int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType, void *param); -SSchJob *schAcquireJob(int64_t refId); +int32_t schAcquireJob(int64_t refId, SSchJob **ppJob); int32_t schReleaseJob(int64_t refId); void schFreeFlowCtrl(SSchJob *pJob); int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); @@ -578,7 +583,7 @@ int32_t schJobFetchRows(SSchJob *pJob); int32_t schJobFetchRowsA(SSchJob *pJob); int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList); -char *schDumpEpSet(SEpSet *pEpSet); +int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes); char *schGetOpStr(SCH_OP_TYPE type); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index a8e2e79aee..58a0706223 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -1165,7 +1165,9 @@ int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_ int8_t status = 0; SSchTask *pTask = NULL; - SSchJob *pJob = schAcquireJob(rId); + SSchJob *pJob = NULL; + + (void)schAcquireJob(rId, &pJob); if (NULL == pJob) { qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); SCH_ERR_RET(TSDB_CODE_QRY_JOB_NOT_EXIST); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index d0dc04d6b4..08a8f684f5 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -113,7 +113,7 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs } atomic_store_ptr(&pJob->fetchRes, rsp); - atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows)); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows)); if (rsp->completed) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); @@ -166,13 +166,27 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SCH_LOCK(SCH_WRITE, &pJob->resLock); if (NULL == pJob->execRes.res) { pJob->execRes.res = (void*)taosArrayInit(batchRsp.nRsps, POINTER_BYTES); + if (NULL == pJob->execRes.res) { + code = terrno; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + tDecoderClear(&coder); + SCH_ERR_JRET(code); + } + pJob->execRes.msgType = TDMT_VND_CREATE_TABLE; } for (int32_t i = 0; i < batchRsp.nRsps; ++i) { SVCreateTbRsp *rsp = batchRsp.pRsps + i; if (rsp->pMeta) { - taosArrayPush((SArray*)pJob->execRes.res, &rsp->pMeta); + if (NULL == taosArrayPush((SArray*)pJob->execRes.res, &rsp->pMeta)) { + code = terrno; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + tDecoderClear(&coder); + SCH_ERR_JRET(code); + } } if (TSDB_CODE_SUCCESS != rsp->code) { @@ -264,7 +278,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SCH_ERR_JRET(code); } - atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); int32_t createTbRspNum = taosArrayGetSize(rsp->aCreateTbRsp); SCH_TASK_DLOG("submit succeed, affectedRows:%d, createTbRspNum:%d", rsp->affectedRows, createTbRspNum); @@ -275,7 +289,12 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SSubmitRsp2 *sum = pJob->execRes.res; sum->affectedRows += rsp->affectedRows; if (sum->aCreateTbRsp) { - taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp); + if (NULL == taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp)) { + code = terrno; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + SCH_ERR_JRET(code); + } + taosArrayDestroy(rsp->aCreateTbRsp); } else { TSWAP(sum->aCreateTbRsp, rsp->aCreateTbRsp); @@ -313,10 +332,14 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SDecoder coder = {0}; SVDeleteRsp rsp = {0}; tDecoderInit(&coder, msg, msgSize); - tDecodeSVDeleteRsp(&coder, &rsp); + if (tDecodeSVDeleteRsp(&coder, &rsp) < 0) { + code = terrno; + tDecoderClear(&coder); + SCH_ERR_JRET(code); + } tDecoderClear(&coder); - atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); SCH_TASK_DLOG("delete succeed, affectedRows:%" PRId64, rsp.affectedRows); } @@ -351,7 +374,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SCH_ERR_JRET(schSaveJobExecRes(pJob, &rsp)); - atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); taosMemoryFreeClear(msg); @@ -479,7 +502,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); // called if drop task rsp received code - rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error if (pMsg) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -501,7 +524,7 @@ int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchCallbackParamHeader *head = (SSchCallbackParamHeader *)param; - rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error qDebug("handle %p is broken", pMsg->handle); @@ -531,7 +554,7 @@ int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) { if (code) { qError("hb rsp error:%s", tstrerror(code)); - rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error SCH_ERR_JRET(code); } @@ -585,9 +608,14 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo param->head.isHbParam = true; SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + taosMemoryFree(param); + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum: %d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } param->nodeEpId.nodeId = addr->nodeId; SEp *pEp = SCH_GET_CUR_EP(addr); - strcpy(param->nodeEpId.ep.fqdn, pEp->fqdn); + TAOS_STRCPY(param->nodeEpId.ep.fqdn, pEp->fqdn); param->nodeEpId.ep.port = pEp->port; param->pTrans = trans->pTrans; *pParam = param; @@ -712,7 +740,7 @@ int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { int32_t code = 0; - memcpy(pDst, pSrc, sizeof(SRpcCtx)); + TAOS_MEMCPY(pDst, pSrc, sizeof(SRpcCtx)); pDst->brokenVal.val = NULL; pDst->args = NULL; @@ -760,7 +788,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; - memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); + TAOS_MEMCPY(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { @@ -877,7 +905,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(dst, pSrc, sizeof(*dst)); + TAOS_MEMCPY(dst, pSrc, sizeof(*dst)); *pDst = (SSchCallbackParamHeader *)dst; return TSDB_CODE_SUCCESS; @@ -889,7 +917,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(dst, pSrc, sizeof(*dst)); + TAOS_MEMCPY(dst, pSrc, sizeof(*dst)); *pDst = (SSchCallbackParamHeader *)dst; return TSDB_CODE_SUCCESS; @@ -904,7 +932,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pDst, pSrc, sizeof(*pSrc)); + TAOS_MEMCPY(pDst, pSrc, sizeof(*pSrc)); pDst->param = NULL; SCH_ERR_JRET(schCloneCallbackParam(pSrc->param, (SSchCallbackParamHeader **)&pDst->param)); @@ -948,6 +976,10 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery if (isHb && persistHandle && trans->pHandle == 0) { trans->pHandle = rpcAllocHandle(); + if (NULL == trans->pHandle) { + SCH_TASK_ELOG("rpcAllocHandle failed, code:%x", terrno); + SCH_ERR_JRET(terrno); + } } if (pJob && pTask) { @@ -1000,7 +1032,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { req.header.vgId = nodeEpId->nodeId; req.sId = schMgmt.sId; - memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); + TAOS_MEMCPY(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); SCH_LOCK(SCH_READ, &schMgmt.hbLock); SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId)); @@ -1013,7 +1045,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { SCH_LOCK(SCH_WRITE, &hb->lock); code = schCloneHbRpcCtx(&hb->rpcCtx, &rpcCtx); - memcpy(&trans, &hb->trans, sizeof(trans)); + TAOS_MEMCPY(&trans, &hb->trans, sizeof(trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); @@ -1039,7 +1071,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { SQueryNodeAddr addr = {.nodeId = nodeEpId->nodeId}; addr.epSet.inUse = 0; addr.epSet.numOfEps = 1; - memcpy(&addr.epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); + TAOS_MEMCPY(&addr.epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); code = schAsyncSendMsg(NULL, NULL, &trans, &addr, msgType, msg, msgSize, true, &rpcCtx); msg = NULL; @@ -1064,6 +1096,11 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get condidateAddr, candidateIdx %d, totalNum: %d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_JRET(terrno); + } + isCandidateAddr = true; SCH_TASK_DLOG("target candidateIdx %d, epInUse %d/%d", pTask->candidateIdx, addr->epSet.inUse, addr->epSet.numOfEps); @@ -1082,7 +1119,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(msg, pTask->msg, msgSize); + TAOS_MEMCPY(msg, pTask->msg, msgSize); break; } @@ -1098,13 +1135,21 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.msg = pTask->msg; req.source = pJob->source; msgSize = tSerializeSVDeleteReq(NULL, 0, &req); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSVDeleteReq failed, code:%x", terrno); + SCH_ERR_JRET(terrno); + } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - tSerializeSVDeleteReq(msg, msgSize, &req); + msgSize = tSerializeSVDeleteReq(msg, msgSize, &req); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSVDeleteReq second failed, code:%x", terrno); + SCH_ERR_JRET(terrno); + } break; } case TDMT_SCH_QUERY: @@ -1221,7 +1266,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.sId = schMgmt.sId; req.header.vgId = addr->nodeId; req.epId.nodeId = addr->nodeId; - memcpy(&req.epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); + TAOS_MEMCPY(&req.epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index d37393137f..f24ee74101 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -43,7 +43,7 @@ int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) { SCH_RET(schProcessOnJobFailure(pJob, (param ? *(int32_t*)param : 0))); break; case JOB_TASK_STATUS_DROP: - schProcessOnJobDropped(pJob, *(int32_t*)param); + (void)schProcessOnJobDropped(pJob, *(int32_t*)param); // ignore error if (taosRemoveRef(schMgmt.jobRef, pJob->refId)) { SCH_JOB_ELOG("remove job from job list failed, refId:0x%" PRIx64, pJob->refId); @@ -65,7 +65,8 @@ _return: } int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) { - SSchJob* pJob = schAcquireJob(jobId); + SSchJob* pJob = NULL; + (void)schAcquireJob(jobId, &pJob); if (NULL == pJob) { qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); SCH_ERR_RET(TSDB_CODE_SCH_JOB_NOT_EXISTS); @@ -90,7 +91,7 @@ int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq code = pJob->errCode; } - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); // ignore error return code; } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index c317a63ce1..6dd6aa9aae 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -165,7 +165,7 @@ int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, v SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execId)); } - schUpdateTaskExecNode(pJob, pTask, handle, execId); + SCH_ERR_RET(schUpdateTaskExecNode(pJob, pTask, handle, execId)); if ((execId != pTask->execId || execId <= pTask->failedExecId) || pTask->waitRetry) { // ignore it SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, @@ -297,7 +297,11 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { for (int32_t i = 0; i < parentNum; ++i) { SSchTask *parent = *(SSchTask **)taosArrayGet(pTask->parents, i); - + if (NULL == parent) { + SCH_TASK_ELOG("fail to get task %d parent, parentNum: %d", i, parentNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_LOCK(SCH_WRITE, &parent->planLock); SDownstreamSourceNode source = { .type = QUERY_NODE_DOWNSTREAM_SOURCE, @@ -308,9 +312,14 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { .fetchMsgType = SCH_FETCH_TYPE(pTask), .localExec = SCH_IS_LOCAL_EXEC_TASK(pJob, pTask), }; - qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); + code = qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); + if (TSDB_CODE_SUCCESS != code) { + SCH_TASK_ELOG("qSetSubplanExecutionNode failed, groupId: %d", pTask->plan->id.groupId); + } SCH_UNLOCK(SCH_WRITE, &parent->planLock); + SCH_ERR_RET(code); + int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1); if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { @@ -413,16 +422,16 @@ void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask) { schDropTaskOnExecNode(pJob, pTask); if (pTask->delayTimer) { - taosTmrStopA(&pTask->delayTimer); + (void)taosTmrStopA(&pTask->delayTimer); // ignore error } taosHashClear(pTask->execNodes); - schRemoveTaskFromExecList(pJob, pTask); + (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error schDeregisterTaskHb(pJob, pTask); taosMemoryFreeClear(pTask->msg); pTask->msgLen = 0; pTask->lastMsgType = 0; pTask->childReady = 0; - memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); + TAOS_MEMSET(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); } int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { @@ -443,11 +452,21 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_ERR_JRET(schUpdateTaskCandidateAddr(pJob, pTask, pData->pEpSet)); } else if (SYNC_SELF_LEADER_REDIRECT_ERROR(rspCode)) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d, code:%s", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps, pEp->fqdn, pEp->port, tstrerror(rspCode)); } else { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_SWITCH_EPSET(addr); SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); } @@ -476,7 +495,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 for (int32_t i = 0; i < childrenNum; ++i) { SSchTask *pChild = taosArrayGetP(pTask->children, i); SCH_LOCK_TASK(pChild); - schDoTaskRedirect(pJob, pChild, NULL, rspCode); + (void)schDoTaskRedirect(pJob, pChild, NULL, rspCode); // error handled internal SCH_UNLOCK_TASK(pChild); } @@ -494,18 +513,23 @@ int32_t schResetTaskSetLevelInfo(SSchJob *pJob, SSchTask *pTask) { atomic_load_32(&pLevel->taskExecDoneNum), atomic_load_32(&pLevel->taskLaunchedNum)); if (SCH_GET_TASK_STATUS(pTask) >= JOB_TASK_STATUS_PART_SUCC) { - atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); } - atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); int32_t childrenNum = taosArrayGetSize(pTask->children); for (int32_t i = 0; i < childrenNum; ++i) { SSchTask *pChild = taosArrayGetP(pTask->children, i); + if (NULL == pChild) { + SCH_TASK_ELOG("fail to get the %dth child, childrenNum:%d", i, childrenNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_LOCK_TASK(pChild); pLevel = pChild->level; - atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); - atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); SCH_UNLOCK_TASK(pChild); } @@ -711,9 +735,9 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo } int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { - atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); + (void)atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); - schRemoveTaskFromExecList(pJob, pTask); + (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -724,6 +748,11 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_DATA_BIND_TASK(pTask)) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_SWITCH_EPSET(addr); } else { SCH_ERR_RET(schSwitchTaskCandidateAddr(pJob, pTask)); @@ -743,6 +772,11 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { for (int32_t i = 0; i < nodeNum; ++i) { SQueryNodeLoad *nload = taosArrayGet(pJob->nodeList, i); + if (NULL == nload) { + SCH_TASK_ELOG("fail to get the %dth node in nodeList, nodeNum:%d", i, nodeNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SQueryNodeAddr *naddr = &nload->addr; if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { @@ -810,6 +844,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { } int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) { + int32_t code = TSDB_CODE_SUCCESS; if (NULL == pTask->candidateAddrs || 1 != taosArrayGetSize(pTask->candidateAddrs)) { SCH_TASK_ELOG("not able to update cndidate addr, addr num %d", (int32_t)(pTask->candidateAddrs ? taosArrayGetSize(pTask->candidateAddrs) : 0)); @@ -817,18 +852,27 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSe } SQueryNodeAddr *pAddr = taosArrayGet(pTask->candidateAddrs, 0); + if (NULL == pAddr) { + SCH_TASK_ELOG("fail to get task 0th condidataAddr, totalNum:%d", (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } - char *origEpset = schDumpEpSet(&pAddr->epSet); - char *newEpset = schDumpEpSet(pEpSet); + char *origEpset = NULL; + char *newEpset = NULL; + + SCH_ERR_RET(schDumpEpSet(&pAddr->epSet, &origEpset)); + SCH_ERR_JRET(schDumpEpSet(pEpSet, &newEpset)); SCH_TASK_DLOG("update task target node %d epset from %s to %s", pAddr->nodeId, origEpset, newEpset); + TAOS_MEMCPY(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet)); + +_return: + taosMemoryFree(origEpset); taosMemoryFree(newEpset); - memcpy(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet)); - - return TSDB_CODE_SUCCESS; + return code; } int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) { @@ -877,7 +921,6 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { } int32_t size = (int32_t)taosHashGetSize(pTask->execNodes); - if (size <= 0) { SCH_TASK_DLOG("task has no execNodes, no need to drop it, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); return; @@ -889,7 +932,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { if (nodeInfo->handle) { SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); void *pExecId = taosHashGetKey(nodeInfo, NULL); - schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, pExecId); + (void)schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, pExecId); // ignore error and continue SCH_TASK_DLOG("start to drop task's %dth execNode", i); } else { @@ -939,6 +982,11 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { for (int32_t i = 0; i < taskNum; ++i) { STaskStatus *pStatus = taosArrayGet(pStatusList, i); + if (NULL == pStatus) { + qError("fail to get the %dth task status in hb rsp, taskNum:%d", i, taskNum); + continue; + } + int32_t code = 0; qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, @@ -983,10 +1031,15 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { for (int32_t i = 0; i < resNum; ++i) { SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i); + if (NULL == localRsp) { + qError("fail to get the %dth LOCAL explain rsp msg, total:%d", i, resNum); + continue; + } qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); - pJob = schAcquireJob(localRsp->rId); + pJob = NULL; + (void)schAcquireJob(localRsp->rId, &pJob); if (NULL == pJob) { qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, localRsp->tId, localRsp->rId); @@ -996,7 +1049,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { int8_t status = 0; if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("will not do further processing cause of job status %s", jobTaskStatusStr(status)); - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } @@ -1006,7 +1059,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { code = schProcessExplainRsp(pJob, pTask, &localRsp->rsp); } - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, localRsp->tId, code); @@ -1022,6 +1075,11 @@ _return: for (int32_t i = 0; i < resNum; ++i) { SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i); + if (NULL == localRsp) { + qError("in _return fail to get the %dth LOCAL explain rsp msg, total:%d", i, resNum); + continue; + } + tFreeSExplainRsp(&localRsp->rsp); } @@ -1076,6 +1134,9 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); + if (NULL == explainRes) { + SCH_ERR_RET(terrno); + } } SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, @@ -1097,7 +1158,9 @@ _return: int32_t schLaunchTaskImpl(void *param) { SSchTaskCtx *pCtx = (SSchTaskCtx *)param; - SSchJob *pJob = schAcquireJob(pCtx->jobRid); + SSchJob *pJob = NULL; + + (void)schAcquireJob(pCtx->jobRid, &pJob); if (NULL == pJob) { qDebug("job refId 0x%" PRIx64 " already not exist", pCtx->jobRid); taosMemoryFree(param); @@ -1113,7 +1176,7 @@ int32_t schLaunchTaskImpl(void *param) { int8_t status = 0; int32_t code = 0; - atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); + (void)atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); pTask->execId++; pTask->retryTimes++; pTask->waitRetry = false; @@ -1160,7 +1223,7 @@ _return: SCH_UNLOCK_TASK(pTask); } - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); taosMemoryFree(param); @@ -1178,7 +1241,7 @@ int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { param->asyncLaunch = true; - taosAsyncExec(schLaunchTaskImpl, param, NULL); + SCH_ERR_RET(taosAsyncExec(schLaunchTaskImpl, param, NULL)); } else { SCH_ERR_RET(schLaunchTaskImpl(param)); } @@ -1252,7 +1315,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer); + (void)taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer); return TSDB_CODE_SUCCESS; } @@ -1283,7 +1346,7 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { SCH_LOCK_TASK(pTask); if (pTask->delayTimer) { - taosTmrStopA(&pTask->delayTimer); + (void)taosTmrStopA(&pTask->delayTimer); } schDropTaskOnExecNode(pJob, pTask); SCH_UNLOCK_TASK(pTask); @@ -1327,6 +1390,9 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); + if (NULL == explainRes) { + SCH_ERR_RET(terrno); + } } SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 689c98d395..811890dde5 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -22,9 +22,14 @@ #include "tref.h" #include "trpc.h" -FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { +FORCE_INLINE int32_t schAcquireJob(int64_t refId, SSchJob** ppJob) { qDebug("sch acquire jobId:0x%" PRIx64, refId); - return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); + *ppJob = (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); + if (NULL == *ppJob) { + return terrno; + } + + return TSDB_CODE_SUCCESS; } FORCE_INLINE int32_t schReleaseJob(int64_t refId) { @@ -36,15 +41,16 @@ FORCE_INLINE int32_t schReleaseJob(int64_t refId) { return taosReleaseRef(schMgmt.jobRef, refId); } -char *schDumpEpSet(SEpSet *pEpSet) { +int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes) { + *ppRes = NULL; if (NULL == pEpSet) { - return NULL; + return TSDB_CODE_SUCCESS; } int32_t maxSize = 1024; char *str = taosMemoryMalloc(maxSize); if (NULL == str) { - return NULL; + return terrno; } int32_t n = 0; @@ -54,7 +60,8 @@ char *schDumpEpSet(SEpSet *pEpSet) { n += snprintf(str + n, maxSize - n, "[%s:%d]", pEp->fqdn, pEp->port); } - return str; + *ppRes = str; + return TSDB_CODE_SUCCESS; } char *schGetOpStr(SCH_OP_TYPE type) { @@ -73,7 +80,7 @@ char *schGetOpStr(SCH_OP_TYPE type) { } void schFreeHbTrans(SSchHbTrans *pTrans) { - rpcReleaseHandle((void *)pTrans->trans.pHandleId, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle((void *)pTrans->trans.pHandleId, TAOS_CONN_CLIENT); schFreeRpcCtx(&pTrans->rpcCtx); } @@ -86,7 +93,7 @@ void schCleanClusterHb(void *pTrans) { if (hb->trans.pTrans == pTrans) { SQueryNodeEpId *pEpId = taosHashGetKey(hb, NULL); schFreeHbTrans(hb); - taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); + (void)taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); } hb = taosHashIterate(schMgmt.hbConnections, hb); @@ -109,7 +116,7 @@ int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *ep int64_t taskNum = atomic_load_64(&hb->taskNum); if (taskNum <= 0) { schFreeHbTrans(hb); - taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + (void)taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); } SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); @@ -165,7 +172,7 @@ int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId * break; } - atomic_add_fetch_64(&hb->taskNum, 1); + (void)atomic_add_fetch_64(&hb->taskNum, 1); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); @@ -178,12 +185,17 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { } SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + return; + } + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; SEp *pEp = SCH_GET_CUR_EP(addr); - strcpy(epId.ep.fqdn, pEp->fqdn); + TAOS_STRCPY(epId.ep.fqdn, pEp->fqdn); epId.ep.port = pEp->port; SCH_LOCK(SCH_READ, &schMgmt.hbLock); @@ -197,7 +209,7 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { int64_t taskNum = atomic_sub_fetch_64(&hb->taskNum, 1); if (0 == taskNum) { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); - schRemoveHbConnection(pJob, pTask, &epId); + (void)schRemoveHbConnection(pJob, pTask, &epId); } else { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); } @@ -211,12 +223,17 @@ int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { } SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + return TSDB_CODE_SCH_INTERNAL_ERROR; + } + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; SEp *pEp = SCH_GET_CUR_EP(addr); - strcpy(epId.ep.fqdn, pEp->fqdn); + TAOS_STRCPY(epId.ep.fqdn, pEp->fqdn); epId.ep.port = pEp->port; SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId)); @@ -240,7 +257,7 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { } SCH_LOCK(SCH_WRITE, &hb->lock); - memcpy(&hb->trans, trans, sizeof(*trans)); + TAOS_MEMCPY(&hb->trans, trans, sizeof(*trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); @@ -256,7 +273,7 @@ void schCloseJobRef(void) { } if (schMgmt.jobRef >= 0) { - taosCloseRef(schMgmt.jobRef); + (void)taosCloseRef(schMgmt.jobRef); schMgmt.jobRef = -1; } } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 8f85e066cd..45e0ecf738 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -169,7 +169,8 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) { return; } - SSchJob *pJob = schAcquireJob(*jobId); + SSchJob *pJob = NULL; + (void)schAcquireJob(*jobId, &pJob); if (NULL == pJob) { qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); return; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 78e876f82c..f906e3ec3e 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -75,9 +75,7 @@ int32_t schtStartFetch = 0; void schtInitLogFile() { const char *defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; - rpcInit(); tsAsyncLog = 0; - rpcInit(); qDebugFlag = 159; strcpy(tsLogDir, TD_LOG_DIR_PATH); @@ -136,8 +134,13 @@ int32_t schtBuildSubmitRspMsg(uint32_t *msize, void **rspMsg) { tEncodeSize(tEncodeSSubmitRsp2, &submitRsp, msgSize, ret); void *msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + return terrno; + } tEncoderInit(&ec, (uint8_t *)msg, msgSize); - tEncodeSSubmitRsp2(&ec, &submitRsp); + if (tEncodeSSubmitRsp2(&ec, &submitRsp) < 0) { + return -1; + } tEncoderClear(&ec); *rspMsg = msg; @@ -152,11 +155,26 @@ void schtBuildQueryDag(SQueryPlan *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); + if (NULL == dag->pSubplans) { + return; + } SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == scan) { + return; + } SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == merge) { + return; + } SSubplan *scanPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == scanPlan) { + return; + } SSubplan *mergePlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == mergePlan) { + return; + } scanPlan->id.queryId = qId; scanPlan->id.groupId = 0x0000000000000002; @@ -170,7 +188,13 @@ void schtBuildQueryDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); + if (NULL == scanPlan->pParents) { + return; + } scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + if (NULL == scanPlan->pNode) { + return; + } scanPlan->msgType = TDMT_SCH_QUERY; mergePlan->id.queryId = qId; @@ -181,21 +205,33 @@ void schtBuildQueryDag(SQueryPlan *dag) { mergePlan->execNode.epSet.numOfEps = 0; mergePlan->pChildren = nodesMakeList(); + if (NULL == mergePlan->pChildren) { + return; + } mergePlan->pParents = NULL; mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE); + if (NULL == mergePlan->pNode) { + return; + } mergePlan->msgType = TDMT_SCH_QUERY; merge->pNodeList = nodesMakeList(); + if (NULL == merge->pNodeList) { + return; + } scan->pNodeList = nodesMakeList(); + if (NULL == scan->pNodeList) { + return; + } - nodesListAppend(merge->pNodeList, (SNode *)mergePlan); - nodesListAppend(scan->pNodeList, (SNode *)scanPlan); + (void)nodesListAppend(merge->pNodeList, (SNode *)mergePlan); + (void)nodesListAppend(scan->pNodeList, (SNode *)scanPlan); - nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); - nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); + (void)nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); + (void)nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); - nodesListAppend(dag->pSubplans, (SNode *)merge); - nodesListAppend(dag->pSubplans, (SNode *)scan); + (void)nodesListAppend(dag->pSubplans, (SNode *)merge); + (void)nodesListAppend(dag->pSubplans, (SNode *)scan); } void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { @@ -205,18 +241,42 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); + if (NULL == dag->pSubplans) { + return; + } SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == scan) { + return; + } SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == merge) { + return; + } SSubplan *mergePlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == mergePlan) { + return; + } merge->pNodeList = nodesMakeList(); + if (NULL == merge->pNodeList) { + return; + } scan->pNodeList = nodesMakeList(); + if (NULL == scan->pNodeList) { + return; + } mergePlan->pChildren = nodesMakeList(); + if (NULL == mergePlan->pChildren) { + return; + } for (int32_t i = 0; i < scanPlanNum; ++i) { SSubplan *scanPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == scanPlan) { + return; + } scanPlan->id.queryId = qId; scanPlan->id.groupId = 0x0000000000000002; scanPlan->id.subplanId = 0x0000000000000003 + i; @@ -233,13 +293,19 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); + if (NULL == scanPlan->pParents) { + return; + } scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + if (NULL == scanPlan->pNode) { + return; + } scanPlan->msgType = TDMT_SCH_QUERY; - nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); - nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); + (void)nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); + (void)nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); - nodesListAppend(scan->pNodeList, (SNode *)scanPlan); + (void)nodesListAppend(scan->pNodeList, (SNode *)scanPlan); } mergePlan->id.queryId = qId; @@ -251,12 +317,15 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { mergePlan->pParents = NULL; mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE); + if (NULL == mergePlan->pNode) { + return; + } mergePlan->msgType = TDMT_SCH_QUERY; - nodesListAppend(merge->pNodeList, (SNode *)mergePlan); + (void)nodesListAppend(merge->pNodeList, (SNode *)mergePlan); - nodesListAppend(dag->pSubplans, (SNode *)merge); - nodesListAppend(dag->pSubplans, (SNode *)scan); + (void)nodesListAppend(dag->pSubplans, (SNode *)merge); + (void)nodesListAppend(dag->pSubplans, (SNode *)scan); } void schtFreeQueryDag(SQueryPlan *dag) {} @@ -267,10 +336,22 @@ void schtBuildInsertDag(SQueryPlan *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); + if (NULL == dag->pSubplans) { + return; + } SNodeListNode *inserta = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == inserta) { + return; + } inserta->pNodeList = nodesMakeList(); + if (NULL == inserta->pNodeList) { + return; + } SSubplan *insertPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == insertPlan) { + return; + } insertPlan->id.queryId = qId; insertPlan->id.groupId = 0x0000000000000003; @@ -286,13 +367,22 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan->pParents = NULL; insertPlan->pNode = NULL; insertPlan->pDataSink = (SDataSinkNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT); + if (NULL == insertPlan->pDataSink) { + return; + } ((SDataInserterNode *)insertPlan->pDataSink)->size = 1; ((SDataInserterNode *)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1); + if (NULL == ((SDataInserterNode *)insertPlan->pDataSink)->pData) { + return; + } insertPlan->msgType = TDMT_VND_SUBMIT; - nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); + (void)nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); insertPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == insertPlan) { + return; + } insertPlan->id.queryId = qId; insertPlan->id.groupId = 0x0000000000000003; @@ -308,22 +398,31 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan->pParents = NULL; insertPlan->pNode = NULL; insertPlan->pDataSink = (SDataSinkNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT); + if (NULL == insertPlan->pDataSink) { + return; + } ((SDataInserterNode *)insertPlan->pDataSink)->size = 1; ((SDataInserterNode *)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1); + if (NULL == ((SDataInserterNode *)insertPlan->pDataSink)->pData) { + return; + } insertPlan->msgType = TDMT_VND_SUBMIT; - nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); + (void)nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); - nodesListAppend(dag->pSubplans, (SNode *)inserta); + (void)nodesListAppend(dag->pSubplans, (SNode *)inserta); } int32_t schtPlanToString(const SSubplan *subplan, char **str, int32_t *len) { *str = (char *)taosMemoryCalloc(1, 20); + if (NULL == *str) { + return -1; + } *len = 20; return 0; } -void schtExecNode(SSubplan *subplan, uint64_t groupId, SQueryNodeAddr *ep) {} +int32_t schtExecNode(SSubplan *subplan, uint64_t groupId, SQueryNodeAddr *ep) { return 0; } void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) {} @@ -431,24 +530,27 @@ void *schtSendRsp(void *param) { taosMsleep(1); } - pJob = schAcquireJob(job); - + code = schAcquireJob(job, &pJob); + if (code) { + return NULL; + } + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildSubmitRspMsg(&msg.len, &rmsg); + (void)schtBuildSubmitRspMsg(&msg.len, &rmsg); msg.msgType = TDMT_VND_SUBMIT_RSP; msg.pData = rmsg; - schHandleResponseMsg(pJob, task, task->execId, &msg, 0); + (void)schHandleResponseMsg(pJob, task, task->execId, &msg, 0); pIter = taosHashIterate(pJob->execTasks, pIter); } - schReleaseJob(job); + (void)schReleaseJob(job); schtJobDone = true; @@ -457,20 +559,25 @@ void *schtSendRsp(void *param) { void *schtCreateFetchRspThread(void *param) { int64_t job = *(int64_t *)param; - SSchJob *pJob = schAcquireJob(job); + SSchJob *pJob = NULL; + (void)schAcquireJob(job, &pJob); + if (NULL == pJob) { + return NULL; + } + taosSsleep(1); int32_t code = 0; SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildFetchRspMsg(&msg.len, &rmsg); + (void)schtBuildFetchRspMsg(&msg.len, &rmsg); msg.msgType = TDMT_SCH_MERGE_FETCH_RSP; msg.pData = rmsg; code = schHandleResponseMsg(pJob, pJob->fetchTask, pJob->fetchTask->execId, &msg, 0); - schReleaseJob(job); + (void)schReleaseJob(job); assert(code == 0); return NULL; @@ -488,12 +595,17 @@ void *schtFetchRspThread(void *aa) { taosUsleep(100); param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); - + if (NULL == param) { + return NULL; + } param->queryId = schtQueryId; param->taskId = schtFetchTaskId; int32_t code = 0; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); + if (NULL == rsp) { + return NULL; + } rsp->completed = 1; rsp->numOfRows = 10; @@ -549,12 +661,17 @@ void *schtRunJobThread(void *aa) { schtBuildQueryDag(dag); SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad)); - + if (NULL == qnodeList) { + assert(0); + } + SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + if (NULL == taosArrayPush(qnodeList, &load)) { + assert(0); + } queryDone = 0; @@ -572,7 +689,9 @@ void *schtRunJobThread(void *aa) { code = schedulerExecJob(&req, &queryJobRefId); assert(code == 0); - pJob = schAcquireJob(queryJobRefId); + pJob = NULL; + code = schAcquireJob(queryJobRefId, &pJob); + if (NULL == pJob) { taosArrayDestroy(qnodeList); schtFreeQueryDag(dag); @@ -580,16 +699,24 @@ void *schtRunJobThread(void *aa) { } execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == execTasks) { + assert(0); + } void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; schtFetchTaskId = task->taskId - 1; - taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task)); + if (taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task))) { + assert(0); + } pIter = taosHashIterate(pJob->execTasks, pIter); } param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); + if (NULL == param) { + assert(0); + } param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -601,7 +728,9 @@ void *schtRunJobThread(void *aa) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + if (schtBuildQueryRspMsg(&msg.len, &rmsg)) { + assert(0); + } msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -612,6 +741,9 @@ void *schtRunJobThread(void *aa) { } param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); + if (NULL == param) { + assert(0); + } param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -622,7 +754,9 @@ void *schtRunJobThread(void *aa) { param->taskId = task->taskId - 1; SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + if (schtBuildQueryRspMsg(&msg.len, &rmsg)) { + assert(0); + } msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -728,7 +862,9 @@ TEST(queryTest, normalCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); + SSchJob *pJob = NULL; + code = schAcquireJob(job, &pJob); + ASSERT_EQ(code, 0); void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { @@ -839,8 +975,10 @@ TEST(queryTest, readyFirstCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); - + SSchJob *pJob = NULL; + code = schAcquireJob(job, &pJob); + ASSERT_EQ(code, 0); + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; @@ -956,7 +1094,9 @@ TEST(queryTest, flowCtrlCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); + SSchJob *pJob = NULL; + code = schAcquireJob(job, &pJob); + ASSERT_EQ(code, 0); while (!queryDone) { void *pIter = taosHashIterate(pJob->execTasks, NULL); @@ -1094,13 +1234,17 @@ TEST(otherTest, otherCase) { schReleaseJob(0); schFreeRpcCtx(NULL); - ASSERT_EQ(schDumpEpSet(NULL), (char *)NULL); + char* ep = NULL; + ASSERT_EQ(schDumpEpSet(NULL, &ep), TSDB_CODE_SUCCESS); ASSERT_EQ(strcmp(schGetOpStr(SCH_OP_NULL), "NULL"), 0); ASSERT_EQ(strcmp(schGetOpStr((SCH_OP_TYPE)100), "UNKNOWN"), 0); } int main(int argc, char **argv) { schtInitLogFile(); + if (rpcInit()) { + assert(0); + } taosSeedRand(taosGetTimestampSec()); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 620f5f1248..3ea3e814ed 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -310,7 +310,7 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { taosArrayDestroy(pInfo->pList); pInfo->pList = NULL; - + if (pInfo->checkRspTmr != NULL) { /*bool ret = */ taosTmrStop(pInfo->checkRspTmr); pInfo->checkRspTmr = NULL; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 617adaa016..1959180a3e 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1162,10 +1162,10 @@ void streamClearChkptReadyMsg(SActiveCheckpointInfo* pActiveInfo) { static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId, int32_t downstreamNodeId) { stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); - bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); - clearBufferedDispatchMsg(pTask); - int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs; + bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); + + clearBufferedDispatchMsg(pTask); // put data into inputQ of current task is also allowed if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { @@ -1189,13 +1189,24 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId return 0; } -static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, const char* id) { +static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, int32_t* pNotRsp, const char* id) { int32_t numOfRsp = 0; bool alreadySet = false; bool updated = false; + bool allRsp = false; + *pNotRsp = 0; taosThreadMutexLock(&pMsgInfo->lock); - for (int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) { + int32_t numOfDispatchBranch = taosArrayGetSize(pMsgInfo->pSendInfo); + + for(int32_t i = 0; i < numOfDispatchBranch; ++i) { + SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, i); + if (pEntry->rspTs != -1) { + numOfRsp += 1; + } + } + + for (int32_t j = 0; j < numOfDispatchBranch; ++j) { SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j); if (pEntry->nodeId == vgId) { ASSERT(!alreadySet); @@ -1203,18 +1214,20 @@ static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int3 pEntry->status = code; alreadySet = true; updated = true; - stDebug("s-task:%s record the rsp recv, ts:%" PRId64 " code:%d, idx:%d", id, now, code, j); - } - - if (pEntry->rspTs != -1) { numOfRsp += 1; + + stDebug("s-task:%s record the rsp recv, ts:%" PRId64 " code:%d, idx:%d, total recv:%d/%d", id, now, code, j, + numOfRsp, numOfDispatchBranch); } } - taosThreadMutexUnlock(&pMsgInfo->lock); - ASSERT(updated); + *pNotRsp = numOfDispatchBranch - numOfRsp; + allRsp = (numOfRsp == numOfDispatchBranch); - return numOfRsp; + taosThreadMutexUnlock(&pMsgInfo->lock); + + ASSERT(updated); + return allRsp; } bool isDispatchRspTimeout(SDispatchEntry* pEntry, int64_t now) { @@ -1240,7 +1253,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int32_t vgId = pTask->pMeta->vgId; SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo; int64_t now = taosGetTimestampMs(); - int32_t totalRsp = 0; + bool allRsp = false; + int32_t notRsp = 0; taosThreadMutexLock(&pMsgInfo->lock); int32_t msgId = pMsgInfo->msgId; @@ -1269,18 +1283,18 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (code == TSDB_CODE_STREAM_TASK_NOT_EXIST) { // destination task does not exist, not retry anymore stError("s-task:%s failed to dispatch msg to task:0x%x(vgId:%d), msgId:%d no retry, since task destroyed already", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId, msgId); - totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, id); + allRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, ¬Rsp, id); } else { stError("s-task:%s failed to dispatch msgId:%d to task:0x%x(vgId:%d), code:%s, add to retry list", id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, tstrerror(code)); - totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, code, now, id); + allRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, code, now, ¬Rsp, id); } } else { // code == 0 if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { pTask->inputq.status = TASK_INPUT_STATUS__BLOCKED; // block the input of current task, to push pressure to upstream - totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, pRsp->inputStatus, now, id); + allRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, pRsp->inputStatus, now, ¬Rsp, id); stTrace("s-task:%s inputQ of downstream task:0x%x(vgId:%d) is full, wait for retry dispatch", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); } else { @@ -1292,7 +1306,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); } - totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, id); + allRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, ¬Rsp, id); { bool delayDispatch = (pMsgInfo->dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); @@ -1317,13 +1331,11 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } } - int32_t notRsp = taosArrayGetSize(pMsgInfo->pSendInfo) - totalRsp; if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { - if (notRsp > 0) { + if (!allRsp) { stDebug( "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, " - "waiting " - "for %d rsp", + "waiting for %d rsp", id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code), notRsp); } else { stDebug( @@ -1337,7 +1349,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } // all msg rsp already, continue - if (notRsp == 0) { + if (allRsp) { ASSERT(pTask->outputq.status == TASK_OUTPUT_STATUS__WAIT); // we need to re-try send dispatch msg to downstream tasks diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 221250ba74..da30caad98 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -263,6 +263,7 @@ void tFreeStreamTask(SStreamTask* pTask) { tDeleteSchemaWrapper(pTask->outputInfo.tbSink.pSchemaWrapper); taosMemoryFree(pTask->outputInfo.tbSink.pTSchema); tSimpleHashCleanup(pTask->outputInfo.tbSink.pTblInfo); + tDeleteSchemaWrapper(pTask->outputInfo.tbSink.pTagSchema); } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { taosArrayDestroy(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos); } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 0adb7050c6..61b7e02a4c 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -18,6 +18,7 @@ #include "tencode.h" #include "tstreamUpdate.h" #include "ttime.h" +#include "tutil.h" #define DEFAULT_FALSE_POSITIVE 0.01 #define DEFAULT_BUCKET_SIZE 131072 @@ -34,7 +35,8 @@ static int64_t adjustExpEntries(int64_t entries) { return TMIN(DEFAULT_EXPECTED_ENTRIES, entries); } int compareKeyTs(void* pTs1, void* pTs2, void* pPkVal, __compar_fn_t cmpPkFn) { - return compareInt64Val(pTs1, pTs2);; + return compareInt64Val(pTs1, pTs2); + ; } int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpPkFn) { @@ -49,7 +51,7 @@ int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpP int32_t getKeyBuff(TSKEY ts, int64_t tbUid, void* pVal, int32_t len, char* buff) { *(TSKEY*)buff = ts; - memcpy(buff+ sizeof(TSKEY), &tbUid, sizeof(int64_t)); + memcpy(buff + sizeof(TSKEY), &tbUid, sizeof(int64_t)); if (len == 0) { return sizeof(TSKEY) + sizeof(int64_t); } @@ -66,26 +68,40 @@ int32_t getValueBuff(TSKEY ts, char* pVal, int32_t len, char* buff) { return sizeof(TSKEY) + len; } -void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { +int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pInfo->numSBFs < count) { count = pInfo->numSBFs; } for (uint64_t i = 0; i < count; ++i) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - SScalableBf *tsSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); - taosArrayPush(pInfo->pTsSBFs, &tsSBF); + SScalableBf* tsSBF = NULL; + code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &tsSBF); + TSDB_CHECK_CODE(code, lino, _error); + void* res = taosArrayPush(pInfo->pTsSBFs, &tsSBF); + if (!res) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } } + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static void clearItemHelper(void *p) { - SScalableBf **pBf = p; +static void clearItemHelper(void* p) { + SScalableBf** pBf = p; tScalableBfDestroy(*pBf); } -void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) { +void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count) { if (count < pInfo->numSBFs) { for (uint64_t i = 0; i < count; ++i) { - SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0); + SScalableBf* pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0); tScalableBfDestroy(pTsSBFs); taosArrayRemove(pInfo->pTsSBFs, 0); } @@ -124,14 +140,19 @@ static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t w return watermark; } -SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen) { - return updateInfoInit(pInterval->interval, pInterval->precision, watermark, igUp, pkType, pkLen); +int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, + SUpdateInfo** ppInfo) { + return updateInfoInit(pInterval->interval, pInterval->precision, watermark, igUp, pkType, pkLen, ppInfo); } -SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen) { - SUpdateInfo *pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); +int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, + SUpdateInfo** ppInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SUpdateInfo* pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); if (pInfo == NULL) { - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } pInfo->pTsBuckets = NULL; pInfo->pTsSBFs = NULL; @@ -145,17 +166,19 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma bfSize = (uint64_t)(pInfo->watermark / pInfo->interval); pInfo->numSBFs = bfSize; - pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void *)); + pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void*)); if (pInfo->pTsSBFs == NULL) { updateInfoDestroy(pInfo); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } windowSBfAdd(pInfo, bfSize); pInfo->pTsBuckets = taosArrayInit(DEFAULT_BUCKET_SIZE, sizeof(TSKEY)); if (pInfo->pTsBuckets == NULL) { updateInfoDestroy(pInfo); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } TSKEY dumy = 0; @@ -167,31 +190,54 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK); + if (!pInfo->pMap) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } pInfo->maxDataVersion = 0; pInfo->pkColLen = pkLen; pInfo->pkColType = pkType; pInfo->pKeyBuff = taosMemoryCalloc(1, sizeof(TSKEY) + sizeof(int64_t) + pkLen); + if (!pInfo->pKeyBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } pInfo->pValueBuff = taosMemoryCalloc(1, sizeof(TSKEY) + pkLen); + if (!pInfo->pValueBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } if (pkLen != 0) { pInfo->comparePkRowFn = compareKeyTsAndPk; - pInfo->comparePkCol = getKeyComparFunc(pkType, TSDB_ORDER_ASC);; + pInfo->comparePkCol = getKeyComparFunc(pkType, TSDB_ORDER_ASC); + ; } else { pInfo->comparePkRowFn = compareKeyTs; pInfo->comparePkCol = NULL; } - return pInfo; + (*ppInfo) = pInfo; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { +static int32_t getSBf(SUpdateInfo* pInfo, TSKEY ts, SScalableBf** ppSBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (ts <= 0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->minTS < 0) { pInfo->minTS = (TSKEY)(ts / pInfo->interval * pInfo->interval); } int64_t index = (int64_t)((ts - pInfo->minTS) / pInfo->interval); if (index < 0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } if (index >= pInfo->numSBFs) { uint64_t count = index + 1 - pInfo->numSBFs; @@ -199,22 +245,31 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { windowSBfAdd(pInfo, count); index = pInfo->numSBFs - 1; } - SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index); + SScalableBf* res = taosArrayGetP(pInfo->pTsSBFs, index); if (res == NULL) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - res = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); + code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res); + TSDB_CHECK_CODE(code, lino, _end); + taosArrayPush(pInfo->pTsSBFs, &res); } - return res; + (*ppSBf) = res; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid) { - void *pVal = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); +bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) { + void* pVal = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); if (pVal || taosHashGetSize(pInfo->pMap) >= DEFAULT_MAP_SIZE) return true; return false; } -TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) { +TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) { + int32_t code = TSDB_CODE_SUCCESS; if (pBlock == NULL || pBlock->info.rows == 0) return INT64_MIN; TSKEY maxTs = INT64_MIN; void* pPkVal = NULL; @@ -223,14 +278,14 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p int32_t len = 0; int64_t tbUid = pBlock->info.id.uid; - SColumnInfoData *pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol); - SColumnInfoData *pPkDataInfo = NULL; + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol); + SColumnInfoData* pPkDataInfo = NULL; if (primaryKeyCol >= 0) { pPkDataInfo = taosArrayGet(pBlock->pDataBlock, primaryKeyCol); } for (int32_t i = 0; i < pBlock->info.rows; i++) { - TSKEY ts = ((TSKEY *)pColDataInfo->pData)[i]; + TSKEY ts = ((TSKEY*)pColDataInfo->pData)[i]; if (maxTs < ts) { maxTs = ts; if (primaryKeyCol >= 0) { @@ -238,17 +293,22 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p maxLen = colDataGetRowLength(pPkDataInfo, i); } } - SScalableBf *pSBf = getSBf(pInfo, ts); + SScalableBf* pSBf = NULL; + int32_t code = getSBf(pInfo, ts, &pSBf); + if (code != TSDB_CODE_SUCCESS) { + uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } if (pSBf) { if (primaryKeyCol >= 0) { pPkVal = colDataGetData(pPkDataInfo, i); len = colDataGetRowLength(pPkDataInfo, i); } int32_t buffLen = getKeyBuff(ts, tbUid, pPkVal, len, pInfo->pKeyBuff); + // we don't care whether the data is updated or not tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen); } } - void *pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); + void* pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); if (pMaxTs == NULL || pInfo->comparePkRowFn(pMaxTs, &maxTs, pMaxPkVal, pInfo->comparePkCol) == -1) { int32_t valueLen = getValueBuff(maxTs, pMaxPkVal, maxLen, pInfo->pValueBuff); taosHashPut(pInfo->pMap, &tbUid, sizeof(int64_t), pInfo->pValueBuff, valueLen); @@ -256,14 +316,14 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p return maxTs; } -bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { +bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { int32_t res = TSDB_CODE_FAILED; int32_t buffLen = 0; buffLen = getKeyBuff(ts, tableId, pPkVal, len, pInfo->pKeyBuff); - void* *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); + void** pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; - TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index); + TSKEY maxTs = *(TSKEY*)taosArrayGet(pInfo->pTsBuckets, index); if (ts < maxTs - pInfo->watermark) { // this window has been closed. if (pInfo->pCloseWinSBF) { @@ -277,10 +337,15 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* p return true; } - SScalableBf *pSBf = getSBf(pInfo, ts); + SScalableBf* pSBf = NULL; + int32_t code = getSBf(pInfo, ts, &pSBf); + if (code != TSDB_CODE_SUCCESS) { + uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } int32_t size = taosHashGetSize(pInfo->pMap); - if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1 )) { + if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || + (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1)) { int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff); taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen); // pSBf may be a null pointer @@ -309,7 +374,7 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* p return true; } -void updateInfoDestroy(SUpdateInfo *pInfo) { +void updateInfoDestroy(SUpdateInfo* pInfo) { if (pInfo == NULL) { return; } @@ -317,7 +382,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { uint64_t size = taosArrayGetSize(pInfo->pTsSBFs); for (uint64_t i = 0; i < size; i++) { - SScalableBf *pSBF = taosArrayGetP(pInfo->pTsSBFs, i); + SScalableBf* pSBF = taosArrayGetP(pInfo->pTsSBFs, i); tScalableBfDestroy(pSBF); } @@ -329,15 +394,19 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { taosMemoryFree(pInfo); } -void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) { +void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo) { if (pInfo->pCloseWinSBF) { return; } int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - pInfo->pCloseWinSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); + int32_t code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &pInfo->pCloseWinSBF); + if (code != TSDB_CODE_SUCCESS) { + pInfo->pCloseWinSBF = NULL; + uError("%s failed to add close window SBF since %s", __func__, tstrerror(code)); + } } -void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { +void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo) { if (!pInfo || !pInfo->pCloseWinSBF) { return; } @@ -345,62 +414,124 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { pInfo->pCloseWinSBF = NULL; } -int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) { +int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (!pInfo) { - return 0; + return TSDB_CODE_SUCCESS; } SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; + if (tStartEncode(&encoder) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t size = taosArrayGetSize(pInfo->pTsBuckets); - if (tEncodeI32(&encoder, size) < 0) return -1; - for (int32_t i = 0; i < size; i++) { - TSKEY *pTs = (TSKEY *)taosArrayGet(pInfo->pTsBuckets, i); - if (tEncodeI64(&encoder, *pTs) < 0) return -1; + if (tEncodeI32(&encoder, size) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } - if (tEncodeU64(&encoder, pInfo->numBuckets) < 0) return -1; + for (int32_t i = 0; i < size; i++) { + TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->pTsBuckets, i); + if (tEncodeI64(&encoder, *pTs) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + } + + if (tEncodeU64(&encoder, pInfo->numBuckets) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t sBfSize = taosArrayGetSize(pInfo->pTsSBFs); - if (tEncodeI32(&encoder, sBfSize) < 0) return -1; + if (tEncodeI32(&encoder, sBfSize) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } for (int32_t i = 0; i < sBfSize; i++) { - SScalableBf *pSBf = taosArrayGetP(pInfo->pTsSBFs, i); - if (tScalableBfEncode(pSBf, &encoder) < 0) return -1; + SScalableBf* pSBf = taosArrayGetP(pInfo->pTsSBFs, i); + if (tScalableBfEncode(pSBf, &encoder) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } } - if (tEncodeU64(&encoder, pInfo->numSBFs) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->interval) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->watermark) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->minTS) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->numSBFs) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI64(&encoder, pInfo->interval) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI64(&encoder, pInfo->watermark) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI64(&encoder, pInfo->minTS) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } - if (tScalableBfEncode(pInfo->pCloseWinSBF, &encoder) < 0) return -1; + if (tScalableBfEncode(pInfo->pCloseWinSBF, &encoder) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t mapSize = taosHashGetSize(pInfo->pMap); - if (tEncodeI32(&encoder, mapSize) < 0) return -1; - void *pIte = NULL; + if (tEncodeI32(&encoder, mapSize) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pInfo->pMap, pIte)) != NULL) { - void *key = taosHashGetKey(pIte, &keyLen); - if (tEncodeU64(&encoder, *(uint64_t *)key) < 0) return -1; + void* key = taosHashGetKey(pIte, &keyLen); + if (tEncodeU64(&encoder, *(uint64_t*)key) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t valueSize = taosHashGetValueSize(pIte); - if (tEncodeBinary(&encoder, (const uint8_t *)pIte, valueSize) < 0) return -1; + if (tEncodeBinary(&encoder, (const uint8_t*)pIte, valueSize) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } } - if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } - if (tEncodeI32(&encoder, pInfo->pkColLen) < 0) return -1; - if (tEncodeI8(&encoder, pInfo->pkColType) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->pkColLen) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI8(&encoder, pInfo->pkColType) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } tEndEncode(&encoder); int32_t tlen = encoder.pos; tEncoderClear(&encoder); - return tlen; + *pLen = tlen; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { +int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; ASSERT(pInfo); SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -419,10 +550,12 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { int32_t sBfSize = 0; if (tDecodeI32(&decoder, &sBfSize) < 0) return -1; - pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void *)); + pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void*)); for (int32_t i = 0; i < sBfSize; i++) { - SScalableBf *pSBf = tScalableBfDecode(&decoder); - if (!pSBf) return -1; + SScalableBf* pSBf = NULL; + code = tScalableBfDecode(&decoder, &pSBf); + TSDB_CHECK_CODE(code, lino, _error); + taosArrayPush(pInfo->pTsSBFs, &pSBf); } @@ -430,15 +563,20 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { if (tDecodeI64(&decoder, &pInfo->interval) < 0) return -1; if (tDecodeI64(&decoder, &pInfo->watermark) < 0) return -1; if (tDecodeI64(&decoder, &pInfo->minTS) < 0) return -1; - pInfo->pCloseWinSBF = tScalableBfDecode(&decoder); + + code = tScalableBfDecode(&decoder, &pInfo->pCloseWinSBF); + if (code != TSDB_CODE_SUCCESS) { + pInfo->pCloseWinSBF = NULL; + code = TSDB_CODE_SUCCESS; + } int32_t mapSize = 0; if (tDecodeI32(&decoder, &mapSize) < 0) return -1; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); pInfo->pMap = taosHashInit(mapSize, hashFn, true, HASH_NO_LOCK); uint64_t uid = 0; - void* pVal = NULL; - int32_t valSize = 0; + void* pVal = NULL; + int32_t valSize = 0; for (int32_t i = 0; i < mapSize; i++) { if (tDecodeU64(&decoder, &uid) < 0) return -1; if (tDecodeBinary(&decoder, (uint8_t**)&pVal, &valSize) < 0) return -1; @@ -454,7 +592,8 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { pInfo->pValueBuff = taosMemoryCalloc(1, sizeof(TSKEY) + pInfo->pkColLen); if (pInfo->pkColLen != 0) { pInfo->comparePkRowFn = compareKeyTsAndPk; - pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC);; + pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC); + ; } else { pInfo->comparePkRowFn = compareKeyTs; pInfo->comparePkCol = NULL; @@ -463,11 +602,16 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { tEndDecode(&decoder); tDecoderClear(&decoder); - return 0; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { - TSKEY *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); +bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { + TSKEY* pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); bool res = true; if (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == 1) { res = false; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 3d8588a55a..009854b45b 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -74,16 +74,16 @@ typedef struct SRaftStore { TdThreadMutex mutex; } SRaftStore; -typedef struct SSyncHbTimerData { +struct SSyncHbTimerData { int64_t syncNodeRid; SSyncTimer* pTimer; SRaftId destId; uint64_t logicClock; int64_t execTime; int64_t rid; -} SSyncHbTimerData; +}; -typedef struct SSyncTimer { +struct SSyncTimer { void* pTimer; TAOS_TMR_CALLBACK timerCb; uint64_t logicClock; @@ -92,7 +92,7 @@ typedef struct SSyncTimer { int64_t timeStamp; SRaftId destId; int64_t hbDataRid; -} SSyncTimer; +}; typedef struct SElectTimerParam { uint64_t logicClock; @@ -106,7 +106,7 @@ typedef struct SPeerState { int64_t lastSendTime; } SPeerState; -typedef struct SSyncNode { +struct SSyncNode { // init by SSyncInfo SyncGroupId vgId; SRaftCfg raftCfg; @@ -116,7 +116,7 @@ typedef struct SSyncNode { // sync io SSyncLogBuffer* pLogBuf; - SWal* pWal; + struct SWal* pWal; const SMsgCb* msgcb; int32_t (*syncSendMSg)(const SEpSet* pEpSet, SRpcMsg* pMsg); int32_t (*syncEqMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg); @@ -234,7 +234,7 @@ typedef struct SSyncNode { bool isStart; -} SSyncNode; +}; // open/close -------------- SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index e62ca7d69a..acb9bd20f3 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -103,7 +103,7 @@ typedef void* queue[2]; #define TRANS_MAGIC_NUM 0x5f375a86 #define TRANS_NOVALID_PACKET(src) ((src) != TRANS_MAGIC_NUM ? 1 : 0) -typedef SRpcMsg STransMsg; +typedef struct SRpcMsg STransMsg; typedef SRpcCtx STransCtx; typedef SRpcCtxVal STransCtxVal; typedef SRpcInfo STrans; diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 150faab571..1cf4ecf72e 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -22,26 +22,30 @@ #define UNIT_NUM_BITS 64ULL #define UNIT_ADDR_NUM_BITS 6ULL -static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { +static FORCE_INLINE bool setBit(uint64_t* buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; uint64_t old = buf[unitIndex]; buf[unitIndex] |= (1ULL << (index % UNIT_NUM_BITS)); return buf[unitIndex] != old; } -static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) { +static FORCE_INLINE bool getBit(uint64_t* buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); return buf[unitIndex] & mask; } -SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) { +int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF) { + int32_t code = 0; + int32_t lino = 0; if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); + SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); if (pBF == NULL) { - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pBF->expectedEntries = expectedEntries; pBF->errorRate = errorRate; @@ -61,12 +65,19 @@ SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) { pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); if (pBF->buffer == NULL) { tBloomFilterDestroy(pBF); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } - return pBF; + (*ppBF) = pBF; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { +int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) { ASSERT(!tBloomFilterIsFull(pBF)); bool hasChange = false; const register uint64_t size = pBF->numBits; @@ -82,7 +93,7 @@ int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { return TSDB_CODE_FAILED; } -int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { +int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len) { uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len); bool hasChange = false; @@ -99,7 +110,7 @@ int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { return TSDB_CODE_FAILED; } -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { +int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) { const register uint64_t size = pBF->numBits; uint64_t cbHash = hash1; for (uint32_t i = 0; i < pBF->hashFunctions; ++i) { @@ -111,7 +122,7 @@ int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t return TSDB_CODE_FAILED; } -void tBloomFilterDestroy(SBloomFilter *pBF) { +void tBloomFilterDestroy(SBloomFilter* pBF) { if (pBF == NULL) { return; } @@ -119,41 +130,70 @@ void tBloomFilterDestroy(SBloomFilter *pBF) { taosMemoryFree(pBF); } -int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder) { +int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder) { if (tEncodeU32(pEncoder, pBF->hashFunctions) < 0) return -1; if (tEncodeU64(pEncoder, pBF->expectedEntries) < 0) return -1; if (tEncodeU64(pEncoder, pBF->numUnits) < 0) return -1; if (tEncodeU64(pEncoder, pBF->numBits) < 0) return -1; if (tEncodeU64(pEncoder, pBF->size) < 0) return -1; for (uint64_t i = 0; i < pBF->numUnits; i++) { - uint64_t *pUnits = (uint64_t *)pBF->buffer; + uint64_t* pUnits = (uint64_t*)pBF->buffer; if (tEncodeU64(pEncoder, pUnits[i]) < 0) return -1; } if (tEncodeDouble(pEncoder, pBF->errorRate) < 0) return -1; return 0; } -SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder) { - SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); +int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF) { + int32_t code = 0; + int32_t lino = 0; + SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); + if (!pBF) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } pBF->buffer = NULL; - if (tDecodeU32(pDecoder, &pBF->hashFunctions) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->expectedEntries) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->numUnits) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->numBits) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->size) < 0) goto _error; + if (tDecodeU32(pDecoder, &pBF->hashFunctions) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->expectedEntries) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->numUnits) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->numBits) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->size) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); for (int32_t i = 0; i < pBF->numUnits; i++) { - uint64_t *pUnits = (uint64_t *)pBF->buffer; - if (tDecodeU64(pDecoder, pUnits + i) < 0) goto _error; + uint64_t* pUnits = (uint64_t*)pBF->buffer; + if (tDecodeU64(pDecoder, pUnits + i) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + } + if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) goto _error; pBF->hashFn1 = HASH_FUNCTION_1; pBF->hashFn2 = HASH_FUNCTION_2; - return pBF; + (*ppBF) = pBF; + return TSDB_CODE_SUCCESS; _error: tBloomFilterDestroy(pBF); - return NULL; + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + return code; } -bool tBloomFilterIsFull(const SBloomFilter *pBF) { return pBF->size >= pBF->expectedEntries; } +bool tBloomFilterIsFull(const SBloomFilter* pBF) { return pBF->size >= pBF->expectedEntries; } diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 7af794546b..4a4dfd2653 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -17,6 +17,7 @@ #include "tscalablebf.h" #include "taoserror.h" +#include "tutil.h" #define DEFAULT_GROWTH 2 #define DEFAULT_TIGHTENING_RATIO 0.5 @@ -24,82 +25,120 @@ #define SBF_INVALID -1 #define SBF_VALID 0 -static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate); +static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate, + SBloomFilter** ppNormalBf); -SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) { +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; const uint32_t defaultSize = 8; if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (pSBf == NULL) { - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; pSBf->status = SBF_VALID; pSBf->numBits = 0; - pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *)); - if (tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO) == NULL) { + pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void*)); + if (!pSBf->bfArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } + + SBloomFilter* pNormalBf = NULL; + code = tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + if (code != TSDB_CODE_SUCCESS) { tScalableBfDestroy(pSBf); - return NULL; + TSDB_CHECK_CODE(code, lino, _error); } pSBf->growth = DEFAULT_GROWTH; pSBf->hashFn1 = HASH_FUNCTION_1; pSBf->hashFn2 = HASH_FUNCTION_2; - return pSBf; + (*ppSBf) = pSBf; + return TSDB_CODE_SUCCESS; + +_error: + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + return code; } -int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { +int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pSBf->status == SBF_INVALID) { - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + int32_t size = taosArrayGetSize(pSBf->bfArray); + SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); + if (!pNormalBf) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } - int32_t size = taosArrayGetSize(pSBf->bfArray); - SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); - ASSERT(pNormalBf); if (tBloomFilterIsFull(pNormalBf)) { - pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, - pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); - if (pNormalBf == NULL) { + code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, + pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + if (code != TSDB_CODE_SUCCESS) { pSBf->status = SBF_INVALID; - return TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } } return tBloomFilterPut(pNormalBf, keyBuf, len); + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { +int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pSBf->status == SBF_INVALID) { - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); - int32_t size = taosArrayGetSize(pSBf->bfArray); + int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 2; i >= 0; --i) { if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } } - SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); + SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); ASSERT(pNormalBf); if (tBloomFilterIsFull(pNormalBf)) { - pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, - pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); - if (pNormalBf == NULL) { + code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, + pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + if (code != TSDB_CODE_SUCCESS) { pSBf->status = SBF_INVALID; - return TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } } return tBloomFilterPutHash(pNormalBf, h1, h2); + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) { +int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len) { if (pSBf->status == SBF_INVALID) { return TSDB_CODE_FAILED; } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); - int32_t size = taosArrayGetSize(pSBf->bfArray); + int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 1; i >= 0; --i) { if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; @@ -108,24 +147,35 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 return TSDB_CODE_SUCCESS; } -static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate) { +static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate, + SBloomFilter** ppNormalBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - SBloomFilter *pNormalBf = tBloomFilterInit(expectedEntries, errorRate); - if (pNormalBf == NULL) { - return NULL; - } + SBloomFilter* pNormalBf = NULL; + code = tBloomFilterInit(expectedEntries, errorRate, &pNormalBf); + TSDB_CHECK_CODE(code, lino, _error); + if (taosArrayPush(pSBf->bfArray, &pNormalBf) == NULL) { tBloomFilterDestroy(pNormalBf); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pSBf->numBits += pNormalBf->numBits; - return pNormalBf; + (*ppNormalBf) = pNormalBf; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -void tScalableBfDestroy(SScalableBf *pSBf) { +void tScalableBfDestroy(SScalableBf* pSBf) { if (pSBf == NULL) { return; } @@ -135,7 +185,7 @@ void tScalableBfDestroy(SScalableBf *pSBf) { taosMemoryFree(pSBf); } -int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { +int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder) { if (!pSBf) { if (tEncodeI32(pEncoder, 0) < 0) return -1; return 0; @@ -143,7 +193,7 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { int32_t size = taosArrayGetSize(pSBf->bfArray); if (tEncodeI32(pEncoder, size) < 0) return -1; for (int32_t i = 0; i < size; i++) { - SBloomFilter *pBF = taosArrayGetP(pSBf->bfArray, i); + SBloomFilter* pBF = taosArrayGetP(pSBf->bfArray, i); if (tBloomFilterEncode(pBF, pEncoder) < 0) return -1; } if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1; @@ -153,30 +203,61 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { return 0; } -SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { - SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); +int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + if (!pSBf) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } pSBf->hashFn1 = HASH_FUNCTION_1; pSBf->hashFn2 = HASH_FUNCTION_2; pSBf->bfArray = NULL; int32_t size = 0; - if (tDecodeI32(pDecoder, &size) < 0) goto _error; - if (size == 0) { - tScalableBfDestroy(pSBf); - return NULL; + if (tDecodeI32(pDecoder, &size) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - pSBf->bfArray = taosArrayInit(size * 2, sizeof(void *)); + if (size == 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + pSBf->bfArray = taosArrayInit(size * 2, POINTER_BYTES); + if (!pSBf->bfArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } + for (int32_t i = 0; i < size; i++) { - SBloomFilter *pBF = tBloomFilterDecode(pDecoder); - if (!pBF) goto _error; + SBloomFilter* pBF = NULL; + code = tBloomFilterDecode(pDecoder, &pBF); + TSDB_CHECK_CODE(code, lino, _error); taosArrayPush(pSBf->bfArray, &pBF); } - if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error; - if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) goto _error; - if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) goto _error; - if (tDecodeI8(pDecoder, &pSBf->status) < 0) goto _error; - return pSBf; + if (tDecodeU32(pDecoder, &pSBf->growth) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeI8(pDecoder, &pSBf->status) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + (*ppSBf) = pSBf; _error: - tScalableBfDestroy(pSBf); - return NULL; + + if (code != TSDB_CODE_SUCCESS) { + tScalableBfDestroy(pSBf); + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index c51de3c8a4..9d83cbf086 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -8,12 +8,15 @@ using namespace std; TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { int64_t ts1 = 1650803518000; - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 0)); - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 1)); - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, -0.1)); - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(0, 0.01)); + SBloomFilter* pBFTmp = NULL; + GTEST_ASSERT_NE(0, tBloomFilterInit(100, 0, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(100, 1, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(100, -0.1, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(0, 0.01, &pBFTmp)); - SBloomFilter *pBF1 = tBloomFilterInit(100, 0.005); + SBloomFilter *pBF1 = NULL; + int32_t code = tBloomFilterInit(100, 0.005, &pBF1); + GTEST_ASSERT_EQ(0, code); GTEST_ASSERT_EQ(pBF1->numBits, 1152); GTEST_ASSERT_EQ(pBF1->numUnits, 1152 / 64); int64_t count = 0; @@ -25,16 +28,19 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { } ASSERT_TRUE(tBloomFilterIsFull(pBF1)); - SBloomFilter *pBF2 = tBloomFilterInit(1000 * 10000, 0.1); + SBloomFilter* pBF2 = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(1000 * 10000, 0.1, &pBF2)); GTEST_ASSERT_EQ(pBF2->numBits, 47925312); GTEST_ASSERT_EQ(pBF2->numUnits, 47925312 / 64); - SBloomFilter *pBF3 = tBloomFilterInit(10000 * 10000, 0.001); + SBloomFilter* pBF3 = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(10000 * 10000, 0.001, &pBF3)); GTEST_ASSERT_EQ(pBF3->numBits, 1437758784); GTEST_ASSERT_EQ(pBF3->numUnits, 1437758784 / 64); int64_t size = 10000; - SBloomFilter *pBF4 = tBloomFilterInit(size, 0.001); + SBloomFilter* pBF4 = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(size, 0.001, &pBF4)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; GTEST_ASSERT_EQ(tBloomFilterPut(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); @@ -42,17 +48,17 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { ASSERT_TRUE(!tBloomFilterIsFull(pBF4)); for (int64_t i = 0; i < 1000; i++) { - int64_t ts = i + ts1; - uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); - uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); + int64_t ts = i + ts1; + uint64_t h1 = (uint64_t)pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); + uint64_t h2 = (uint64_t)pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_FAILED); } for (int64_t i = 2000; i < 3000; i++) { - int64_t ts = i + ts1; - uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); - uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); - GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS); + int64_t ts = i + ts1; + uint64_t h1 = (uint64_t)pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); + uint64_t h2 = (uint64_t)pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); + GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS); } tBloomFilterDestroy(pBF1); @@ -64,12 +70,14 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int64_t ts1 = 1650803518000; - GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 0)); - GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 1)); - GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, -0.1)); - GTEST_ASSERT_EQ(NULL, tScalableBfInit(0, 0.01)); + SScalableBf* tsSBF = NULL; + GTEST_ASSERT_NE(0, tScalableBfInit(100, 0, &tsSBF)); + GTEST_ASSERT_NE(0, tScalableBfInit(100, 1, &tsSBF)); + GTEST_ASSERT_NE(0, tScalableBfInit(100, -0.1, &tsSBF)); + GTEST_ASSERT_NE(0, tScalableBfInit(0, 0.01, &tsSBF)); - SScalableBf *pSBF1 = tScalableBfInit(100, 0.01); + SScalableBf* pSBF1 = NULL; + GTEST_ASSERT_EQ(0, tScalableBfInit(100, 0.01, &pSBF1)); GTEST_ASSERT_EQ(pSBF1->numBits, 1152); int64_t count = 0; int64_t index = 0; @@ -108,7 +116,7 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int32_t aSize = taosArrayGetSize(pSBF1->bfArray); int64_t totalBits = 0; for (int64_t i = 0; i < aSize; i++) { - SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF1->bfArray, i); + SBloomFilter* pBF = (SBloomFilter*)taosArrayGetP(pSBF1->bfArray, i); ASSERT_TRUE(tBloomFilterIsFull(pBF)); totalBits += pBF->numBits; } @@ -120,7 +128,8 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { } int64_t size = 10000; - SScalableBf *pSBF4 = tScalableBfInit(size, 0.001); + SScalableBf* pSBF4 = NULL; + GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, &pSBF4)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; GTEST_ASSERT_EQ(tScalableBfPut(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); diff --git a/tests/army/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py index 77a22b9256..6fc8e858a3 100644 --- a/tests/army/cmdline/fullopt.py +++ b/tests/army/cmdline/fullopt.py @@ -28,13 +28,13 @@ from frame import * class TDTestCase(TBase): updatecfgDict = { - 'queryMaxConcurrentTables': '2K', - 'streamMax': '1M', + 'queryMaxConcurrentTables': '2K', + 'streamMax': '1M', 'totalMemoryKB': '1G', 'streamMax': '1P', 'streamBufferSize':'1T', 'slowLogScope':"query" - } + } def insertData(self): tdLog.info(f"insert data.") @@ -53,7 +53,7 @@ class TDTestCase(TBase): def doTaos(self): tdLog.info(f"check taos command options...") - + # local command options = [ "DebugFlag 143", @@ -71,7 +71,6 @@ class TDTestCase(TBase): "smlTsDefaultName tsdef", "serverPort 6030", "timezone tz", - "tempDir /var/tmp" ] # exec for option in options: @@ -83,7 +82,7 @@ class TDTestCase(TBase): # help rets = etool.runBinFile("taos", "--help") self.checkListNotEmpty(rets) - # b r w s + # b r w s sql = f"select * from {self.db}.{self.stb} limit 10" rets = etool.runBinFile("taos", f'-B -r -w 100 -s "{sql}" ') self.checkListNotEmpty(rets) @@ -124,16 +123,16 @@ class TDTestCase(TBase): tdLog.info(f"check taosd command options...") idx = 1 # dnode1 cfg = sc.dnodeCfgPath(idx) - + # -s sdb = "./sdb.json" eos.delFile(sdb) etool.exeBinFile("taosd", f"-s -c {cfg}") - + # -C etool.exeBinFile("taosd", "-C") - # -k + # -k etool.exeBinFile("taosd", "-k", False) # -V rets = etool.runBinFile("taosd", "-V") @@ -153,7 +152,7 @@ class TDTestCase(TBase): sc.dnodeStop(idx) etool.exeBinFile("taosd", "-a http://192.168.1.10") - #exe + #exe etool.exeBinFile("taosd", f"-E abc -c {cfg}", False) sc.dnodeStop(idx) etool.exeBinFile("taosd", f"-e def -c {cfg}", False) @@ -182,7 +181,7 @@ class TDTestCase(TBase): # do action self.doTaosd() - + tdLog.success(f"{__file__} successfully executed") diff --git a/tests/script/tsim/insert/update2.sim b/tests/script/tsim/insert/update2.sim index d2b5b05267..8ea767da0e 100644 --- a/tests/script/tsim/insert/update2.sim +++ b/tests/script/tsim/insert/update2.sim @@ -120,7 +120,8 @@ endi if $data01 != 2 then return -1 endi -if $data02 != 1 then +if $data02 != 1 then + print expect 1 , actual $data02 return -1 endi if $data03 != 1 then diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 2338674ae3..a3830d2518 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -121,7 +121,8 @@ int smlProcess_json1_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -149,7 +150,8 @@ int smlProcess_json1_Test() { char *sql3[1] = {0}; for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); - strncpy(sql3[i], sql2[i], 1023); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, @@ -176,7 +178,8 @@ int smlProcess_json1_Test() { char *sql5[1] = {0}; for (int i = 0; i < 1; i++) { sql5[i] = taosMemoryCalloc(1, 1024); - strncpy(sql5[i], sql4[i], 1023); + ASSERT(sql5[i] != NULL); + (void)strncpy(sql5[i], sql4[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, @@ -215,7 +218,8 @@ int smlProcess_json2_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -249,7 +253,8 @@ int smlProcess_json3_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -286,7 +291,8 @@ int smlProcess_json_tag_not_same_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -312,7 +318,8 @@ int smlProcess_json_tag_not_same_Test() { char *sql3[1] = {0}; for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); - strncpy(sql3[i], sql2[i], 1023); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, @@ -338,7 +345,8 @@ int smlProcess_json_tag_not_same_Test() { char *sql5[1] = {0}; for (int i = 0; i < 1; i++) { sql5[i] = taosMemoryCalloc(1, 1024); - strncpy(sql5[i], sql4[i], 1023); + ASSERT(sql5[i] != NULL); + (void)strncpy(sql5[i], sql4[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, @@ -1147,7 +1155,8 @@ int sml_19221_Test() { taos_free_result(pRes); char *tmp = (char *)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql[0], strlen(sql[0])); + ASSERT(tmp != NULL); + (void)memcpy(tmp, sql[0], strlen(sql[0])); *(char *)(tmp + 44) = 0; int32_t totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, @@ -1218,7 +1227,8 @@ int sml_ts3116_Test() { taos_free_result(pRes); int32_t totalRows = 0; char *tmp = (char *)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql, strlen(sql)); + ASSERT(tmp != NULL); + (void)memcpy(tmp, sql, strlen(sql)); totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); @@ -1235,7 +1245,8 @@ int sml_ts3116_Test() { taos_free_result(pRes); tmp = (char *)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql1, strlen(sql1)); + ASSERT(tmp != NULL); + (void)memcpy(tmp, sql1, strlen(sql1)); totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); @@ -2049,7 +2060,8 @@ int sml_td29373_Test() { char *sql3[1] = {0}; for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); - strncpy(sql3[i], sql2[i], 1023); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL,