diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index b45064ce08..47a56d3175 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -49,6 +49,7 @@ table_option: { 7. Escape character "\`" can be used to avoid the conflict between table names and reserved keywords, above rules will be bypassed when using escape character on table names, but the upper limit for the name length is still valid. The table names specified using escape character are case sensitive. For example \`aBc\` and \`abc\` are different table names but `abc` and `aBc` are same table names because they are both converted to `abc` internally. Only ASCII visible characters can be used with escape character. +8. For the details of using `ENCODE` and `COMPRESS`, please refer to [Encode and Compress for Column](../compress). **Parameter description** diff --git a/docs/en/12-taos-sql/04-stable.md b/docs/en/12-taos-sql/04-stable.md index ad179a5fe3..e7101eaba6 100644 --- a/docs/en/12-taos-sql/04-stable.md +++ b/docs/en/12-taos-sql/04-stable.md @@ -13,17 +13,29 @@ create_definition: col_name column_definition column_definition: - type_name + type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type'] + +table_options: + table_option ... + +table_option: { + COMMENT 'string_value' + | SMA(col_name [, col_name] ...) + | TTL value +} + ``` **More explanations** -- Each supertable can have a maximum of 4096 columns, including tags. The minimum number of columns is 3: a timestamp column used as the key, one tag column, and one data column. -- The TAGS keyword defines the tag columns for the supertable. The following restrictions apply to tag columns: +1. Each supertable can have a maximum of 4096 columns, including tags. The minimum number of columns is 3: a timestamp column used as the key, one tag column, and one data column. +2. Since version 3.3.0.0, besides the timestamp, you can specify another column as primary key using `PRIMARY KEY` keyword, the column specified using `primary key` must be type of integer or varchar. +2. The TAGS keyword defines the tag columns for the supertable. The following restrictions apply to tag columns: - A tag column can use the TIMESTAMP data type, but the values in the column must be fixed numbers. Timestamps including formulae, such as "now + 10s", cannot be stored in a tag column. - The name of a tag column cannot be the same as the name of any other column. - The name of a tag column cannot be a reserved keyword. - Each supertable must contain between 1 and 128 tags. The total length of the TAGS keyword cannot exceed 16 KB. -- For more information about table parameters, see Create a Table. +3. Regarding how to use `ENCODE` and `COMPRESS`, please refer to [Encode and Compress for Column](../compress). +3. For more information about table parameters, see [Create a Table](../table). ## View a Supertable diff --git a/docs/en/21-tdinternal/08-compress.md b/docs/en/12-taos-sql/31-compress.md similarity index 100% rename from docs/en/21-tdinternal/08-compress.md rename to docs/en/12-taos-sql/31-compress.md diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index f50551b5de..5020296422 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -729,6 +729,57 @@ The charset that takes effect is UTF-8. | Value Range | -1: none message is compressed; 0: all messages are compressed; N (N>0): messages exceeding N bytes are compressed | | Default | -1 | +### fPrecision + +| Attribute | Description | +| -------- | -------------------------------- | +| Application | Server Only | +| Meaning | Compression precision for float data type | +| Value Range | 0.1 ~ 0.00000001 | +| Default | 0.00000001 | +| Note | The floating value below this setting will be cut off | + +### dPrecision + +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Server Only | +| Meaning | Compression precision for double data type | +| Value Range | 0.1 ~ 0.0000000000000001 | +| Default | 0.0000000000000001 | +| Note | The floating value below this setting will be cut off | + +### lossyColumn + +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Server Only | +| Meaning | Enable TSZ lossy compression for float and/or double | +| Value Range | float, double | +| Default | none: disable TSZ lossy compression | + +**补充说明** +1. It's only available since 3.2.0.0 version, and can't downgrade to previous version once upgrading to 3.2.0.0 and enabling this parameter +2. TSZ compression algorithm compresses data based on data prediction technique, so it's more suitable for data with specific pattern +3. TSZ compression algorithm may take longer time but it has better compression ratio, so it's suitable when you have enough CPU resources and are more sensitive to disk occupation +4. Example: enable TSZ for both float and double +```shell +lossyColumns float|double +``` +5. After configuring, taosd service needs to restarted. After restarting, if you see the following output in taosd logfile, it means the function has been enabled +```sql + 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double +``` + +### ifAdtFse + +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Server Only | +| Meaning | Replace HUFFMAN with FSE in TSZ, FSE is faster when compressing but slower when uncompressing | +| Value Range | 0: Use HUFFMAN, 1: Use FSE | +| Default | 0: Use HUFFMAN | + ## Other Parameters diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index f5ff1b83ef..f665aa2277 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -49,6 +49,7 @@ table_option: { 6. 使用数据类型 BINARY/NCHAR/GEOMETRY，需指定其最长的字节数，如 BINARY(20)，表示 20 字节。 7. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一， 例如：\`aBc\` 和 \`abc\` 是不同的表名，但是 abc 和 aBc 是相同的表名。 +8. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考[按列压缩](../compress) **参数说明** diff --git a/docs/zh/12-taos-sql/04-stable.md b/docs/zh/12-taos-sql/04-stable.md index 9c6de9e220..5ad2f4a19d 100644 --- a/docs/zh/12-taos-sql/04-stable.md +++ b/docs/zh/12-taos-sql/04-stable.md @@ -13,17 +13,28 @@ create_definition: col_name column_definition column_definition: - type_name + type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type'] + +table_options: + table_option ... + +table_option: { + COMMENT 'string_value' + | SMA(col_name [, col_name] ...) + | TTL value +} ``` **使用说明** -- 超级表中列的最大个数为 4096，需要注意，这里的 4096 是包含 TAG 列在内的，最小个数为 3，包含一个时间戳主键、一个 TAG 列和一个数据列。 -- TAGS语法指定超级表的标签列，标签列需要遵循以下约定： +1. 超级表中列的最大个数为 4096，需要注意，这里的 4096 是包含 TAG 列在内的，最小个数为 3，包含一个时间戳主键、一个 TAG 列和一个数据列。 +2. 除时间戳主键列之外，还可以通过 PRIMARY KEY 关键字指定第二列为额外的主键列。被指定为主键列的第二列必须为整型或字符串类型（varchar） +3. TAGS语法指定超级表的标签列，标签列需要遵循以下约定： - TAGS 中的 TIMESTAMP 列写入数据时需要提供给定值，而暂不支持四则运算，例如 NOW + 10s 这类表达式。 - TAGS 列名不能与其他列名相同。 - TAGS 列名不能为预留关键字。 - TAGS 最多允许 128 个，至少 1 个，总长度不超过 16 KB。 -- 关于表参数的详细说明，参见 CREATE TABLE 中的介绍。 +4. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考 [按列压缩](../compress) +5. 关于 table_option 中的参数说明，请参考 [建表 SQL 说明](../table) ## 查看超级表 diff --git a/docs/zh/21-tdinternal/08-compress.md b/docs/zh/12-taos-sql/31-compress.md similarity index 98% rename from docs/zh/21-tdinternal/08-compress.md rename to docs/zh/12-taos-sql/31-compress.md index dfec63c212..0f2b260832 100644 --- a/docs/zh/21-tdinternal/08-compress.md +++ b/docs/zh/12-taos-sql/31-compress.md @@ -1,5 +1,6 @@ --- title: 可配置压缩算法 +sidebar_label: 可配置压缩 description: 可配置压缩算法 --- diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 814b543859..9dced94799 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -784,6 +784,57 @@ charset 的有效值是 UTF-8。 | 取值范围 | -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩 | | 缺省值 | -1 | +### fPrecision +FLOAT 类型压缩精度控制： + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 设置 float 类型浮点数压缩精度 | +| 取值范围 | 0.1 ~ 0.00000001 | +| 缺省值 | 0.00000001 | +| 补充说明 | 小于此值的浮点数尾数部分将被截取 | + +### dPrecision +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 设置 double 类型浮点数压缩精度 | +| 取值范围 | 0.1 ~ 0.0000000000000001 | +| 缺省值 | 0.0000000000000001 | +| 补充说明 | 小于此值的浮点数尾数部分将被截取 | + +### lossyColumn + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 对 float 和/或 double 类型启用 TSZ 有损压缩 | +| 取值范围 | float, double | +| 缺省值 | none：表示关闭无损压缩 | + +**补充说明** +1. 在 3.2.0.0 ~ 3.3.0.0（不包含）版本生效，启用该参数后不能回退到升级前的版本 +2. TSZ 压缩算法是通过数据预测技术完成的压缩，所以更适合有规律变化的数据 +3. TSZ 压缩时间会更长一些，如果您的服务器 CPU 空闲多，存储空间小的情况下适合选用 +4. 示例：对 float 和 double 类型都启用有损压缩 +```shell +lossyColumns float|double +``` +5. 配置需重启服务生效，重启如果在 taosd 日志中看到以下内容，表明配置已生效： +```sql + 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double +``` + +### ifAdtFse + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 在启用 TSZ 有损压缩时，使用 FSE 算法替换 HUFFMAN 算法， FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法 | +| 取值范围 | 0：关闭 1：打开 | +| 缺省值 | 0：关闭 | + ## 3.0 中有效的配置参数列表 | # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | @@ -922,8 +973,5 @@ charset 的有效值是 UTF-8。 | 73 | probeSeconds | 是 | 否 | 3.0 行为未知 | | 74 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | | 75 | probeInterval | 是 | 否 | 3.0 行为未知 | -| 76 | lossyColumns | 是 | 否 | 3.0 行为未知 | -| 77 | fPrecision | 是 | 否 | 3.0 行为未知 | -| 78 | dPrecision | 是 | 否 | 3.0 行为未知 | | 79 | maxRange | 是 | 否 | 3.0 行为未知 | | 80 | range | 是 | 否 | 3.0 行为未知 | diff --git a/docs/zh/21-tdinternal/07-tsz.md b/docs/zh/21-tdinternal/07-tsz.md deleted file mode 100644 index db1a340ab8..0000000000 --- a/docs/zh/21-tdinternal/07-tsz.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: TSZ 压缩算法 -description: TDengine 对浮点数进行高效压缩的算法 ---- - -TSZ 压缩算法是 TDengine 为浮点数据类型提供的可选压缩算法，可以实现浮点数有损至无损全状态压缩，相比默认压缩算法， TSZ 压缩算法压缩率更高，即使切至无损状态，压缩率也会比默认压缩高一倍。 - -## 适合场景 - -- TSZ 压缩算法是通过数据预测技术完成的压缩，所以更适合有规律变化的数据 -- TSZ 压缩时间会更长一些，如果您的服务器 CPU 空闲多，存储空间小的情况下适合选用 - -## 使用步骤 -- TDengine 支持版本为 3.2.0.0 或以上 -- 开启选项 - 在 taos.cfg 配置中增加以下内容，即可开启 TSZ 压缩算法，功能打开后，会替换默认算法。 - 以下表示字段类型是 float 及 double 类型都使用此压缩算法，也可以单独只配置一个 - -```sql - lossyColumns float|double -``` - -- 配置需重启服务生效 -- Taosd 日志输出以下内容，表明功能已生效： - -```sql - 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double -``` - -## 配置参数 - -### fPrecision -FLOAT 类型精度控制： - -| 属性 | 说明 | -| -------- | -------------------------------- | -| 适用范围 | 服务器端 | -| 含义 | 设置 float 类型浮点数压缩精度 | -| 取值范围 | 0.1 ~ 0.00000001 | -| 缺省值 | 0.00000001 | -| 补充说明 | 小于此值的浮点数尾数部分将被截取 | - - - -### dPrecision -DOUBLE 类型精度控制： - -| 属性 | 说明 | -| -------- | -------------------------------- | -| 适用范围 | 服务器端 | -| 含义 | 设置 double 类型浮点数压缩精度 | -| 取值范围 | 0.1 ~ 0.0000000000000001 | -| 缺省值 | 0.0000000000000001 | -| 补充说明 | 小于此值的浮点数尾数部分将被截取 | - - -### ifAdtFse -TSZ 压缩中可选择的算法 FSE，默认为 HUFFMAN： - -| 属性 | 说明 | -| -------- | -------------------------------- | -| 适用范围 | 服务器端 | -| 含义 | 使用 FSE 算法替换 HUFFMAN 算法， FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法 | -| 取值范围 | 0：关闭 1：打开 | -| 缺省值 | 0：关闭 | - - -## 注意事项 -- 打开 TSZ 后生成的存储数据格式，回退至 3.2.0.0 之前的版本，数据将不能被识别 diff --git a/include/common/cos_cp.h b/include/common/cos_cp.h index fd778b1a1d..29532c0265 100644 --- a/include/common/cos_cp.h +++ b/include/common/cos_cp.h @@ -60,8 +60,8 @@ typedef struct { } SCheckpoint; int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint); -void cos_cp_close(TdFilePtr fd); -void cos_cp_remove(char const* filepath); +int32_t cos_cp_close(TdFilePtr fd); +int32_t cos_cp_remove(char const* filepath); int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint); int32_t cos_cp_dump(SCheckpoint* checkpoint); diff --git a/include/common/tmisce.h b/include/common/tmisce.h index 267ca814d4..39274e807c 100644 --- a/include/common/tmisce.h +++ b/include/common/tmisce.h @@ -31,7 +31,7 @@ typedef struct SCorEpSet { int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t len); int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp); -void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); +int32_t addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2); void epsetAssign(SEpSet* dst, const SEpSet* pSrc); diff --git a/include/common/tname.h b/include/common/tname.h index 6a89d2a6be..f0a68f28c4 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -81,7 +81,7 @@ typedef struct { // uint64_t uid; // child table uid, may be useful } RandTableName; -void buildChildTableName(RandTableName* rName); +int32_t buildChildTableName(RandTableName* rName); #ifdef __cplusplus } diff --git a/include/libs/audit/audit.h b/include/libs/audit/audit.h index dd3df27866..4fa69f1b4f 100644 --- a/include/libs/audit/audit.h +++ b/include/libs/audit/audit.h @@ -50,7 +50,7 @@ typedef struct { int32_t auditInit(const SAuditCfg *pCfg); void auditCleanup(); -void auditSend(SJson *pJson); +int32_t auditSend(SJson *pJson); void auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, char *detail, int32_t len); void auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index edb6ec9cbf..0a33798f0c 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -381,7 +381,7 @@ typedef struct SStateStore { void** ppVal, int32_t* pVLen); int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); - SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); + int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); @@ -389,12 +389,12 @@ typedef struct SStateStore { void (*updateInfoDestroy)(SUpdateInfo* pInfo); void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count); - void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); + int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); - SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); + int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); - int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); + int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen); int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo); SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 3ac357055e..fe0faa28ad 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -149,10 +149,10 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void qDestroyBoundColInfo(void* pInfo); -SQuery* smlInitHandle(); +int32_t smlInitHandle(SQuery** query); int32_t smlBuildRow(STableDataCxt* pTableCxt); int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index); -STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); +int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt); void clearColValArraySml(SArray* pCols); int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, @@ -163,8 +163,8 @@ int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCr int numFields, bool needChangeLength, char* errstr, int32_t errstrLen); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); -SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); -SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap); +int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut); +int32_t serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap, SArray** pOut); void destoryCatalogReq(SCatalogReq *pCatalogReq); bool isPrimaryKeyImpl(SNode* pExpr); int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo); diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 1e2ccf8705..8ab3b898ca 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -23,8 +23,6 @@ extern "C" { #include "catalog.h" #include "planner.h" -extern tsem_t schdRspSem; - typedef struct SQueryProfileSummary { int64_t startTs; // Object created and added into the message queue int64_t endTs; // the timestamp when the task is completed @@ -101,8 +99,6 @@ void schedulerFreeJob(int64_t* job, int32_t errCode); void schedulerDestroy(void); -void schdExecCallback(SExecResult* pResult, void* param, int32_t code); - #ifdef __cplusplus } #endif diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index b6569ea771..068f0a282e 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -235,6 +235,8 @@ typedef struct { int64_t stbUid; char stbFullName[TSDB_TABLE_FNAME_LEN]; SSchemaWrapper* pSchemaWrapper; + SSchemaWrapper* pTagSchema; + bool autoCreateCtb; void* vnode; // not available to encoder and decoder FTbSink* tbSinkFunc; STSchema* pTSchema; diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 632a82743f..e1b33de429 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -15,29 +15,29 @@ #ifndef _TSTREAMUPDATE_H_ #define _TSTREAMUPDATE_H_ +#include "storageapi.h" #include "taosdef.h" #include "tarray.h" #include "tcommon.h" #include "tmsg.h" -#include "storageapi.h" #ifdef __cplusplus extern "C" { #endif -SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); -SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); -TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); -bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); -bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid); -void updateInfoDestroy(SUpdateInfo *pInfo); -void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); -void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); -int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); -int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); -void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count); -void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); -bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); +int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); +int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); +TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); +bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); +bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid); +void updateInfoDestroy(SUpdateInfo* pInfo); +void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo); +void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo); +int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen); +int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo); +void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count); +int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count); +bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); #ifdef __cplusplus } diff --git a/include/os/osMath.h b/include/os/osMath.h index 3c05d15397..edbfa935ce 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -32,9 +32,9 @@ extern "C" { #define TSWAP(a, b) \ do { \ char *__tmp = (char*)alloca(sizeof(a)); \ - memcpy(__tmp, &(a), sizeof(a)); \ - memcpy(&(a), &(b), sizeof(a)); \ - memcpy(&(b), __tmp, sizeof(a)); \ + (void)memcpy(__tmp, &(a), sizeof(a)); \ + (void)memcpy(&(a), &(b), sizeof(a)); \ + (void)memcpy(&(b), __tmp, sizeof(a)); \ } while (0) #ifdef WINDOWS diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 683d10e926..c6a5ce27c4 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -53,6 +53,9 @@ void taosPrintBackTrace(); void taosMemoryTrim(int32_t size); void *taosMemoryMallocAlign(uint32_t alignment, int64_t size); +#define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n)) +#define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n)) + #define taosMemoryFreeClear(ptr) \ do { \ if (ptr) { \ diff --git a/include/os/osString.h b/include/os/osString.h index ac7dd7eda8..c834eb8e65 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -55,10 +55,12 @@ typedef enum { M2C = 0, C2M } ConvType; #define tstrncpy(dst, src, size) \ do { \ - strncpy((dst), (src), (size)); \ + (void)strncpy((dst), (src), (size)); \ (dst)[(size)-1] = 0; \ } while (0) +#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) + char *tstrdup(const char *src); int32_t taosUcs4len(TdUcs4 *ucs4); int64_t taosStr2int64(const char *str); diff --git a/include/util/talgo.h b/include/util/talgo.h index 29c51cc16b..dda213ae5a 100644 --- a/include/util/talgo.h +++ b/include/util/talgo.h @@ -52,7 +52,7 @@ typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, const void * @param param * @param comparFn */ -void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn); +int32_t taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn); /** * Non-recursive quick sort. @@ -98,8 +98,8 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size * @param maxroot: if heap is max root heap * @return */ -void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, - __ext_compar_fn_t compar, char *buf, bool maxroot); +int32_t taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, + __ext_compar_fn_t compar, char *buf, bool maxroot); /** * sort heap to make sure it is a max/min root heap @@ -114,7 +114,8 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const * @param maxroot: if heap is max root heap * @return */ -void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, bool maxroot); +int32_t taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, + bool maxroot); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 900d3eedad..053bb20e2a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -942,6 +942,11 @@ int32_t taosGetErrSize(); // UTIL #define TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x6000) +// AUDIT +#define TSDB_CODE_AUDIT_NOT_FORMAT_TO_JSON TAOS_DEF_ERROR_CODE(0, 0x6100) +#define TSDB_CODE_AUDIT_FAIL_SEND_AUDIT_RECORD TAOS_DEF_ERROR_CODE(0, 0x6101) +#define TSDB_CODE_AUDIT_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0x6102) + #ifdef __cplusplus } #endif diff --git a/include/util/tarray.h b/include/util/tarray.h index cef09e40f8..0d01c9248f 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -251,7 +251,7 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t * @return */ -void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); +int32_t taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode); void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver); diff --git a/include/util/tbloomfilter.h b/include/util/tbloomfilter.h index 23bf5aaafb..f0450cb862 100644 --- a/include/util/tbloomfilter.h +++ b/include/util/tbloomfilter.h @@ -19,6 +19,8 @@ #include "os.h" #include "tencode.h" #include "thash.h" +#include "tlog.h" +#include "tutil.h" #ifdef __cplusplus extern "C" { @@ -35,19 +37,19 @@ typedef struct SBloomFilter { uint64_t size; _hash_fn_t hashFn1; _hash_fn_t hashFn2; - void *buffer; + void* buffer; double errorRate; } SBloomFilter; -SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate); -int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2); -int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len); -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2); -void tBloomFilterDestroy(SBloomFilter *pBF); -void tBloomFilterDump(const SBloomFilter *pBF); -bool tBloomFilterIsFull(const SBloomFilter *pBF); -int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder); -SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder); +int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF); +int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2); +int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len); +int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t h1, uint64_t h2); +void tBloomFilterDestroy(SBloomFilter* pBF); +void tBloomFilterDump(const SBloomFilter* pBF); +bool tBloomFilterIsFull(const SBloomFilter* pBF); +int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder); +int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF); #ifdef __cplusplus } diff --git a/include/util/tencode.h b/include/util/tencode.h index b8da040689..9b9f8af1b6 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -18,6 +18,7 @@ #include "tcoding.h" #include "tlist.h" +#include "tutil.h" #ifdef __cplusplus extern "C" { @@ -46,38 +47,8 @@ typedef struct { SDecoderNode* dStack; } SDecoder; -#define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL) -#define tGet(TYPE, BUF, VAL) (VAL) = ((TYPE*)(BUF))[0] - -#define tRPut16(PDEST, PSRC) \ - ((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[1]; \ - ((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[0]; - -#define tRPut32(PDEST, PSRC) \ - ((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[3]; \ - ((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[2]; \ - ((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[1]; \ - ((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[0]; - -#define tRPut64(PDEST, PSRC) \ - ((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[7]; \ - ((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[6]; \ - ((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[5]; \ - ((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[4]; \ - ((uint8_t*)(PDEST))[4] = ((uint8_t*)(PSRC))[3]; \ - ((uint8_t*)(PDEST))[5] = ((uint8_t*)(PSRC))[2]; \ - ((uint8_t*)(PDEST))[6] = ((uint8_t*)(PSRC))[1]; \ - ((uint8_t*)(PDEST))[7] = ((uint8_t*)(PSRC))[0]; - -#define tRGet16 tRPut16 -#define tRGet32 tRPut32 -#define tRGet64 tRPut64 - -#define TD_CODER_POS(CODER) ((CODER)->pos) -#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) -#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) -#define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) -#define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos) +#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) +#define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos) #define tEncodeSize(E, S, SIZE, RET) \ do { \ @@ -100,6 +71,7 @@ void tEncoderInit(SEncoder* pCoder, uint8_t* data, uint32_t size); void tEncoderClear(SEncoder* pCoder); int32_t tStartEncode(SEncoder* pCoder); void tEndEncode(SEncoder* pCoder); +static int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint32_t size); static int32_t tEncodeU8(SEncoder* pCoder, uint8_t val); static int32_t tEncodeI8(SEncoder* pCoder, int8_t val); static int32_t tEncodeU16(SEncoder* pCoder, uint16_t val); @@ -127,6 +99,7 @@ void tDecoderClear(SDecoder* SDecoder); int32_t tStartDecode(SDecoder* pCoder); void tEndDecode(SDecoder* pCoder); static bool tDecodeIsEnd(SDecoder* pCoder); +static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size); static int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val); static int32_t tDecodeI8(SDecoder* pCoder, int8_t* val); static int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val); @@ -149,97 +122,65 @@ static int32_t tDecodeCStr(SDecoder* pCoder, char** val); static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); /* ------------------------ IMPL ------------------------ */ -#define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \ - if ((CODER)->data) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(VAL))) return -1; \ - tPut(TYPE, TD_CODER_CURRENT(CODER), (VAL)); \ - } \ - TD_CODER_MOVE_POS(CODER, sizeof(VAL)); \ - return 0; - -#define TD_ENCODE_VARIANT_MACRO(CODER, VAL) \ - while ((VAL) >= ENCODE_LIMIT) { \ - if ((CODER)->data) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ - TD_CODER_CURRENT(CODER)[0] = ((VAL) | ENCODE_LIMIT) & 0xff; \ - } \ - \ - (VAL) >>= 7; \ - TD_CODER_MOVE_POS(CODER, 1); \ - } \ - \ - if ((CODER)->data) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ - TD_CODER_CURRENT(CODER)[0] = (uint8_t)(VAL); \ - } \ - TD_CODER_MOVE_POS(CODER, 1); \ - return 0; - -#define TD_DECODE_MACRO(CODER, PVAL, TYPE, BITS) \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(*(PVAL)))) return -1; \ - tGet(TYPE, TD_CODER_CURRENT(CODER), *(PVAL)); \ - TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL))); \ - return 0; - -#define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \ - int32_t i = 0; \ - *(PVAL) = 0; \ - for (;;) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ - TYPE tval = TD_CODER_CURRENT(CODER)[0]; \ - if (tval < ENCODE_LIMIT) { \ - *(PVAL) |= (tval << (7 * i)); \ - TD_CODER_MOVE_POS(pCoder, 1); \ - break; \ - } else { \ - *(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \ - i++; \ - TD_CODER_MOVE_POS(pCoder, 1); \ - } \ - } \ - \ - return 0; - -// 8 -static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) { +static FORCE_INLINE int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint32_t size) { if (pCoder->data) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1; - tPut(uint8_t, TD_CODER_CURRENT(pCoder), val); + if (pCoder->pos + size > pCoder->size) { + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + } + memcpy(pCoder->data + pCoder->pos, val, size); } - TD_CODER_MOVE_POS(pCoder, sizeof(val)); + + pCoder->pos += size; return 0; } -static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) { - if (pCoder->data) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1; - tPut(int8_t, TD_CODER_CURRENT(pCoder), val); - } - TD_CODER_MOVE_POS(pCoder, sizeof(val)); - return 0; +static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); } +static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); } +static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) { + return tEncodeFixed(pCoder, &val, sizeof(val)); +} +static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) { + return tEncodeFixed(pCoder, &val, sizeof(val)); +} +static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) { + return tEncodeFixed(pCoder, &val, sizeof(val)); +} +static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) { + return tEncodeFixed(pCoder, &val, sizeof(val)); +} +static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) { + return tEncodeFixed(pCoder, &val, sizeof(val)); +} +static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) { + return tEncodeFixed(pCoder, &val, sizeof(val)); +} +static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) { + while (val >= ENCODE_LIMIT) { + TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff)); + val >>= 7; + } + return tEncodeU8(pCoder, val); } - -// 16 -static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) { TD_ENCODE_MACRO(pCoder, val, uint16_t, 16); } -static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) { TD_ENCODE_MACRO(pCoder, val, int16_t, 16); } -// 32 -static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) { TD_ENCODE_MACRO(pCoder, val, uint32_t, 32); } -static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) { TD_ENCODE_MACRO(pCoder, val, int32_t, 32); } -// 64 -static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) { TD_ENCODE_MACRO(pCoder, val, uint64_t, 64); } -static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) { TD_ENCODE_MACRO(pCoder, val, int64_t, 64); } -// 16v -static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); } static FORCE_INLINE int32_t tEncodeI16v(SEncoder* pCoder, int16_t val) { return tEncodeU16v(pCoder, ZIGZAGE(int16_t, val)); } -// 32v -static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); } +static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) { + while (val >= ENCODE_LIMIT) { + TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff)); + val >>= 7; + } + return tEncodeU8(pCoder, val); +} static FORCE_INLINE int32_t tEncodeI32v(SEncoder* pCoder, int32_t val) { return tEncodeU32v(pCoder, ZIGZAGE(int32_t, val)); } -// 64v -static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); } +static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) { + while (val >= ENCODE_LIMIT) { + TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff)); + val >>= 7; + } + return tEncodeU8(pCoder, val); +} static FORCE_INLINE int32_t tEncodeI64v(SEncoder* pCoder, int64_t val) { return tEncodeU64v(pCoder, ZIGZAGE(int64_t, val)); } @@ -265,14 +206,16 @@ static FORCE_INLINE int32_t tEncodeDouble(SEncoder* pCoder, double val) { } static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len) { - if (tEncodeU32v(pCoder, len) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeU32v(pCoder, len)); if (len) { if (pCoder->data) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, len)) return -1; - memcpy(TD_CODER_CURRENT(pCoder), val, len); + if (pCoder->pos + len > pCoder->size) { + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + } + memcpy(pCoder->data + pCoder->pos, val, len); } - TD_CODER_MOVE_POS(pCoder, len); + pCoder->pos += len; } return 0; } @@ -286,70 +229,137 @@ static FORCE_INLINE int32_t tEncodeCStr(SEncoder* pCoder, const char* val) { } /* ------------------------ FOR DECODER ------------------------ */ -// 8 -static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1; - tGet(uint8_t, TD_CODER_CURRENT(pCoder), *val); - TD_CODER_MOVE_POS(pCoder, sizeof(*val)); +static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size) { + if (pCoder->pos + size > pCoder->size) { + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + } else if (val) { + memcpy(val, pCoder->data + pCoder->pos, size); + } + pCoder->pos += size; return 0; } -static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1; - tGet(int8_t, TD_CODER_CURRENT(pCoder), *val); - TD_CODER_MOVE_POS(pCoder, sizeof(*val)); - return 0; +static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); } +static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) { return tDecodeFixed(pCoder, val, sizeof(*val)); } + // 16 -static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) { TD_DECODE_MACRO(pCoder, val, uint16_t, 16); } -static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) { TD_DECODE_MACRO(pCoder, val, int16_t, 16); } +static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); +} +static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); +} // 32 -static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) { TD_DECODE_MACRO(pCoder, val, uint32_t, 32); } -static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) { TD_DECODE_MACRO(pCoder, val, int32_t, 32); } +static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); +} +static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); +} // 64 -static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) { TD_DECODE_MACRO(pCoder, val, uint64_t, 64); } -static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) { TD_DECODE_MACRO(pCoder, val, int64_t, 64); } +static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); +} +static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); +} // 16v static FORCE_INLINE int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val) { - TD_DECODE_VARIANT_MACRO(pCoder, val, uint16_t); + uint8_t byte; + uint16_t tval = 0; + for (int32_t i = 0;; i++) { + TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte)); + if (byte < ENCODE_LIMIT) { + tval |= (((uint16_t)byte) << (7 * i)); + break; + } else { + tval |= ((((uint16_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i)); + } + } + + if (val) { + *val = tval; + } + + return 0; } static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) { uint16_t tval; - if (tDecodeU16v(pCoder, &tval) < 0) { - return -1; + TAOS_CHECK_RETURN(tDecodeU16v(pCoder, &tval)); + + if (val) { + *val = ZIGZAGD(int16_t, tval); } - if (val) *val = ZIGZAGD(int16_t, tval); + return 0; } // 32v static FORCE_INLINE int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val) { - TD_DECODE_VARIANT_MACRO(pCoder, val, uint32_t); + uint8_t byte; + uint32_t tval = 0; + for (int32_t i = 0;; i++) { + TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte)); + if (byte < ENCODE_LIMIT) { + tval |= (((uint32_t)byte) << (7 * i)); + break; + } else { + tval |= ((((uint32_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i)); + } + } + + if (val) { + *val = tval; + } + + return 0; } static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) { uint32_t tval; - if (tDecodeU32v(pCoder, &tval) < 0) { - return -1; + TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &tval)); + + if (val) { + *val = ZIGZAGD(int32_t, tval); } - if (val) *val = ZIGZAGD(int32_t, tval); + return 0; } // 64v static FORCE_INLINE int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val) { - TD_DECODE_VARIANT_MACRO(pCoder, val, uint64_t); + uint8_t byte; + uint64_t tval = 0; + for (int32_t i = 0;; i++) { + TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte)); + if (byte < ENCODE_LIMIT) { + tval |= (((uint64_t)byte) << (7 * i)); + break; + } else { + tval |= ((((uint64_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i)); + } + } + + if (val) { + *val = tval; + } + + return 0; } static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) { uint64_t tval; - if (tDecodeU64v(pCoder, &tval) < 0) { - return -1; + TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &tval)); + + if (val) { + *val = ZIGZAGD(int64_t, tval); } - if (val) *val = ZIGZAGD(int64_t, tval); + return 0; } @@ -359,11 +369,11 @@ static FORCE_INLINE int32_t tDecodeFloat(SDecoder* pCoder, float* val) { float f; } v; - if (tDecodeU32(pCoder, &(v.ui)) < 0) { - return -1; - } + TAOS_CHECK_RETURN(tDecodeU32(pCoder, &(v.ui))); - *val = v.f; + if (val) { + *val = v.f; + } return 0; } @@ -373,30 +383,36 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) { double d; } v; - if (tDecodeU64(pCoder, &(v.ui)) < 0) { - return -1; - } + TAOS_CHECK_RETURN(tDecodeU64(pCoder, &(v.ui))); - *val = v.d; + if (val) { + *val = v.d; + } return 0; } static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) { uint32_t length = 0; - if (tDecodeU32v(pCoder, &length) < 0) return -1; - if (len) *len = length; - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; - if (val) { - *val = (uint8_t*)TD_CODER_CURRENT(pCoder); + TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &length)); + if (len) { + *len = length; } - TD_CODER_MOVE_POS(pCoder, length); + if (pCoder->pos + length > pCoder->size) { + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + } + + if (val) { + *val = pCoder->data + pCoder->pos; + } + + pCoder->pos += length; return 0; } static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) { - if (tDecodeBinary(pCoder, (uint8_t**)val, len) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeBinary(pCoder, (uint8_t**)val, len)); (*len) -= 1; return 0; } @@ -409,7 +425,7 @@ static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) { static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { char* pStr; uint32_t len; - if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeCStrAndLen(pCoder, &pStr, &len)); memcpy(val, pStr, len + 1); return 0; @@ -417,16 +433,22 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) { uint64_t length = 0; - if (tDecodeU64v(pCoder, &length) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &length)); if (length) { if (len) *len = length; - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; - *val = taosMemoryMalloc(length); - if (*val == NULL) return -1; - memcpy(*val, TD_CODER_CURRENT(pCoder), length); + if (pCoder->pos + length > pCoder->size) { + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + } - TD_CODER_MOVE_POS(pCoder, length); + *val = taosMemoryMalloc(length); + if (*val == NULL) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(*val, pCoder->data + pCoder->pos, length); + + pCoder->pos += length; } else { *val = NULL; } @@ -435,16 +457,20 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uin static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, uint32_t* len) { uint32_t length = 0; - if (tDecodeU32v(pCoder, &length) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &length)); if (length) { if (len) *len = length; - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; + if (pCoder->pos + length > pCoder->size) { + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + } *val = taosMemoryMalloc(length); - if (*val == NULL) return -1; - memcpy(*val, TD_CODER_CURRENT(pCoder), length); + if (*val == NULL) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + memcpy(*val, pCoder->data + pCoder->pos, length); - TD_CODER_MOVE_POS(pCoder, length); + pCoder->pos += length; } else { *val = NULL; } @@ -452,7 +478,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, u } static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SDecoder* pCoder, char** val, uint64_t* len) { - if (tDecodeBinaryAlloc(pCoder, (void**)val, len) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeBinaryAlloc(pCoder, (void**)val, len)); (*len) -= 1; return 0; } diff --git a/include/util/tqueue.h b/include/util/tqueue.h index bed218ac1b..f7eaf794b0 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -73,36 +73,36 @@ struct STaosQnode { char item[]; }; -STaosQueue *taosOpenQueue(); -void taosCloseQueue(STaosQueue *queue); -void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); -void *taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize); -void taosFreeQitem(void *pItem); -int32_t taosWriteQitem(STaosQueue *queue, void *pItem); -int32_t taosReadQitem(STaosQueue *queue, void **ppItem); -bool taosQueueEmpty(STaosQueue *queue); -void taosUpdateItemSize(STaosQueue *queue, int32_t items); -int32_t taosQueueItemSize(STaosQueue *queue); -int64_t taosQueueMemorySize(STaosQueue *queue); -void taosSetQueueCapacity(STaosQueue *queue, int64_t size); -void taosSetQueueMemoryCapacity(STaosQueue *queue, int64_t mem); +int32_t taosOpenQueue(STaosQueue **queue); +void taosCloseQueue(STaosQueue *queue); +void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); +int32_t taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize, void **item); +void taosFreeQitem(void *pItem); +int32_t taosWriteQitem(STaosQueue *queue, void *pItem); +int32_t taosReadQitem(STaosQueue *queue, void **ppItem); +bool taosQueueEmpty(STaosQueue *queue); +void taosUpdateItemSize(STaosQueue *queue, int32_t items); +int32_t taosQueueItemSize(STaosQueue *queue); +int64_t taosQueueMemorySize(STaosQueue *queue); +void taosSetQueueCapacity(STaosQueue *queue, int64_t size); +void taosSetQueueMemoryCapacity(STaosQueue *queue, int64_t mem); -STaosQall *taosAllocateQall(); -void taosFreeQall(STaosQall *qall); -int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall); -int32_t taosGetQitem(STaosQall *qall, void **ppItem); -void taosResetQitems(STaosQall *qall); -int32_t taosQallItemSize(STaosQall *qall); -int64_t taosQallMemSize(STaosQall *qll); -int64_t taosQallUnAccessedItemSize(STaosQall *qall); -int64_t taosQallUnAccessedMemSize(STaosQall *qall); +int32_t taosAllocateQall(STaosQall **qall); +void taosFreeQall(STaosQall *qall); +int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall); +int32_t taosGetQitem(STaosQall *qall, void **ppItem); +void taosResetQitems(STaosQall *qall); +int32_t taosQallItemSize(STaosQall *qall); +int64_t taosQallMemSize(STaosQall *qll); +int64_t taosQallUnAccessedItemSize(STaosQall *qall); +int64_t taosQallUnAccessedMemSize(STaosQall *qall); -STaosQset *taosOpenQset(); -void taosCloseQset(STaosQset *qset); -void taosQsetThreadResume(STaosQset *qset); -int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle); -void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue); -int32_t taosGetQueueNumber(STaosQset *qset); +int32_t taosOpenQset(STaosQset **qset); +void taosCloseQset(STaosQset *qset); +void taosQsetThreadResume(STaosQset *qset); +int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle); +void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue); +int32_t taosGetQueueNumber(STaosQset *qset); int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo); int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo); diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index d3ce2eb23b..f82493aca2 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -23,22 +23,22 @@ extern "C" { #endif typedef struct SScalableBf { - SArray *bfArray; // array of bloom filters - uint32_t growth; - uint64_t numBits; - uint32_t maxBloomFilters; - int8_t status; + SArray* bfArray; // array of bloom filters + uint32_t growth; + uint64_t numBits; + uint32_t maxBloomFilters; + int8_t status; _hash_fn_t hashFn1; _hash_fn_t hashFn2; } SScalableBf; -SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate); -int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); -int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); -int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); -void tScalableBfDestroy(SScalableBf *pSBf); -int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder); -SScalableBf *tScalableBfDecode(SDecoder *pDecoder); +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf); +int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len); +int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len); +int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len); +void tScalableBfDestroy(SScalableBf* pSBf); +int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder); +int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf); #ifdef __cplusplus } diff --git a/include/util/tutil.h b/include/util/tutil.h index d1fc11d0fe..31ce34f667 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -21,6 +21,7 @@ #include "tdef.h" #include "thash.h" #include "tmd5.h" +#include "tutil.h" #ifdef __cplusplus extern "C" { diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 122914fd34..e901f86b41 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -82,7 +82,7 @@ extern "C" { #define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH)) -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) +#define MOVE_FORWARD_ONE(sql, len) ((void)memmove((void *)((sql)-1), (sql), len)) #define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ for (int i = 1; i < keyLen; ++i) { \ @@ -211,19 +211,19 @@ extern int64_t smlFactorS[]; typedef int32_t (*_equal_fn_sml)(const void *, const void *); -SSmlHandle *smlBuildSmlInfo(TAOS *taos); +int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle); void smlDestroyInfo(SSmlHandle *info); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); -int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); -bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); +void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); +int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); -SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); -SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); +int32_t smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen, SSmlTableInfo** tInfo); +int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta** sMeta); int32_t smlSetCTableName(SSmlTableInfo *oneTable); -void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo); -STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen); +int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo); +int32_t smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen, STableMeta **pTableMeta); int32_t is_same_child_table_telnet(const void *a, const void *b); int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); int32_t smlClearForRerun(SSmlHandle *info); @@ -236,7 +236,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); -SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement); +int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta); bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv); int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements); diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index ae2a57ba97..304f18cd68 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -1,13 +1,13 @@ #include "clientMonitor.h" -#include "clientLog.h" -#include "os.h" -#include "tmisce.h" -#include "ttime.h" -#include "ttimer.h" -#include "tglobal.h" -#include "tqueue.h" #include "cJSON.h" #include "clientInt.h" +#include "clientLog.h" +#include "os.h" +#include "tglobal.h" +#include "tmisce.h" +#include "tqueue.h" +#include "ttime.h" +#include "ttimer.h" SRWLatch monitorLock; void* monitorTimer; @@ -20,41 +20,41 @@ STaosQueue* monitorQueue; SHashObj* monitorSlowLogHash; char tmpSlowLogPath[PATH_MAX] = {0}; -static int32_t getSlowLogTmpDir(char* tmpPath, int32_t size){ +static int32_t getSlowLogTmpDir(char* tmpPath, int32_t size) { if (tsTempDir == NULL) { return -1; } int ret = snprintf(tmpPath, size, "%s/tdengine_slow_log/", tsTempDir); - if (ret < 0){ + if (ret < 0) { tscError("failed to get tmp path ret:%d", ret); return ret; } return 0; } -static void processFileInTheEnd(TdFilePtr pFile, char* path){ - if(pFile == NULL){ +static void processFileInTheEnd(TdFilePtr pFile, char* path) { + if (pFile == NULL) { return; } - if(taosFtruncateFile(pFile, 0) != 0){ + if (taosFtruncateFile(pFile, 0) != 0) { tscError("failed to truncate file:%s, errno:%d", path, errno); return; } - if(taosUnLockFile(pFile) != 0){ + if (taosUnLockFile(pFile) != 0) { tscError("failed to unlock file:%s, errno:%d", path, errno); return; } - if(taosCloseFile(&(pFile)) != 0){ + if (taosCloseFile(&(pFile)) != 0) { tscError("failed to close file:%s, errno:%d", path, errno); return; } - if(taosRemoveFile(path) != 0){ + if (taosRemoveFile(path) != 0) { tscError("failed to remove file:%s, errno:%d", path, errno); return; } } -static void destroySlowLogClient(void* data){ +static void destroySlowLogClient(void* data) { if (data == NULL) { return; } @@ -63,40 +63,40 @@ static void destroySlowLogClient(void* data){ taosMemoryFree(slowLogClient); } -static void destroyMonitorClient(void* data){ +static void destroyMonitorClient(void* data) { if (data == NULL) { return; } MonitorClient* pMonitor = *(MonitorClient**)data; - if(pMonitor == NULL){ + if (pMonitor == NULL) { return; } taosTmrStopA(&pMonitor->timer); taosHashCleanup(pMonitor->counters); taos_collector_registry_destroy(pMonitor->registry); -// taos_collector_destroy(pMonitor->colector); + // taos_collector_destroy(pMonitor->colector); taosMemoryFree(pMonitor); } -static void monitorFreeSlowLogData(void *paras) { +static void monitorFreeSlowLogData(void* paras) { MonitorSlowLogData* pData = (MonitorSlowLogData*)paras; if (pData == NULL) { return; } taosMemoryFreeClear(pData->data); - if (pData->type == SLOW_LOG_READ_BEGINNIG){ + if (pData->type == SLOW_LOG_READ_BEGINNIG) { taosMemoryFree(pData->fileName); } } -static void monitorFreeSlowLogDataEx(void *paras) { +static void monitorFreeSlowLogDataEx(void* paras) { monitorFreeSlowLogData(paras); taosMemoryFree(paras); } static SAppInstInfo* getAppInstByClusterId(int64_t clusterId) { - void *p = taosHashGet(appInfo.pInstMapByClusterId, &clusterId, LONG_BYTES); - if(p == NULL){ + void* p = taosHashGet(appInfo.pInstMapByClusterId, &clusterId, LONG_BYTES); + if (p == NULL) { tscError("failed to get app inst, clusterId:%" PRIx64, clusterId); return NULL; } @@ -111,21 +111,25 @@ static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); } - if(param != NULL){ + if (param != NULL) { MonitorSlowLogData* p = (MonitorSlowLogData*)param; - if(code != 0){ + if (code != 0) { tscError("failed to send slow log:%s, clusterId:%" PRIx64, p->data, p->clusterId); } - MonitorSlowLogData tmp = {.clusterId = p->clusterId, .type = p->type, .fileName = p->fileName, - .pFile= p->pFile, .offset = p->offset, .data = NULL}; - if(monitorPutData2MonitorQueue(tmp) == 0){ + MonitorSlowLogData tmp = {.clusterId = p->clusterId, + .type = p->type, + .fileName = p->fileName, + .pFile = p->pFile, + .offset = p->offset, + .data = NULL}; + if (monitorPutData2MonitorQueue(tmp) == 0) { p->fileName = NULL; } } return code; } -static int32_t sendReport(void* pTransporter, SEpSet *epSet, char* pCont, MONITOR_TYPE type, void* param) { +static int32_t sendReport(void* pTransporter, SEpSet* epSet, char* pCont, MONITOR_TYPE type, void* param) { SStatisReq sStatisReq; sStatisReq.pCont = pCont; sStatisReq.contLen = strlen(pCont); @@ -167,11 +171,11 @@ FAILED: return -1; } -static void generateClusterReport(taos_collector_registry_t* registry, void* pTransporter, SEpSet *epSet) { +static void generateClusterReport(taos_collector_registry_t* registry, void* pTransporter, SEpSet* epSet) { char ts[50] = {0}; sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); char* pCont = (char*)taos_collector_registry_bridge_new(registry, ts, "%" PRId64, NULL); - if(NULL == pCont) { + if (NULL == pCont) { tscError("generateClusterReport failed, get null content."); return; } @@ -190,8 +194,8 @@ static void reportSendProcess(void* param, void* tmrId) { } MonitorClient* pMonitor = (MonitorClient*)param; - SAppInstInfo* pInst = getAppInstByClusterId(pMonitor->clusterId); - if(pInst == NULL){ + SAppInstInfo* pInst = getAppInstByClusterId(pMonitor->clusterId); + if (pInst == NULL) { taosRUnLockLatch(&monitorLock); return; } @@ -202,15 +206,15 @@ static void reportSendProcess(void* param, void* tmrId) { taosRUnLockLatch(&monitorLock); } -static void sendAllCounter(){ +static void sendAllCounter() { MonitorClient** ppMonitor = NULL; while ((ppMonitor = taosHashIterate(monitorSlowLogHash, ppMonitor))) { MonitorClient* pMonitor = *ppMonitor; - if (pMonitor == NULL){ + if (pMonitor == NULL) { continue; } SAppInstInfo* pInst = getAppInstByClusterId(pMonitor->clusterId); - if(pInst == NULL){ + if (pInst == NULL) { taosHashCancelIterate(monitorCounterHash, ppMonitor); break; } @@ -231,46 +235,48 @@ void monitorCreateClient(int64_t clusterId) { } pMonitor->clusterId = clusterId; char clusterKey[32] = {0}; - if(snprintf(clusterKey, sizeof(clusterKey), "%"PRId64, clusterId) < 0){ + if (snprintf(clusterKey, sizeof(clusterKey), "%" PRId64, clusterId) < 0) { tscError("failed to create cluster key"); goto fail; } pMonitor->registry = taos_collector_registry_new(clusterKey); - if(pMonitor->registry == NULL){ + if (pMonitor->registry == NULL) { tscError("failed to create registry"); goto fail; } pMonitor->colector = taos_collector_new(clusterKey); - if(pMonitor->colector == NULL){ + if (pMonitor->colector == NULL) { tscError("failed to create collector"); goto fail; } taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector); - pMonitor->counters = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pMonitor->counters = + (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pMonitor->counters == NULL) { tscError("failed to create monitor counters"); goto fail; } -// taosHashSetFreeFp(pMonitor->counters, destroyCounter); + // taosHashSetFreeFp(pMonitor->counters, destroyCounter); - if(taosHashPut(monitorCounterHash, &clusterId, LONG_BYTES, &pMonitor, POINTER_BYTES) != 0){ + if (taosHashPut(monitorCounterHash, &clusterId, LONG_BYTES, &pMonitor, POINTER_BYTES) != 0) { tscError("failed to put monitor client to hash"); goto fail; } SAppInstInfo* pInst = getAppInstByClusterId(clusterId); - if(pInst == NULL){ + if (pInst == NULL) { tscError("failed to get app instance by cluster id"); pMonitor = NULL; goto fail; } - pMonitor->timer = taosTmrStart(reportSendProcess, pInst->monitorParas.tsMonitorInterval * 1000, (void*)pMonitor, monitorTimer); - if(pMonitor->timer == NULL){ + pMonitor->timer = + taosTmrStart(reportSendProcess, pInst->monitorParas.tsMonitorInterval * 1000, (void*)pMonitor, monitorTimer); + if (pMonitor->timer == NULL) { tscError("failed to start timer"); goto fail; } - tscInfo("[monitor] monitorCreateClient for %"PRIx64 "finished %p.", clusterId, pMonitor); + tscInfo("[monitor] monitorCreateClient for %" PRIx64 "finished %p.", clusterId, pMonitor); } taosWUnLockLatch(&monitorLock); if (-1 != atomic_val_compare_exchange_32(&monitorFlag, -1, 0)) { @@ -283,7 +289,8 @@ fail: taosWUnLockLatch(&monitorLock); } -void monitorCreateClientCounter(int64_t clusterId, const char* name, const char* help, size_t label_key_count, const char** label_keys) { +void monitorCreateClientCounter(int64_t clusterId, const char* name, const char* help, size_t label_key_count, + const char** label_keys) { taosWLockLatch(&monitorLock); MonitorClient** ppMonitor = (MonitorClient**)taosHashGet(monitorCounterHash, &clusterId, LONG_BYTES); if (ppMonitor == NULL || *ppMonitor == NULL) { @@ -291,16 +298,16 @@ void monitorCreateClientCounter(int64_t clusterId, const char* name, const char* goto end; } taos_counter_t* newCounter = taos_counter_new(name, help, label_key_count, label_keys); - if (newCounter == NULL) - return; - MonitorClient* pMonitor = *ppMonitor; + if (newCounter == NULL) return; + MonitorClient* pMonitor = *ppMonitor; taos_collector_add_metric(pMonitor->colector, newCounter); - if(taosHashPut(pMonitor->counters, name, strlen(name), &newCounter, POINTER_BYTES) != 0){ + if (taosHashPut(pMonitor->counters, name, strlen(name), &newCounter, POINTER_BYTES) != 0) { tscError("failed to put counter to monitor"); taos_counter_destroy(newCounter); goto end; } - tscInfo("[monitor] monitorCreateClientCounter %"PRIx64"(%p):%s : %p.", pMonitor->clusterId, pMonitor, name, newCounter); + tscInfo("[monitor] monitorCreateClientCounter %" PRIx64 "(%p):%s : %p.", pMonitor->clusterId, pMonitor, name, + newCounter); end: taosWUnLockLatch(&monitorLock); @@ -315,18 +322,18 @@ void monitorCounterInc(int64_t clusterId, const char* counterName, const char** MonitorClient** ppMonitor = (MonitorClient**)taosHashGet(monitorCounterHash, &clusterId, LONG_BYTES); if (ppMonitor == NULL || *ppMonitor == NULL) { - tscError("monitorCounterInc not found pMonitor %"PRId64, clusterId); + tscError("monitorCounterInc not found pMonitor %" PRId64, clusterId); goto end; } MonitorClient* pMonitor = *ppMonitor; taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, counterName, strlen(counterName)); if (ppCounter == NULL || *ppCounter == NULL) { - tscError("monitorCounterInc not found pCounter %"PRIx64":%s.", clusterId, counterName); + tscError("monitorCounterInc not found pCounter %" PRIx64 ":%s.", clusterId, counterName); goto end; } taos_counter_inc(*ppCounter, label_values); - tscDebug("[monitor] monitorCounterInc %"PRIx64"(%p):%s", pMonitor->clusterId, pMonitor, counterName); + tscDebug("[monitor] monitorCounterInc %" PRIx64 "(%p):%s", pMonitor->clusterId, pMonitor, counterName); end: taosWUnLockLatch(&monitorLock); @@ -339,13 +346,13 @@ const char* monitorResultStr(SQL_RESULT_CODE code) { static void monitorThreadFuncUnexpectedStopped(void) { atomic_store_32(&slowLogFlag, -1); } -static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char *tmpPath){ +static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char* tmpPath) { TdFilePtr pFile = NULL; - void* tmp = taosHashGet(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES); - if (tmp == NULL){ + void* tmp = taosHashGet(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES); + if (tmp == NULL) { char path[PATH_MAX] = {0}; char clusterId[32] = {0}; - if (snprintf(clusterId, sizeof(clusterId), "%" PRIx64, slowLogData->clusterId) < 0){ + if (snprintf(clusterId, sizeof(clusterId), "%" PRIx64, slowLogData->clusterId) < 0) { tscError("failed to generate clusterId:%" PRIx64, slowLogData->clusterId); return; } @@ -358,8 +365,8 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char *tmpP return; } - SlowLogClient *pClient = taosMemoryCalloc(1, sizeof(SlowLogClient)); - if (pClient == NULL){ + SlowLogClient* pClient = taosMemoryCalloc(1, sizeof(SlowLogClient)); + if (pClient == NULL) { tscError("failed to allocate memory for slow log client"); taosCloseFile(&pFile); return; @@ -368,58 +375,58 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char *tmpP strcpy(pClient->path, path); pClient->offset = 0; pClient->pFile = pFile; - if (taosHashPut(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES, &pClient, POINTER_BYTES) != 0){ + if (taosHashPut(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES, &pClient, POINTER_BYTES) != 0) { tscError("failed to put clusterId:%" PRId64 " to hash table", slowLogData->clusterId); taosCloseFile(&pFile); taosMemoryFree(pClient); return; } - if(taosLockFile(pFile) < 0){ + if (taosLockFile(pFile) < 0) { tscError("failed to lock file:%p since %s", pFile, terrstr()); return; } - }else{ + } else { pFile = (*(SlowLogClient**)tmp)->pFile; } - if(taosLSeekFile(pFile, 0, SEEK_END) < 0){ + if (taosLSeekFile(pFile, 0, SEEK_END) < 0) { tscError("failed to seek file:%p code: %d", pFile, errno); return; } - if (taosWriteFile(pFile, slowLogData->data, strlen(slowLogData->data) + 1) < 0){ + if (taosWriteFile(pFile, slowLogData->data, strlen(slowLogData->data) + 1) < 0) { tscError("failed to write len to file:%p since %s", pFile, terrstr()); } - tscDebug("[monitor] write slow log to file:%p, clusterId:%"PRIx64, pFile, slowLogData->clusterId); + tscDebug("[monitor] write slow log to file:%p, clusterId:%" PRIx64, pFile, slowLogData->clusterId); } -static char* readFile(TdFilePtr pFile, int64_t *offset, int64_t size){ - tscDebug("[monitor] readFile slow begin pFile:%p, offset:%"PRId64 ", size:%"PRId64, pFile, *offset, size); - if(taosLSeekFile(pFile, *offset, SEEK_SET) < 0){ +static char* readFile(TdFilePtr pFile, int64_t* offset, int64_t size) { + tscDebug("[monitor] readFile slow begin pFile:%p, offset:%" PRId64 ", size:%" PRId64, pFile, *offset, size); + if (taosLSeekFile(pFile, *offset, SEEK_SET) < 0) { tscError("failed to seek file:%p code: %d", pFile, errno); return NULL; } ASSERT(size > *offset); - char* pCont = NULL; + char* pCont = NULL; int64_t totalSize = 0; if (size - *offset >= SLOW_LOG_SEND_SIZE_MAX) { - pCont = taosMemoryCalloc(1, 4 + SLOW_LOG_SEND_SIZE_MAX); //4 reserved for [] + pCont = taosMemoryCalloc(1, 4 + SLOW_LOG_SEND_SIZE_MAX); // 4 reserved for [] totalSize = 4 + SLOW_LOG_SEND_SIZE_MAX; - }else{ + } else { pCont = taosMemoryCalloc(1, 4 + (size - *offset)); totalSize = 4 + (size - *offset); } - if(pCont == NULL){ + if (pCont == NULL) { tscError("failed to allocate memory for slow log, size:%" PRId64, totalSize); return NULL; } - char* buf = pCont; + char* buf = pCont; strcat(buf++, "["); int64_t readSize = taosReadFile(pFile, buf, SLOW_LOG_SEND_SIZE_MAX); if (readSize <= 0) { - if (readSize < 0){ + if (readSize < 0) { tscError("failed to read len from file:%p since %s", pFile, terrstr()); } taosMemoryFree(pCont); @@ -427,24 +434,24 @@ static char* readFile(TdFilePtr pFile, int64_t *offset, int64_t size){ } totalSize = 0; - while(1){ + while (1) { size_t len = strlen(buf); - totalSize += (len+1); + totalSize += (len + 1); if (totalSize > readSize || len == 0) { - *(buf-1) = ']'; + *(buf - 1) = ']'; *buf = '\0'; break; } - buf[len] = ','; // replace '\0' with ',' + buf[len] = ','; // replace '\0' with ',' buf += (len + 1); - *offset += (len+1); + *offset += (len + 1); } - tscDebug("[monitor] readFile slow log end, data:%s, offset:%"PRId64, pCont, *offset); + tscDebug("[monitor] readFile slow log end, data:%s, offset:%" PRId64, pCont, *offset); return pCont; } -static int64_t getFileSize(char* path){ +static int64_t getFileSize(char* path) { int64_t fileSize = 0; if (taosStatFile(path, &fileSize, NULL, NULL) < 0) { return -1; @@ -453,13 +460,14 @@ static int64_t getFileSize(char* path){ return fileSize; } -static int32_t sendSlowLog(int64_t clusterId, char* data, TdFilePtr pFile, int64_t offset, SLOW_LOG_QUEUE_TYPE type, char* fileName, void* pTransporter, SEpSet *epSet){ - if (data == NULL){ +static int32_t sendSlowLog(int64_t clusterId, char* data, TdFilePtr pFile, int64_t offset, SLOW_LOG_QUEUE_TYPE type, + char* fileName, void* pTransporter, SEpSet* epSet) { + if (data == NULL) { taosMemoryFree(fileName); return -1; } MonitorSlowLogData* pParam = taosMemoryMalloc(sizeof(MonitorSlowLogData)); - if(pParam == NULL){ + if (pParam == NULL) { taosMemoryFree(data); taosMemoryFree(fileName); return -1; @@ -473,121 +481,124 @@ static int32_t sendSlowLog(int64_t clusterId, char* data, TdFilePtr pFile, int64 return sendReport(pTransporter, epSet, data, MONITOR_TYPE_SLOW_LOG, pParam); } -static int32_t monitorReadSend(int64_t clusterId, TdFilePtr pFile, int64_t* offset, int64_t size, SLOW_LOG_QUEUE_TYPE type, char* fileName){ +static int32_t monitorReadSend(int64_t clusterId, TdFilePtr pFile, int64_t* offset, int64_t size, + SLOW_LOG_QUEUE_TYPE type, char* fileName) { SAppInstInfo* pInst = getAppInstByClusterId(clusterId); - if(pInst == NULL){ + if (pInst == NULL) { tscError("failed to get app instance by clusterId:%" PRId64, clusterId); return -1; } SEpSet ep = getEpSet_s(&pInst->mgmtEp); - char* data = readFile(pFile, offset, size); - return sendSlowLog(clusterId, data, (type == SLOW_LOG_READ_BEGINNIG ? pFile : NULL), *offset, type, fileName, pInst->pTransporter, &ep); + char* data = readFile(pFile, offset, size); + return sendSlowLog(clusterId, data, (type == SLOW_LOG_READ_BEGINNIG ? pFile : NULL), *offset, type, fileName, + pInst->pTransporter, &ep); } -static void monitorSendSlowLogAtBeginning(int64_t clusterId, char** fileName, TdFilePtr pFile, int64_t offset){ - int64_t size = getFileSize(*fileName); - if(size <= offset){ +static void monitorSendSlowLogAtBeginning(int64_t clusterId, char** fileName, TdFilePtr pFile, int64_t offset) { + int64_t size = getFileSize(*fileName); + if (size <= offset) { processFileInTheEnd(pFile, *fileName); tscDebug("[monitor] monitorSendSlowLogAtBeginning delete file:%s", *fileName); - }else{ + } else { int32_t code = monitorReadSend(clusterId, pFile, &offset, size, SLOW_LOG_READ_BEGINNIG, *fileName); - tscDebug("[monitor] monitorSendSlowLogAtBeginning send slow log clusterId:%"PRId64",ret:%d", clusterId, code); + tscDebug("[monitor] monitorSendSlowLogAtBeginning send slow log clusterId:%" PRId64 ",ret:%d", clusterId, code); *fileName = NULL; } } -static void monitorSendSlowLogAtRunning(int64_t clusterId){ +static void monitorSendSlowLogAtRunning(int64_t clusterId) { void* tmp = taosHashGet(monitorSlowLogHash, &clusterId, LONG_BYTES); - if (tmp == NULL){ + if (tmp == NULL) { return; } SlowLogClient* pClient = (*(SlowLogClient**)tmp); - if (pClient == NULL){ + if (pClient == NULL) { return; } - int64_t size = getFileSize(pClient->path); - if(size <= pClient->offset){ - if(taosFtruncateFile(pClient->pFile, 0) < 0){ + int64_t size = getFileSize(pClient->path); + if (size <= pClient->offset) { + if (taosFtruncateFile(pClient->pFile, 0) < 0) { tscError("failed to truncate file:%p code: %d", pClient->pFile, errno); } tscDebug("[monitor] monitorSendSlowLogAtRunning truncate file to 0 file:%p", pClient->pFile); pClient->offset = 0; - }else{ + } else { int32_t code = monitorReadSend(clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_RUNNING, NULL); - tscDebug("[monitor] monitorSendSlowLogAtRunning send slow log clusterId:%"PRId64",ret:%d", clusterId, code); + tscDebug("[monitor] monitorSendSlowLogAtRunning send slow log clusterId:%" PRId64 ",ret:%d", clusterId, code); } } static bool monitorSendSlowLogAtQuit(int64_t clusterId) { void* tmp = taosHashGet(monitorSlowLogHash, &clusterId, LONG_BYTES); - if (tmp == NULL){ + if (tmp == NULL) { return true; } SlowLogClient* pClient = (*(SlowLogClient**)tmp); - if (pClient == NULL){ + if (pClient == NULL) { return true; } int64_t size = getFileSize(pClient->path); - if(size <= pClient->offset){ + if (size <= pClient->offset) { processFileInTheEnd(pClient->pFile, pClient->path); pClient->pFile = NULL; tscInfo("[monitor] monitorSendSlowLogAtQuit remove file:%s", pClient->path); - if((--quitCnt) == 0){ + if ((--quitCnt) == 0) { return true; } - }else{ + } else { int32_t code = monitorReadSend(clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_QUIT, NULL); - tscDebug("[monitor] monitorSendSlowLogAtQuit send slow log clusterId:%"PRId64",ret:%d", clusterId, code); + tscDebug("[monitor] monitorSendSlowLogAtQuit send slow log clusterId:%" PRId64 ",ret:%d", clusterId, code); } return false; } -static void monitorSendAllSlowLogAtQuit(){ +static void monitorSendAllSlowLogAtQuit() { void* pIter = NULL; while ((pIter = taosHashIterate(monitorSlowLogHash, pIter))) { SlowLogClient* pClient = (*(SlowLogClient**)pIter); - if(pClient == NULL) { + if (pClient == NULL) { continue; } int64_t size = getFileSize(pClient->path); - if(size <= pClient->offset){ + if (size <= pClient->offset) { processFileInTheEnd(pClient->pFile, pClient->path); pClient->pFile = NULL; - }else if(pClient->offset == 0){ + } else if (pClient->offset == 0) { int64_t* clusterId = (int64_t*)taosHashGetKey(pIter, NULL); - int32_t code = monitorReadSend(*clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_QUIT, NULL); - tscDebug("[monitor] monitorSendAllSlowLogAtQuit send slow log clusterId:%"PRId64",ret:%d", *clusterId, code); - if (code == 0){ - quitCnt ++; + int32_t code = monitorReadSend(*clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_QUIT, NULL); + tscDebug("[monitor] monitorSendAllSlowLogAtQuit send slow log clusterId:%" PRId64 ",ret:%d", *clusterId, code); + if (code == 0) { + quitCnt++; } } } } -static void processFileRemoved(SlowLogClient* pClient){ +static void processFileRemoved(SlowLogClient* pClient) { taosUnLockFile(pClient->pFile); taosCloseFile(&(pClient->pFile)); - TdFilePtr pFile = taosOpenFile(pClient->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_READ | TD_FILE_TRUNC); + TdFilePtr pFile = + taosOpenFile(pClient->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_READ | TD_FILE_TRUNC); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tscError("failed to open file:%s since %s", pClient->path, terrstr()); - }else{ + } else { pClient->pFile = pFile; } } -static void monitorSendAllSlowLog(){ +static void monitorSendAllSlowLog() { int64_t t = taosGetMonoTimestampMs(); - void* pIter = NULL; + void* pIter = NULL; while ((pIter = taosHashIterate(monitorSlowLogHash, pIter))) { int64_t* clusterId = (int64_t*)taosHashGetKey(pIter, NULL); SAppInstInfo* pInst = getAppInstByClusterId(*clusterId); SlowLogClient* pClient = (*(SlowLogClient**)pIter); - if (pClient == NULL){ + if (pClient == NULL) { taosHashCancelIterate(monitorSlowLogHash, pIter); return; } - if (t - pClient->lastCheckTime > pInst->monitorParas.tsMonitorInterval * 1000){ + if (t - pClient->lastCheckTime > pInst->monitorParas.tsMonitorInterval * 1000) { pClient->lastCheckTime = t; } else { continue; @@ -595,35 +606,35 @@ static void monitorSendAllSlowLog(){ if (pInst != NULL && pClient->offset == 0) { int64_t size = getFileSize(pClient->path); - if(size <= 0){ - if(size < 0){ + if (size <= 0) { + if (size < 0) { tscError("[monitor] monitorSendAllSlowLog failed to get file size:%s, err:%d", pClient->path, errno); - if(errno == ENOENT){ + if (errno == ENOENT) { processFileRemoved(pClient); } } continue; } int32_t code = monitorReadSend(*clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_RUNNING, NULL); - tscDebug("[monitor] monitorSendAllSlowLog send slow log clusterId:%"PRId64",ret:%d", *clusterId, code); + tscDebug("[monitor] monitorSendAllSlowLog send slow log clusterId:%" PRId64 ",ret:%d", *clusterId, code); } } } -static void monitorSendAllSlowLogFromTempDir(int64_t clusterId){ +static void monitorSendAllSlowLogFromTempDir(int64_t clusterId) { SAppInstInfo* pInst = getAppInstByClusterId((int64_t)clusterId); - if(pInst == NULL || !pInst->monitorParas.tsEnableMonitor){ + if (pInst == NULL || !pInst->monitorParas.tsEnableMonitor) { tscInfo("[monitor] monitor is disabled, skip send slow log"); return; } char namePrefix[PATH_MAX] = {0}; - if (snprintf(namePrefix, sizeof(namePrefix), "%s%"PRIx64, TD_TMP_FILE_PREFIX, clusterId) < 0) { + if (snprintf(namePrefix, sizeof(namePrefix), "%s%" PRIx64, TD_TMP_FILE_PREFIX, clusterId) < 0) { tscError("failed to generate slow log file name prefix"); return; } - char tmpPath[PATH_MAX] = {0}; + char tmpPath[PATH_MAX] = {0}; if (getSlowLogTmpDir(tmpPath, sizeof(tmpPath)) < 0) { return; } @@ -639,11 +650,9 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId){ continue; } - char *name = taosGetDirEntryName(de); - if (strcmp(name, ".") == 0 || - strcmp(name, "..") == 0 || - strstr(name, namePrefix) == NULL) { - tscInfo("skip file:%s, for cluster id:%"PRIx64, name, clusterId); + char* name = taosGetDirEntryName(de); + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0 || strstr(name, namePrefix) == NULL) { + tscInfo("skip file:%s, for cluster id:%" PRIx64, name, clusterId); continue; } @@ -659,7 +668,7 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId){ taosCloseFile(&pFile); continue; } - char *tmp = taosStrdup(filename); + char* tmp = taosStrdup(filename); monitorSendSlowLogAtBeginning(clusterId, &tmp, pFile, 0); taosMemoryFree(tmp); } @@ -667,7 +676,7 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId){ taosCloseDir(&pDir); } -static void* monitorThreadFunc(void *param){ +static void* monitorThreadFunc(void* param) { setThreadName("client-monitor-slowlog"); #ifdef WINDOWS @@ -680,18 +689,18 @@ static void* monitorThreadFunc(void *param){ return NULL; } tscDebug("monitorThreadFunc start"); - int64_t quitTime = 0; + int64_t quitTime = 0; while (1) { if (atomic_load_32(&slowLogFlag) > 0 > 0) { - if(quitCnt == 0){ + if (quitCnt == 0) { monitorSendAllSlowLogAtQuit(); - if(quitCnt == 0){ + if (quitCnt == 0) { tscInfo("monitorThreadFunc quit since no slow log to send"); break; } quitTime = taosGetMonoTimestampMs(); } - if(taosGetMonoTimestampMs() - quitTime > 500){ //quit at most 500ms + if (taosGetMonoTimestampMs() - quitTime > 500) { // quit at most 500ms tscInfo("monitorThreadFunc quit since timeout"); break; } @@ -700,18 +709,19 @@ static void* monitorThreadFunc(void *param){ MonitorSlowLogData* slowLogData = NULL; taosReadQitem(monitorQueue, (void**)&slowLogData); if (slowLogData != NULL) { - if (slowLogData->type == SLOW_LOG_READ_BEGINNIG){ - if(slowLogData->pFile != NULL){ - monitorSendSlowLogAtBeginning(slowLogData->clusterId, &(slowLogData->fileName), slowLogData->pFile, slowLogData->offset); - }else{ + if (slowLogData->type == SLOW_LOG_READ_BEGINNIG) { + if (slowLogData->pFile != NULL) { + monitorSendSlowLogAtBeginning(slowLogData->clusterId, &(slowLogData->fileName), slowLogData->pFile, + slowLogData->offset); + } else { monitorSendAllSlowLogFromTempDir(slowLogData->clusterId); } - } else if(slowLogData->type == SLOW_LOG_WRITE){ + } else if (slowLogData->type == SLOW_LOG_WRITE) { monitorWriteSlowLog2File(slowLogData, tmpSlowLogPath); - } else if(slowLogData->type == SLOW_LOG_READ_RUNNING){ + } else if (slowLogData->type == SLOW_LOG_READ_RUNNING) { monitorSendSlowLogAtRunning(slowLogData->clusterId); - } else if(slowLogData->type == SLOW_LOG_READ_QUIT){ - if(monitorSendSlowLogAtQuit(slowLogData->clusterId)){ + } else if (slowLogData->type == SLOW_LOG_READ_QUIT) { + if (monitorSendSlowLogAtQuit(slowLogData->clusterId)) { tscInfo("monitorThreadFunc quit since all slow log sended"); monitorFreeSlowLogData(slowLogData); taosFreeQitem(slowLogData); @@ -757,8 +767,11 @@ static void tscMonitorStop() { } int32_t monitorInit() { + int32_t code; + tscInfo("[monitor] tscMonitor init"); - monitorCounterHash = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + monitorCounterHash = + (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); if (monitorCounterHash == NULL) { tscError("failed to create monitorCounterHash"); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -766,7 +779,8 @@ int32_t monitorInit() { } taosHashSetFreeFp(monitorCounterHash, destroyMonitorClient); - monitorSlowLogHash = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + monitorSlowLogHash = + (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); if (monitorSlowLogHash == NULL) { tscError("failed to create monitorSlowLogHash"); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -781,7 +795,7 @@ int32_t monitorInit() { return -1; } - if (getSlowLogTmpDir(tmpSlowLogPath, sizeof(tmpSlowLogPath)) < 0){ + if (getSlowLogTmpDir(tmpSlowLogPath, sizeof(tmpSlowLogPath)) < 0) { terrno = TSDB_CODE_TSC_INTERNAL_ERROR; return -1; } @@ -798,14 +812,15 @@ int32_t monitorInit() { return -1; } - monitorQueue = taosOpenQueue(); - if(monitorQueue == NULL){ + code = taosOpenQueue(&monitorQueue); + if (code) { + terrno = code; tscError("open queue error since %s", terrstr()); return -1; } taosInitRWLatch(&monitorLock); - if (tscMonitortInit() != 0){ + if (tscMonitortInit() != 0) { return -1; } return 0; @@ -828,21 +843,26 @@ void monitorClose() { taosWUnLockLatch(&monitorLock); } -int32_t monitorPutData2MonitorQueue(MonitorSlowLogData data){ +int32_t monitorPutData2MonitorQueue(MonitorSlowLogData data) { + int32_t code; + MonitorSlowLogData* slowLogData; + if (atomic_load_32(&slowLogFlag) == -2) { tscError("[monitor] slow log thread is exiting"); return -1; } - MonitorSlowLogData* slowLogData = taosAllocateQitem(sizeof(MonitorSlowLogData), DEF_QITEM, 0); - if (slowLogData == NULL) { + + code = taosAllocateQitem(sizeof(MonitorSlowLogData), DEF_QITEM, 0, (void**)&slowLogData); + if (code) { tscError("[monitor] failed to allocate slow log data"); - return -1; + return terrno = code; } *slowLogData = data; - tscDebug("[monitor] write slow log to queue, clusterId:%"PRIx64 " type:%s, data:%s", slowLogData->clusterId, queueTypeStr[slowLogData->type], slowLogData->data); - if (taosWriteQitem(monitorQueue, slowLogData) == 0){ + tscDebug("[monitor] write slow log to queue, clusterId:%" PRIx64 " type:%s, data:%s", slowLogData->clusterId, + queueTypeStr[slowLogData->type], slowLogData->data); + if (taosWriteQitem(monitorQueue, slowLogData) == 0) { tsem2_post(&monitorSem); - }else{ + } else { monitorFreeSlowLogData(slowLogData); taosFreeQitem(slowLogData); } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index ce404092b0..f9fff732e9 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1116,9 +1116,9 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { if (taosHashGetSize(pVgroupHashmap) == 0) { goto end; } - SArray* pBufArray = serializeVgroupsCreateTableBatch(pVgroupHashmap); - if (NULL == pBufArray) { - code = TSDB_CODE_OUT_OF_MEMORY; + SArray* pBufArray = NULL; + code = serializeVgroupsCreateTableBatch(pVgroupHashmap, &pBufArray); + if (TSDB_CODE_SUCCESS != code) { goto end; } @@ -1264,9 +1264,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { if (taosHashGetSize(pVgroupHashmap) == 0) { goto end; } - SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap); - if (NULL == pBufArray) { - code = TSDB_CODE_OUT_OF_MEMORY; + SArray* pBufArray = NULL; + code = serializeVgroupsDropTableBatch(pVgroupHashmap, &pBufArray); + if (TSDB_CODE_SUCCESS != code) { goto end; } @@ -1572,9 +1572,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat } // uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid); - pQuery = smlInitHandle(); - if (pQuery == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1656,9 +1655,9 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha if (code != TSDB_CODE_SUCCESS) { goto end; } - pQuery = smlInitHandle(); - if (pQuery == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1755,12 +1754,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - pQuery = smlInitHandle(); - if (pQuery == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { SET_ERROR_MSG("init sml handle failed"); goto end; } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); @@ -1903,12 +1902,12 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - pQuery = smlInitHandle(); - if (pQuery == NULL) { + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { SET_ERROR_MSG("init sml handle failed"); - code = TSDB_CODE_OUT_OF_MEMORY; goto end; } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 7e63dcd36a..00445b4d12 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -110,11 +110,13 @@ int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL}; static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char *pTabName, AUTH_TYPE type) { SUserAuthInfo pAuth = {0}; - snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user); + (void)snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user); if (NULL == pTabName) { - tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb)); + if (tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb)) != 0){ + return TSDB_CODE_SML_INVALID_DATA; + } } else { - toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); + (void)toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); //ignore } pAuth.type = type; @@ -129,17 +131,19 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char : code; } -int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { - if (pBuf->buf) { - memset(pBuf->buf, 0, pBuf->len); - if (msg1) strncat(pBuf->buf, msg1, pBuf->len); - int32_t left = pBuf->len - strlen(pBuf->buf); - if (left > 2 && msg2) { - strncat(pBuf->buf, ":", left - 1); - strncat(pBuf->buf, msg2, left - 2); - } +void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { + if (pBuf->buf == NULL) { + return; + } + (void)memset(pBuf->buf, 0, pBuf->len); + if (msg1) { + (void)strncat(pBuf->buf, msg1, pBuf->len); + } + int32_t left = pBuf->len - strlen(pBuf->buf); + if (left > 2 && msg2) { + (void)strncat(pBuf->buf, ":", left - 1); + (void)strncat(pBuf->buf, msg2, left - 2); } - return TSDB_CODE_SML_INVALID_DATA; } int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) { @@ -161,10 +165,10 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) { +int32_t smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen, SSmlTableInfo** tInfo) { SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); if (!tag) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } tag->sTableName = measure; @@ -173,14 +177,12 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measu tag->cols = taosArrayInit(numRows, POINTER_BYTES); if (tag->cols == NULL) { uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; + taosMemoryFree(tag); + return TSDB_CODE_OUT_OF_MEMORY; } - return tag; - -cleanup: - taosMemoryFree(tag); - return NULL; + *tInfo = tag; + return TSDB_CODE_SUCCESS; } void smlBuildTsKv(SSmlKv *kv, int64_t ts) { @@ -191,34 +193,50 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts) { kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; } -SSmlSTableMeta *smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement) { - SSmlSTableMeta *sMeta = NULL; +static void smlDestroySTableMeta(void *para) { + SSmlSTableMeta *meta = *(SSmlSTableMeta **)para; + taosHashCleanup(meta->tagHash); + taosHashCleanup(meta->colHash); + (void)taosArrayDestroy(meta->tags); + (void)taosArrayDestroy(meta->cols); + taosMemoryFreeClear(meta->tableMeta); + taosMemoryFree(meta); +} + +int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta) { + int32_t code = TSDB_CODE_SUCCESS; char *measure = currElement->measure; int measureLen = currElement->measureLen; if (currElement->measureEscaped) { measure = (char *)taosMemoryMalloc(measureLen); - memcpy(measure, currElement->measure, measureLen); + if (measure == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(measure, currElement->measure, measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); smlStrReplace(measure, measureLen); } - STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + STableMeta *pTableMeta = NULL; + code = smlGetMeta(info, measure, measureLen, &pTableMeta); if (currElement->measureEscaped) { taosMemoryFree(measure); } - if (pTableMeta == NULL) { + if (code != TSDB_CODE_SUCCESS) { info->dataFormat = false; info->reRun = true; - terrno = TSDB_CODE_SUCCESS; - return sMeta; + return code; } - sMeta = smlBuildSTableMeta(info->dataFormat); - if (sMeta == NULL) { + code = smlBuildSTableMeta(info->dataFormat, sMeta); + if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pTableMeta); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return sMeta; + return code; + } + (*sMeta)->tableMeta = pTableMeta; + code = taosHashPut(info->superTables, currElement->measure, currElement->measureLen, sMeta, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS){ + smlDestroySTableMeta(*sMeta); + return code; } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { SSchema *col = pTableMeta->schema + i; SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type}; @@ -232,40 +250,38 @@ SSmlSTableMeta *smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currEleme } if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(sMeta->cols, &kv); + if(taosArrayPush((*sMeta)->cols, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } else { - taosArrayPush(sMeta->tags, &kv); + if(taosArrayPush((*sMeta)->tags, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } } - return sMeta; + return TSDB_CODE_SUCCESS; } bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { // cnt begin 0, add ts so + 2 if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } // bind data int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1); if (unlikely(ret != TSDB_CODE_SUCCESS)) { uDebug("smlBuildCol error, retry"); - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } if (cnt >= taosArrayGetSize(info->maxColKVs)) { - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt); - + if (maxKV == NULL){ + goto END; + } if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } if (unlikely(IS_VAR_DATA_TYPE(kv->type) && kv->length > maxKV->length)) { @@ -273,6 +289,11 @@ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { info->needModifySchema = true; } return true; + +END: + info->dataFormat = false; + info->reRun = true; + return false; } bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { @@ -285,6 +306,9 @@ bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); + if (maxKV == NULL) { + goto END; + } if (unlikely(!IS_SAME_KEY)) { goto END; } @@ -306,8 +330,8 @@ int32_t smlJoinMeasureTag(SSmlLineInfo *elements) { if (elements->measureTag == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(elements->measureTag, elements->measure, elements->measureLen); - memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); + (void)memcpy(elements->measureTag, elements->measure, elements->measureLen); + (void)memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); elements->measureTagsLen = elements->measureLen + elements->tagsLen; return TSDB_CODE_SUCCESS; } @@ -325,14 +349,14 @@ static bool smlIsPKTable(STableMeta *pTableMeta){ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { bool isSameMeasure = IS_SAME_SUPER_TABLE; if (isSameMeasure) { - return 0; + return TSDB_CODE_SUCCESS; } SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); SSmlSTableMeta *sMeta = NULL; if (unlikely(tmp == NULL)) { - sMeta = smlBuildSuperTableInfo(info, elements); - if (sMeta == NULL) return -1; + int32_t code = smlBuildSuperTableInfo(info, elements, &sMeta); + if (code != 0) return code; } else { sMeta = *tmp; } @@ -342,39 +366,59 @@ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { info->maxColKVs = sMeta->cols; if(smlIsPKTable(sMeta->tableMeta)){ - terrno = TSDB_CODE_SML_NOT_SUPPORT_PK; - return -1; + return TSDB_CODE_SML_NOT_SUPPORT_PK; } return 0; } int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { + int32_t code = TSDB_CODE_SUCCESS; SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); SSmlTableInfo *tinfo = NULL; if (unlikely(oneTable == NULL)) { - tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); - if (unlikely(!tinfo)) { - return TSDB_CODE_OUT_OF_MEMORY; + code = smlBuildTableInfo(1, elements->measure, elements->measureLen, &tinfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); + if(code != 0){ + smlDestroyTableInfo(&tinfo); + return code; } - taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); tinfo->tags = taosArrayDup(info->preLineTagKV, NULL); + if(tinfo->tags == NULL){ + smlDestroyTableInfo(&tinfo); + return TSDB_CODE_OUT_OF_MEMORY; + } for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i); + if(kv == NULL){ + smlDestroyTableInfo(&tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } if (kv->keyEscaped) kv->key = NULL; if (kv->valueEscaped) kv->value = NULL; } - smlSetCTableName(tinfo); - getTableUid(info, elements, tinfo); + code = smlSetCTableName(tinfo); + if (code != TSDB_CODE_SUCCESS){ + smlDestroyTableInfo(&tinfo); + return code; + } + code = getTableUid(info, elements, tinfo); + if (code != TSDB_CODE_SUCCESS){ + smlDestroyTableInfo(&tinfo); + return code; + } if (info->dataFormat) { info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { + code = smlInitTableDataCtx(info->pQuery, info->currSTableMeta, &tinfo->tableDataCtx); + if (code != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; + return code; } } } else { @@ -404,9 +448,16 @@ int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv * uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); if (elements->colArray == NULL) { elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + if (elements->colArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (taosArrayPush(elements->colArray, kvTs) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + if (taosArrayPush(elements->colArray, kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(elements->colArray, kvTs); - taosArrayPush(elements->colArray, kv); } info->preLine = *elements; @@ -442,6 +493,9 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + if(tag == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } totalNameLen += tag->length; } if (totalNameLen < TSDB_TABLE_NAME_LEN) { @@ -449,12 +503,15 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { } } if (autoChildName) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - strncat(childTableName, tag->value, tag->length); + if(tag == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } + (void)strncat(childTableName, tag->value, tag->length); if (i != taosArrayGetSize(tags) - 1) { - strcat(childTableName, tsSmlAutoChildTableNameDelimiter); + (void)strcat(childTableName, tsSmlAutoChildTableNameDelimiter); } } if (tsSmlDot2Underline) { @@ -466,10 +523,13 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + if(tag == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + (void)strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); } @@ -483,60 +543,60 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { } int32_t smlSetCTableName(SSmlTableInfo *oneTable) { - smlParseTableName(oneTable->tags, oneTable->childTableName); + int32_t code = smlParseTableName(oneTable->tags, oneTable->childTableName); + if(code != TSDB_CODE_SUCCESS){ + return code; + } if (strlen(oneTable->childTableName) == 0) { SArray *dst = taosArrayDup(oneTable->tags, NULL); + if (dst == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } ASSERT(oneTable->sTableNameLen < TSDB_TABLE_NAME_LEN); char superName[TSDB_TABLE_NAME_LEN] = {0}; RandTableName rName = {dst, NULL, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; if (tsSmlDot2Underline) { - memcpy(superName, oneTable->sTableName, oneTable->sTableNameLen); + (void)memcpy(superName, oneTable->sTableName, oneTable->sTableNameLen); smlStrReplace(superName, oneTable->sTableNameLen); rName.stbFullName = superName; } else { rName.stbFullName = oneTable->sTableName; } - buildChildTableName(&rName); - taosArrayDestroy(dst); + code = buildChildTableName(&rName); + if (code != TSDB_CODE_SUCCESS){ + return code; + } + (void)taosArrayDestroy(dst); } return TSDB_CODE_SUCCESS; } -void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { +int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { char key[TSDB_TABLE_NAME_LEN * 2 + 1] = {0}; size_t nLen = strlen(tinfo->childTableName); - memcpy(key, currElement->measure, currElement->measureLen); + (void)memcpy(key, currElement->measure, currElement->measureLen); if (tsSmlDot2Underline) { smlStrReplace(key, currElement->measureLen); } - memcpy(key + currElement->measureLen + 1, tinfo->childTableName, nLen); + (void)memcpy(key + currElement->measureLen + 1, tinfo->childTableName, nLen); void *uid = taosHashGet(info->tableUids, key, currElement->measureLen + 1 + nLen); // use \0 as separator for stable name and child table name if (uid == NULL) { tinfo->uid = info->uid++; - taosHashPut(info->tableUids, key, currElement->measureLen + 1 + nLen, &tinfo->uid, sizeof(uint64_t)); + return taosHashPut(info->tableUids, key, currElement->measureLen + 1 + nLen, &tinfo->uid, sizeof(uint64_t)); } else { tinfo->uid = *(uint64_t *)uid; } + return TSDB_CODE_SUCCESS; } -static void smlDestroySTableMeta(void *para) { - SSmlSTableMeta *meta = *(SSmlSTableMeta **)para; - taosHashCleanup(meta->tagHash); - taosHashCleanup(meta->colHash); - taosArrayDestroy(meta->tags); - taosArrayDestroy(meta->cols); - taosMemoryFreeClear(meta->tableMeta); - taosMemoryFree(meta); -} - -SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { +int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); if (!meta) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } if (unlikely(!isDataFormat)) { @@ -564,14 +624,15 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { uError("SML:smlBuildSTableMeta failed to allocate memory"); goto cleanup; } - return meta; + *sMeta = meta; + return TSDB_CODE_SUCCESS; cleanup: smlDestroySTableMeta(meta); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } -bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { +int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; @@ -743,8 +804,8 @@ bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { return true; } -STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen) { - STableMeta *pTableMeta = NULL; +int32_t smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen, STableMeta **pTableMeta) { + *pTableMeta = NULL; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); @@ -754,14 +815,14 @@ STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen conn.requestId = info->pRequest->requestId; conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, measure, measureLen); + (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + (void)memcpy(pName.tname, measure, measureLen); - int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, pTableMeta); if (code != TSDB_CODE_SUCCESS) { - return NULL; + return code; } - return pTableMeta; + return TSDB_CODE_SUCCESS; } static int64_t smlGenId() { @@ -803,7 +864,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm *action = SCHEMA_ACTION_ADD_COLUMN; } } - return 0; + return TSDB_CODE_SUCCESS; } #define BOUNDARY 1024 @@ -838,6 +899,9 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH for (int j = 0; j < taosArrayGetSize(cols); ++j) { if (j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); if (code != TSDB_CODE_SUCCESS) { return code; @@ -846,6 +910,9 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } if(taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL){ return TSDB_CODE_PAR_DUPLICATED_COLUMN; } @@ -854,10 +921,18 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH } static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { + int32_t code = TSDB_CODE_SUCCESS; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (hashTmp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } int32_t i = 0; for (; i < length; i++) { - taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); + code = taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); + if (code != 0){ + goto END; + } } if (isTag) { @@ -867,13 +942,19 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool } for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (kv == NULL){ + code = TSDB_CODE_SML_INVALID_DATA; + goto END; + } if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { - taosHashCleanup(hashTmp); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto END; } } + +END: taosHashCleanup(hashTmp); - return 0; + return code; } static int32_t getBytes(uint8_t type, int32_t length) { @@ -889,6 +970,9 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO SArray *results, int32_t numOfCols, bool isTag) { for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } ESchemaAction action = SCHEMA_ACTION_NULL; int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); if (code != 0) { @@ -898,8 +982,10 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO SField field = {0}; field.type = kv->type; field.bytes = getBytes(kv->type, kv->length); - memcpy(field.name, kv->key, kv->keyLen); - taosArrayPush(results, &field); + (void)memcpy(field.name, kv->key, kv->keyLen); + if (taosArrayPush(results, &field) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); if (index == NULL) { @@ -909,6 +995,9 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO uint16_t newIndex = *index; if (isTag) newIndex -= numOfCols; SField *field = (SField *)taosArrayGet(results, newIndex); + if (field == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } field->bytes = getBytes(kv->type, kv->length); } } @@ -917,6 +1006,9 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO int32_t len = 0; for (int j = 0; j < taosArrayGetSize(results); ++j) { SField *field = taosArrayGet(results, j); + if (field == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } len += field->bytes; } if (len > maxLen) { @@ -936,18 +1028,29 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, // put front for free pReq.numOfColumns = taosArrayGetSize(pColumns); + pReq.pTags = pTags; + pReq.numOfTags = taosArrayGetSize(pTags); + pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions)); + if (pReq.pColumns == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (int32_t i = 0; i < pReq.numOfColumns; ++i) { - SField *pField = taosArrayGet(pColumns, i); + SField *pField = taosArrayGet(pColumns, i); + if (pField == NULL){ + code = TSDB_CODE_SML_INVALID_DATA; + goto end; + } SFieldWithOptions fieldWithOption = {0}; setFieldWithOptions(&fieldWithOption, pField); setDefaultOptionsForField(&fieldWithOption); - taosArrayPush(pReq.pColumns, &fieldWithOption); + if (taosArrayPush(pReq.pColumns, &fieldWithOption) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } - pReq.numOfTags = taosArrayGetSize(pTags); - pReq.pTags = pTags; - if (action == SCHEMA_ACTION_CREATE_STABLE) { pReq.colVer = 1; pReq.tagVer = 1; @@ -968,6 +1071,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; } else { uError("SML:0x%" PRIx64 " invalid action:%d", info->id, action); + code = TSDB_CODE_SML_INVALID_DATA; goto end; } @@ -987,38 +1091,52 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SField field = {0}; field.type = TSDB_DATA_TYPE_NCHAR; field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - strcpy(field.name, tsSmlTagName); - taosArrayPush(pReq.pTags, &field); + tstrncpy(field.name, tsSmlTagName, sizeof(field.name)); + if (taosArrayPush(pReq.pTags, &field) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } pReq.commentLen = -1; pReq.igExists = true; - tNameExtractFullName(pName, pReq.name); + (void)tNameExtractFullName(pName, pReq.name); pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); + if (pCmdMsg.msgLen < 0){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); if (NULL == pCmdMsg.pMsg) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery; - memset(&pQuery, 0, sizeof(pQuery)); + if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) < 0){ + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pCmdMsg.pMsg); + goto end; + } + + SQuery pQuery = {0}; + (void)memset(&pQuery, 0, sizeof(pQuery)); pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; pQuery.stableQuery = true; - launchQueryImpl(pRequest, &pQuery, true, NULL); + (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // no need to check return value if (pRequest->code == TSDB_CODE_SUCCESS) { - catalogRemoveTableMeta(info->pCatalog, pName); + code = catalogRemoveTableMeta(info->pCatalog, pName); + if (code != TSDB_CODE_SUCCESS){ + goto end; + } } code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); end: destroyRequest(pRequest); @@ -1053,11 +1171,15 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { size_t superTableLen = 0; void *superTable = taosHashGetKey(tmp, &superTableLen); char *measure = taosMemoryMalloc(superTableLen); - memcpy(measure, superTable, superTableLen); + if (measure == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + (void)memcpy(measure, superTable, superTableLen); PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); smlStrReplace(measure, superTableLen); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, measure, superTableLen); + (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + (void)memcpy(pName.tname, measure, superTableLen); taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); @@ -1069,23 +1191,32 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); + if (pColumns == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); + if (pTags == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + (void)taosArrayDestroy(pColumns); + goto end; + } code = smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList tag1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList col1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); - taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pColumns); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; @@ -1107,9 +1238,16 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (hashTmp == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (uint16_t i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + if (code != 0){ + goto end; + } } ESchemaAction action = SCHEMA_ACTION_NULL; @@ -1126,46 +1264,64 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { action); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + if (pColumns == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - + if (pTags == NULL){ + (void)taosArrayDestroy(pColumns); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { SField field = {0}; field.type = pTableMeta->schema[i].type; field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); + tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); + if (taosArrayPush(pColumns, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } else { - taosArrayPush(pTags, &field); + if (taosArrayPush(pTags, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } } code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, pTableMeta->tableInfo.numOfColumns, true); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } if (taosArrayGetSize(pTags) + pTableMeta->tableInfo.numOfColumns > TSDB_MAX_COLUMNS) { uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } if (taosArrayGetSize(pTags) > TSDB_MAX_TAGS) { uError("SML:0x%" PRIx64 " too many tags than 128", info->id); code = TSDB_CODE_PAR_INVALID_TAGS_NUM; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pColumns); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; @@ -1185,7 +1341,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { taosHashClear(hashTmp); for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } } action = SCHEMA_ACTION_NULL; code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false); @@ -1201,18 +1360,36 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { action); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + if (pColumns == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - + if (pTags == NULL){ + (void)taosArrayDestroy(pColumns); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { SField field = {0}; field.type = pTableMeta->schema[i].type; field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); + tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); + if (taosArrayPush(pColumns, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } else { - taosArrayPush(pTags, &field); + if (taosArrayPush(pTags, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } } @@ -1220,21 +1397,21 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { pTableMeta->tableInfo.numOfColumns, false); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } if (taosArrayGetSize(pColumns) + pTableMeta->tableInfo.numOfTags > TSDB_MAX_COLUMNS) { uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pColumns); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; @@ -1284,13 +1461,13 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema); - return 0; + return TSDB_CODE_SUCCESS; end: taosHashCancelIterate(info->superTables, tmp); taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); - catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); + (void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, tstrerror(code), info->dataFormat, info->needModifySchema); @@ -1301,9 +1478,14 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols terrno = 0; for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); if (ret == 0) { - taosArrayPush(metaArray, kv); + if (taosArrayPush(metaArray, kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } @@ -1317,10 +1499,16 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg, SHashObj* checkDuplicate) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); if (index) { SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); + if (value == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } + if (isTag) { if (kv->length > value->length) { value->length = kv->length; @@ -1345,7 +1533,9 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols int16_t size = tmp; int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); if (ret == 0) { - taosArrayPush(metaArray, kv); + if(taosArrayPush(metaArray, kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } @@ -1365,7 +1555,7 @@ void smlDestroyTableInfo(void *para) { taosHashCleanup(kvHash); } - taosArrayDestroy(tag->cols); + (void)taosArrayDestroy(tag->cols); taosArrayDestroyEx(tag->tags, freeSSmlKv); taosMemoryFree(tag); } @@ -1391,13 +1581,13 @@ void smlDestroyInfo(SSmlHandle *info) { cJSON *tags = (cJSON *)taosArrayGetP(info->tagJsonArray, i); cJSON_Delete(tags); } - taosArrayDestroy(info->tagJsonArray); + (void)taosArrayDestroy(info->tagJsonArray); for (int i = 0; i < taosArrayGetSize(info->valueJsonArray); i++) { cJSON *value = (cJSON *)taosArrayGetP(info->valueJsonArray, i); cJSON_Delete(value); } - taosArrayDestroy(info->valueJsonArray); + (void)taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); @@ -1418,21 +1608,22 @@ void smlDestroyInfo(SSmlHandle *info) { taosMemoryFreeClear(info); } -SSmlHandle *smlBuildSmlInfo(TAOS *taos) { +int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { int32_t code = TSDB_CODE_SUCCESS; SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); if (NULL == info) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } if (taos != NULL) { info->taos = acquireTscObj(*(int64_t *)taos); if (info->taos == NULL) { - goto cleanup; + code = TSDB_CODE_TSC_DISCONNECTED; + goto FAILED; } code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto cleanup; + goto FAILED; } } @@ -1440,27 +1631,37 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (info->pVgHash == NULL || info->childTables == NULL || info->superTables == NULL || info->tableUids == NULL) { + uError("create SSmlHandle hash obj failed"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto FAILED; + } taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); info->id = smlGenId(); - info->pQuery = smlInitHandle(); + code = smlInitHandle(&info->pQuery); + if (code != TSDB_CODE_SUCCESS){ + goto FAILED; + } info->dataFormat = true; info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); - if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables || NULL == info->tableUids) { - uError("create SSmlHandle failed"); - goto cleanup; + if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) { + uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); + code = TSDB_CODE_OUT_OF_MEMORY; + goto FAILED; } - return info; + *handle = info; + return code; -cleanup: +FAILED: smlDestroyInfo(info); - return NULL; + return code; } static int32_t smlPushCols(SArray *colsArray, SArray *cols) { @@ -1471,15 +1672,26 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (kv == NULL){ + taosHashCleanup(kvHash); + return TSDB_CODE_SML_INVALID_DATA; + } terrno = 0; - taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + int32_t code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { taosHashCleanup(kvHash); return TSDB_CODE_PAR_DUPLICATED_COLUMN; } + if (code != TSDB_CODE_SUCCESS){ + taosHashCleanup(kvHash); + return code; + } } - taosArrayPush(colsArray, &kvHash); + if (taosArrayPush(colsArray, &kvHash) == NULL) { + taosHashCleanup(kvHash); + return TSDB_CODE_OUT_OF_MEMORY; + } return TSDB_CODE_SUCCESS; } @@ -1542,9 +1754,10 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { } else { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); - SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); - if (meta == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + SSmlSTableMeta *meta = NULL; + ret = smlBuildSTableMeta(info->dataFormat, &meta); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } ret = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); if (ret != TSDB_CODE_SUCCESS) { @@ -1572,11 +1785,17 @@ static int32_t smlInsertData(SSmlHandle *info) { if (info->pRequest->dbList == NULL) { info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); + if (info->pRequest->dbList == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } char *data = (char *)taosArrayReserve(info->pRequest->dbList, 1); + if (data == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - tNameGetFullDbName(&pName, data); + (void)tNameGetFullDbName(&pName, data); //ignore SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); while (oneTable) { @@ -1584,18 +1803,26 @@ static int32_t smlInsertData(SSmlHandle *info) { int measureLen = tableData->sTableNameLen; char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); - memcpy(measure, tableData->sTableName, tableData->sTableNameLen); + if (measure == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(measure, tableData->sTableName, tableData->sTableNameLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); smlStrReplace(measure, measureLen); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, measure, measureLen); + (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + (void)memcpy(pName.tname, measure, measureLen); if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); + if (info->pRequest->tableList == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (taosArrayPush(info->pRequest->tableList, &pName) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(info->pRequest->tableList, &pName); - strcpy(pName.tname, tableData->childTableName); + tstrncpy(pName.tname, tableData->childTableName, sizeof(pName.tname)); SRequestConnInfo conn = {0}; conn.pTrans = info->taos->pAppInfo->pTransporter; @@ -1618,7 +1845,13 @@ static int32_t smlInsertData(SSmlHandle *info) { taosHashCancelIterate(info->childTables, oneTable); return code; } - taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); + code = taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); + if (code != TSDB_CODE_SUCCESS){ + uError("SML:0x%" PRIx64 " taosHashPut failed. table name: %s", info->id, tableData->childTableName); + taosMemoryFree(measure); + taosHashCancelIterate(info->childTables, oneTable); + return code; + } SSmlSTableMeta **pMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); @@ -1655,9 +1888,10 @@ static int32_t smlInsertData(SSmlHandle *info) { info->cost.insertRpcTime = taosGetTimestampUs(); SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; - atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); + (void)atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); // no need to check return code + + (void)launchQueryImpl(info->pRequest, info->pQuery, true, NULL); // no need to check return code - launchQueryImpl(info->pRequest, info->pQuery, true, NULL); uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code, tstrerror(info->pRequest->code)); @@ -1691,7 +1925,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); } - memset(&info->preLine, 0, sizeof(SSmlLineInfo)); + (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; @@ -1721,7 +1955,7 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi if (*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)) { char *print = taosMemoryCalloc(*len + 1, 1); - memcpy(print, *tmp, *len); + (void)memcpy(print, *tmp, *len); uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines, info->protocol, *len, print); taosMemoryFree(print); @@ -1777,7 +2011,11 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char if (code != TSDB_CODE_SUCCESS) { if (rawLine != NULL) { char *print = taosMemoryCalloc(len + 1, 1); - memcpy(print, tmp, len); + if (print == NULL) { + uError("SML:0x%" PRIx64 " smlParseLine failed. out of memory", info->id); + return code; + } + (void)memcpy(print, tmp, len); uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print); taosMemoryFree(print); } else { @@ -1880,7 +2118,7 @@ void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawL uError("malloc %d for sml sql failed", rlen + 1); return; } - memcpy(sql, p, rlen); + (void)memcpy(sql, p, rlen); sql[rlen] = 0; request->sqlstr = sql; @@ -1892,7 +2130,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, int protocol, int precision, int32_t ttl, int64_t reqid) { int32_t code = TSDB_CODE_SUCCESS; if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; + uError("SML:taos_schemaless_insert error taos is null"); return NULL; } SRequestObj *request = NULL; @@ -1905,11 +2143,13 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, return NULL; } - info = smlBuildSmlInfo(taos); - if (info == NULL) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error SSmlHandle is null"); - return (TAOS_RES *)request; + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + code = smlBuildSmlInfo(taos, &info); + if (code != TSDB_CODE_SUCCESS) { + request->code = code; + smlBuildInvalidDataMsg(&msg, "init SSmlHandle failed", NULL); + uError("SML:taos_schemaless_insert error SSmlHandle is null, err msg:%s", tstrerror(code)); + goto end; } info->pRequest = request; info->pRequest->pQuery = info->pQuery; @@ -1922,7 +2162,6 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, smlSetReqSQL(request, lines, rawLine, rawLineEnd); - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); @@ -1961,7 +2200,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, break; } taosMsleep(100); - refreshMeta(request->pTscObj, request); + (void)refreshMeta(request->pTscObj, request); //ignore return code,try again uInfo("SML:%" PRIx64 " retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code, tstrerror(code)); smlDestroyInfo(info); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 845884d5ac..14d6a95e22 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -29,29 +29,29 @@ (start)++; \ } -static char *smlJsonGetObj(char *payload) { +static int32_t smlJsonGetObj(char **payload) { int leftBracketCnt = 0; bool isInQuote = false; - while (*payload) { - if (*payload == '"' && *(payload - 1) != '\\') { + while (**payload) { + if (**payload == '"' && *((*payload) - 1) != '\\') { isInQuote = !isInQuote; - } else if (!isInQuote && unlikely(*payload == '{')) { + } else if (!isInQuote && unlikely(**payload == '{')) { leftBracketCnt++; - payload++; + (*payload)++; continue; - } else if (!isInQuote && unlikely(*payload == '}')) { + } else if (!isInQuote && unlikely(**payload == '}')) { leftBracketCnt--; - payload++; + (*payload)++; if (leftBracketCnt == 0) { - return payload; + return 0; } else if (leftBracketCnt < 0) { - return NULL; + return -1; } continue; } - payload++; + (*payload)++; } - return NULL; + return -1; } int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { @@ -99,8 +99,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { offset[index++] = *start - sTmp; element->timestamp = (*start); if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->timestampLen = tmp - (*start); *start = tmp; } @@ -127,8 +128,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { offset[index++] = *start - sTmp; element->cols = (*start); if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->colsLen = tmp - (*start); *start = tmp; } @@ -153,8 +155,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { JUMP_JSON_SPACE((*start)) offset[index++] = *start - sTmp; element->tags = (*start); - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->tagsLen = tmp - (*start); *start = tmp; } @@ -209,8 +212,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { (*start) += offset[index++]; element->timestamp = *start; if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->timestampLen = tmp - (*start); *start = tmp; } @@ -227,8 +231,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { (*start) += offset[index++]; element->cols = *start; if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->colsLen = tmp - (*start); *start = tmp; } @@ -244,8 +249,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { } else if ((*start)[1] == 't' && (*start)[2] == 'a') { (*start) += offset[index++]; element->tags = (*start); - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->tagsLen = tmp - (*start); *start = tmp; } @@ -261,7 +267,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { uError("elements != %d", OTD_JSON_FIELDS_NUM); return TSDB_CODE_TSC_INVALID_JSON; } - return 0; + return TSDB_CODE_SUCCESS; } static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { @@ -466,7 +472,11 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { break; } case cJSON_String: { - smlConvertJSONString(kv, "binary", root); + int32_t ret = smlConvertJSONString(kv, "binary", root); + if (ret != TSDB_CODE_SUCCESS) { + uError("OTD:Failed to parse binary value from JSON Obj"); + return ret; + } break; } case cJSON_Object: { @@ -492,20 +502,17 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ int32_t tagNum = cJSON_GetArraySize(tags); if (unlikely(tagNum == 0)) { uError("SML:Tag should not be empty"); - terrno = TSDB_CODE_TSC_INVALID_JSON; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } for (int32_t i = 0; i < tagNum; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); if (unlikely(tag == NULL)) { - terrno = TSDB_CODE_TSC_INVALID_JSON; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } size_t keyLen = strlen(tag->string); if (unlikely(IS_INVALID_COL_LEN(keyLen))) { uError("OTD:Tag key length is 0 or too large than 64"); - terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - return -1; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } // add kv to SSmlKv @@ -516,19 +523,19 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ // value int32_t ret = smlParseValueFromJSON(tag, &kv); if (unlikely(ret != TSDB_CODE_SUCCESS)) { - terrno = ret; - return -1; + return ret; + } + if (taosArrayPush(preLineKV, &kv) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(preLineKV, &kv); if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - terrno = TSDB_CODE_SUCCESS; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } cnt++; } - return 0; + return TSDB_CODE_SUCCESS; } static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { @@ -536,12 +543,18 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo if(info->dataFormat){ ret = smlProcessSuperTable(info, elements); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } } ret = smlProcessTagJson(info, tags); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } ret = smlJoinMeasureTag(elements); if(ret != 0){ @@ -703,6 +716,9 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo // Parse tags bool needFree = info->dataFormat; elements->tags = cJSON_PrintUnformatted(tagsJson); + if (elements->tags == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } elements->tagsLen = strlen(elements->tags); if (is_same_child_table_telnet(elements, &info->preLine) != 0) { ret = smlParseTagsFromJSON(info, tagsJson, elements); @@ -729,7 +745,8 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo // notice!!! put ts back to tag to ensure get meta->precision int64_t ts = smlParseTSFromJSON(info, tsJson); if (unlikely(ts < 0)) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + char* tmp = cJSON_PrintUnformatted(tsJson); + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); return TSDB_CODE_INVALID_TIMESTAMP; } SSmlKv kvTs = {0}; @@ -823,7 +840,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * } if (ret != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_VALUE; + return ret; } if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; @@ -844,18 +861,23 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * cJSON *valueJson = cJSON_Parse(elements->cols); if (unlikely(valueJson == NULL)) { uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols); + elements->cols[elements->colsLen] = tmp; return TSDB_CODE_TSC_INVALID_JSON; } - taosArrayPush(info->tagJsonArray, &valueJson); + if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){ + cJSON_Delete(valueJson); + elements->cols[elements->colsLen] = tmp; + return TSDB_CODE_OUT_OF_MEMORY; + } ret = smlParseValueFromJSONObj(valueJson, &kv); if (ret != TSDB_CODE_SUCCESS) { - uError("SML:Failed to parse value from JSON Obj:%s", elements->cols); + uError("SML:0x%" PRIx64 " Failed to parse value from JSON Obj:%s", info->id, elements->cols); elements->cols[elements->colsLen] = tmp; return TSDB_CODE_TSC_INVALID_VALUE; } elements->cols[elements->colsLen] = tmp; } else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { - uError("SML:cols invalidate:%s", elements->cols); + uError("SML:0x%" PRIx64 " cols invalidate:%s", info->id, elements->cols); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -870,7 +892,11 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * return TSDB_CODE_TSC_INVALID_JSON; } - taosArrayPush(info->tagJsonArray, &tagsJson); + if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){ + cJSON_Delete(tagsJson); + uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id); + return TSDB_CODE_OUT_OF_MEMORY; + } ret = smlParseTagsFromJSON(info, tagsJson, elements); if (unlikely(ret)) { uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); @@ -937,7 +963,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { return ret; } info->lines = (SSmlLineInfo *)tmp; - memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); + (void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); } ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); if ((info->lines + cnt)->measure == NULL) break; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 7535cbfd0c..3456b7df17 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -103,7 +103,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return code; } char* tmp = taosMemoryCalloc(pVal->length, 1); - memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length); taosMemoryFree(tmp); if (code != TSDB_CODE_SUCCESS) { @@ -149,7 +152,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { if(data == NULL){ return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); + (void)memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); pVal->value = data; } @@ -212,8 +215,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(IS_EQUAL(*sql,escapeChar))) { keyLen = *sql - key; @@ -230,8 +232,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); - terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - return -1; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } // parse value @@ -245,8 +246,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ break; } else if (unlikely(IS_EQUAL(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { @@ -261,24 +261,28 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ if (unlikely(valueLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - return -1; + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } if (keyEscaped) { char *tmp = (char *)taosMemoryMalloc(keyLen); - memcpy(tmp, key, keyLen); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, key, keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); key = tmp; } if (valueEscaped) { char *tmp = (char *)taosMemoryMalloc(valueLen); - memcpy(tmp, value, valueLen); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, value, valueLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen); value = tmp; } @@ -289,11 +293,12 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; - taosArrayPush(preLineKV, &kv); + if(taosArrayPush(preLineKV, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - terrno = TSDB_CODE_SUCCESS; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } cnt++; @@ -302,7 +307,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ } (*sql)++; } - return 0; + return TSDB_CODE_SUCCESS; } static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { @@ -315,13 +320,19 @@ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if(info->dataFormat){ ret = smlProcessSuperTable(info, elements); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } } ret = smlProcessTagLine(info, sql, sqlEnd); if(ret != 0){ - return terrno; + if (info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } return smlProcessChildTable(info, elements); @@ -410,7 +421,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (keyEscaped) { char *tmp = (char *)taosMemoryMalloc(kv.keyLen); - memcpy(tmp, key, kv.keyLen); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, key, kv.keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen); kv.key = tmp; kv.keyEscaped = keyEscaped; @@ -418,7 +432,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (valueEscaped) { char *tmp = (char *)taosMemoryMalloc(kv.length); - memcpy(tmp, kv.value, kv.length); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, kv.value, kv.length); PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY); if(kv.type == TSDB_DATA_TYPE_VARBINARY){ @@ -437,8 +454,13 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } else { if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); + if (currElement->colArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(currElement->colArray, &kv); // reserve for timestamp } cnt++; diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index bc0e560178..a25ef4ec95 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -84,8 +84,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ while (sql < sqlEnd) { if (unlikely(*sql == SPACE)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(*sql == EQUAL)) { keyLen = sql - key; @@ -97,8 +96,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ if (unlikely(IS_INVALID_COL_LEN(keyLen))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); - terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - return -1; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } // parse value @@ -111,8 +109,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ } if (unlikely(*sql == EQUAL)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } sql++; } @@ -120,13 +117,11 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ if (unlikely(valueLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); - terrno = TSDB_CODE_TSC_INVALID_VALUE; - return -1; + return TSDB_CODE_TSC_INVALID_VALUE; } if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - return -1; + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } SSmlKv kv = {.key = key, @@ -136,14 +131,15 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ .length = valueLen, .keyEscaped = false, .valueEscaped = false}; - taosArrayPush(preLineKV, &kv); + if (taosArrayPush(preLineKV, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - terrno = TSDB_CODE_SUCCESS; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } cnt++; } - return 0; + return TSDB_CODE_SUCCESS; } static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) { @@ -156,13 +152,19 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS if(info->dataFormat){ ret = smlProcessSuperTable(info, elements); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } } ret = smlProcessTagTelnet(info, data, sqlEnd); if(ret != 0){ - return terrno; + if (info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } ret = smlJoinMeasureTag(elements); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 3c6ef00bf4..c363686343 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -733,13 +733,17 @@ end: } static void generateTimedTask(int64_t refId, int32_t type) { - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + tmq_t* tmq; + int8_t* pTaskType; + int32_t code; + + tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) return; - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); - if (pTaskType != NULL){ + code = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0, (void**)&pTaskType); + if (code == TSDB_CODE_SUCCESS) { *pTaskType = type; - if (taosWriteQitem(tmq->delayedTask, pTaskType) == 0){ + if (taosWriteQitem(tmq->delayedTask, pTaskType) == 0) { tsem2_post(&tmq->rspSem); } } @@ -877,7 +881,7 @@ void tmqSendHbReq(void* param, void* tmrId) { OVER: tDestroySMqHbReq(&req); - if(tmrId != NULL){ + if (tmrId != NULL) { taosTmrReset(tmqSendHbReq, DEFAULT_HEARTBEAT_INTERVAL, param, tmqMgmt.timer, &tmq->hbLiveTimer); } taosReleaseRef(tmqMgmt.rsetId, refId); @@ -890,7 +894,12 @@ static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) { } int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { - STaosQall* qall = taosAllocateQall(); + STaosQall* qall; + int32_t code; + + code = taosAllocateQall(&qall); + if (code) return code; + taosReadAllQitems(pTmq->delayedTask, qall); int32_t numOfItems = taosQallItemSize(qall); @@ -914,7 +923,8 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, pTmq->autoCommitInterval / 1000.0); - taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->commitTimer); + taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, + &pTmq->commitTimer); } else { tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType); } @@ -977,14 +987,14 @@ void tmqClearUnhandleMsg(tmq_t* tmq) { } int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { - if(param == NULL) { + if (param == NULL) { return code; } SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; pParam->rspErr = code; - if(pMsg){ + if (pMsg) { taosMemoryFree(pMsg->pEpSet); } tsem_post(&pParam->rspSem); @@ -1057,13 +1067,13 @@ void tmqFreeImpl(void* handle) { taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl); taos_close_internal(tmq->pTscObj); - if(tmq->commitTimer) { + if (tmq->commitTimer) { taosTmrStopA(&tmq->commitTimer); } - if(tmq->epTimer) { + if (tmq->epTimer) { taosTmrStopA(&tmq->epTimer); } - if(tmq->hbLiveTimer) { + if (tmq->hbLiveTimer) { taosTmrStopA(&tmq->hbLiveTimer); } taosMemoryFree(tmq); @@ -1101,6 +1111,8 @@ void tmqMgmtClose(void) { if (errstr != NULL) snprintf(errstr, errstrLen, MSG); tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { + int32_t code; + if (conf == NULL) { SET_ERROR_MSG_TMQ("configure is null") return NULL; @@ -1124,12 +1136,31 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass; pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); - pTmq->mqueue = taosOpenQueue(); - pTmq->delayedTask = taosOpenQueue(); - pTmq->qall = taosAllocateQall(); + code = taosOpenQueue(&pTmq->mqueue); + if (code) { + terrno = code; + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + SET_ERROR_MSG_TMQ("open queue failed") + goto _failed; + } - if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL || - conf->groupId[0] == 0) { + code = taosOpenQueue(&pTmq->delayedTask); + if (code) { + terrno = code; + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + SET_ERROR_MSG_TMQ("open delayed task queue failed") + goto _failed; + } + + code = taosAllocateQall(&pTmq->qall); + if (code) { + terrno = code; + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + SET_ERROR_MSG_TMQ("allocate qall failed") + goto _failed; + } + + if (conf->groupId[0] == 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty") @@ -1315,7 +1346,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { // init ep timer tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer); // init auto commit timer - tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer); + tmq->commitTimer = + taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer); FAIL: taosArrayDestroyP(req.topicNames, taosMemoryFree); @@ -1372,20 +1404,20 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (pParam == NULL || pMsg == NULL) { goto FAIL; } - int64_t refId = pParam->refId; - int32_t vgId = pParam->vgId; - uint64_t requestId = pParam->requestId; + int64_t refId = pParam->refId; + int32_t vgId = pParam->vgId; + uint64_t requestId = pParam->requestId; tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { code = TSDB_CODE_TMQ_CONSUMER_CLOSED; goto FAIL; } - SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0); - if (pRspWrapper == NULL) { + SMqPollRspWrapper* pRspWrapper; + code = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0, (void**)&pRspWrapper); + if (code) { tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d, since out of memory", tmq->consumerId, vgId); taosReleaseRef(tmqMgmt.rsetId, refId); - code = TSDB_CODE_OUT_OF_MEMORY; goto FAIL; } @@ -2575,7 +2607,7 @@ end: } int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { - if(param == NULL) return code; + if (param == NULL) return code; SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); if (tmq == NULL) { @@ -2600,9 +2632,9 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { doUpdateLocalEp(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } else { - SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0); - if (pWrapper == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + SMqAskEpRspWrapper* pWrapper; + code = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0, (void**)&pWrapper); + if (code) { goto END; } @@ -2620,13 +2652,13 @@ END: FAIL: if (pParam->sync) { SAskEpInfo* pInfo = pParam->pParam; - if(pInfo) { + if (pInfo) { pInfo->code = code; tsem_post(&pInfo->sem); } } - if(pMsg){ + if (pMsg) { taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pData); } @@ -2636,11 +2668,11 @@ FAIL: int32_t syncAskEp(tmq_t* pTmq) { SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); - if(pInfo == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pInfo == NULL) return TSDB_CODE_OUT_OF_MEMORY; tsem_init(&pInfo->sem, 0, 0); int32_t code = askEp(pTmq, pInfo, true, false); - if(code == 0){ + if (code == 0) { tsem_wait(&pInfo->sem); code = pInfo->code; } @@ -2778,7 +2810,7 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { } static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { - if(param == NULL) { + if (param == NULL) { return code; } SMqVgWalInfoParam* pParam = param; @@ -2812,7 +2844,7 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { tsem_post(&pCommon->rsp); } - if(pMsg){ + if (pMsg) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); } @@ -3248,7 +3280,7 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); } - if(param == NULL) { + if (param == NULL) { return code; } SMqSeekParam* pParam = param; diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 054698fff0..77079ed863 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -40,13 +40,16 @@ TEST(testCase, smlParseInfluxString_Test) { msgBuf.len = 256; SSmlLineInfo elements = {0}; - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, 0); info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; // case 1 char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *sql = (char *)taosMemoryCalloc(256, 1); - memcpy(sql, tmp, strlen(tmp) + 1); + ASSERT_NE(sql, nullptr); + (void)memcpy(sql, tmp, strlen(tmp) + 1); int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); @@ -62,22 +65,22 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 2 false tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 4 tag is null tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); @@ -92,13 +95,13 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 5 tag is null tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql + 1); @@ -112,36 +115,36 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 6 tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; - smlClearForRerun(info); + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; + ASSERT_EQ(smlClearForRerun(info), 0); // case 7 tmp = " st , "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 8 false tmp = ", st , "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; taosMemoryFree(sql); smlDestroyInfo(info); @@ -228,7 +231,9 @@ TEST(testCase, smlParseCols_Error_Test) { "escape_test,tag1=\"tag1_value\",tag2=\"tag2_value\" col0=\"co\"l\"0_value\",col1=\"col1_value\" 1680918783010000000" }; - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) { @@ -238,13 +243,14 @@ TEST(testCase, smlParseCols_Error_Test) { msgBuf.len = 256; int32_t len = strlen(data[i]); char *sql = (char *)taosMemoryCalloc(256, 1); - memcpy(sql, data[i], len + 1); + ASSERT_NE(sql, nullptr); + (void)memcpy(sql, data[i], len + 1); SSmlLineInfo elements = {0}; int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); // printf("i:%d\n", i); ASSERT_NE(ret, TSDB_CODE_SUCCESS); taosMemoryFree(sql); - taosArrayDestroy(elements.colArray); + (void)taosArrayDestroy(elements.colArray); } smlDestroyInfo(info); } @@ -254,7 +260,9 @@ TEST(testCase, smlParseCols_Test) { SSmlMsgBuf msgBuf; msgBuf.buf = msg; msgBuf.len = 256; - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; SSmlLineInfo elements = {0}; @@ -267,7 +275,8 @@ TEST(testCase, smlParseCols_Test) { "cboolf=f,cnch_=l\"iuwq\" 1626006833639000000"; int32_t len = strlen(data); char *sql = (char *)taosMemoryCalloc(1024, 1); - memcpy(sql, data, len + 1); + ASSERT_NE(sql, nullptr); + (void)memcpy(sql, data, len + 1); int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); int32_t size = taosArrayGetSize(elements.colArray); @@ -275,6 +284,7 @@ TEST(testCase, smlParseCols_Test) { // binary SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); @@ -285,6 +295,7 @@ TEST(testCase, smlParseCols_Test) { // nchar kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); @@ -293,6 +304,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -301,6 +313,7 @@ TEST(testCase, smlParseCols_Test) { // double kv = (SSmlKv *)taosArrayGet(elements.colArray, 4); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); @@ -310,6 +323,7 @@ TEST(testCase, smlParseCols_Test) { // float kv = (SSmlKv *)taosArrayGet(elements.colArray, 5); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); @@ -319,6 +333,7 @@ TEST(testCase, smlParseCols_Test) { // float kv = (SSmlKv *)taosArrayGet(elements.colArray, 6); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); @@ -328,6 +343,7 @@ TEST(testCase, smlParseCols_Test) { // tiny int kv = (SSmlKv *)taosArrayGet(elements.colArray, 7); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); @@ -336,6 +352,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned tiny int kv = (SSmlKv *)taosArrayGet(elements.colArray, 8); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); @@ -344,6 +361,7 @@ TEST(testCase, smlParseCols_Test) { // small int kv = (SSmlKv *)taosArrayGet(elements.colArray, 9); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); @@ -352,6 +370,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned smallint kv = (SSmlKv *)taosArrayGet(elements.colArray, 10); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); @@ -360,6 +379,7 @@ TEST(testCase, smlParseCols_Test) { // int kv = (SSmlKv *)taosArrayGet(elements.colArray, 11); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); @@ -368,6 +388,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned int kv = (SSmlKv *)taosArrayGet(elements.colArray, 12); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); @@ -376,6 +397,7 @@ TEST(testCase, smlParseCols_Test) { // bigint kv = (SSmlKv *)taosArrayGet(elements.colArray, 13); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); @@ -384,6 +406,7 @@ TEST(testCase, smlParseCols_Test) { // bigint kv = (SSmlKv *)taosArrayGet(elements.colArray, 14); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); ASSERT_EQ(kv->keyLen, 2); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); @@ -392,6 +415,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned bigint kv = (SSmlKv *)taosArrayGet(elements.colArray, 15); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); @@ -400,6 +424,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 16); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); ASSERT_EQ(kv->keyLen, 9); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -408,6 +433,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 17); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -416,6 +442,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 18); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -424,13 +451,14 @@ TEST(testCase, smlParseCols_Test) { // nchar kv = (SSmlKv *)taosArrayGet(elements.colArray, 19); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 4); ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); - taosArrayDestroy(elements.colArray); + (void)taosArrayDestroy(elements.colArray); taosMemoryFree(sql); smlDestroyInfo(info); } @@ -471,7 +499,9 @@ TEST(testCase, smlParseNumber_Test) { } TEST(testCase, smlParseTelnetLine_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->dataFormat = false; info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); @@ -509,7 +539,9 @@ TEST(testCase, smlParseTelnetLine_error_Test) { } TEST(testCase, smlParseTelnetLine_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->dataFormat = false; info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); @@ -528,7 +560,9 @@ TEST(testCase, smlParseTelnetLine_Test) { } //TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { -// SSmlHandle *info = smlBuildSmlInfo(NULL); +// SSmlHandle *info = nullptr; +// int32_t code = smlBuildSmlInfo(nullptr, &info); +// ASSERT_EQ(code, TSDB_CODE_SUCCESS); // info->protocol = TSDB_SML_JSON_PROTOCOL; // ASSERT_NE(info, nullptr); // @@ -568,7 +602,7 @@ TEST(testCase, smlParseNumber_performance_Test) { for (int j = 0; j < 10000000; ++j) { kv.value = str[i]; kv.length = strlen(str[i]); - smlParseNumber(&kv, &msgBuf); + (void)smlParseNumber(&kv, &msgBuf); //ignore result } printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1); printf("\n"); @@ -576,7 +610,7 @@ TEST(testCase, smlParseNumber_performance_Test) { for (int j = 0; j < 10000000; ++j) { kv.value = str[i]; kv.length = strlen(str[i]); - smlParseNumberOld(&kv, &msgBuf); + (void)smlParseNumberOld(&kv, &msgBuf); //ignore result } printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); printf("\n\n"); diff --git a/source/common/src/cos.c b/source/common/src/cos.c index fe6e20c47f..2d3eb3f92b 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -3,6 +3,7 @@ #include "cos.h" #include "cos_cp.h" #include "tdef.h" +#include "tutil.h" extern char tsS3Endpoint[]; extern char tsS3AccessKeyId[]; @@ -40,7 +41,7 @@ int32_t s3Begin() { if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) { uError("Failed to initialize libs3: %s\n", S3_get_status_name(status)); - return -1; + TAOS_RETURN(TSDB_CODE_FAILED); } protocolG = !tsS3Https; @@ -48,12 +49,12 @@ int32_t s3Begin() { uriStyleG = S3UriStyleVirtualHost; } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } void s3End() { S3_deinitialize(); } -int32_t s3Init() { return 0; /*s3Begin();*/ } +int32_t s3Init() { TAOS_RETURN(TSDB_CODE_SUCCESS); /*s3Begin();*/ } void s3CleanUp() { /*s3End();*/ } @@ -61,7 +62,7 @@ void s3CleanUp() { /*s3End();*/ static int32_t s3ListBucket(char const *bucketname); int32_t s3CheckCfg() { - int32_t code = 0; + int32_t code = 0, lino = 0; if (!tsS3Enabled) { fprintf(stderr, "s3 not configured.\n"); @@ -71,7 +72,7 @@ int32_t s3CheckCfg() { code = s3Begin(); if (code != 0) { fprintf(stderr, "failed to initialize s3.\n"); - goto _exit; + TAOS_CHECK_GOTO(code, &lino, _exit); } // test put @@ -91,20 +92,17 @@ int32_t s3CheckCfg() { TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); if (!fp) { - code = TAOS_SYSTEM_ERROR(errno); fprintf(stderr, "failed to open test file: %s.\n", path); // uError("ERROR: %s Failed to open %s", __func__, path); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) { - code = TAOS_SYSTEM_ERROR(errno); fprintf(stderr, "failed to write test file: %s.\n", path); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } if (taosFsyncFile(fp) < 0) { - code = TAOS_SYSTEM_ERROR(errno); fprintf(stderr, "failed to fsync test file: %s.\n", path); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } taosCloseFile(&fp); @@ -112,7 +110,7 @@ int32_t s3CheckCfg() { code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16); if (code != 0) { fprintf(stderr, "put object %s : failed.\n", objectname[0]); - goto _exit; + TAOS_CHECK_GOTO(code, &lino, _exit); } fprintf(stderr, "put object %s: success.\n\n", objectname[0]); @@ -121,7 +119,7 @@ int32_t s3CheckCfg() { code = s3ListBucket(tsS3BucketName); if (code != 0) { fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName); - goto _exit; + TAOS_CHECK_GOTO(code, &lino, _exit); } fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName); @@ -134,7 +132,7 @@ int32_t s3CheckCfg() { code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock); if (code != 0) { fprintf(stderr, "get object %s : failed.\n", objectname[0]); - goto _exit; + TAOS_CHECK_GOTO(code, &lino, _exit); } char buf[7] = {0}; memcpy(buf, pBlock, c_len); @@ -147,14 +145,14 @@ int32_t s3CheckCfg() { code = s3DeleteObjects(objectname, 1); if (code != 0) { fprintf(stderr, "delete object %s : failed.\n", objectname[0]); - goto _exit; + TAOS_CHECK_GOTO(code, &lino, _exit); } fprintf(stderr, "delete object %s: success.\n\n", objectname[0]); s3End(); _exit: - return code; + TAOS_RETURN(code); } static int should_retry() { @@ -169,7 +167,7 @@ static int should_retry() { } */ - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status, @@ -202,7 +200,6 @@ static S3Status responsePropertiesCallbackNull(const S3ResponseProperties *prope } static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { - //(void)callbackData; TS3SizeCBD *cbd = callbackData; if (properties->contentLength > 0) { cbd->content_length = properties->contentLength; @@ -249,7 +246,7 @@ static int32_t s3ListBucket(char const *bucketname) { SArray *objectArray = getListByPrefix("s3"); if (objectArray == NULL) { - return -1; + TAOS_RETURN(TSDB_CODE_FAILED); } const char **object_name = TARRAY_DATA(objectArray); @@ -262,7 +259,7 @@ static int32_t s3ListBucket(char const *bucketname) { taosArrayDestroyEx(objectArray, s3FreeObjectKey); - return code; + TAOS_RETURN(code); } typedef struct growbuffer { @@ -607,13 +604,13 @@ static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char c code = TAOS_SYSTEM_ERROR(EIO); } - return code; + TAOS_RETURN(code); } static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name, int64_t contentLength, S3PutProperties *put_prop, put_object_callback_data *data) { - int32_t code = 0; + int32_t code = 0, lino = 0; uint64_t totalContentLength = contentLength; uint64_t todoContentLength = contentLength; UploadManager manager = {0}; @@ -640,6 +637,9 @@ static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, cha {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *)); + if (!manager.etags) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } manager.next_etags_pos = 0; do { S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager); @@ -647,8 +647,7 @@ static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, cha if (manager.upload_id == 0 || manager.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } upload: @@ -672,8 +671,7 @@ upload: } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry()); if (partData.put_object_data.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg); - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } contentLength -= chunk_size; todoContentLength -= chunk_size; @@ -686,8 +684,7 @@ upload: int n; for (i = 0; i < totalSeq; i++) { if (!manager.etags[i]) { - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } n = snprintf(buf, sizeof(buf), "%d" @@ -704,11 +701,13 @@ upload: } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } -clean: +_exit: + if (code) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } if (manager.upload_id) { taosMemoryFree(manager.upload_id); } @@ -718,13 +717,13 @@ clean: growbuffer_destroy(manager.gb); taosMemoryFree(manager.etags); - return code; + TAOS_RETURN(code); } static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime, char const *object_name, int64_t contentLength, S3PutProperties *put_prop, put_object_callback_data *data) { - int32_t code = 0; + int32_t code = 0, lino = 0; uint64_t totalContentLength = contentLength; // uint64_t todoContentLength = contentLength; @@ -744,13 +743,16 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const SCheckpoint cp = {0}; cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart)); + if (!cp.parts) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } if (taosCheckExistFile(file_cp_path)) { if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) { manager.upload_id = strdup(cp.upload_id); need_init_upload = false; } else { - cos_cp_remove(file_cp_path); + TAOS_CHECK_GOTO(cos_cp_remove(file_cp_path), &lino, _exit); } } @@ -763,16 +765,14 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const if (manager.upload_id == 0 || manager.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size); } if (cos_cp_open(file_cp_path, &cp)) { - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } int part_num = 0; @@ -791,6 +791,10 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *)); + if (!manager.etags) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } + manager.next_etags_pos = 0; upload: @@ -802,8 +806,7 @@ upload: if (i > 0 && cp.parts[i - 1].completed) { if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } } @@ -827,25 +830,23 @@ upload: } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry()); if (partData.put_object_data.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg); - code = TAOS_SYSTEM_ERROR(EIO); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); //(void)cos_cp_dump(&cp); - goto clean; } if (!manager.etags[seq - 1]) { - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0); - (void)cos_cp_dump(&cp); + TAOS_CHECK_GOTO(cos_cp_dump(&cp), &lino, _exit); contentLength -= chunk_size; // todoContentLength -= chunk_size; } - cos_cp_close(cp.thefile); + TAOS_CHECK_GOTO(cos_cp_close(cp.thefile), &lino, _exit); cp.thefile = 0; int size = 0; @@ -869,20 +870,23 @@ upload: } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } - cos_cp_remove(file_cp_path); + TAOS_CHECK_GOTO(cos_cp_remove(file_cp_path), &lino, _exit); -clean: +_exit: /* if (parts) { taosMemoryFree(parts); } */ + if (code) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (cp.thefile) { - cos_cp_close(cp.thefile); + (void)cos_cp_close(cp.thefile); } if (cp.parts) { taosMemoryFree(cp.parts); @@ -899,7 +903,7 @@ clean: taosMemoryFree(manager.etags); growbuffer_destroy(manager.gb); - return code; + TAOS_RETURN(code); } int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) { @@ -917,15 +921,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w put_object_callback_data data = {0}; if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); uError("ERROR: %s Failed to stat file %s: ", __func__, file); - return code; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { - code = TAOS_SYSTEM_ERROR(errno); uError("ERROR: %s Failed to open file %s: ", __func__, file); - return code; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = @@ -956,7 +958,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w growbuffer_destroy(data.gb); } - return code; + TAOS_RETURN(code); } int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { @@ -974,22 +976,19 @@ int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int put_object_callback_data data = {0}; if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); uError("ERROR: %s Failed to stat file %s: ", __func__, file); - return code; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } contentLength = size; if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { - code = TAOS_SYSTEM_ERROR(errno); uError("ERROR: %s Failed to open file %s: ", __func__, file); - return code; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) { taosCloseFile(&data.infileFD); - code = TAOS_SYSTEM_ERROR(errno); - return code; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = @@ -1016,7 +1015,7 @@ int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int growbuffer_destroy(data.gb); } - return code; + TAOS_RETURN(code); } typedef struct list_bucket_callback_data { @@ -1130,11 +1129,11 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); - code = -1; + code = TSDB_CODE_FAILED; } } - return code; + TAOS_RETURN(code); } void s3DeleteObjectsByPrefix(const char *prefix) { @@ -1187,17 +1186,19 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, if (cbd.status != S3StatusOK) { uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg); - return TAOS_SYSTEM_ERROR(EIO); + + TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO)); } if (check && cbd.buf_pos != size) { uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg); - return TAOS_SYSTEM_ERROR(EIO); + + TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO)); } *ppBlock = cbd.buf; - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static S3Status getObjectCallback(int bufferSize, const char *buffer, void *callbackData) { @@ -1218,9 +1219,8 @@ int32_t s3GetObjectToFile(const char *object_name, const char *fileName) { TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("[s3] open file error, errno:%d, fileName:%s", terrno, fileName); - return -1; + uError("[s3] open file error, errno:%d, fileName:%s", TAOS_SYSTEM_ERROR(errno), fileName); + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } TS3GetData cbd = {0}; @@ -1232,16 +1232,17 @@ int32_t s3GetObjectToFile(const char *object_name, const char *fileName) { if (cbd.status != S3StatusOK) { uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); taosCloseFile(&pFile); - return TAOS_SYSTEM_ERROR(EIO); + TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO)); } taosCloseFile(&pFile); - return 0; + + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { SArray *objectArray = getListByPrefix(prefix); - if (objectArray == NULL) return -1; + if (objectArray == NULL) TAOS_RETURN(TSDB_CODE_FAILED); for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) { char *object = taosArrayGetP(objectArray, i); @@ -1255,7 +1256,7 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { } if (s3GetObjectToFile(object, fileName) != 0) { taosArrayDestroyEx(objectArray, s3FreeObjectKey); - return -1; + TAOS_RETURN(TSDB_CODE_FAILED); } } taosArrayDestroyEx(objectArray, s3FreeObjectKey); @@ -1279,7 +1280,7 @@ long s3Size(const char *object_name) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); - return -1; + TAOS_RETURN(TSDB_CODE_FAILED); } size = cbd.content_length; @@ -1306,7 +1307,7 @@ int32_t s3Init() { // set log output, default stderr cos_log_set_output(NULL); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } void s3CleanUp() { cos_http_io_deinitialize(); } @@ -1363,10 +1364,10 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) { cos_pool_destroy(p); if (s->code != 200) { - return code = s->code; + TAOS_RETURN(s->code); } - return code; + TAOS_RETURN(code); } int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) { @@ -1405,10 +1406,10 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_ cos_pool_destroy(p); if (s->code != 200) { - return code = s->code; + TAOS_RETURN(s->code); } - return code; + TAOS_RETURN(code); } void s3DeleteObjectsByPrefix(const char *prefix_str) { @@ -1465,7 +1466,7 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) { cos_warn_log("delete objects failed\n"); } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } bool s3Exists(const char *object_name) { @@ -1547,7 +1548,7 @@ bool s3Get(const char *object_name, const char *path) { int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) { (void)check; - int32_t code = 0; + int32_t code = 0, lino = 0; cos_pool_t *p = NULL; int is_cname = 0; cos_status_t *s = NULL; @@ -1588,10 +1589,9 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_ s = cos_get_object_to_buffer(options, &bucket, &object, headers, NULL, &download_buffer, &resp_headers); log_status(s); if (!cos_status_is_ok(s)) { - vError("s3: %d(%s)", s->code, s->error_msg); - vError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); - code = TAOS_SYSTEM_ERROR(EIO); - return code; + uError("s3: %d(%s)", s->code, s->error_msg); + uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO)); } // print_headers(resp_headers); @@ -1601,6 +1601,10 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_ cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) { len += cos_buf_size(content); } // char *buf = cos_pcalloc(p, (apr_size_t)(len + 1)); char *buf = taosMemoryCalloc(1, (apr_size_t)(len)); + if (!buf) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } + // buf[len] = '\0'; cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) { size = cos_buf_size(content); @@ -1609,12 +1613,13 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_ } // cos_warn_log("Download data=%s", buf); +_exit: //销毁内存池 cos_pool_destroy(p); *ppBlock = buf; - return code; + TAOS_RETURN(code); } typedef struct { diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index 509f0a3a1c..d7be47ce45 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -9,42 +9,35 @@ int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) { TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH); if (!fd) { - code = TAOS_SYSTEM_ERROR(errno); - uError("ERROR: %s Failed to open %s", __func__, cp_path); - return code; + uError("%s Failed to open %s", __func__, cp_path); + TAOS_CHECK_RETURN(TAOS_SYSTEM_ERROR(errno)); } checkpoint->thefile = fd; - return code; + TAOS_RETURN(code); } -void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); } -void cos_cp_remove(char const* filepath) { (void)taosRemoveFile(filepath); } +int32_t cos_cp_close(TdFilePtr fd) { return taosCloseFile(&fd); } +int32_t cos_cp_remove(char const* filepath) { return taosRemoveFile(filepath); } static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) { - int32_t code = 0; + int32_t code = 0, lino = 0; cJSON const* item2 = NULL; cJSON* json = cJSON_Parse(cp_body); if (NULL == json) { - code = TSDB_CODE_FILE_CORRUPTED; - uError("ERROR: %s Failed to parse json", __func__); - goto _exit; + TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit); } cJSON const* item = cJSON_GetObjectItem(json, "ver"); if (!cJSON_IsNumber(item) || item->valuedouble != 1) { - code = TSDB_CODE_FILE_CORRUPTED; - uError("ERROR: %s Failed to parse json ver: %f", __func__, item->valuedouble); - goto _exit; + TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit); } item = cJSON_GetObjectItem(json, "type"); if (!cJSON_IsNumber(item)) { - code = TSDB_CODE_FILE_CORRUPTED; - uError("ERROR: %s Failed to parse json", __func__); - goto _exit; + TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit); } cp->cp_type = item->valuedouble; @@ -155,40 +148,36 @@ static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) { } _exit: + if (code) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } if (json) cJSON_Delete(json); if (cp_body) taosMemoryFree(cp_body); - - return code; + TAOS_RETURN(code); } int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) { - int32_t code = 0; + int32_t code = 0, lino = 0; + char* cp_body = NULL; TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ); if (!fd) { - code = TAOS_SYSTEM_ERROR(errno); - uError("ERROR: %s Failed to open %s", __func__, filepath); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } int64_t size = -1; - code = taosStatFile(filepath, &size, NULL, NULL); - if (code) { - uError("ERROR: %s Failed to stat %s", __func__, filepath); - goto _exit; - } + TAOS_CHECK_GOTO(taosStatFile(filepath, &size, NULL, NULL), &lino, _exit); - char* cp_body = taosMemoryMalloc(size + 1); + cp_body = taosMemoryMalloc(size + 1); + if (!cp_body) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } int64_t n = taosReadFile(fd, cp_body, size); if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - uError("ERROR: %s Failed to read %s", __func__, filepath); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } else if (n != size) { - code = TSDB_CODE_FILE_CORRUPTED; - uError("ERROR: %s Failed to read %s %" PRId64 "/%" PRId64, __func__, filepath, n, size); - goto _exit; + TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit); } taosCloseFile(&fd); cp_body[size] = '\0'; @@ -196,184 +185,161 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) { return cos_cp_parse_body(cp_body, checkpoint); _exit: + if (code) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } if (fd) { taosCloseFile(&fd); } if (cp_body) { taosMemoryFree(cp_body); } - - return code; + TAOS_RETURN(code); } static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) { - int32_t code = 0; + int32_t code = 0, lino = 0; char* data = cJSON_PrintUnformatted(json); - if (NULL == data) { - return TSDB_CODE_OUT_OF_MEMORY; + if (!data) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } TdFilePtr fp = checkpoint->thefile; if (taosFtruncateFile(fp, 0) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } if (taosLSeekFile(fp, 0, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } if (taosWriteFile(fp, data, strlen(data)) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } if (taosFsyncFile(fp) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } _exit: + if (code) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } taosMemoryFree(data); - return code; + TAOS_RETURN(code); } int32_t cos_cp_dump(SCheckpoint* cp) { - int32_t code = 0; - int32_t lino = 0; + int32_t code = 0, lino = 0; cJSON* ojson = NULL; cJSON* json = cJSON_CreateObject(); - if (!json) return TSDB_CODE_OUT_OF_MEMORY; + if (!json) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (COS_CP_TYPE_UPLOAD == cp->cp_type) { ojson = cJSON_AddObjectToObject(json, "file"); if (!ojson) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } } else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) { ojson = cJSON_AddObjectToObject(json, "object"); if (!ojson) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } } ojson = cJSON_AddObjectToObject(json, "cpparts"); if (!ojson) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts"); if (!ajson) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } for (int i = 0; i < cp->part_num; ++i) { cJSON* item = cJSON_CreateObject(); if (!item) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } cJSON_AddItemToArray(ajson, item); if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } } code = cos_cp_save_json(json, cp); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(code, &lino, _exit); _exit: if (code) { uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } cJSON_Delete(json); - return code; + TAOS_RETURN(code); } void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {} diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 142e75dbcb..965246326a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2492,7 +2492,10 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha RandTableName rname = { .tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname}; - buildChildTableName(&rname); + int32_t code = buildChildTableName(&rname); + if(code != TSDB_CODE_SUCCESS){ + return code; + } taosArrayDestroy(tags); if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 119438fbd6..7538b504e3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -504,7 +504,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index d2b52b201e..7b349e91b0 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -15,9 +15,9 @@ #define _DEFAULT_SOURCE #include "tmisce.h" +#include "tdatablock.h" #include "tglobal.h" #include "tjson.h" -#include "tdatablock.h" int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { pEp->port = 0; @@ -34,18 +34,26 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { pEp->port = tsServerPort; } + if (pEp->port <= 0) { + return TSDB_CODE_INVALID_PARA; + } + return 0; } -void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) { +int32_t addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) { if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) { - return; + return TSDB_CODE_INVALID_PARA; } int32_t index = pEpSet->numOfEps; + if (index >= sizeof(pEpSet->eps) / sizeof(pEpSet->eps[0])) { + return TSDB_CODE_OUT_OF_RANGE; + } tstrncpy(pEpSet->eps[index].fqdn, fqdn, tListLen(pEpSet->eps[index].fqdn)); pEpSet->eps[index].port = port; pEpSet->numOfEps += 1; + return 0; } bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) { @@ -131,75 +139,96 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet) { return ep; } -int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t bufLen) { - int len = snprintf(pBuf, bufLen, "epset:{"); - if (len < 0) { - return -1; - } +int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) { + int32_t ret = 0; + int32_t nwrite = 0; - for (int _i = 0; (_i < pEpSet->numOfEps) && (bufLen > len); _i++) { + nwrite = snprintf(pBuf + nwrite, cap, "epset:{"); + if (nwrite <= 0 || nwrite >= cap) { + return TSDB_CODE_OUT_OF_BUFFER; + } + cap -= nwrite; + + for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) { int32_t ret = 0; if (_i == pEpSet->numOfEps - 1) { - ret = snprintf(pBuf + len, bufLen - len, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); + ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); } else { - ret = snprintf(pBuf + len, bufLen - len, "%d. %s:%d, ", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); + ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d, ", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); } - if (ret < 0) { - return -1; + if (ret <= 0 || ret >= cap) { + return TSDB_CODE_OUT_OF_BUFFER; } - len += ret; + nwrite += ret; + cap -= ret; } - if (len < bufLen) { - /*len += */snprintf(pBuf + len, bufLen - len, "}, inUse:%d", pEpSet->inUse); + if (cap <= 0) { + return TSDB_CODE_OUT_OF_BUFFER; } + ret = snprintf(pBuf + nwrite, cap, "}, inUse:%d", pEpSet->inUse); + if (ret <= 0 || ret >= cap) { + return TSDB_CODE_OUT_OF_BUFFER; + } return TSDB_CODE_SUCCESS; } int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) { - SJson* pJson = tjsonCreateObject(); - if (pJson == NULL) return -1; + int32_t code = 0; + SJson* pJson = tjsonCreateObject(); + if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; + char tmp[4096] = {0}; - tjsonAddDoubleToObject(pJson, "reportVersion", 1); + TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "reportVersion", 1), NULL, _exit); - tjsonAddIntegerToObject(pJson, "clusterId", clusterId); - tjsonAddIntegerToObject(pJson, "startTime", startTime); + TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "clusterId", clusterId), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "startTime", startTime), NULL, _exit); // Do NOT invoke the taosGetFqdn here. // this function may be invoked when memory exception occurs,so we should assume that it is running in a memory locked // environment. The lock operation by taosGetFqdn may cause this program deadlock. - tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn), NULL, _exit); - tjsonAddIntegerToObject(pJson, "pid", taosGetPId()); + TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "pid", taosGetPId()), NULL, _exit); - taosGetAppName(tmp, NULL); - tjsonAddStringToObject(pJson, "appName", tmp); + code = taosGetAppName(tmp, NULL); + if (code != 0) { + code = TAOS_SYSTEM_ERROR(errno); + TAOS_CHECK_GOTO(code, NULL, _exit); + } + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "appName", tmp), NULL, _exit); if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) { - tjsonAddStringToObject(pJson, "os", tmp); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "os", tmp), NULL, _exit); + } else { + // do nothing } float numOfCores = 0; if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) { - tjsonAddStringToObject(pJson, "cpuModel", tmp); - tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "cpuModel", tmp), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores), NULL, _exit); } else { - tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores); + TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores), NULL, _exit); } - snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); - tjsonAddStringToObject(pJson, "memory", tmp); + int32_t nBytes = snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); + if (nBytes <= 9 || nBytes >= sizeof(tmp)) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_RANGE, NULL, _exit); + } + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "memory", tmp), NULL, _exit); - tjsonAddStringToObject(pJson, "version", version); - tjsonAddStringToObject(pJson, "buildInfo", buildinfo); - tjsonAddStringToObject(pJson, "gitInfo", gitinfo); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "version", version), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "buildInfo", buildinfo), NULL, _exit); - tjsonAddIntegerToObject(pJson, "crashSig", signum); - tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "gitInfo", gitinfo), NULL, _exit); + + TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashSig", signum), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()), NULL, _exit); #ifdef _TD_DARWIN_64 taosLogTraceToBuf(tmp, sizeof(tmp), 4); @@ -209,26 +238,47 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t taosLogTraceToBuf(tmp, sizeof(tmp), 8); #endif - tjsonAddStringToObject(pJson, "stackInfo", tmp); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "stackInfo", tmp), NULL, _exit); char* pCont = tjsonToString(pJson); + if (pCont == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_CHECK_GOTO(code, NULL, _exit); + goto _exit; + } + tjsonDelete(pJson); - *pMsg = pCont; - - return TSDB_CODE_SUCCESS; + pJson = NULL; +_exit: + tjsonDelete(pJson); + TAOS_RETURN(code); } int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { - SConfig* pConf = taosGetCfg(); + int32_t code = 0; + SConfig* pConf = taosGetCfg(); + if (pConf == NULL) { + return TSDB_CODE_INVALID_CFG; + } + int32_t numOfRows = 0; int32_t col = startCol; SConfigItem* pItem = NULL; + SConfigIter* pIter = NULL; - blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)); - SConfigIter* pIter = cfgCreateIter(pConf); + int8_t locked = 0; + + TAOS_CHECK_GOTO(blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)), NULL, _exit); + + pIter = cfgCreateIter(pConf); + if (pIter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_CHECK_GOTO(code, NULL, _exit); + } cfgLock(pConf); + locked = 1; while ((pItem = cfgNextIter(pIter)) != NULL) { col = startCol; @@ -236,29 +286,44 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { // GRANT_CFG_SKIP; char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, col++); - colDataSetVal(pColInfo, numOfRows, name, false); + if (pColInfo == NULL) { + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_CHECK_GOTO(code, NULL, _exit); + } + + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, name, false), NULL, _exit); char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t valueLen = 0; cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); varDataSetLen(value, valueLen); + pColInfo = taosArrayGet(pBlock->pDataBlock, col++); - colDataSetVal(pColInfo, numOfRows, value, false); + if (pColInfo == NULL) { + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_CHECK_GOTO(code, NULL, _exit); + } + + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, value, false), NULL, _exit); char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen); varDataSetLen(scope, valueLen); + pColInfo = taosArrayGet(pBlock->pDataBlock, col++); - colDataSetVal(pColInfo, numOfRows, scope, false); + if (pColInfo == NULL) { + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_CHECK_GOTO(code, NULL, _exit); + } + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit); numOfRows++; } - - cfgUnLock(pConf); - pBlock->info.rows = numOfRows; - +_exit: + if (locked) cfgUnLock(pConf); cfgDestroyIter(pIter); - return TSDB_CODE_SUCCESS; + TAOS_RETURN(code); } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 4fe2beb6aa..d817fac292 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -98,7 +98,7 @@ SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName int32_t tNameExtractFullName(const SName* name, char* dst) { // invalid full name format, abort if (!tNameIsValid(name)) { - return -1; + return TSDB_CODE_INVALID_PARA; } int32_t len = snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); @@ -154,8 +154,7 @@ int32_t tNameGetDbName(const SName* name, char* dst) { const char* tNameGetDbNameP(const SName* name) { return &name->dbname[0]; } int32_t tNameGetFullDbName(const SName* name, char* dst) { - snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); - return 0; + return snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); } bool tNameIsEmpty(const SName* name) { return name->type == 0 || name->acctId == 0; } @@ -293,17 +292,19 @@ static int compareKv(const void* p1, const void* p2) { /* * use stable name and tags to grearate child table name */ -void buildChildTableName(RandTableName* rName) { +int32_t buildChildTableName(RandTableName* rName) { SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen); if (sb.buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return; + return TSDB_CODE_OUT_OF_MEMORY; } taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { taosStringBuilderAppendChar(&sb, ','); SSmlKv* tagKv = taosArrayGet(rName->tags, j); + if(tagKv == NULL) { + return TSDB_CODE_SML_INVALID_DATA; + } taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendChar(&sb, '='); if (IS_VAR_DATA_TYPE(tagKv->type)) { @@ -323,8 +324,9 @@ void buildChildTableName(RandTableName* rName) { rName->ctbShortName[0] = 't'; rName->ctbShortName[1] = '_'; for (int i = 0; i < 16; i++) { - sprintf(temp, "%02x", context.digest[i]); - strcat(rName->ctbShortName, temp); + (void)sprintf(temp, "%02x", context.digest[i]); + (void)strcat(rName->ctbShortName, temp); } taosStringBuilderDestroy(&sb); + return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index e5c32f9a43..f28de4695c 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -153,6 +153,8 @@ int32_t mmPutMsgToFetchQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { + int32_t code; + SSingleWorker *pWorker = NULL; switch (qtype) { case WRITE_QUEUE: @@ -181,13 +183,15 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { } if (pWorker == NULL) return -1; - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); - if (pMsg == NULL) return -1; + SRpcMsg *pMsg; + code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg); + if (code) return code; memcpy(pMsg, pRpc, sizeof(SRpcMsg)); pRpc->pCont = NULL; - dTrace("msg:%p, is created and will put into %s queue, type:%s len:%d", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType), pRpc->contLen); - int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg); + dTrace("msg:%p, is created and will put into %s queue, type:%s len:%d", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType), + pRpc->contLen); + code = mmPutMsgToWorker(pMgmt, pWorker, pMsg); if (code != 0) { dTrace("msg:%p, is freed", pMsg); rpcFreeCont(pMsg->pCont); diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 5c635ff5ea..b443cbc351 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -58,8 +58,11 @@ int32_t qmPutNodeMsgToFetchQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); - if (pMsg == NULL) return -1; + int32_t code; + SRpcMsg *pMsg; + + code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg); + if (code) return code; memcpy(pMsg, pRpc, sizeof(SRpcMsg)); pRpc->pCont = NULL; diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index 1c57685414..e356380039 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -126,8 +126,11 @@ void smStopWorker(SSnodeMgmt *pMgmt) { } int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); - if (pMsg == NULL) { + int32_t code; + SRpcMsg *pMsg; + + code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg); + if (code) { rpcFreeCont(pRpc->pCont); pRpc->pCont = NULL; return -1; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 0415340be9..b7d35030b8 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -311,6 +311,7 @@ int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { + int32_t code; if (pRpc->contLen < sizeof(SMsgHead)) { dError("invalid rpc msg with no msg head at pCont. pRpc:%p, type:%s, len:%d", pRpc, TMSG_INFO(pRpc->msgType), pRpc->contLen); @@ -320,8 +321,9 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { } EQItype itype = APPLY_QUEUE == qtype ? DEF_QITEM : RPC_QITEM; - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), itype, pRpc->contLen); - if (pMsg == NULL) { + SRpcMsg *pMsg; + code = taosAllocateQitem(sizeof(SRpcMsg), itype, pRpc->contLen, (void **)&pMsg); + if (code) { rpcFreeCont(pRpc->pCont); pRpc->pCont = NULL; return -1; @@ -335,7 +337,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { memcpy(pMsg, pRpc, sizeof(SRpcMsg)); pRpc->pCont = NULL; - int32_t code = vmPutMsgToQueue(pMgmt, pMsg, qtype); + code = vmPutMsgToQueue(pMgmt, pMsg, qtype); if (code != 0) { dTrace("msg:%p, is freed", pMsg); rpcFreeCont(pMsg->pCont); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 99d641ff3f..bf35319fae 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -210,8 +210,8 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { pRpc->info.wrapper = pWrapper; EQItype itype = IsReq(pRpc) ? RPC_QITEM : DEF_QITEM; // rsp msg is not restricted by tsRpcQueueMemoryUsed - pMsg = taosAllocateQitem(sizeof(SRpcMsg), itype, pRpc->contLen); - if (pMsg == NULL) goto _OVER; + code = taosAllocateQitem(sizeof(SRpcMsg), itype, pRpc->contLen, (void **)&pMsg); + if (code) goto _OVER; memcpy(pMsg, pRpc, sizeof(SRpcMsg)); dGTrace("msg:%p, is created, type:%s handle:%p len:%d", pMsg, TMSG_INFO(pRpc->msgType), pMsg->info.handle, diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index a6002d2d65..d2173d431e 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -61,7 +61,7 @@ struct SRSmaQTaskInfoItem { int32_t len; int8_t type; int64_t suid; - void * qTaskInfo; + void *qTaskInfo; }; static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { @@ -185,7 +185,7 @@ int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) { void *pIter = NULL; while ((pIter = taosHashIterate(pStore->uidHash, pIter))) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - SArray * pTbUids = *(SArray **)pIter; + SArray *pTbUids = *(SArray **)pIter; if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids, isAdd) != TSDB_CODE_SUCCESS) { taosHashCancelIterate(pStore->uidHash, pIter); @@ -213,7 +213,7 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui } SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SHashObj * infoHash = NULL; + SHashObj *infoHash = NULL; if (!pStat || !(infoHash = RSMA_INFO_HASH(pStat))) { terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; @@ -264,11 +264,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat int8_t idx) { if ((param->qmsgLen > 0) && param->qmsg[idx]) { SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); - SRetention * pRetention = SMA_RETENTION(pSma); - STsdbCfg * pTsdbCfg = SMA_TSDB_CFG(pSma); - SVnode * pVnode = pSma->pVnode; + SRetention *pRetention = SMA_RETENTION(pSma); + STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); + SVnode *pVnode = pSma->pVnode; char taskInfDir[TSDB_FILENAME_LEN] = {0}; - void * pStreamState = NULL; + void *pStreamState = NULL; // set the backend of stream state tdRSmaQTaskInfoGetFullPath(pVnode, pRSmaInfo->suid, idx + 1, pVnode->pTfs, taskInfDir); @@ -371,6 +371,8 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat * @return int32_t */ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName) { + int32_t code; + if ((param->qmsgLen[0] == 0) && (param->qmsgLen[1] == 0)) { smaDebug("vgId:%d, no qmsg1/qmsg2 for rollup table %s %" PRIi64, SMA_VID(pSma), tbName, suid); return TSDB_CODE_SUCCESS; @@ -383,7 +385,7 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con } #endif - SSmaEnv * pEnv = SMA_RSMA_ENV(pSma); + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; @@ -410,8 +412,13 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con pRSmaInfo->suid = suid; T_REF_INIT_VAL(pRSmaInfo, 1); - if (!(pRSmaInfo->queue = taosOpenQueue()) || !(pRSmaInfo->qall = taosAllocateQall()) || - tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0 || + code = taosOpenQueue(&pRSmaInfo->queue); + if (code) goto _err; + + code = taosAllocateQall(&pRSmaInfo->qall); + if (code) goto _err; + + if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0 || tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 1) < 0) { goto _err; } @@ -682,8 +689,8 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma int32_t code = 0; int32_t lino = 0; SSDataBlock *output = NULL; - SArray * pResList = pItem->pResList; - STSchema * pTSchema = pInfo->pTSchema; + SArray *pResList = pItem->pResList; + STSchema *pTSchema = pInfo->pTSchema; int64_t suid = pInfo->suid; while (1) { @@ -742,7 +749,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } } - STsdb * sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); + STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); SSubmitReq2 *pReq = NULL; if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) { @@ -804,11 +811,13 @@ _exit: */ static int32_t tdExecuteRSmaImplAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid) { + int32_t code; int32_t size = RSMA_EXEC_MSG_HLEN + len; // header + payload - void * qItem = taosAllocateQitem(size, DEF_QITEM, 0); + void *qItem; - if (!qItem) { - return TSDB_CODE_FAILED; + code = taosAllocateQitem(size, DEF_QITEM, 0, (void **)&qItem); + if (code) { + return code; } void *pItem = qItem; @@ -883,7 +892,7 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int64_t version, int32_t inputType, SRSmaInfo *pInfo, ERsmaExecType type, int8_t level) { int32_t idx = level - 1; - void * qTaskInfo = RSMA_INFO_QTASK(pInfo, idx); + void *qTaskInfo = RSMA_INFO_QTASK(pInfo, idx); SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx); if (!qTaskInfo) { @@ -923,7 +932,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { int32_t code = 0; int32_t lino = 0; - SSmaEnv * pEnv = SMA_RSMA_ENV(pSma); + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = NULL; SRSmaInfo *pRSmaInfo = NULL; @@ -1078,8 +1087,8 @@ _err: static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { int32_t code = 0; int32_t lino = 0; - SVnode * pVnode = pSma->pVnode; - SArray * suidList = NULL; + SVnode *pVnode = pSma->pVnode; + SArray *suidList = NULL; STbUidStore uidStore = {0}; SMetaReader mr = {0}; tb_uid_t suid = 0; @@ -1207,7 +1216,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { int32_t code = 0; int32_t lino = 0; int32_t nTaskInfo = 0; - SSma * pSma = pRSmaStat->pSma; + SSma *pSma = pRSmaStat->pSma; SVnode *pVnode = pSma->pVnode; if (taosHashGetSize(pInfoHash) <= 0) { @@ -1240,7 +1249,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { do { int32_t nStreamFlushed = 0; int32_t nSleep = 0; - void * infoHash = NULL; + void *infoHash = NULL; while (true) { while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; @@ -1282,7 +1291,7 @@ _checkpoint: SStreamMeta *pMeta = NULL; int64_t checkpointId = taosGetTimestampNs(); bool checkpointBuilt = false; - void * infoHash = NULL; + void *infoHash = NULL; while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; if (RSMA_INFO_IS_DEL(pRSmaInfo)) { @@ -1355,10 +1364,10 @@ _exit: * @param tmrId */ static void tdRSmaFetchTrigger(void *param, void *tmrId) { - SRSmaRef * pRSmaRef = NULL; - SSma * pSma = NULL; - SRSmaStat * pStat = NULL; - SRSmaInfo * pRSmaInfo = NULL; + SRSmaRef *pRSmaRef = NULL; + SSma *pSma = NULL; + SRSmaStat *pStat = NULL; + SRSmaInfo *pRSmaInfo = NULL; SRSmaInfoItem *pItem = NULL; if (!(pRSmaRef = taosHashGet(smaMgmt.refHash, ¶m, POINTER_BYTES))) { @@ -1526,7 +1535,7 @@ _err: } static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SArray *pSubmitArr, ERsmaExecType type) { - void * msg = NULL; + void *msg = NULL; int8_t resume = 0; int32_t nSubmit = 0; int32_t nDelete = 0; @@ -1557,7 +1566,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA _resume_delete: version = RSMA_EXEC_MSG_VER(msg); if ((terrno = tqExtractDelDataBlock(RSMA_EXEC_MSG_BODY(msg), RSMA_EXEC_MSG_LEN(msg), version, - &packData.pDataBlock, 1))) { + &packData.pDataBlock, 1))) { taosFreeQitem(msg); goto _err; } @@ -1630,11 +1639,11 @@ _err: int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { int32_t code = 0; int32_t lino = 0; - SVnode * pVnode = pSma->pVnode; - SSmaEnv * pEnv = SMA_RSMA_ENV(pSma); + SVnode *pVnode = pSma->pVnode; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SHashObj * infoHash = NULL; - SArray * pSubmitArr = NULL; + SHashObj *infoHash = NULL; + SArray *pSubmitArr = NULL; bool isFetchAll = false; if (!pRSmaStat || !(infoHash = RSMA_INFO_HASH(pRSmaStat))) { diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 1c8f102725..3efc653f64 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -46,6 +46,7 @@ static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, static SArray* createDefaultTagColName(); static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, int64_t gid, bool newSubTableRule); +static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo); int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, const char* pIdStr, bool newSubTableRule) { @@ -270,6 +271,14 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); taosArrayPush(reqs.pArray, pCreateTbReq); + + STableSinkInfo* pInfo = NULL; + bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, gid, &pInfo); + if (!alreadyCached) { + code = doCreateSinkInfo(pCreateTbReq->name, &pInfo); + doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, pTask->id.idStr); + } + tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name); } @@ -634,6 +643,18 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI return TSDB_CODE_SUCCESS; } +int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo) { + int32_t nameLen = strlen(pDstTableName); + (*pInfo) = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen + 1); + if (*pInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pInfo)->name.len = nameLen; + memcpy((*pInfo)->name.data, pDstTableName, nameLen); + return TSDB_CODE_SUCCESS; +} + int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName, SSubmitTbData* pTableData) { uint64_t groupId = pDataBlock->info.id.groupId; @@ -670,22 +691,15 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) && !alreadyAddGroupId(dstTableName, groupId) && groupId != 0) { tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName); - if(pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER){ + if (pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER) { buildCtbNameAddGroupId(NULL, dstTableName, groupId); - }else if(pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER && stbFullName) { + } else if (pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER && stbFullName) { buildCtbNameAddGroupId(stbFullName, dstTableName, groupId); } } } - int32_t nameLen = strlen(dstTableName); - pTableSinkInfo = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen + 1); - if (pTableSinkInfo == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pTableSinkInfo->name.len = nameLen; - memcpy(pTableSinkInfo->name.data, dstTableName, nameLen); + int32_t code = doCreateSinkInfo(dstTableName, &pTableSinkInfo); tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName); } @@ -693,7 +707,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat pTableData->uid = pTableSinkInfo->uid; if (pTableData->uid == 0) { - tqTrace("s-task:%s cached tableInfo uid is invalid, acquire it from meta", id); + tqTrace("s-task:%s cached tableInfo:%s uid is invalid, acquire it from meta", id, pTableSinkInfo->name.data); return doWaitForDstTableCreated(pVnode, pTask, pTableSinkInfo, dstTableName, &pTableData->uid); } else { tqTrace("s-task:%s set the dstTable uid from cache:%" PRId64, id, pTableData->uid); @@ -711,24 +725,33 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat if (metaGetTableEntryByName(&mr, dstTableName) < 0) { metaReaderClear(&mr); - tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName); + if (pTask->outputInfo.tbSink.autoCreateCtb) { + tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName); - SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal)); + SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal)); - pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; - pTableData->pCreateTbReq = - buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray, - pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); - taosArrayDestroy(pTagArray); + pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + pTableData->pCreateTbReq = + buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray, + pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); + taosArrayDestroy(pTagArray); - if (pTableData->pCreateTbReq == NULL) { - tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno)); - taosMemoryFree(pTableSinkInfo); - return terrno; + if (pTableData->pCreateTbReq == NULL) { + tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName, + tstrerror(terrno)); + taosMemoryFree(pTableSinkInfo); + return terrno; + } + + pTableSinkInfo->uid = 0; + doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); + } else { + metaReaderClear(&mr); + + tqError("s-task:%s vgId:%d dst-table:%s not auto-created, and not create in tsdb, discard data", id, + vgId, dstTableName); + return TSDB_CODE_TDB_TABLE_NOT_EXIST; } - - pTableSinkInfo->uid = 0; - doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); } else { bool isValid = isValidDstChildTable(&mr, vgId, dstTableName, suid); if (!isValid) { @@ -774,16 +797,34 @@ int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_ } void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { - const SArray* pBlocks = (const SArray*)data; - SVnode* pVnode = (SVnode*)vnode; - int64_t suid = pTask->outputInfo.tbSink.stbUid; - char* stbFullName = pTask->outputInfo.tbSink.stbFullName; - STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; - int32_t vgId = TD_VID(pVnode); - int32_t numOfBlocks = taosArrayGetSize(pBlocks); - int32_t code = TSDB_CODE_SUCCESS; - const char* id = pTask->id.idStr; - int64_t earlyTs = tsdbGetEarliestTs(pVnode->pTsdb); + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->outputInfo.tbSink.stbUid; + char* stbFullName = pTask->outputInfo.tbSink.stbFullName; + STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; + int32_t vgId = TD_VID(pVnode); + int32_t numOfBlocks = taosArrayGetSize(pBlocks); + int32_t code = TSDB_CODE_SUCCESS; + const char* id = pTask->id.idStr; + int64_t earlyTs = tsdbGetEarliestTs(pVnode->pTsdb); + STaskOutputInfo* pOutputInfo = &pTask->outputInfo; + + if (pTask->outputInfo.tbSink.pTagSchema == NULL) { + SMetaReader mer1 = {0}; + metaReaderDoInit(&mer1, pVnode->pMeta, META_READER_LOCK); + + code = metaReaderGetTableEntryByUid(&mer1, pOutputInfo->tbSink.stbUid); + pOutputInfo->tbSink.pTagSchema = tCloneSSchemaWrapper(&mer1.me.stbEntry.schemaTag); + metaReaderClear(&mer1); + + SSchemaWrapper* pTagSchema = pOutputInfo->tbSink.pTagSchema; + SSchema* pCol1 = &pTagSchema->pSchema[0]; + if (pTagSchema->nCols == 1 && pCol1->type == TSDB_DATA_TYPE_UBIGINT && strcmp(pCol1->name, "group_id") == 0) { + pOutputInfo->tbSink.autoCreateCtb = true; + } else { + pOutputInfo->tbSink.autoCreateCtb = false; + } + } bool onlySubmitData = hasOnlySubmitData(pBlocks, numOfBlocks); if (!onlySubmitData) { @@ -815,6 +856,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP}; code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d s-task:%s dst-table not exist, stb:%s discard stream results", vgId, id, stbFullName); continue; } @@ -868,6 +910,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { if (index == NULL) { // no data yet, append it code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d dst-table gid:%" PRId64 " not exist, discard stream results", vgId, groupId); continue; } @@ -929,11 +972,6 @@ bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks) { } int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id) { - if (tSimpleHashGetSize(pSinkTableMap) > MAX_CACHE_TABLE_INFO_NUM) { - taosMemoryFreeClear(pTableSinkInfo); // too many items, failed to cache it - return TSDB_CODE_FAILED; - } - int32_t code = tSimpleHashPut(pSinkTableMap, &groupId, sizeof(uint64_t), &pTableSinkInfo, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pTableSinkInfo); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index eb1da51b45..16c523a50e 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -164,7 +164,8 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, taosWUnLockLatch(&pTq->lock); } - tOffsetCopy(&dataRsp.common.reqOffset, pOffset); // reqOffset represents the current date offset, may be changed if wal not exists + tOffsetCopy(&dataRsp.common.reqOffset, + pOffset); // reqOffset represents the current date offset, may be changed if wal not exists code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); end : { @@ -178,25 +179,25 @@ end : { } } -#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC, DELETE_FUNC) \ - SDecoder decoder = {0};\ - TYPE req = {0}; \ - void* data = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)); \ - int32_t len = pHead->bodyLen - sizeof(SMsgHead); \ - tDecoderInit(&decoder, data, len); \ - if (DECODE_FUNC(&decoder, &req) == 0 && (req.source & TD_REQ_FROM_TAOX) != 0) { \ - tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, jump meta for, vgId:%d offset %" PRId64 " msgType %d", \ - pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); \ - fetchVer++; \ - DELETE_FUNC(&req); \ - tDecoderClear(&decoder); \ - continue; \ - } \ - DELETE_FUNC(&req); \ +#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC, DELETE_FUNC) \ + SDecoder decoder = {0}; \ + TYPE req = {0}; \ + void* data = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)); \ + int32_t len = pHead->bodyLen - sizeof(SMsgHead); \ + tDecoderInit(&decoder, data, len); \ + if (DECODE_FUNC(&decoder, &req) == 0 && (req.source & TD_REQ_FROM_TAOX) != 0) { \ + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, jump meta for, vgId:%d offset %" PRId64 \ + " msgType %d", \ + pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); \ + fetchVer++; \ + DELETE_FUNC(&req); \ + tDecoderClear(&decoder); \ + continue; \ + } \ + DELETE_FUNC(&req); \ tDecoderClear(&decoder); -static void tDeleteCommon(void* parm) { -} +static void tDeleteCommon(void* parm) {} static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { @@ -313,9 +314,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); continue; } - int32_t tLen = sizeof(SMqRspHead) + len; - void* tBuf = taosMemoryCalloc(1, tLen); - void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead)); + int32_t tLen = sizeof(SMqRspHead) + len; + void* tBuf = taosMemoryCalloc(1, tLen); + void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead)); SEncoder encoder = {0}; tEncoderInit(&encoder, metaBuff, len); code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp); @@ -374,13 +375,13 @@ end: } int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) { - int32_t code = 0; + int32_t code = 0; STqOffsetVal reqOffset = {0}; tOffsetCopy(&reqOffset, &pRequest->reqOffset); // reset the offset if needed if (IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type)) { - bool blockReturned = false; + bool blockReturned = false; code = extractResetOffsetVal(&reqOffset, pTq, pHandle, pRequest, pMsg, &blockReturned); if (code != 0) { goto END; @@ -392,7 +393,7 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ } } else if (reqOffset.type == 0) { // use the consumer specified offset uError("req offset type is 0"); - code = TSDB_CODE_TMQ_INVALID_MSG; + code = TSDB_CODE_TMQ_INVALID_MSG; goto END; } @@ -416,8 +417,8 @@ static void initMqRspHead(SMqRspHead* pMsgHead, int32_t type, int32_t epoch, int pMsgHead->walever = ever; } -int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqBatchMetaRsp* pRsp, - int32_t vgId) { +int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, + const SMqBatchMetaRsp* pRsp, int32_t vgId) { int32_t len = 0; int32_t code = 0; tEncodeSize(tEncodeMqBatchMetaRsp, pRsp, len, code); @@ -444,8 +445,8 @@ int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SM SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0}; tmsgSendRsp(&resp); - tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type: batch meta, size:%ld offset type:%d", vgId, - pReq->consumerId, pReq->epoch, taosArrayGetSize(pRsp->batchMetaReq), pRsp->rspOffset.type); + tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type: batch meta, size:%ld offset type:%d", + vgId, pReq->consumerId, pReq->epoch, taosArrayGetSize(pRsp->batchMetaReq), pRsp->rspOffset.type); return 0; } @@ -527,6 +528,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp, } int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type) { + int32_t code; SDecoder* pCoder = &(SDecoder){0}; SDeleteRes* pRes = &(SDeleteRes){0}; @@ -570,11 +572,11 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void* taosArrayDestroy(pRes->uidList); if (type == 0) { - *pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); - if (*pRefBlock == NULL) { + code = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0, pRefBlock); + if (code) { blockDataCleanup(pDelBlock); taosMemoryFree(pDelBlock); - return TSDB_CODE_OUT_OF_MEMORY; + return code; } ((SStreamRefDataBlock*)(*pRefBlock))->type = STREAM_INPUT__REF_DATA_BLOCK; @@ -614,7 +616,7 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b continue; } - STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask == NULL) { tqError("vgId:%d failed to acquire task:0x%x in retrieving progress", pMeta->vgId, pId->taskId); diff --git a/source/libs/audit/src/auditMain.c b/source/libs/audit/src/auditMain.c index aa3b669c1b..7a8de49abe 100644 --- a/source/libs/audit/src/auditMain.c +++ b/source/libs/audit/src/auditMain.c @@ -32,8 +32,8 @@ char* tsAuditBatchUri = "/audit-batch"; int32_t auditInit(const SAuditCfg *pCfg) { tsAudit.cfg = *pCfg; tsAudit.records = taosArrayInit(0, sizeof(SAuditRecord *)); - taosThreadMutexInit(&tsAudit.lock, NULL); - return 0; + if(tsAudit.records == NULL) return TSDB_CODE_OUT_OF_MEMORY; + return taosThreadMutexInit(&tsAudit.lock, NULL); } static FORCE_INLINE void auditDeleteRecord(SAuditRecord * record) { diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 9547fa12f3..6db3c780e2 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -929,7 +929,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, i SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore, SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr, SStorageAPI* pApi, int32_t tsIndex); -void initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, +int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic); void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins); void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); diff --git a/source/libs/executor/inc/streamexecutorInt.h b/source/libs/executor/inc/streamexecutorInt.h index 8b61b93601..c6053cc96e 100644 --- a/source/libs/executor/inc/streamexecutorInt.h +++ b/source/libs/executor/inc/streamexecutorInt.h @@ -20,6 +20,7 @@ extern "C" { #endif #include "executorInt.h" +#include "tutil.h" void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type); bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo); diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 15288c4406..aa377cd922 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -130,9 +130,11 @@ static int32_t getStatus(SDataDeleterHandle* pDeleter) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; - SDataDeleterBuf* pBuf = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM, 0); - if (NULL == pBuf) { - return TSDB_CODE_OUT_OF_MEMORY; + SDataDeleterBuf* pBuf; + + int32_t code = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM, 0, (void**)&pBuf); + if (code) { + return code; } if (!allocBuf(pDeleter, pInput, pBuf)) { @@ -227,7 +229,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { } taosCloseQueue(pDeleter->pDataBlocks); taosThreadMutexDestroy(&pDeleter->mutex); - + taosMemoryFree(pDeleter->pManager); return TSDB_CODE_SUCCESS; } @@ -270,12 +272,11 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData deleter->pParam = pParam; deleter->status = DS_BUF_EMPTY; deleter->queryEnd = false; - deleter->pDataBlocks = taosOpenQueue(); - taosThreadMutexInit(&deleter->mutex, NULL); - if (NULL == deleter->pDataBlocks) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = taosOpenQueue(&deleter->pDataBlocks); + if (code) { goto _end; } + taosThreadMutexInit(&deleter->mutex, NULL); *pHandle = deleter; return code; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 297c87ab40..3981cedd3f 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -104,7 +104,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn } int32_t dataLen = blockEncode(pInput->pData, pHandle->pCompressBuf, numOfCols); - int32_t len = tsCompressString(pHandle->pCompressBuf, dataLen, 1, pEntry->data, pBuf->allocSize, ONE_STAGE_COMP, NULL, 0); + int32_t len = + tsCompressString(pHandle->pCompressBuf, dataLen, 1, pEntry->data, pBuf->allocSize, ONE_STAGE_COMP, NULL, 0); if (len < dataLen) { pEntry->compressed = 1; pEntry->dataLen = len; @@ -168,9 +169,11 @@ static int32_t getStatus(SDataDispatchHandle* pDispatcher) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { int32_t code = 0; SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM, 0); - if (NULL == pBuf) { - return TSDB_CODE_OUT_OF_MEMORY; + SDataDispatchBuf* pBuf; + + code = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM, 0, (void**)&pBuf); + if (code) { + return code; } if (!allocBuf(pDispatcher, pInput, pBuf)) { @@ -228,7 +231,6 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRow ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows); } - static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; if (NULL == pDispatcher->nextOutput.pData) { @@ -291,6 +293,8 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) { } int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { + int32_t code; + SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); if (NULL == dispatcher) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -309,7 +313,11 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD dispatcher->pSchema = pDataSink->pInputDataBlockDesc; dispatcher->status = DS_BUF_EMPTY; dispatcher->queryEnd = false; - dispatcher->pDataBlocks = taosOpenQueue(); + code = taosOpenQueue(&dispatcher->pDataBlocks); + if (code) { + terrno = code; + goto _return; + } taosThreadMutexInit(&dispatcher->mutex, NULL); if (NULL == dispatcher->pDataBlocks) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index f0fddc28fc..c7236a88e8 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -17,6 +17,7 @@ #include "function.h" #include "os.h" #include "tname.h" +#include "tutil.h" #include "tdatablock.h" #include "tmsg.h" @@ -1357,11 +1358,13 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosMemoryFreeClear(param); } -void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { +int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI; if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - return; + return code; } SStreamScanInfo* pScanInfo = downstream->info; @@ -1369,8 +1372,9 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup pScanInfo->pPartScalarSup = pExpr; pScanInfo->pPartTbnameSup = pTbnameExpr; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); + code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); } + return code; } SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) { @@ -1421,6 +1425,7 @@ void freePartItem(void* ptr) { SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1511,14 +1516,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); - initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); + code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); + TSDB_CHECK_CODE(code, lino, _error); + return pOperator; _error: pTaskInfo->code = code; destroyStreamPartitionOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index f6c5118c88..678d8d569e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2483,27 +2483,54 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl } } -int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { - int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); - len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup); - *pBuff = taosMemoryCalloc(1, len); - void* buf = *pBuff; - encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup); - pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo); - return len; +int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t len = 0; + code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len); + TSDB_CHECK_CODE(code, lino, _end); + + len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup); + *pBuff = taosMemoryCalloc(1, len); + if (!(*pBuff)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + void* buf = *pBuff; + encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup); + int32_t tmp = 0; + code = pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo, &tmp); + TSDB_CHECK_CODE(code, lino, _end); + + *pLen = len; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (!pInfo->pState) { return; } if (needSaveStreamOperatorInfo(&pInfo->basic)) { void* pBuf = NULL; - int32_t len = streamScanOperatorEncode(pInfo, &pBuf); + int32_t len = 0; + code = streamScanOperatorEncode(pInfo, &pBuf, &len); + TSDB_CHECK_CODE(code, lino, _end); pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len); taosMemoryFree(pBuf); saveStreamOperatorStateComplete(&pInfo->basic); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } // other properties are recovered from the execution plan @@ -3101,17 +3128,35 @@ static void destroyStreamScanOperatorInfo(void* param) { } void streamScanReleaseState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamScanInfo* pInfo = pOperator->info; + void* pBuff = NULL; if (!pInfo->pState) { return; } if (!pInfo->pUpdateInfo) { return; } - int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); - void* pBuff = taosMemoryCalloc(1, len); - pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo); + int32_t len = 0; + code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len); + TSDB_CHECK_CODE(code, lino, _end); + + pBuff = taosMemoryCalloc(1, len); + if (!pBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; +TSDB_CHECK_CODE(code, lino, _end); + } + + int32_t tmp = 0; + code = pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo, &tmp); + TSDB_CHECK_CODE(code, lino, _end); + pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), pBuff, len); +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } taosMemoryFree(pBuff); } diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 857d048457..14f2dfbada 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -68,9 +68,7 @@ void destroyStreamCountAggOperatorInfo(void* param) { taosMemoryFreeClear(param); } -bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { - return pAggSup->windowCount != pAggSup->windowSliding; -} +bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; } void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin, SBuffInfo* pBuffInfo) { @@ -89,13 +87,14 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, ASSERT(pBuffInfo->pCur); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); + (void**)&pCurWin->winInfo.pStatePos, &size); if (code == TSDB_CODE_FAILED) { pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, (void**)&pCurWin->winInfo.pStatePos, &size); } } else { - pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount); + pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, + pAggSup->windowCount); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, (void**)&pCurWin->winInfo.pStatePos, &size); if (code == TSDB_CODE_FAILED) { @@ -107,15 +106,16 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pBuffInfo->rebuildWindow = true; } } else { - code = pAggSup->stateStore.streamStateCountWinAddIfNotExist( - pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size); + code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, + pAggSup->windowCount, + (void**)&pCurWin->winInfo.pStatePos, &size); } if (code == TSDB_CODE_SUCCESS) { pCurWin->winInfo.isOutput = true; } - pCurWin->pWindowCount= - (COUNT_TYPE*) ((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE))); + pCurWin->pWindowCount = + (COUNT_TYPE*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE))); if (*pCurWin->pWindowCount == pAggSup->windowCount) { pBuffInfo->rebuildWindow = true; @@ -133,7 +133,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; - int32_t num = 0; + int32_t num = 0; for (int32_t i = start; i < rows; i++) { if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) { num++; @@ -181,7 +181,7 @@ void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, ESt pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, pKey); } SSessionKey tmpKey = {.groupId = pKey->groupId, .win.ekey = INT64_MIN, .win.skey = INT64_MIN}; - int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); if (code != TSDB_CODE_SUCCESS) { pAggSup->stateStore.streamStateFreeCur(pCur); return; @@ -239,7 +239,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2); SStreamStateCur* pCur = NULL; - COUNT_TYPE slidingRows = 0; + COUNT_TYPE slidingRows = 0; for (int32_t i = 0; i < rows;) { if (pInfo->ignoreExpiredData && @@ -269,7 +269,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } getCountWinRange(pAggSup, &curWin.winInfo.sessionWin, STREAM_DELETE_DATA, &range); range.win.skey = TMIN(startTsCols[i], range.win.skey); - range.win.ekey = TMAX(startTsCols[rows-1], range.win.ekey); + range.win.ekey = TMAX(startTsCols[rows - 1], range.win.ekey); uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL); break; @@ -289,8 +289,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(curWin.winInfo, pStUpdated); if (code != TSDB_CODE_SUCCESS) { - qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), - tstrerror(code)); + qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); break; } } @@ -397,13 +396,14 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato int32_t mapSize = 0; buf = taosDecodeFixedI32(buf, &mapSize); for (int32_t i = 0; i < mapSize; i++) { - SSessionKey key = {0}; + SSessionKey key = {0}; SCountWindowInfo curWin = {0}; buf = decodeSSessionKey(buf, &key); SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL}; setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, + sizeof(SResultWindowInfo)); } // 2.twAggSup @@ -441,9 +441,9 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; SRowBuffPos* pPos = NULL; - int32_t size = 0; + int32_t size = 0; for (int32_t i = 0; i < pBlock->info.rows; i++) { - SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]}; + SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]}; SStreamStateCur* pCur = NULL; if (isSlidingCountWindow(pAggSup)) { pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &key, pAggSup->windowCount); @@ -452,7 +452,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { } while (1) { SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN}; - int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void**)&pPos, &size); if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) { pAggSup->stateStore.streamStateFreeCur(pCur); break; @@ -482,7 +482,7 @@ void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SAr SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]}; while (1) { SSessionKey curWin = {0}; - int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); + int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); if (code == TSDB_CODE_FAILED) { break; } @@ -511,11 +511,11 @@ void deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHa } static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { - SExprSupp* pSup = &pOperator->exprSupp; - SStreamCountAggOperatorInfo* pInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExprSupp* pSup = &pOperator->exprSupp; + SStreamCountAggOperatorInfo* pInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; qDebug("stask:%s %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -635,7 +635,7 @@ void streamCountReloadState(SOperatorInfo* pOperator) { int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME, strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size); - TSKEY ts = *(TSKEY*)pBuf; + TSKEY ts = *(TSKEY*)pBuf; pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); taosMemoryFree(pBuf); @@ -650,11 +650,13 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) { SCountWinodwPhysiNode* pCountNode = (SCountWinodwPhysiNode*)pPhyNode; int32_t numOfCols = 0; - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamCountAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamCountAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pOperator->pTaskInfo = pTaskInfo; @@ -664,18 +666,14 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); } SExprSupp* pExpSup = &pOperator->exprSupp; SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); pInfo->twAggSup = (STimeWindowAggSupp){ .waterMark = pCountNode->window.watermark, @@ -686,12 +684,11 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys }; pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId; - code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, - pTaskInfo->streamInfo.pState, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, - &pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, + sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, + GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); + TSDB_CHECK_CODE(code, lino, _error); + pInfo->streamAggSup.windowCount = pCountNode->windowCount; pInfo->streamAggSup.windowSliding = pCountNode->windowSliding; @@ -709,7 +706,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->dataVersion = 0; pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); if (!pInfo->historyWins) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); @@ -735,8 +733,12 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState); if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); + TSDB_CHECK_CODE(code, lino, _error); } return pOperator; @@ -747,6 +749,6 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } - diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 1809d0cc67..e6ff4fe242 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -28,9 +28,9 @@ #include "tlog.h" #include "ttime.h" -#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT) -#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState" -#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint" +#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT) +#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState" +#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint" typedef struct SEventWinfowFlag { bool startFlag; @@ -86,10 +86,11 @@ void destroyStreamEventOperatorInfo(void* param) { void setEventWindowFlag(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo) { char* pFlagInfo = (char*)pWinInfo->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize); - pWinInfo->pWinFlag = (SEventWinfowFlag*) pFlagInfo; + pWinInfo->pWinFlag = (SEventWinfowFlag*)pFlagInfo; } -void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, SEventWindowInfo* pWinInfo) { +void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, + SEventWindowInfo* pWinInfo) { pWinInfo->winInfo.sessionWin = *pKey; pWinInfo->winInfo.pStatePos = pPos; setEventWindowFlag(pAggSup, pWinInfo); @@ -122,22 +123,23 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI pCurWin->winInfo.sessionWin.groupId = groupId; pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.ekey = ts; - SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin); - SSessionKey leftWinKey = {.groupId = groupId}; - void* pVal = NULL; - int32_t len = 0; + SStreamStateCur* pCur = + pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin); + SSessionKey leftWinKey = {.groupId = groupId}; + void* pVal = NULL; + int32_t len = 0; code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len); - if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win) ) { + if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) { bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0); setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin); - if(inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag) ) { + if (inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag)) { pCurWin->winInfo.isOutput = !isWindowIncomplete(pCurWin); goto _end; } } pAggSup->stateStore.streamStateFreeCur(pCur); pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); - SSessionKey rightWinKey = {.groupId = groupId}; + SSessionKey rightWinKey = {.groupId = groupId}; code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len); bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0); if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) { @@ -149,7 +151,7 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI } } - SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; + SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len); setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin); pCurWin->pWinFlag->startFlag = start; @@ -164,7 +166,7 @@ _end: if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pNextWinKey); } - if(pCurWin->winInfo.pStatePos->needFree) { + if (pCurWin->winInfo.pStatePos->needFree) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin); } pAggSup->stateStore.streamStateFreeCur(pCur); @@ -173,14 +175,14 @@ _end: } int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey, - TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, SSHashObj* pResultRows, - SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { + TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, + SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { *pRebuild = false; - if (!pWinInfo->pWinFlag->startFlag && !(starts[start]) ) { + if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) { return 1; } - TSKEY maxTs = INT64_MAX; + TSKEY maxTs = INT64_MAX; STimeWindow* pWin = &pWinInfo->winInfo.sessionWin.win; if (pWinInfo->pWinFlag->endFlag) { maxTs = pWin->ekey + 1; @@ -219,7 +221,7 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); if (ends[i]) { - if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey ) { + if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey) { *pRebuild = true; } return i + 1 - start; @@ -230,26 +232,28 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - int32_t winNum = 0; - SStreamEventAggOperatorInfo* pInfo = pOperator->info; - SResultRow* pCurResult = NULL; - int32_t numOfOutput = pOperator->exprSupp.numOfExprs; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + int32_t winNum = 0; + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + SResultRow* pCurResult = NULL; + int32_t numOfOutput = pOperator->exprSupp.numOfExprs; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; while (1) { if (!pCurWin->pWinFlag->startFlag || pCurWin->pWinFlag->endFlag) { break; } SEventWindowInfo nextWinInfo = {0}; getNextSessionWinInfo(pAggSup, pStUpdated, &pCurWin->winInfo, &nextWinInfo.winInfo); - if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) { + if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || + !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) { releaseOutputBuf(pAggSup->pState, nextWinInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); break; } setEventWindowFlag(pAggSup, &nextWinInfo); - compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, pStDeleted, false); + compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, + pStDeleted, false); pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag; winNum++; } @@ -275,7 +279,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t winRows = 0; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SColumnInfoData* pColStart = NULL; - SColumnInfoData* pColEnd = NULL; + SColumnInfoData* pColEnd = NULL; pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; @@ -290,7 +294,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl return; } - SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; + SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), + .pDataBlock = pSDataBlock->pDataBlock}; code = filterSetDataFromSlotId(pInfo->pStartCondInfo, ¶mStart); if (code != TSDB_CODE_SUCCESS) { qError("set data from start slotId error."); @@ -299,7 +304,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t statusStart = 0; filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart); - SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; + SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), + .pDataBlock = pSDataBlock->pDataBlock}; code = filterSetDataFromSlotId(pInfo->pEndCondInfo, ¶mEnd); if (code != TSDB_CODE_SUCCESS) { qError("set data from end slotId error."); @@ -320,20 +326,23 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t winIndex = 0; bool allEqual = true; SEventWindowInfo curWin = {0}; - SSessionKey nextWinKey = {0}; - setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, &nextWinKey); + SSessionKey nextWinKey = {0}; + setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, + &nextWinKey); setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); bool rebuild = false; - winRows = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, rows, i, - pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); + winRows = + updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, + rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); ASSERT(winRows >= 1); if (rebuild) { uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, - &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); + &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin); - if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && !isWindowIncomplete(&curWin)) { + if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && + !isWindowIncomplete(&curWin)) { saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); } releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); @@ -603,7 +612,8 @@ void streamEventReleaseState(SOperatorInfo* pOperator) { char* pBuff = taosMemoryCalloc(1, resSize); memcpy(pBuff, pInfo->historyWins->pData, winSize); memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); - qDebug("===stream=== event window operator relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins)); + qDebug("===stream=== event window operator relase state. save result count:%d", + (int32_t)taosArrayGetSize(pInfo->historyWins)); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_STATE_NAME, strlen(STREAM_EVENT_OP_STATE_NAME), pBuff, resSize); pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); @@ -647,7 +657,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, i); getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo); - //event window has been deleted + // event window has been deleted if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) { continue; } @@ -659,7 +669,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, - curInfo.winInfo.sessionWin.groupId); + curInfo.winInfo.sessionWin.groupId); if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { saveSessionOutputBuf(pAggSup, &curInfo.winInfo); } @@ -693,6 +703,7 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys SStreamEventWinodwPhysiNode* pEventNode = (SStreamEventWinodwPhysiNode*)pPhyNode; int32_t tsSlotId = ((SColumnNode*)pEventNode->window.pTspk)->slotId; int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamEventAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamEventAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -783,20 +794,18 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState); - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); code = filterInitFromNode((SNode*)pEventNode->pStartCond, &pInfo->pStartCondInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); + code = filterInitFromNode((SNode*)pEventNode->pEndCond, &pInfo->pEndCondInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); return pOperator; @@ -804,5 +813,6 @@ _error: destroyStreamEventOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index d3f90c3134..96c6712866 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -28,14 +28,18 @@ #include "tlog.h" #include "ttime.h" -#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) -#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) -#define IS_NORMAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) +#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) +#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) +#define IS_NORMAL_INTERVAL_OP(op) \ + ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || \ + (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) -#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) -#define IS_NORMAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) +#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) +#define IS_NORMAL_SESSION_OP(op) \ + ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \ + (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) -#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) +#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) #define DEAULT_DELETE_MARK INT64_MAX #define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState" @@ -263,14 +267,16 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa if (chIds) { int32_t childId = getChildIndex(pBlock); if (pInvalidWins) { - qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, winRes.groupId, childId); + qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, + winRes.groupId, childId); taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0); } SArray* chArray = *(void**)chIds; int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ); if (index != -1) { - qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey, winGpId, childId); + qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey, + winGpId, childId); getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); continue; } @@ -390,7 +396,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin } void destroyFlusedPos(void* pRes) { - SRowBuffPos* pPos = (SRowBuffPos*) pRes; + SRowBuffPos* pPos = (SRowBuffPos*)pRes; if (!pPos->needFree && !pPos->pRowBuff) { taosMemoryFreeClear(pPos->pKey); taosMemoryFree(pPos); @@ -488,8 +494,11 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = - pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); + int32_t code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, + pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at %d since %s", __func__, __LINE__, tstrerror(code)); + } } pScanInfo->interval = pInfo->interval; @@ -624,17 +633,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB blockDataUpdateTsWindow(pBlock, 0); } -static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, SArray* pPullWins, - int32_t numOfCh, SOperatorInfo* pOperator) { +static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, + SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); - TSKEY* tsData = (TSKEY*)pStartCol->pData; - SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - TSKEY* tsEndData = (TSKEY*)pEndCol->pData; - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; - int32_t chId = getChildIndex(pBlock); - bool res = false; + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + TSKEY* tsData = (TSKEY*)pStartCol->pData; + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + TSKEY* tsEndData = (TSKEY*)pEndCol->pData; + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; + int32_t chId = getChildIndex(pBlock); + bool res = false; for (int32_t i = 0; i < pBlock->info.rows; i++) { TSKEY winTs = tsData[i]; while (winTs <= tsEndData[i]) { @@ -650,7 +659,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina // pull data is over taosArrayDestroy(chArray); taosHashRemove(pMap, &winRes, sizeof(SWinKey)); - res =true; + res = true; qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts); void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey)); @@ -663,7 +672,8 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request - qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d", winRes.ts, winRes.groupId, numOfCh); + qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d", + winRes.ts, winRes.groupId, numOfCh); if (IS_MID_INTERVAL_OP(pOperator)) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; taosArrayPush(pInfo->pMidPullDatas, &winRes); @@ -671,7 +681,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina taosArrayPush(pInfo->pDelWins, &winRes); addPullWindow(pMap, &winRes, numOfCh); if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); } qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh); } @@ -690,7 +700,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i for (int32_t i = 0; i < size; i++) { SWinKey* winKey = taosArrayGet(wins, i); STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval); - void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); + void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); if (!chIds) { SPullWindowInfo pull = { .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; @@ -698,7 +708,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild); if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap,winKey, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0); } qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild); } @@ -751,7 +761,8 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat if (pBlock->info.id.groupId == 0) { pBlock->info.id.groupId = groupId; void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, false) < 0) { + if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, + false) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); @@ -829,9 +840,7 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN return startPos; } -bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { - return pInfo->primaryPkIndex != -1; -} +bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; } static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId, SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { @@ -852,23 +861,22 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; - void* pPkVal = NULL; - int32_t pkLen = 0; + void* pPkVal = NULL; + int32_t pkLen = 0; SColumnInfoData* pPkColDataInfo = NULL; if (hasSrcPrimaryKeyCol(&pInfo->basic)) { pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); } if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) { - qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 - ",maxKey %" PRId64, - pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); + qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64, + pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex); // timestamp of the data is incorrect if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64, - pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); + pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); } } @@ -916,7 +924,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild); if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); } } } else { @@ -962,7 +970,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat }; if (pInfo->destHasPrimaryKey && code == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) { - tSimpleHashPut(pDeletedMap,&key, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { @@ -1365,7 +1373,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); + SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL; doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); if (IS_FINAL_INTERVAL_OP(pOperator)) { @@ -1399,7 +1407,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE) { - if(!IS_FINAL_INTERVAL_OP(pOperator)) { + if (!IS_FINAL_INTERVAL_OP(pOperator)) { pInfo->recvRetrive = true; copyDataBlock(pInfo->pMidRetriveRes, pBlock); pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE; @@ -1525,8 +1533,8 @@ void streamIntervalReloadState(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL && pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) { - int32_t size = 0; - void* pBuf = NULL; + int32_t size = 0; + void* pBuf = NULL; int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); if (code == 0) { @@ -1600,7 +1608,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pInfo->pState, &pTaskInfo->storageAPI.functionStore); + pInfo->pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1755,27 +1763,37 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num } } -void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, - STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) { +int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, + STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) { SStreamPartitionOperatorInfo* pScanInfo = downstream->info; pScanInfo->tsColIndex = tsColIndex; } if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic); - return; + code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic); + return code; } SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pState = pAggSup->pState; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, - pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); + code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, + pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, + &pScanInfo->pUpdateInfo); + TSDB_CHECK_CODE(code, lino, _end); } pScanInfo->twAggSup = *pTwSup; pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo; pBasic->primaryPkIndex = pScanInfo->primaryKeyIndex; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static TSKEY sesionTs(void* pKey) { @@ -2148,8 +2166,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData TSKEY* endTsCols = (int64_t*)pEndTsCol->pData; - void* pPkVal = NULL; - int32_t pkLen = 0; + void* pPkVal = NULL; + int32_t pkLen = 0; SColumnInfoData* pPkColDataInfo = NULL; if (hasSrcPrimaryKeyCol(&pInfo->basic)) { pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); @@ -2454,8 +2472,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa pBlock->info.id.groupId = pKey->groupId; void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, - &tbname, false) < 0) { + if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, + false) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); @@ -2935,7 +2953,7 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -2968,7 +2986,7 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -3018,6 +3036,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; int32_t numOfCols = 0; int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t lino = 0; SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -3112,8 +3131,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); + TSDB_CHECK_CODE(code, lino, _error); } return pOperator; @@ -3124,6 +3147,7 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } @@ -3574,7 +3598,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (!allEqual) { uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, - &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); + &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); @@ -3681,9 +3705,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); - pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, - pAggSup->stateKeySize, compareStateKey, - (void**)&winfo.pStatePos, &pAggSup->resultRowSize); + pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateKeySize, + compareStateKey, (void**)&winfo.pStatePos, + &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } @@ -3877,7 +3901,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, - strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); qDebug("===stream=== reload state. get result count:%d", num); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; @@ -3941,12 +3965,13 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey); int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _error; + TSDB_CHECK_CODE(code, lino, _error); } pInfo->stateCol = extractColumnFromColumnNode(pColNode); @@ -3955,9 +3980,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); } pInfo->twAggSup = (STimeWindowAggSupp){ @@ -3984,9 +4007,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, type, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pSeDeleted = tSimpleHashInit(64, hashFn); pInfo->pDelIterator = NULL; @@ -3999,7 +4021,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->dataVersion = 0; pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); if (!pInfo->historyWins) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } if (pHandle) { pInfo->isHistoryOp = pHandle->fillHistory; @@ -4026,17 +4049,20 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState); - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); + return pOperator; _error: destroyStreamStateOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } @@ -4203,11 +4229,12 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, }; - pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark, - .calTrigger = pIntervalPhyNode->window.triggerType, - .maxTs = INT64_MIN, - .minTs = INT64_MAX, - .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)}; + pInfo->twAggSup = + (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark, + .calTrigger = pIntervalPhyNode->window.triggerType, + .maxTs = INT64_MIN, + .minTs = INT64_MAX, + .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)}; ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); @@ -4335,7 +4362,7 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS .ts = nextWin.skey, .groupId = groupId, }; - void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey)); + void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey)); int32_t index = -1; SArray* chArray = NULL; int32_t chId = 0; @@ -4407,8 +4434,8 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) { int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { - SWinKey* winKey = taosArrayGet(wins, i); - void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey)); + SWinKey* winKey = taosArrayGet(wins, i); + void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey)); if (!chIds) { addPullWindow(pMidPullMap, winKey, numOfChild); qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild); @@ -4611,5 +4638,6 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { void setStreamOperatorCompleted(SOperatorInfo* pOperator) { setOperatorCompleted(pOperator); - qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); + qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), + getStreamOpName(pOperator->operatorType), pOperator->status); } diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h index 55ea71a368..99e303e51c 100644 --- a/source/libs/parser/inc/parTranslater.h +++ b/source/libs/parser/inc/parTranslater.h @@ -47,7 +47,7 @@ typedef struct STranslateContext { SNode* pPostRoot; } STranslateContext; -bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode); +int32_t biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRet); int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect); int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt); int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput); @@ -57,4 +57,4 @@ int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, c } #endif -#endif /*_TD_PARSER_TRANS_H_*/ \ No newline at end of file +#endif /*_TD_PARSER_TRANS_H_*/ diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 149e02ad3d..c53180eb87 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -509,7 +509,7 @@ cmd ::= SHOW USERS FULL. cmd ::= SHOW USER PRIVILEGES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); } cmd ::= SHOW db_kind_opt(A) DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); - setShowKind(pCxt, pCxt->pRootNode, A); + (void)setShowKind(pCxt, pCxt->pRootNode, A); } cmd ::= SHOW table_kind_db_name_cond_opt(A) TABLES like_pattern_opt(B). { pCxt->pRootNode = createShowTablesStmt(pCxt, A, B, OP_TYPE_LIKE); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 7c2f504a4f..d61d80a691 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -101,7 +101,7 @@ static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken, pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); } else { strncpy(pPassword, pPasswordToken->z, pPasswordToken->n); - strdequote(pPassword); + (void)strdequote(pPassword); if (strtrim(pPassword) <= 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY); } else if (invalidPassword(pPassword)) { @@ -126,8 +126,8 @@ static int32_t parseEndpoint(SAstCreateContext* pCxt, const SToken* pEp, char* p char ep[TSDB_FQDN_LEN + 1 + 5] = {0}; COPY_STRING_FORM_ID_TOKEN(ep, pEp); - strdequote(ep); - strtrim(ep); + (void)strdequote(ep); + (void)strtrim(ep); if (NULL == pPort) { strcpy(pFqdn, ep); return TSDB_CODE_SUCCESS; @@ -371,7 +371,7 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* val->literal = strndup(pLiteral->z, pLiteral->n); if (TK_NK_ID != pLiteral->type && TK_TIMEZONE != pLiteral->type && (IS_VAR_DATA_TYPE(dataType) || TSDB_DATA_TYPE_TIMESTAMP == dataType)) { - trimString(pLiteral->z, pLiteral->n, val->literal, pLiteral->n); + (void)trimString(pLiteral->z, pLiteral->n, val->literal, pLiteral->n); } CHECK_OUT_OF_MEM(val->literal); val->node.resType.type = dataType; @@ -784,7 +784,10 @@ SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLitera return NULL; } } - taosArrayPush(pCxt->pPlaceholderValues, &val); + if (NULL == taosArrayPush(pCxt->pPlaceholderValues, &val)) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + } + CHECK_PARSER_STATUS(pCxt); return (SNode*)val; } @@ -905,7 +908,8 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; } - nodesListMakeAppend(&func->pParameterList, pExpr); + pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); + CHECK_PARSER_STATUS(pCxt); return (SNode*)func; } @@ -923,8 +927,10 @@ SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2) { CHECK_OUT_OF_MEM(list); list->pNodeList = nodesMakeList(); CHECK_OUT_OF_MEM(list->pNodeList); - nodesListAppend(list->pNodeList, p1); - nodesListAppend(list->pNodeList, p2); + pCxt->errCode = nodesListAppend(list->pNodeList, p1); + CHECK_PARSER_STATUS(pCxt); + pCxt->errCode = nodesListAppend(list->pNodeList, p2); + CHECK_PARSER_STATUS(pCxt); return (SNode*)list; } @@ -1121,7 +1127,8 @@ SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode) { CHECK_OUT_OF_MEM(groupingSet); groupingSet->groupingSetType = GP_TYPE_NORMAL; groupingSet->pParameterList = nodesMakeList(); - nodesListAppend(groupingSet->pParameterList, pNode); + pCxt->errCode = nodesListAppend(groupingSet->pParameterList, pNode); + CHECK_PARSER_STATUS(pCxt); return (SNode*)groupingSet; } @@ -1842,7 +1849,7 @@ SNode* createCreateSubTableFromFileClause(SAstCreateContext* pCxt, bool ignoreEx pStmt->ignoreExists = ignoreExists; pStmt->pSpecificTags = pSpecificTags; if (TK_NK_STRING == pFilePath->type) { - trimString(pFilePath->z, pFilePath->n, pStmt->filePath, PATH_MAX); + (void)trimString(pFilePath->z, pFilePath->n, pStmt->filePath, PATH_MAX); } else { strncpy(pStmt->filePath, pFilePath->z, pFilePath->n); } @@ -2087,7 +2094,7 @@ SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, S pDbName = createIdentifierValueNode(pCxt, &option.dbName); } SNode* pStmt = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, pDbName, pTbName, tableCondType); - setShowKind(pCxt, pStmt, option.kind); + (void)setShowKind(pCxt, pStmt, option.kind); return pStmt; } @@ -2408,9 +2415,9 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const } else { pStmt->dnodeId = -1; } - trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); + (void)trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); if (NULL != pValue) { - trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); + (void)trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); } return (SNode*)pStmt; } @@ -2591,9 +2598,9 @@ SNode* createAlterClusterStmt(SAstCreateContext* pCxt, const SToken* pConfig, co CHECK_PARSER_STATUS(pCxt); SAlterClusterStmt* pStmt = (SAlterClusterStmt*)nodesMakeNode(QUERY_NODE_ALTER_CLUSTER_STMT); CHECK_OUT_OF_MEM(pStmt); - trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); + (void)trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); if (NULL != pValue) { - trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); + (void)trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); } return (SNode*)pStmt; } @@ -2602,9 +2609,9 @@ SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, cons CHECK_PARSER_STATUS(pCxt); SAlterLocalStmt* pStmt = (SAlterLocalStmt*)nodesMakeNode(QUERY_NODE_ALTER_LOCAL_STMT); CHECK_OUT_OF_MEM(pStmt); - trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); + (void)trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); if (NULL != pValue) { - trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); + (void)trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); } return (SNode*)pStmt; } @@ -2869,7 +2876,7 @@ SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId) { CHECK_PARSER_STATUS(pCxt); SKillQueryStmt* pStmt = (SKillQueryStmt*)nodesMakeNode(QUERY_NODE_KILL_QUERY_STMT); CHECK_OUT_OF_MEM(pStmt); - trimString(pQueryId->z, pQueryId->n, pStmt->queryId, sizeof(pStmt->queryId) - 1); + (void)trimString(pQueryId->z, pQueryId->n, pStmt->queryId, sizeof(pStmt->queryId) - 1); return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index db2d34b844..89eaf14f75 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -112,7 +112,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem for (int i = 0; i < tags->numOfBound; ++i) { SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; SSmlKv* kv = taosArrayGet(cols, i); - + if (kv == NULL){ + code = TSDB_CODE_SML_INVALID_DATA; + uError("SML smlBuildTagRow error kv is null"); + goto end; + } if (kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type) { code = TSDB_CODE_SML_INVALID_DATA; @@ -120,7 +124,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem goto end; } - taosArrayPush(*tagName, pTagSchema->name); + if (taosArrayPush(*tagName, pTagSchema->name) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML smlBuildTagRow error push tag name"); + goto end; + } STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; // strcpy(val.colName, pTagSchema->name); if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY || @@ -141,7 +149,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem goto end; } char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + (void)snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); taosMemoryFree(p); code = buildSyntaxErrMsg(msg, buf, kv->value); goto end; @@ -149,9 +157,13 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem val.pData = p; val.nData = output; } else { - memcpy(&val.i64, &(kv->value), kv->length); + (void)memcpy(&val.i64, &(kv->value), kv->length); + } + if (taosArrayPush(pTagArray, &val) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML smlBuildTagRow error push tag array"); + goto end; } - taosArrayPush(pTagArray, &val); } code = tTagNew(pTagArray, 1, false, ppTag); @@ -166,20 +178,19 @@ end: return code; } -STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { - STableDataCxt* pTableCxt = NULL; +int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt) { SVCreateTbReq* pCreateTbReq = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, cxt, false, false); if (ret != TSDB_CODE_SUCCESS) { - return NULL; + return ret; } - ret = initTableColSubmitData(pTableCxt); + ret = initTableColSubmitData(*cxt); if (ret != TSDB_CODE_SUCCESS) { - return NULL; + return ret; } - return pTableCxt; + return TSDB_CODE_SUCCESS; } void clearColValArraySml(SArray* pCols) { @@ -197,6 +208,9 @@ void clearColValArraySml(SArray* pCols) { int32_t smlBuildRow(STableDataCxt* pTableCxt) { SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + if (pRow == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); if (TSDB_CODE_SUCCESS != ret) { return ret; @@ -211,18 +225,22 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 int ret = TSDB_CODE_SUCCESS; SSchema* pColSchema = schema + index; SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); + if (pVal == NULL) { + return TSDB_CODE_SUCCESS; + } SSmlKv* kv = (SSmlKv*)data; if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type) { ret = TSDB_CODE_SML_INVALID_DATA; char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1); if (tmp) { - memcpy(tmp, kv->key, kv->keyLen); + (void)memcpy(tmp, kv->key, kv->keyLen); uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name, pColSchema->name, tDataTypes[pColSchema->type].name); taosMemoryFree(tmp); } else { uError("SML smlBuildCol out of memory"); + ret = TSDB_CODE_OUT_OF_MEMORY; } goto end; } @@ -256,9 +274,9 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { pVal->value.nData = kv->length; pVal->value.pData = taosMemoryMalloc(kv->length); - memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); + (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); } else { - memcpy(&pVal->value.val, &(kv->value), kv->length); + (void)memcpy(&pVal->value.val, &(kv->value), kv->length); } pVal->flag = CV_FLAG_VALUE; @@ -276,8 +294,13 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc SVCreateTbReq* pCreateTblReq = NULL; SArray* tagName = NULL; - insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); - int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); + int ret = insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); + if (ret != TSDB_CODE_SUCCESS) { + buildInvalidOperationMsg(&pBuf, "init bound cols error"); + goto end; + } + + ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound tags error"); goto end; @@ -299,7 +322,11 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc ttl); pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); - memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); + if (pCreateTblReq->ctb.stbName == NULL){ + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + (void)memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); if (dataFormat) { STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, @@ -345,11 +372,18 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc for (int32_t r = 0; r < rowNum; ++r) { void* rowData = taosArrayGetP(cols, r); - + if (rowData == NULL) { + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } // 1. set the parsed value from sql string for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); + if (pVal == NULL) { + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); if (p == NULL) { continue; @@ -388,14 +422,22 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { pVal->value.nData = kv->length; pVal->value.pData = taosMemoryMalloc(kv->length); - memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); + if (NULL == pVal->value.pData) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); } else { - memcpy(&pVal->value.val, &(kv->value), kv->length); + (void)memcpy(&pVal->value.val, &(kv->value), kv->length); } pVal->flag = CV_FLAG_VALUE; } SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + if (NULL == pRow) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); if (TSDB_CODE_SUCCESS != ret) { buildInvalidOperationMsg(&pBuf, "tRowBuild error"); @@ -415,11 +457,12 @@ end: return ret; } -SQuery* smlInitHandle() { +int32_t smlInitHandle(SQuery** query) { + *query = NULL; SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); if (NULL == pQuery) { uError("create pQuery error"); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->haveResultSet = false; @@ -428,14 +471,22 @@ SQuery* smlInitHandle() { if (NULL == stmt) { uError("create SVnodeModifyOpStmt error"); qDestroyQuery(pQuery); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (stmt->pTableBlockHashObj == NULL){ + uError("create pTableBlockHashObj error"); + qDestroyQuery(pQuery); + nodesDestroyNode((SNode*)stmt); + return TSDB_CODE_OUT_OF_MEMORY; + } stmt->freeHashFunc = insDestroyTableDataCxtHashMap; stmt->freeArrayFunc = insDestroyVgroupDataCxtList; pQuery->pRoot = (SNode*)stmt; - return pQuery; + *query = pQuery; + + return TSDB_CODE_SUCCESS; } int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9b1c8096aa..97a6239a6f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -384,10 +384,13 @@ static bool hasSameTableAlias(SArray* pTables) { } static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { - size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel); + int32_t code = 0; + size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel); if (currTotalLevel > pCxt->currLevel) { SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); - taosArrayPush(pTables, &pTable); + if (NULL == taosArrayPush(pTables, &pTable)) { + return TSDB_CODE_OUT_OF_MEMORY; + } if (hasSameTableAlias(pTables)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS, "Not unique table/alias: '%s'", ((STableNode*)pTable)->tableAlias); @@ -399,18 +402,22 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { return TSDB_CODE_OUT_OF_MEMORY; } if (pCxt->currLevel == currTotalLevel) { - taosArrayPush(pTables, &pTable); + (void)taosArrayPush(pTables, &pTable); if (hasSameTableAlias(pTables)) { - taosArrayDestroy(pTables); + (void)taosArrayDestroy(pTables); return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS, "Not unique table/alias: '%s'", ((STableNode*)pTable)->tableAlias); } } - taosArrayPush(pCxt->pNsLevel, &pTables); + if (NULL == taosArrayPush(pCxt->pNsLevel, &pTables)) { + code = TSDB_CODE_OUT_OF_MEMORY; + (void)taosArrayDestroy(pTables); + break; + } ++currTotalLevel; } while (currTotalLevel <= pCxt->currLevel); } - return TSDB_CODE_SUCCESS; + return code; } static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) { @@ -421,13 +428,16 @@ static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) { static int32_t collectUseDatabase(const SName* pName, SHashObj* pDbs) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); return collectUseDatabaseImpl(dbFName, pDbs); } static int32_t collectUseTable(const SName* pName, SHashObj* pTable) { char fullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pName, fullName); + int32_t code = tNameExtractFullName(pName, fullName); + if (TSDB_CODE_SUCCESS != code) { + return code; + } return taosHashPut(pTable, fullName, strlen(fullName), pName, sizeof(SName)); } @@ -537,7 +547,7 @@ static int32_t refreshGetTableMeta(STranslateContext* pCxt, const char* pDbName, STableMeta** pMeta) { SParseContext* pParCxt = pCxt->pParseCxt; SName name; - toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); + (void)toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); int32_t code = TSDB_CODE_SUCCESS; if (pParCxt->async) { code = getTableMetaFromCache(pCxt->pMetaCache, &name, pMeta); @@ -559,7 +569,7 @@ static int32_t refreshGetTableMeta(STranslateContext* pCxt, const char* pDbName, static int32_t getDBVgInfoImpl(STranslateContext* pCxt, const SName* pName, SArray** pVgInfo) { SParseContext* pParCxt = pCxt->pParseCxt; char fullDbName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pName, fullDbName); + (void)tNameGetFullDbName(pName, fullDbName); int32_t code = collectUseDatabaseImpl(fullDbName, pCxt->pDbs); if (TSDB_CODE_SUCCESS == code) { if (pParCxt->async) { @@ -581,9 +591,10 @@ static int32_t getDBVgInfoImpl(STranslateContext* pCxt, const SName* pName, SArr static int32_t getDBVgInfo(STranslateContext* pCxt, const char* pDbName, SArray** pVgInfo) { SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName)); + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName)); + if (TSDB_CODE_SUCCESS != code) return code; char dbFname[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&name, dbFname); + (void)tNameGetFullDbName(&name, dbFname); return getDBVgInfoImpl(pCxt, &name, pVgInfo); } @@ -642,10 +653,11 @@ static int32_t getDBCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo SParseContext* pParCxt = pCxt->pParseCxt; SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName)); + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName)); + if (TSDB_CODE_SUCCESS != code) return code; char dbFname[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&name, dbFname); - int32_t code = collectUseDatabaseImpl(dbFname, pCxt->pDbs); + (void)tNameGetFullDbName(&name, dbFname); + code = collectUseDatabaseImpl(dbFname, pCxt->pDbs); if (TSDB_CODE_SUCCESS == code) { if (pParCxt->async) { code = getDbCfgFromCache(pCxt->pMetaCache, dbFname, pInfo); @@ -801,7 +813,7 @@ static int32_t resetHighLevelTranslateNamespace(STranslateContext* pCxt) { } for (int32_t i = size - 1; i >= pCxt->currLevel; --i) { - taosArrayDestroy(taosArrayGetP(pCxt->pNsLevel, i)); + (void)taosArrayDestroy(taosArrayGetP(pCxt->pNsLevel, i)); } taosArrayPopTailBatch(pCxt->pNsLevel, levelNum); @@ -818,9 +830,9 @@ static int32_t resetTranslateNamespace(STranslateContext* pCxt) { if (NULL != pCxt->pNsLevel) { size_t size = taosArrayGetSize(pCxt->pNsLevel); for (size_t i = 0; i < size; ++i) { - taosArrayDestroy(taosArrayGetP(pCxt->pNsLevel, i)); + (void)taosArrayDestroy(taosArrayGetP(pCxt->pNsLevel, i)); } - taosArrayDestroy(pCxt->pNsLevel); + (void)taosArrayDestroy(pCxt->pNsLevel); } pCxt->pNsLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); if (NULL == pCxt->pNsLevel) { @@ -833,9 +845,9 @@ static void destroyTranslateContext(STranslateContext* pCxt) { if (NULL != pCxt->pNsLevel) { size_t size = taosArrayGetSize(pCxt->pNsLevel); for (size_t i = 0; i < size; ++i) { - taosArrayDestroy(taosArrayGetP(pCxt->pNsLevel, i)); + (void)taosArrayDestroy(taosArrayGetP(pCxt->pNsLevel, i)); } - taosArrayDestroy(pCxt->pNsLevel); + (void)taosArrayDestroy(pCxt->pNsLevel); } if (NULL != pCxt->pCmdMsg) { @@ -1024,66 +1036,83 @@ static bool isBlockTimeLineAlignedQuery(SNode* pStmt) { return false; } -SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesNum) { +int32_t buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesNum, SNodeList**ppOut) { + *ppOut = NULL; SNodeList* pPartitionList = NULL; SNode* pNode = NULL; if (pOrderList->length <= nodesNum) { - return NULL; + return TSDB_CODE_SUCCESS; } pNode = nodesListGetNode(pOrderList, nodesNum); SOrderByExprNode* pOrder = (SOrderByExprNode*)pNode; if (!isPrimaryKeyImpl(pOrder->pExpr)) { - return NULL; + return TSDB_CODE_SUCCESS; } + int32_t code = 0; for (int32_t i = 0; i < nodesNum; ++i) { pNode = nodesListGetNode(pOrderList, i); pOrder = (SOrderByExprNode*)pNode; - nodesListMakeStrictAppend(&pPartitionList, nodesCloneNode(pOrder->pExpr)); + code = nodesListMakeStrictAppend(&pPartitionList, nodesCloneNode(pOrder->pExpr)); + if (TSDB_CODE_SUCCESS != code) { + break; + } } + if(TSDB_CODE_SUCCESS == code) + *ppOut = pPartitionList; - return pPartitionList; + return code; } -static bool isTimeLineAlignedQuery(SNode* pStmt) { +static int32_t isTimeLineAlignedQuery(SNode* pStmt, bool* pRes) { + int32_t code = TSDB_CODE_SUCCESS; SSelectStmt* pSelect = (SSelectStmt*)pStmt; if (!isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { - return false; + *pRes = false; + return code; } if (QUERY_NODE_SELECT_STMT == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery; if (pSelect->pPartitionByList) { if (!pSub->timeLineFromOrderBy && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) { - return true; + *pRes = true; + return code; } if (pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) { - SNodeList* pPartitionList = - buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length); - bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList); - nodesDestroyList(pPartitionList); + SNodeList* pPartitionList = NULL; + code = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length, &pPartitionList); + if (TSDB_CODE_SUCCESS == code) { + bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList); + nodesDestroyList(pPartitionList); - if (match) { - return true; + if (match) { + *pRes = true; + return code; + } } } } } - if (QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { SSetOperator* pSub = (SSetOperator*)((STempTableNode*)pSelect->pFromTable)->pSubquery; if (pSelect->pPartitionByList && pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) { - SNodeList* pPartitionList = - buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length); - bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList); - nodesDestroyList(pPartitionList); + SNodeList* pPartitionList = NULL; + code = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length, &pPartitionList); + if (TSDB_CODE_SUCCESS == code) { + bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList); + nodesDestroyList(pPartitionList); - if (match) { - return true; + if (match) { + *pRes = true; + return code; + } } } } - return false; + *pRes = false; + return code; } bool isPrimaryKeyImpl(SNode* pExpr) { @@ -1159,16 +1188,17 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p pCol->numOfPKs = pTable->pMeta->tableInfo.numOfPKs; } -static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColumnNode** pColRef) { +static int32_t setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColumnNode** pColRef) { SColumnNode* pCol = *pColRef; if (NULL == pExpr->pAssociation) { pExpr->pAssociation = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SAssociationNode)); + if (!pExpr->pAssociation) return TSDB_CODE_OUT_OF_MEMORY; } SAssociationNode assNode; assNode.pPlace = (SNode**)pColRef; assNode.pAssociationNode = (SNode*)*pColRef; - taosArrayPush(pExpr->pAssociation, &assNode); + (void)taosArrayPush(pExpr->pAssociation, &assNode); strcpy(pCol->tableAlias, pTable->table.tableAlias); pCol->isPrimTs = isPrimaryKeyImpl((SNode*)pExpr); @@ -1186,6 +1216,7 @@ static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColum strcpy(pCol->node.userAlias, pExpr->userAlias); } pCol->node.resType = pExpr->resType; + return TSDB_CODE_SUCCESS; } static void setColumnPrimTs(STranslateContext* pCxt, SColumnNode* pCol, const STableNode* pTable) { @@ -1230,6 +1261,7 @@ static void setColumnPrimTs(STranslateContext* pCxt, SColumnNode* pCol, const ST } static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, bool igTags, SNodeList* pList) { + int32_t code = 0; if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; int32_t nums = pMeta->tableInfo.numOfColumns + @@ -1245,7 +1277,7 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p } setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i - pMeta->tableInfo.numOfColumns), pCol); setColumnPrimTs(pCxt, pCol, pTable); - nodesListAppend(pList, (SNode*)pCol); + code = nodesListStrictAppend(pList, (SNode*)pCol); } } else { STempTableNode* pTempTable = (STempTableNode*)pTable; @@ -1256,12 +1288,15 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p if (NULL == pCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY); } - nodesListAppend(pList, (SNode*)pCol); - SListCell* pCell = nodesListGetCell(pList, LIST_LENGTH(pList) - 1); - setColumnInfoByExpr(pTempTable, (SExprNode*)pNode, (SColumnNode**)&pCell->pNode); + code = nodesListStrictAppend(pList, (SNode*)pCol); + if (TSDB_CODE_SUCCESS == code) { + SListCell* pCell = nodesListGetCell(pList, LIST_LENGTH(pList) - 1); + code = setColumnInfoByExpr(pTempTable, (SExprNode*)pNode, (SColumnNode**)&pCell->pNode); + } + if (TSDB_CODE_SUCCESS != code) break; } } - return TSDB_CODE_SUCCESS; + return code; } static bool isInternalPrimaryKey(const SColumnNode* pCol) { @@ -1294,6 +1329,7 @@ static int32_t findAndSetColumn(STranslateContext* pCxt, SColumnNode** pColRef, } } + int32_t code = 0; if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; if (isInternalPrimaryKey(pCol)) { @@ -1326,16 +1362,20 @@ static int32_t findAndSetColumn(STranslateContext* pCxt, SColumnNode** pColRef, if (*pFound) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AMBIGUOUS_COLUMN, pCol->colName); } - setColumnInfoByExpr(pTempTable, pExpr, pColRef); + code = setColumnInfoByExpr(pTempTable, pExpr, pColRef); + if (TSDB_CODE_SUCCESS != code) { + break; + } *pFound = true; } else if (isPrimaryKeyImpl(pNode) && isInternalPrimaryKey(pCol)) { - setColumnInfoByExpr(pTempTable, pExpr, pColRef); + code = setColumnInfoByExpr(pTempTable, pExpr, pColRef); + if (TSDB_CODE_SUCCESS != code) break; pCol->isPrimTs = true; *pFound = true; } } } - return TSDB_CODE_SUCCESS; + return code; } static EDealRes translateColumnWithPrefix(STranslateContext* pCxt, SColumnNode** pCol) { @@ -1441,11 +1481,17 @@ static void biMakeAliasNameInMD5(char* pExprStr, int32_t len, char* pAlias) { } } -static SNode* biMakeTbnameProjectAstNode(char* funcName, char* tableAlias) { +static int32_t biMakeTbnameProjectAstNode(char* funcName, char* tableAlias, SNode** pOutNode) { + int32_t code = 0; SValueNode* valNode = NULL; if (tableAlias != NULL) { SValueNode* n = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (!n) return TSDB_CODE_OUT_OF_MEMORY; n->literal = tstrdup(tableAlias); + if (!n->literal) { + nodesDestroyNode((SNode*)n); + return TSDB_CODE_OUT_OF_MEMORY; + } n->node.resType.type = TSDB_DATA_TYPE_BINARY; n->node.resType.bytes = strlen(n->literal); n->translate = false; @@ -1453,39 +1499,63 @@ static SNode* biMakeTbnameProjectAstNode(char* funcName, char* tableAlias) { } SFunctionNode* tbNameFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - tstrncpy(tbNameFunc->functionName, "tbname", TSDB_FUNC_NAME_LEN); - if (valNode != NULL) { - nodesListMakeAppend(&tbNameFunc->pParameterList, (SNode*)valNode); + if (!tbNameFunc) { + nodesDestroyNode((SNode*)valNode); + valNode = NULL; + code = TSDB_CODE_OUT_OF_MEMORY; } - snprintf(tbNameFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), (tableAlias) ? "%s.tbname" : "%stbname", - (tableAlias) ? tableAlias : ""); - strncpy(tbNameFunc->node.aliasName, tbNameFunc->functionName, TSDB_COL_NAME_LEN); - - if (funcName == NULL) { - return (SNode*)tbNameFunc; - } else { - SFunctionNode* multiResFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - tstrncpy(multiResFunc->functionName, funcName, TSDB_FUNC_NAME_LEN); - nodesListMakeAppend(&multiResFunc->pParameterList, (SNode*)tbNameFunc); - - if (tsKeepColumnName) { - snprintf(multiResFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), - (tableAlias) ? "%s.tbname" : "%stbname", (tableAlias) ? tableAlias : ""); - strcpy(multiResFunc->node.aliasName, tbNameFunc->functionName); - } else { - snprintf(multiResFunc->node.userAlias, sizeof(multiResFunc->node.userAlias), - tableAlias ? "%s(%s.tbname)" : "%s(%stbname)", funcName, tableAlias ? tableAlias : ""); - biMakeAliasNameInMD5(multiResFunc->node.userAlias, strlen(multiResFunc->node.userAlias), - multiResFunc->node.aliasName); + if (TSDB_CODE_SUCCESS == code) { + tstrncpy(tbNameFunc->functionName, "tbname", TSDB_FUNC_NAME_LEN); + if (valNode != NULL) { + code = nodesListMakeStrictAppend(&tbNameFunc->pParameterList, (SNode*)valNode); + valNode = NULL; } - - return (SNode*)multiResFunc; } + if (TSDB_CODE_SUCCESS == code) { + snprintf(tbNameFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), (tableAlias) ? "%s.tbname" : "%stbname", + (tableAlias) ? tableAlias : ""); + strncpy(tbNameFunc->node.aliasName, tbNameFunc->functionName, TSDB_COL_NAME_LEN); + if (funcName == NULL) { + *pOutNode = (SNode*)tbNameFunc; + return code; + } else { + SFunctionNode* multiResFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (!multiResFunc) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + if (TSDB_CODE_SUCCESS == code) { + tstrncpy(multiResFunc->functionName, funcName, TSDB_FUNC_NAME_LEN); + code = nodesListMakeStrictAppend(&multiResFunc->pParameterList, (SNode*)tbNameFunc); + if (TSDB_CODE_SUCCESS != code) tbNameFunc = NULL; + } + + if (TSDB_CODE_SUCCESS == code) { + if (tsKeepColumnName) { + snprintf(multiResFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), + (tableAlias) ? "%s.tbname" : "%stbname", (tableAlias) ? tableAlias : ""); + strcpy(multiResFunc->node.aliasName, tbNameFunc->functionName); + } else { + snprintf(multiResFunc->node.userAlias, sizeof(multiResFunc->node.userAlias), + tableAlias ? "%s(%s.tbname)" : "%s(%stbname)", funcName, tableAlias ? tableAlias : ""); + biMakeAliasNameInMD5(multiResFunc->node.userAlias, strlen(multiResFunc->node.userAlias), + multiResFunc->node.aliasName); + } + *pOutNode = (SNode*)multiResFunc; + } else { + nodesDestroyNode((SNode*)multiResFunc); + } + } + } + if (TSDB_CODE_SUCCESS != code) nodesDestroyNode((SNode*)tbNameFunc); + return code; } static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt* pSelect, SNode* pNode, SListCell* pSelectListCell) { SNodeList* pTbnameNodeList = nodesMakeList(); + if (!pTbnameNodeList) return TSDB_CODE_OUT_OF_MEMORY; + int32_t code = 0; SFunctionNode* pFunc = (SFunctionNode*)pNode; if (strcasecmp(pFunc->functionName, "last") == 0 || strcasecmp(pFunc->functionName, "last_row") == 0 || @@ -1496,16 +1566,19 @@ static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt if (nodesIsStar(pPara)) { SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); size_t n = taosArrayGetSize(pTables); - for (int32_t i = 0; i < n; ++i) { + for (int32_t i = 0; i < n && TSDB_CODE_SUCCESS == code; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); if (nodeType(pTable) == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)pTable)->pMeta != NULL && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { - SNode* pTbnameNode = biMakeTbnameProjectAstNode(pFunc->functionName, NULL); - nodesListAppend(pTbnameNodeList, pTbnameNode); + SNode* pTbnameNode = NULL; + code = biMakeTbnameProjectAstNode(pFunc->functionName, NULL, &pTbnameNode); + if (TSDB_CODE_SUCCESS == code) + code = nodesListStrictAppend(pTbnameNodeList, pTbnameNode); } } - if (LIST_LENGTH(pTbnameNodeList) > 0) { + if (TSDB_CODE_SUCCESS == code && LIST_LENGTH(pTbnameNodeList) > 0) { nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList); + pTbnameNodeList = NULL; } } else if (nodesIsTableStar(pPara)) { char* pTableAlias = ((SColumnNode*)pPara)->tableAlias; @@ -1514,37 +1587,47 @@ static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt if (TSDB_CODE_SUCCESS == code && nodeType(pTable) == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)pTable)->pMeta != NULL && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { - SNode* pTbnameNode = biMakeTbnameProjectAstNode(pFunc->functionName, pTableAlias); - nodesListAppend(pTbnameNodeList, pTbnameNode); + SNode* pTbnameNode = NULL; + code = biMakeTbnameProjectAstNode(pFunc->functionName, pTableAlias, &pTbnameNode); + if (TSDB_CODE_SUCCESS == code) + code = nodesListStrictAppend(pTbnameNodeList, pTbnameNode); } - if (LIST_LENGTH(pTbnameNodeList) > 0) { + if (TSDB_CODE_SUCCESS == code && LIST_LENGTH(pTbnameNodeList) > 0) { nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList); + pTbnameNodeList = NULL; } } + if (TSDB_CODE_SUCCESS != code) break; } } - return TSDB_CODE_SUCCESS; + nodesDestroyList(pTbnameNodeList); + return code; } // after translate from // before translate select list int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { + int32_t code = TSDB_CODE_SUCCESS; SNode* pNode = NULL; SNodeList* pTbnameNodeList = nodesMakeList(); + if (!pTbnameNodeList) return TSDB_CODE_OUT_OF_MEMORY; WHERE_EACH(pNode, pSelect->pProjectionList) { if (nodesIsStar(pNode)) { SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); size_t n = taosArrayGetSize(pTables); - for (int32_t i = 0; i < n; ++i) { + for (int32_t i = 0; i < n && TSDB_CODE_SUCCESS == code; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); if (nodeType(pTable) == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)pTable)->pMeta != NULL && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { - SNode* pTbnameNode = biMakeTbnameProjectAstNode(NULL, NULL); - nodesListAppend(pTbnameNodeList, pTbnameNode); + SNode* pTbnameNode = NULL; + code = biMakeTbnameProjectAstNode(NULL, NULL, &pTbnameNode); + if (TSDB_CODE_SUCCESS == code) + code = nodesListStrictAppend(pTbnameNodeList, pTbnameNode); } } - if (LIST_LENGTH(pTbnameNodeList) > 0) { + if (TSDB_CODE_SUCCESS == code && LIST_LENGTH(pTbnameNodeList) > 0) { nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList); + pTbnameNodeList = NULL; } } else if (nodesIsTableStar(pNode)) { char* pTableAlias = ((SColumnNode*)pNode)->tableAlias; @@ -1552,38 +1635,46 @@ int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { int32_t code = findTable(pCxt, pTableAlias, &pTable); if (TSDB_CODE_SUCCESS == code && nodeType(pTable) == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)pTable)->pMeta != NULL && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { - SNode* pTbnameNode = biMakeTbnameProjectAstNode(NULL, pTableAlias); - nodesListAppend(pTbnameNodeList, pTbnameNode); + SNode* pTbnameNode = NULL; + code = biMakeTbnameProjectAstNode(NULL, pTableAlias, &pTbnameNode); + if (TSDB_CODE_SUCCESS ==code) + code = nodesListStrictAppend(pTbnameNodeList, pTbnameNode); } - if (LIST_LENGTH(pTbnameNodeList) > 0) { + if (TSDB_CODE_SUCCESS == code && LIST_LENGTH(pTbnameNodeList) > 0) { nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList); + pTbnameNodeList = NULL; } } else if (nodeType(pNode) == QUERY_NODE_FUNCTION) { - biRewriteSelectFuncParamStar(pCxt, pSelect, pNode, cell); + code = biRewriteSelectFuncParamStar(pCxt, pSelect, pNode, cell); } + if (TSDB_CODE_SUCCESS != code) break; WHERE_NEXT; } + nodesDestroyList(pTbnameNodeList); - return TSDB_CODE_SUCCESS; + return code; } -bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) { +int32_t biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRet) { SColumnNode* pCol = (SColumnNode*)(*ppNode); if ((strcasecmp(pCol->colName, "tbname") == 0) && ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable && QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { SFunctionNode* tbnameFuncNode = NULL; - tbnameFuncNode = - (SFunctionNode*)biMakeTbnameProjectAstNode(NULL, (pCol->tableAlias[0] != '\0') ? pCol->tableAlias : NULL); + int32_t code = biMakeTbnameProjectAstNode(NULL, (pCol->tableAlias[0] != '\0') ? pCol->tableAlias : NULL, + (SNode**)&tbnameFuncNode); + if (TSDB_CODE_SUCCESS != code) return code; tbnameFuncNode->node.resType = pCol->node.resType; strcpy(tbnameFuncNode->node.aliasName, pCol->node.aliasName); strcpy(tbnameFuncNode->node.userAlias, pCol->node.userAlias); nodesDestroyNode(*ppNode); *ppNode = (SNode*)tbnameFuncNode; - return true; + *pRet = true; + return TSDB_CODE_SUCCESS; } - return false; + *pRet = false; + return TSDB_CODE_SUCCESS; } int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) { @@ -1625,7 +1716,10 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { if (pCxt->pParseCxt->biMode) { SNode** ppNode = (SNode**)pCol; - if (biRewriteToTbnameFunc(pCxt, ppNode)) { + bool ret; + pCxt->errCode = biRewriteToTbnameFunc(pCxt, ppNode, &ret); + if (TSDB_CODE_SUCCESS != pCxt->errCode) return DEAL_RES_ERROR; + if (ret) { return translateFunction(pCxt, (SFunctionNode**)ppNode); } } @@ -2034,9 +2128,9 @@ static int32_t rewriteCountStarAsCount1(STranslateContext* pCxt, SFunctionNode* pVal->node.resType.type = TSDB_DATA_TYPE_INT; pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; const int32_t val = 1; - nodesSetValueNodeValue(pVal, (void*)&val); + (void)nodesSetValueNodeValue(pVal, (void*)&val); pVal->translate = true; - nodesListErase(pCount->pParameterList, nodesListGetCell(pCount->pParameterList, 0)); + (void)nodesListErase(pCount->pParameterList, nodesListGetCell(pCount->pParameterList, 0)); code = nodesListAppend(pCount->pParameterList, (SNode*)pVal); return code; } @@ -2292,7 +2386,7 @@ static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode** if (code != TSDB_CODE_SUCCESS) { return code; } - translateColumn(pCxt, (SColumnNode**)ppNode); + (void)translateColumn(pCxt, (SColumnNode**)ppNode); return pCxt->errCode; } return TSDB_CODE_SUCCESS; @@ -2307,11 +2401,17 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu "%s function must be used in select statements", pFunc->functionName); } SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; + bool isTimelineAlignedQuery = false; if ((NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && - !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) && - !isTimeLineAlignedQuery(pCxt->pCurrStmt)) || - (NULL != pSelect->pFromTable && QUERY_NODE_JOIN_TABLE == nodeType(pSelect->pFromTable) && - (TIME_LINE_GLOBAL != pSelect->timeLineCurMode && TIME_LINE_MULTI != pSelect->timeLineCurMode))) { + !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery))) { + int32_t code = isTimeLineAlignedQuery(pCxt->pCurrStmt, &isTimelineAlignedQuery); + if (TSDB_CODE_SUCCESS != code) return code; + if (!isTimelineAlignedQuery) + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "%s function requires valid time series input", pFunc->functionName); + } + if (NULL != pSelect->pFromTable && QUERY_NODE_JOIN_TABLE == nodeType(pSelect->pFromTable) && + (TIME_LINE_GLOBAL != pSelect->timeLineCurMode && TIME_LINE_MULTI != pSelect->timeLineCurMode)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "%s function requires valid time series input", pFunc->functionName); } @@ -2494,7 +2594,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { } } -static int32_t rewriteFuncToValue(STranslateContext* pCxt, char* pLiteral, SNode** pNode) { +static int32_t rewriteFuncToValue(STranslateContext* pCxt, char** pLiteral, SNode** pNode) { SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == pVal) { return TSDB_CODE_OUT_OF_MEMORY; @@ -2502,13 +2602,14 @@ static int32_t rewriteFuncToValue(STranslateContext* pCxt, char* pLiteral, SNode strcpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName); strcpy(pVal->node.userAlias, ((SExprNode*)*pNode)->userAlias); pVal->node.resType = ((SExprNode*)*pNode)->resType; - if (NULL == pLiteral) { + if (NULL == pLiteral || NULL == *pLiteral) { pVal->isNull = true; } else { - pVal->literal = pLiteral; + pVal->literal = *pLiteral; if (IS_VAR_DATA_TYPE(pVal->node.resType.type)) { - pVal->node.resType.bytes = strlen(pLiteral); + pVal->node.resType.bytes = strlen(*pLiteral); } + *pLiteral = NULL; } if (DEAL_RES_ERROR != translateValue(pCxt, pVal)) { nodesDestroyNode(*pNode); @@ -2527,7 +2628,9 @@ static int32_t rewriteDatabaseFunc(STranslateContext* pCxt, SNode** pNode) { return TSDB_CODE_OUT_OF_MEMORY; } } - return rewriteFuncToValue(pCxt, pCurrDb, pNode); + int32_t code = rewriteFuncToValue(pCxt, &pCurrDb, pNode); + if (TSDB_CODE_SUCCESS != code) taosMemoryFree(pCurrDb); + return code; } static int32_t rewriteClentVersionFunc(STranslateContext* pCxt, SNode** pNode) { @@ -2535,7 +2638,9 @@ static int32_t rewriteClentVersionFunc(STranslateContext* pCxt, SNode** pNode) { if (NULL == pVer) { return TSDB_CODE_OUT_OF_MEMORY; } - return rewriteFuncToValue(pCxt, pVer, pNode); + int32_t code = rewriteFuncToValue(pCxt, &pVer, pNode); + if (TSDB_CODE_SUCCESS != code) taosMemoryFree(pVer); + return code; } static int32_t rewriteServerVersionFunc(STranslateContext* pCxt, SNode** pNode) { @@ -2543,7 +2648,9 @@ static int32_t rewriteServerVersionFunc(STranslateContext* pCxt, SNode** pNode) if (NULL == pVer) { return TSDB_CODE_OUT_OF_MEMORY; } - return rewriteFuncToValue(pCxt, pVer, pNode); + int32_t code = rewriteFuncToValue(pCxt, &pVer, pNode); + if (TSDB_CODE_SUCCESS != code) taosMemoryFree(pVer); + return code; } static int32_t rewriteServerStatusFunc(STranslateContext* pCxt, SNode** pNode) { @@ -2551,18 +2658,26 @@ static int32_t rewriteServerStatusFunc(STranslateContext* pCxt, SNode** pNode) { return TSDB_CODE_RPC_NETWORK_UNAVAIL; } char* pStatus = taosStrdup((void*)"1"); - return rewriteFuncToValue(pCxt, pStatus, pNode); + int32_t code = rewriteFuncToValue(pCxt, &pStatus, pNode); + if (TSDB_CODE_SUCCESS != code) taosMemoryFree(pStatus); + return code; } static int32_t rewriteUserFunc(STranslateContext* pCxt, SNode** pNode) { char userConn[TSDB_USER_LEN + 1 + TSDB_FQDN_LEN] = {0}; // format 'user@host' int32_t len = snprintf(userConn, sizeof(userConn), "%s@", pCxt->pParseCxt->pUser); - taosGetFqdn(userConn + len); + if (TSDB_CODE_SUCCESS != taosGetFqdn(userConn + len)) { + return terrno; + } char* pUserConn = taosStrdup((void*)userConn); if (NULL == pUserConn) { return TSDB_CODE_OUT_OF_MEMORY; } - return rewriteFuncToValue(pCxt, pUserConn, pNode); + int32_t code = rewriteFuncToValue(pCxt, &pUserConn, pNode); + if (TSDB_CODE_SUCCESS != code) { + taosMemoryFree(pUserConn); + } + return code; } static int32_t rewriteSystemInfoFunc(STranslateContext* pCxt, SNode** pNode) { @@ -2622,7 +2737,7 @@ static int32_t rewriteToColumnAndRetranslate(STranslateContext* pCxt, SNode** pp if (code != TSDB_CODE_SUCCESS) { return code; } - translateColumn(pCxt, (SColumnNode**)ppNode); + (void)translateColumn(pCxt, (SColumnNode**)ppNode); if (pCxt->errCode != TSDB_CODE_SUCCESS) { return generateSyntaxErrMsg(&pCxt->msgBuf, errCode); } else { @@ -2750,7 +2865,9 @@ static int32_t rewriteQueryTimeFunc(STranslateContext* pCxt, int64_t val, SNode* return TSDB_CODE_OUT_OF_MEMORY; } snprintf(pStr, 20, "%" PRId64 "", val); - return rewriteFuncToValue(pCxt, pStr, pNode); + int32_t code = rewriteFuncToValue(pCxt, &pStr, pNode); + if (TSDB_CODE_SUCCESS != code) taosMemoryFree(pStr); + return code; } static int32_t rewriteQstartFunc(STranslateContext* pCxt, SNode** pNode) { @@ -3389,12 +3506,17 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { static EDealRes doCheckGetAggColCoexist(SNode** pNode, void* pContext) { CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; + int32_t code = 0; if (isVectorFunc(*pNode)) { return DEAL_RES_IGNORE_CHILD; } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { pCxt->existCol = true; - nodesListMakeStrictAppend(&pCxt->pColList, *pNode); + code = nodesListMakeStrictAppend(&pCxt->pColList, *pNode); + if (TSDB_CODE_SUCCESS != code) { + pCxt->pTranslateCxt->errCode = code; + return DEAL_RES_ERROR; + } } return DEAL_RES_CONTINUE; } @@ -3563,7 +3685,9 @@ static int32_t addMnodeToVgroupList(const SEpSet* pEpSet, SArray** pVgroupList) } SVgroupInfo vg = {.vgId = MNODE_HANDLE}; memcpy(&vg.epSet, pEpSet, sizeof(SEpSet)); - taosArrayPush(*pVgroupList, &vg); + if (NULL == taosArrayPush(*pVgroupList, &vg)) { + return TSDB_CODE_OUT_OF_MEMORY; + } return TSDB_CODE_SUCCESS; } @@ -3616,7 +3740,9 @@ static int32_t getVnodeSysTableVgroupListImpl(STranslateContext* pCxt, SName* pT if (NULL == *pVgroupList) { code = TSDB_CODE_OUT_OF_MEMORY; } else { - taosArrayPush(*pVgroupList, &vgInfo); + if (NULL == taosArrayPush(*pVgroupList, &vgInfo)) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } } else if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || TSDB_CODE_MND_DB_IN_DROPPING == code) { @@ -3659,7 +3785,7 @@ static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, if (TSDB_CODE_SUCCESS == code) { code = toVgroupsInfo(pVgs, &pRealTable->pVgroupList); } - taosArrayDestroy(pVgs); + (void)taosArrayDestroy(pVgs); return code; } @@ -3670,7 +3796,7 @@ static int32_t setDnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, if (TSDB_CODE_SUCCESS == code) { code = dnodeToVgroupsInfo(pDnodes, &pRealTable->pVgroupList); } - taosArrayDestroy(pDnodes); + (void)taosArrayDestroy(pDnodes); return code; } @@ -3690,7 +3816,7 @@ static int32_t setSuperTableVgroupList(STranslateContext* pCxt, SName* pName, SR if (TSDB_CODE_SUCCESS == code) { code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList); } - taosArrayDestroy(vgroupList); + (void)taosArrayDestroy(vgroupList); return code; } @@ -3785,15 +3911,16 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo for (int32_t i = 0; i < pRealTable->pTsmas->size; ++i) { STableTSMAInfo* pTsma = taosArrayGetP(pRealTable->pTsmas, i); SName tsmaTargetTbName = {0}; - toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); + SVgroupInfo vgInfo = {0}; + bool exists = false; + (void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); int32_t len = snprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); strncpy(tsmaTargetTbName.tname, buf, MD5_OUTPUT_LEN); - collectUseTable(&tsmaTargetTbName, pCxt->pTargetTables); - SVgroupInfo vgInfo = {0}; - bool exists = false; - code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &tsmaTargetTbName, &vgInfo, &exists); + code = collectUseTable(&tsmaTargetTbName, pCxt->pTargetTables); + if (TSDB_CODE_SUCCESS == code) + code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &tsmaTargetTbName, &vgInfo, &exists); if (TSDB_CODE_SUCCESS == code) { ASSERT(exists); if (!pRealTable->tsmaTargetTbVgInfo) { @@ -3810,7 +3937,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo } pVgpsInfo->numOfVgroups = 1; pVgpsInfo->vgroups[0] = vgInfo; - taosArrayPush(pRealTable->tsmaTargetTbVgInfo, &pVgpsInfo); + (void)taosArrayPush(pRealTable->tsmaTargetTbVgInfo, &pVgpsInfo); } else { break; } @@ -3839,7 +3966,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo // ignore table not exists error code = TSDB_CODE_SUCCESS; } - taosArrayPush(pRealTable->tsmaTargetTbInfo, &ctbInfo); + (void)taosArrayPush(pRealTable->tsmaTargetTbInfo, &ctbInfo); } } } @@ -4449,9 +4576,10 @@ int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinPare if (TSDB_VIEW_TABLE == pRealTable->pMeta->tableType && !pCurrSmt->tagScan) { return translateView(pCxt, pTable, &name); } - translateAudit(pCxt, pRealTable, &name); + code = translateAudit(pCxt, pRealTable, &name); #endif - code = setTableVgroupList(pCxt, &name, pRealTable); + if (TSDB_CODE_SUCCESS == code) + code = setTableVgroupList(pCxt, &name, pRealTable); if (TSDB_CODE_SUCCESS == code) { code = setTableIndex(pCxt, &name, pRealTable); } @@ -4579,7 +4707,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { strcpy(pFunc->node.aliasName, pCol->colName); } else { len = snprintf(buf, sizeof(buf) - 1, "%s(%s.%s)", pSrcFunc->functionName, pCol->tableAlias, pCol->colName); - taosCreateMD5Hash(buf, len); + (void)taosCreateMD5Hash(buf, len); strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); len = snprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pCol->colName); // note: userAlias could be truncated here @@ -4587,7 +4715,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { } } else { len = snprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); - taosCreateMD5Hash(buf, len); + (void)taosCreateMD5Hash(buf, len); strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); len = snprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); // note: userAlias could be truncated here @@ -4906,21 +5034,24 @@ static int32_t rewriteProjectAlias(SNodeList* pProjectionList) { static int32_t checkProjectAlias(STranslateContext* pCxt, SNodeList* pProjectionList, SHashObj** pOutput) { SHashObj* pUserAliasSet = taosHashInit(LIST_LENGTH(pProjectionList), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (!pUserAliasSet) return TSDB_CODE_OUT_OF_MEMORY; SNode* pProject = NULL; + int32_t code = TSDB_CODE_SUCCESS; FOREACH(pProject, pProjectionList) { SExprNode* pExpr = (SExprNode*)pProject; if (NULL != taosHashGet(pUserAliasSet, pExpr->userAlias, strlen(pExpr->userAlias))) { taosHashCleanup(pUserAliasSet); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AMBIGUOUS_COLUMN, pExpr->userAlias); } - taosHashPut(pUserAliasSet, pExpr->userAlias, strlen(pExpr->userAlias), &pExpr, POINTER_BYTES); + code = taosHashPut(pUserAliasSet, pExpr->userAlias, strlen(pExpr->userAlias), &pExpr, POINTER_BYTES); + if (TSDB_CODE_SUCCESS != code) break; } - if (NULL == pOutput) { - taosHashCleanup(pUserAliasSet); - } else { + if (TSDB_CODE_SUCCESS == code && NULL != pOutput) { *pOutput = pUserAliasSet; + } else { + taosHashCleanup(pUserAliasSet); } - return TSDB_CODE_SUCCESS; + return code; } static int32_t translateProjectionList(STranslateContext* pCxt, SSelectStmt* pSelect) { @@ -5009,14 +5140,15 @@ static int32_t getQueryTimeRange(STranslateContext* pCxt, SNode* pWhere, STimeWi } SNode* pPrimaryKeyCond = NULL; - filterPartitionCond(&pCond, &pPrimaryKeyCond, NULL, NULL, NULL); + int32_t code = filterPartitionCond(&pCond, &pPrimaryKeyCond, NULL, NULL, NULL); - int32_t code = TSDB_CODE_SUCCESS; - if (NULL != pPrimaryKeyCond) { - bool isStrict = false; - code = getTimeRange(&pPrimaryKeyCond, pTimeRange, &isStrict); - } else { - *pTimeRange = TSWINDOW_INITIALIZER; + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pPrimaryKeyCond) { + bool isStrict = false; + code = getTimeRange(&pPrimaryKeyCond, pTimeRange, &isStrict); + } else { + *pTimeRange = TSWINDOW_INITIALIZER; + } } nodesDestroyNode(pCond); nodesDestroyNode(pPrimaryKeyCond); @@ -5325,7 +5457,9 @@ static EDealRes collectWindowsPseudocolumns(SNode* pNode, void* pContext) { SFunctionNode* pFunc = (SFunctionNode*)pNode; if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType || FUNCTION_TYPE_WDURATION == pFunc->funcType) { - nodesListStrictAppend(pCols, nodesCloneNode(pNode)); + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pCols, nodesCloneNode(pNode))) { + return DEAL_RES_ERROR; + } } } return DEAL_RES_CONTINUE; @@ -5363,26 +5497,37 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType == TSDB_SYSTEM_TABLE) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, "WINDOW"); } - if (QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow) && - ((NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && - !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) && - !isTimeLineAlignedQuery(pCxt->pCurrStmt)) || - (NULL != pSelect->pFromTable && QUERY_NODE_JOIN_TABLE == nodeType(pSelect->pFromTable) && - (TIME_LINE_GLOBAL != pSelect->timeLineCurMode && TIME_LINE_MULTI != pSelect->timeLineCurMode)))) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY); + int32_t code = 0; + if (QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow)) { + if (NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && + !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + bool isTimelineAlignedQuery = false; + code = isTimeLineAlignedQuery(pCxt->pCurrStmt, &isTimelineAlignedQuery); + if (TSDB_CODE_SUCCESS != code) return code; + if (!isTimelineAlignedQuery) return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY); + } + if (NULL != pSelect->pFromTable && QUERY_NODE_JOIN_TABLE == nodeType(pSelect->pFromTable) && + (TIME_LINE_GLOBAL != pSelect->timeLineCurMode && TIME_LINE_MULTI != pSelect->timeLineCurMode)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY); + } } - if (QUERY_NODE_INTERVAL_WINDOW == nodeType(pSelect->pWindow) && - ((NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && - !isBlockTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) && - !isTimeLineAlignedQuery(pCxt->pCurrStmt)) || - (NULL != pSelect->pFromTable && QUERY_NODE_JOIN_TABLE == nodeType(pSelect->pFromTable) && - (TIME_LINE_NONE == pSelect->timeLineCurMode)))) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY); + if (QUERY_NODE_INTERVAL_WINDOW == nodeType(pSelect->pWindow)) { + if (NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && + !isBlockTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + bool isTimelineAlignedQuery = false; + code = isTimeLineAlignedQuery(pCxt->pCurrStmt, &isTimelineAlignedQuery); + if (TSDB_CODE_SUCCESS != code) return code; + if (!isTimelineAlignedQuery) return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY); + } + if (NULL != pSelect->pFromTable && QUERY_NODE_JOIN_TABLE == nodeType(pSelect->pFromTable) && + (TIME_LINE_NONE == pSelect->timeLineCurMode)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY); + } } pCxt->currClause = SQL_CLAUSE_WINDOW; - int32_t code = translateExpr(pCxt, &pSelect->pWindow); + code = translateExpr(pCxt, &pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { code = translateSpecificWindow(pCxt, pSelect); } @@ -5576,9 +5721,12 @@ typedef struct SEqCondTbNameTableInfo { } SEqCondTbNameTableInfo; //[tableAlias.]tbname = tbNamVal -static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, - SArray** ppTabNames) { - if (pOperator->opType != OP_TYPE_EQUAL) return false; +static int32_t isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, + SArray** ppTabNames, bool* pRet) { + if (pOperator->opType != OP_TYPE_EQUAL) { + *pRet = false; + return TSDB_CODE_SUCCESS; + } SFunctionNode* pTbnameFunc = NULL; SValueNode* pValueNode = NULL; if (nodeType(pOperator->pLeft) == QUERY_NODE_FUNCTION && @@ -5592,7 +5740,8 @@ static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOper pTbnameFunc = (SFunctionNode*)pOperator->pRight; pValueNode = (SValueNode*)pOperator->pLeft; } else { - return false; + *pRet = false; + return TSDB_CODE_SUCCESS; } if (LIST_LENGTH(pTbnameFunc->pParameterList) == 0) { @@ -5603,21 +5752,25 @@ static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOper SValueNode* pQualValNode = (SValueNode*)pQualNode; *ppTableAlias = pQualValNode->literal; } else { - return false; + *pRet = false; + return TSDB_CODE_SUCCESS; } *ppTabNames = taosArrayInit(1, sizeof(void*)); - taosArrayPush(*ppTabNames, &(pValueNode->literal)); - return true; + if (!*ppTabNames) return TSDB_CODE_OUT_OF_MEMORY; + (void)taosArrayPush(*ppTabNames, &(pValueNode->literal)); + *pRet = true; + return TSDB_CODE_SUCCESS; } //[tableAlias.]tbname in (value1, value2, ...) -static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, - SArray** ppTbNames) { +static int32_t isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, + SArray** ppTbNames, bool* pRet) { if (pOperator->opType != OP_TYPE_IN) return false; if (nodeType(pOperator->pLeft) != QUERY_NODE_FUNCTION || ((SFunctionNode*)(pOperator->pLeft))->funcType != FUNCTION_TYPE_TBNAME || nodeType(pOperator->pRight) != QUERY_NODE_NODE_LIST) { - return false; + *pRet = false; + return TSDB_CODE_SUCCESS; } SFunctionNode* pTbnameFunc = (SFunctionNode*)pOperator->pLeft; @@ -5629,27 +5782,36 @@ static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOper SValueNode* pQualValNode = (SValueNode*)pQualNode; *ppTableAlias = pQualValNode->literal; } else { - return false; + *pRet = false; + return TSDB_CODE_SUCCESS; } - *ppTbNames = taosArrayInit(1, sizeof(void*)); SNodeListNode* pValueListNode = (SNodeListNode*)pOperator->pRight; + *ppTbNames = taosArrayInit(LIST_LENGTH(pValueListNode->pNodeList), sizeof(void*)); + if (!*ppTbNames) return TSDB_CODE_OUT_OF_MEMORY; SNodeList* pValueNodeList = pValueListNode->pNodeList; SNode* pValNode = NULL; FOREACH(pValNode, pValueNodeList) { if (nodeType(pValNode) != QUERY_NODE_VALUE) { - return false; + *pRet = false; + return TSDB_CODE_SUCCESS; } - taosArrayPush(*ppTbNames, &((SValueNode*)pValNode)->literal); + (void)taosArrayPush(*ppTbNames, &((SValueNode*)pValNode)->literal); } - return true; + *pRet = true; + return TSDB_CODE_SUCCESS; } -static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SEqCondTbNameTableInfo* pInfo) { +static int32_t findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SEqCondTbNameTableInfo* pInfo, bool* pRet) { int32_t code = TSDB_CODE_SUCCESS; char* pTableAlias = NULL; char* pTbNameVal = NULL; - if (isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames) || - isOperatorTbnameInCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames)) { + bool eqTbnameCond = false, tbnameInCond = false; + code = isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames, &eqTbnameCond); + if (TSDB_CODE_SUCCESS == code) { + code = isOperatorTbnameInCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames, &tbnameInCond); + } + if (TSDB_CODE_SUCCESS != code) return code; + if (eqTbnameCond || tbnameInCond) { STableNode* pTable; if (pTableAlias == NULL) { pTable = (STableNode*)((SSelectStmt*)(pCxt->pCurrStmt))->pFromTable; @@ -5659,12 +5821,14 @@ static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWher if (code == TSDB_CODE_SUCCESS && nodeType(pTable) == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)pTable)->pMeta && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { pInfo->pRealTable = (SRealTableNode*)pTable; - return true; + *pRet = true; + return TSDB_CODE_SUCCESS; } - taosArrayDestroy(pInfo->aTbnames); + (void)taosArrayDestroy(pInfo->aTbnames); pInfo->aTbnames = NULL; } - return false; + *pRet = false; + return TSDB_CODE_SUCCESS; } static bool isTableExistInTableTbnames(SArray* aTableTbNames, SRealTableNode* pTable) { @@ -5677,103 +5841,127 @@ static bool isTableExistInTableTbnames(SArray* aTableTbNames, SRealTableNode* pT return false; } -static void findEqualCondTbnameInLogicCondAnd(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { - SNode* pTmpNode = NULL; +static int32_t findEqualCondTbnameInLogicCondAnd(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { + SNode* pTmpNode = NULL; + int32_t code = TSDB_CODE_SUCCESS; FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; - bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); + bool bIsEqTbnameCond; + code = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info, &bIsEqTbnameCond); + if (TSDB_CODE_SUCCESS != code) break; if (bIsEqTbnameCond) { if (!isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) { // TODO: intersect tbNames of same table? speed - taosArrayPush(aTableTbnames, &info); + if (NULL == taosArrayPush(aTableTbnames, &info)) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } } else { - taosArrayDestroy(info.aTbnames); + (void)taosArrayDestroy(info.aTbnames); } } } // TODO: logic cond } + return code; } -static void unionEqualCondTbnamesOfSameTable(SArray* aTableTbnames, SEqCondTbNameTableInfo* pInfo) { +static int32_t unionEqualCondTbnamesOfSameTable(SArray* aTableTbnames, SEqCondTbNameTableInfo* pInfo) { + int32_t code = TSDB_CODE_SUCCESS; bool bFoundTable = false; for (int i = 0; i < taosArrayGetSize(aTableTbnames); ++i) { SEqCondTbNameTableInfo* info = taosArrayGet(aTableTbnames, i); if (info->pRealTable == pInfo->pRealTable) { - taosArrayAddAll(info->aTbnames, pInfo->aTbnames); - taosArrayDestroy(pInfo->aTbnames); + if (NULL == taosArrayAddAll(info->aTbnames, pInfo->aTbnames)) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } + (void)taosArrayDestroy(pInfo->aTbnames); pInfo->aTbnames = NULL; bFoundTable = true; break; } } - if (!bFoundTable) { - taosArrayPush(aTableTbnames, pInfo); + if (TSDB_CODE_SUCCESS == code && !bFoundTable) { + if (NULL == taosArrayPush(aTableTbnames, pInfo)) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } + return code; } -static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { - bool bAllTbName = true; - SNode* pTmpNode = NULL; +static int32_t findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { + int32_t code = TSDB_CODE_SUCCESS; + bool bAllTbName = true; + SNode* pTmpNode = NULL; FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { // TODO: logic cond if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; - bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); + bool bIsEqTbnameCond; + code = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info, &bIsEqTbnameCond); + if (TSDB_CODE_SUCCESS != code) break; if (!bIsEqTbnameCond) { bAllTbName = false; break; } else { - unionEqualCondTbnamesOfSameTable(aTableTbnames, &info); + code = unionEqualCondTbnamesOfSameTable(aTableTbnames, &info); + if (TSDB_CODE_SUCCESS != code) break; } } else { bAllTbName = false; break; } } - if (!bAllTbName) { + if (TSDB_CODE_SUCCESS == code && !bAllTbName) { for (int i = 0; i < taosArrayGetSize(aTableTbnames); ++i) { SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTableTbnames, i); - taosArrayDestroy(pInfo->aTbnames); + (void)taosArrayDestroy(pInfo->aTbnames); pInfo->aTbnames = NULL; } taosArrayClear(aTableTbnames); } + return code; } static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { + int32_t code = TSDB_CODE_SUCCESS; // TODO: optimize nested and/or condition. now only the fist level is processed. if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; - bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pWhere, &info); + bool bIsEqTbnameCond; + code = findEqCondTbNameInOperatorNode(pCxt, pWhere, &info, &bIsEqTbnameCond); + if (TSDB_CODE_SUCCESS != code) return code; if (bIsEqTbnameCond) { - taosArrayPush(aTableTbnames, &info); + if (NULL == taosArrayPush(aTableTbnames, &info)) { + return TSDB_CODE_OUT_OF_MEMORY; + } } } else if (nodeType(pWhere) == QUERY_NODE_LOGIC_CONDITION) { if (((SLogicConditionNode*)pWhere)->condType == LOGIC_COND_TYPE_AND) { - findEqualCondTbnameInLogicCondAnd(pCxt, pWhere, aTableTbnames); + code = findEqualCondTbnameInLogicCondAnd(pCxt, pWhere, aTableTbnames); } else if (((SLogicConditionNode*)pWhere)->condType == LOGIC_COND_TYPE_OR) { - findEqualCondTbnameInLogicCondOr(pCxt, pWhere, aTableTbnames); + code = findEqualCondTbnameInLogicCondOr(pCxt, pWhere, aTableTbnames); } } - return TSDB_CODE_SUCCESS; + return code; } -static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTbnames, const char* dbName, +static void findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTbnames, const char* dbName, int32_t numOfVgroups, SVgroupsInfo* vgsInfo) { int32_t nVgroups = 0; int32_t nTbls = taosArrayGetSize(aTbnames); if (nTbls >= numOfVgroups) { vgsInfo->numOfVgroups = 0; - return TSDB_CODE_SUCCESS; + return; } for (int j = 0; j < nTbls; ++j) { SName snameTb; char* tbName = taosArrayGetP(aTbnames, j); - toName(pCxt->pParseCxt->acctId, dbName, tbName, &snameTb); + (void)toName(pCxt->pParseCxt->acctId, dbName, tbName, &snameTb); SVgroupInfo vgInfo = {0}; bool bExists; int32_t code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &snameTb, &vgInfo, &bExists); @@ -5795,7 +5983,6 @@ static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTbna break; } } - return TSDB_CODE_SUCCESS; } static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* pSelect, SArray* aTables) { @@ -5831,7 +6018,7 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* code = TSDB_CODE_OUT_OF_MEMORY; break; } - taosArrayPush(pTbNames, &pNewTbName); + (void)taosArrayPush(pTbNames, &pNewTbName); sprintf(pNewTbName, "%s.%s_%s", pTsma->dbFName, pTsma->name, pTbName); int32_t len = taosCreateMD5Hash(pNewTbName, strlen(pNewTbName)); } @@ -5842,7 +6029,7 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* if (TSDB_CODE_SUCCESS == code) { findVgroupsFromEqualTbname(pCxt, pTbNames, pInfo->pRealTable->table.dbName, numOfVgs, vgsInfo); if (vgsInfo->numOfVgroups != 0) { - taosArrayPush(pInfo->pRealTable->tsmaTargetTbVgInfo, &vgsInfo); + (void)taosArrayPush(pInfo->pRealTable->tsmaTargetTbVgInfo, &vgsInfo); } else { taosMemoryFree(vgsInfo); } @@ -5864,9 +6051,9 @@ static int32_t setTableVgroupsFromEqualTbnameCond(STranslateContext* pCxt, SSele } for (int i = 0; i < taosArrayGetSize(aTables); ++i) { SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, i); - taosArrayDestroy(pInfo->aTbnames); + (void)taosArrayDestroy(pInfo->aTbnames); } - taosArrayDestroy(aTables); + (void)taosArrayDestroy(aTables); return code; } @@ -5877,7 +6064,7 @@ static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) { code = getQueryTimeRange(pCxt, pSelect->pWhere, &pSelect->timeRange); } if (pSelect->pWhere != NULL && pCxt->pParseCxt->topicQuery == false) { - setTableVgroupsFromEqualTbnameCond(pCxt, pSelect); + code = setTableVgroupsFromEqualTbnameCond(pCxt, pSelect); } return code; } @@ -5939,8 +6126,10 @@ static EDealRes collectTableAlias(SNode* pNode, void* pContext) { *(SSHashObj**)pContext = pHash; } - tSimpleHashPut(*(SSHashObj**)pContext, pCol->tableAlias, strlen(pCol->tableAlias), pCol->tableAlias, - sizeof(pCol->tableAlias)); + if (TSDB_CODE_SUCCESS != tSimpleHashPut(*(SSHashObj**)pContext, pCol->tableAlias, strlen(pCol->tableAlias), pCol->tableAlias, + sizeof(pCol->tableAlias))) { + return DEAL_RES_ERROR; + } return DEAL_RES_CONTINUE; } @@ -6200,8 +6389,9 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect code = checkIsEmptyResult(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { - resetSelectFuncNumWithoutDup(pSelect); - code = checkAggColCoexist(pCxt, pSelect); + code = resetSelectFuncNumWithoutDup(pSelect); + if (TSDB_CODE_SUCCESS == code) + code = checkAggColCoexist(pCxt, pSelect); } /* if (TSDB_CODE_SUCCESS == code) { @@ -6574,6 +6764,7 @@ static int64_t getBigintFromValueNode(SValueNode* pVal) { } static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbReq* pReq) { + int32_t code = TSDB_CODE_SUCCESS; if (NULL != pRetentions) { pReq->pRetensions = taosArrayInit(LIST_LENGTH(pRetentions), sizeof(SRetention)); if (NULL == pReq->pRetensions) { @@ -6588,17 +6779,23 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe pKeep = (SValueNode*)nodesListGetNode(((SNodeListNode*)pNode)->pNodeList, 1); SRetention retention = { .freq = pFreq->datum.i, .freqUnit = pFreq->unit, .keep = pKeep->datum.i, .keepUnit = pKeep->unit}; - taosArrayPush(pReq->pRetensions, &retention); + if (NULL == taosArrayPush(pReq->pRetensions, &retention)) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } + } + if (TSDB_CODE_SUCCESS == code) { + pReq->numOfRetensions = taosArrayGetSize(pReq->pRetensions); } - pReq->numOfRetensions = taosArrayGetSize(pReq->pRetensions); } - return TSDB_CODE_SUCCESS; + return code; } static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt, SCreateDbReq* pReq) { SName name = {0}; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); - tNameGetFullDbName(&name, pReq->db); + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + if (TSDB_CODE_SUCCESS != code) return code; + (void)tNameGetFullDbName(&name, pReq->db); pReq->numOfVgroups = pStmt->pOptions->numOfVgroups; pReq->numOfStables = pStmt->pOptions->singleStable; pReq->buffer = pStmt->pOptions->buffer; @@ -7256,7 +7453,9 @@ static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq) typedef int32_t (*FSerializeFunc)(void* pBuf, int32_t bufLen, void* pReq); static int32_t buildCmdMsg(STranslateContext* pCxt, int16_t msgType, FSerializeFunc func, void* pReq) { - fillCmdSql(pCxt, msgType, pReq); + int32_t code = TSDB_CODE_SUCCESS; + code = fillCmdSql(pCxt, msgType, pReq); + if (TSDB_CODE_SUCCESS != code) return code; pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; @@ -7269,9 +7468,10 @@ static int32_t buildCmdMsg(STranslateContext* pCxt, int16_t msgType, FSerializeF taosMemoryFreeClear(pCxt->pCmdMsg); return TSDB_CODE_OUT_OF_MEMORY; } - func(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, pReq); - - return TSDB_CODE_SUCCESS; + if (-1 == func(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, pReq)) { + code = TSDB_CODE_INVALID_MSG; + } + return code; } static int32_t translateCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt) { @@ -7290,19 +7490,22 @@ static int32_t translateCreateDatabase(STranslateContext* pCxt, SCreateDatabaseS static int32_t translateDropDatabase(STranslateContext* pCxt, SDropDatabaseStmt* pStmt) { SDropDbReq dropReq = {0}; SName name = {0}; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); - tNameGetFullDbName(&name, dropReq.db); + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + if (TSDB_CODE_SUCCESS != code) return code; + (void)tNameGetFullDbName(&name, dropReq.db); dropReq.ignoreNotExists = pStmt->ignoreNotExists; - int32_t code = buildCmdMsg(pCxt, TDMT_MND_DROP_DB, (FSerializeFunc)tSerializeSDropDbReq, &dropReq); + code = buildCmdMsg(pCxt, TDMT_MND_DROP_DB, (FSerializeFunc)tSerializeSDropDbReq, &dropReq); tFreeSDropDbReq(&dropReq); return code; } -static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, SAlterDbReq* pReq) { - SName name = {0}; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); - tNameGetFullDbName(&name, pReq->db); +static int32_t buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, SAlterDbReq* pReq) { + SName name = {0}; + int32_t code = TSDB_CODE_SUCCESS; + code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + if (TSDB_CODE_SUCCESS != code) return code; + (void)tNameGetFullDbName(&name, pReq->db); pReq->buffer = pStmt->pOptions->buffer; pReq->pageSize = -1; pReq->pages = pStmt->pOptions->pages; @@ -7324,7 +7527,7 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, pReq->s3KeepLocal = pStmt->pOptions->s3KeepLocal; pReq->s3Compact = pStmt->pOptions->s3Compact; pReq->withArbitrator = pStmt->pOptions->withArbitrator; - return; + return code; } static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt) { @@ -7358,9 +7561,10 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm } SAlterDbReq alterReq = {0}; - buildAlterDbReq(pCxt, pStmt, &alterReq); + code = buildAlterDbReq(pCxt, pStmt, &alterReq); - code = buildCmdMsg(pCxt, TDMT_MND_ALTER_DB, (FSerializeFunc)tSerializeSAlterDbReq, &alterReq); + if (TSDB_CODE_SUCCESS == code) + code = buildCmdMsg(pCxt, TDMT_MND_ALTER_DB, (FSerializeFunc)tSerializeSAlterDbReq, &alterReq); tFreeSAlterDbReq(&alterReq); return code; } @@ -7368,8 +7572,9 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm static int32_t translateTrimDatabase(STranslateContext* pCxt, STrimDatabaseStmt* pStmt) { STrimDbReq req = {.maxSpeed = pStmt->maxSpeed}; SName name = {0}; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); - tNameGetFullDbName(&name, req.db); + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + if (TSDB_CODE_SUCCESS != code) return code; + (void)tNameGetFullDbName(&name, req.db); return buildCmdMsg(pCxt, TDMT_MND_TRIM_DB, (FSerializeFunc)tSerializeSTrimDbReq, &req); } @@ -7393,13 +7598,17 @@ static int32_t checkColumnOptions(SNodeList* pList) { static int32_t translateS3MigrateDatabase(STranslateContext* pCxt, SS3MigrateDatabaseStmt* pStmt) { SS3MigrateDbReq req = {0}; SName name = {0}; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); - tNameGetFullDbName(&name, req.db); + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + if (TSDB_CODE_SUCCESS != code) return code; + (void)tNameGetFullDbName(&name, req.db); return buildCmdMsg(pCxt, TDMT_MND_S3MIGRATE_DB, (FSerializeFunc)tSerializeSS3MigrateDbReq, &req); } static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calBytes) { *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SFieldWithOptions)); + if (!pArray) return TSDB_CODE_OUT_OF_MEMORY; + + int32_t code = TSDB_CODE_SUCCESS; SNode* pNode; FOREACH(pNode, pList) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; @@ -7422,13 +7631,21 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calB if (pCol->pOptions && ((SColumnOptions*)pCol->pOptions)->bPrimaryKey) { field.flags |= COL_IS_KEY; } - taosArrayPush(*pArray, &field); + if (NULL == taosArrayPush(*pArray, &field)) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } } - return TSDB_CODE_SUCCESS; + if (TSDB_CODE_SUCCESS != code) { + (void)taosArrayDestroy(*pArray); + *pArray = NULL; + } + return code; } static int32_t tagDefNodeToField(SNodeList* pList, SArray** pArray, bool calBytes) { *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField)); + if (!*pArray) return TSDB_CODE_OUT_OF_MEMORY; SNode* pNode; FOREACH(pNode, pList) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; @@ -7444,7 +7661,7 @@ static int32_t tagDefNodeToField(SNodeList* pList, SArray** pArray, bool calByte if (pCol->sma) { field.flags |= COL_SMA_ON; } - taosArrayPush(*pArray, &field); + (void)taosArrayPush(*pArray, &field); } return TSDB_CODE_SUCCESS; } @@ -8143,12 +8360,15 @@ static int32_t buildRollupFuncs(SNodeList* pFuncs, SArray** pArray) { return TSDB_CODE_SUCCESS; } *pArray = taosArrayInit(LIST_LENGTH(pFuncs), TSDB_FUNC_NAME_LEN); + if (!*pArray) return TSDB_CODE_OUT_OF_MEMORY; SNode* pNode; - FOREACH(pNode, pFuncs) { taosArrayPush(*pArray, ((SFunctionNode*)pNode)->functionName); } + FOREACH(pNode, pFuncs) { (void)taosArrayPush(*pArray, ((SFunctionNode*)pNode)->functionName); } return TSDB_CODE_SUCCESS; } static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) { + int32_t code = TSDB_CODE_SUCCESS; + SName tableName; pReq->igExists = pStmt->ignoreExists; pReq->delay1 = pStmt->pOptions->maxDelay1; pReq->delay2 = pStmt->pOptions->maxDelay2; @@ -8161,25 +8381,29 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm pReq->source = TD_REQ_FROM_APP; // columnDefNodeToField(pStmt->pCols, &pReq->pColumns, true); // columnDefNodeToField(pStmt->pTags, &pReq->pTags, true); - columnDefNodeToField(pStmt->pCols, &pReq->pColumns, true); - tagDefNodeToField(pStmt->pTags, &pReq->pTags, true); - pReq->numOfColumns = LIST_LENGTH(pStmt->pCols); - pReq->numOfTags = LIST_LENGTH(pStmt->pTags); - if (pStmt->pOptions->commentNull == false) { - pReq->pComment = taosStrdup(pStmt->pOptions->comment); - if (NULL == pReq->pComment) { - return TSDB_CODE_OUT_OF_MEMORY; + code = columnDefNodeToField(pStmt->pCols, &pReq->pColumns, true); + if (TSDB_CODE_SUCCESS == code) + code = tagDefNodeToField(pStmt->pTags, &pReq->pTags, true); + if (TSDB_CODE_SUCCESS == code) { + pReq->numOfColumns = LIST_LENGTH(pStmt->pCols); + pReq->numOfTags = LIST_LENGTH(pStmt->pTags); + if (pStmt->pOptions->commentNull == false) { + pReq->pComment = taosStrdup(pStmt->pOptions->comment); + if (NULL == pReq->pComment) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pReq->commentLen = strlen(pStmt->pOptions->comment); + } else { + pReq->commentLen = -1; } - pReq->commentLen = strlen(pStmt->pOptions->comment); - } else { - pReq->commentLen = -1; + code = buildRollupFuncs(pStmt->pOptions->pRollupFuncs, &pReq->pFuncs); } - buildRollupFuncs(pStmt->pOptions->pRollupFuncs, &pReq->pFuncs); - pReq->numOfFuncs = taosArrayGetSize(pReq->pFuncs); - - SName tableName; - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); - int32_t code = collectUseTable(&tableName, pCxt->pTables); + if (TSDB_CODE_SUCCESS == code) { + pReq->numOfFuncs = taosArrayGetSize(pReq->pFuncs); + code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); + } + if (TSDB_CODE_SUCCESS == code) + code = collectUseTable(&tableName, pCxt->pTables); if (TSDB_CODE_SUCCESS == code) { code = collectUseTable(&tableName, pCxt->pTargetTables); } @@ -8206,9 +8430,11 @@ static int32_t doTranslateDropSuperTable(STranslateContext* pCxt, const SName* p int32_t code = collectUseTable(pTableName, pCxt->pTargetTables); if (TSDB_CODE_SUCCESS == code) { SMDropStbReq dropReq = {0}; - tNameExtractFullName(pTableName, dropReq.name); - dropReq.igNotExists = ignoreNotExists; - code = buildCmdMsg(pCxt, TDMT_MND_DROP_STB, (FSerializeFunc)tSerializeSMDropStbReq, &dropReq); + code = tNameExtractFullName(pTableName, dropReq.name); + if (TSDB_CODE_SUCCESS == code) { + dropReq.igNotExists = ignoreNotExists; + code = buildCmdMsg(pCxt, TDMT_MND_DROP_STB, (FSerializeFunc)tSerializeSMDropStbReq, &dropReq); + } tFreeSMDropStbReq(&dropReq); } return code; @@ -8230,7 +8456,8 @@ static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableS static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SMAlterStbReq* pAlterReq) { SName tableName; - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pAlterReq->name); + int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pAlterReq->name); + if (TSDB_CODE_SUCCESS != code) return code; pAlterReq->alterType = pStmt->alterType; if (TSDB_ALTER_TABLE_UPDATE_OPTIONS == pStmt->alterType) { @@ -8262,17 +8489,23 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: { TAOS_FIELD field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}; strcpy(field.name, pStmt->colName); - taosArrayPush(pAlterReq->pFields, &field); + if (NULL == taosArrayPush(pAlterReq->pFields, &field)) { + return TSDB_CODE_OUT_OF_MEMORY; + } break; } case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { TAOS_FIELD oldField = {0}; strcpy(oldField.name, pStmt->colName); - taosArrayPush(pAlterReq->pFields, &oldField); + if (NULL == taosArrayPush(pAlterReq->pFields, &oldField)) { + return TSDB_CODE_OUT_OF_MEMORY; + } TAOS_FIELD newField = {0}; strcpy(newField.name, pStmt->newColName); - taosArrayPush(pAlterReq->pFields, &newField); + if (NULL == taosArrayPush(pAlterReq->pFields, &newField)) { + return TSDB_CODE_OUT_OF_MEMORY; + } break; } case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: { @@ -8288,13 +8521,16 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* if (code != TSDB_CODE_SUCCESS) { return code; } - taosArrayPush(pAlterReq->pFields, &field); + if (NULL == taosArrayPush(pAlterReq->pFields, &field)) { + return TSDB_CODE_OUT_OF_MEMORY; + } break; } case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: { - taosArrayDestroy(pAlterReq->pFields); + (void)taosArrayDestroy(pAlterReq->pFields); pAlterReq->pFields = taosArrayInit(1, sizeof(SFieldWithOptions)); + if (!pAlterReq->pFields) return TSDB_CODE_OUT_OF_MEMORY; SFieldWithOptions field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}; // TAOS_FIELD field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}; strcpy(field.name, pStmt->colName); @@ -8313,7 +8549,7 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* return code; } } - taosArrayPush(pAlterReq->pFields, &field); + if (NULL == taosArrayPush(pAlterReq->pFields, &field)) return TSDB_CODE_OUT_OF_MEMORY; break; } default: @@ -8518,10 +8754,12 @@ static int32_t translateAlterSuperTable(STranslateContext* pCxt, SAlterTableStmt static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* pStmt) { SUseDbReq usedbReq = {0}; SName name = {0}; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); - tNameExtractFullName(&name, usedbReq.db); - int32_t code = - getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + if (TSDB_CODE_SUCCESS == code) { + code = tNameExtractFullName(&name, usedbReq.db); + } + if (TSDB_CODE_SUCCESS == code) + code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq); } @@ -8735,20 +8973,23 @@ static int32_t getSmaIndexAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SMCreateSmaReq* pReq) { SName name; - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), pReq->name); - memset(&name, 0, sizeof(SName)); - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), pReq->stb); - pReq->igExists = pStmt->ignoreExists; - pReq->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; - pReq->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; - pReq->offset = (NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0); - pReq->sliding = + int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), pReq->name); + if (TSDB_CODE_SUCCESS == code) { + memset(&name, 0, sizeof(SName)); + code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), pReq->stb); + } + if (TSDB_CODE_SUCCESS == code) { + pReq->igExists = pStmt->ignoreExists; + pReq->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; + pReq->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; + pReq->offset = (NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0); + pReq->sliding = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pReq->interval); - pReq->slidingUnit = + pReq->slidingUnit = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pReq->intervalUnit); + } - int32_t code = TSDB_CODE_SUCCESS; - if (NULL != pStmt->pOptions->pStreamOptions) { + if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pOptions->pStreamOptions) { SStreamOptions* pStreamOpt = (SStreamOptions*)pStmt->pOptions->pStreamOptions; if (NULL != pStreamOpt->pDelay) { code = getTableMaxDelayOption(pCxt, (SValueNode*)pStreamOpt->pDelay, &pReq->maxDelay); @@ -8868,7 +9109,8 @@ int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, SS if (TSDB_CODE_SUCCESS == code) { code = setQuery(&pCxt, pQuery); } - setRefreshMeta(&pCxt, pQuery); + int32_t tmpCode = setRefreshMeta(&pCxt, pQuery); + if (TSDB_CODE_SUCCESS == code) code = tmpCode; destroyTranslateContext(&pCxt); tFreeSMCreateSmaReq(pStmt->pReq); taosMemoryFreeClear(pStmt->pReq); @@ -8882,14 +9124,17 @@ static int32_t buildCreateFullTextReq(STranslateContext* pCxt, SCreateIndexStmt* static int32_t buildCreateTagIndexReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SCreateTagIndexReq* pReq) { SName name; - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), pReq->idxName); - memset(&name, 0, sizeof(SName)); - - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), pReq->stbName); - memset(&name, 0, sizeof(SName)); - - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); - tNameGetFullDbName(&name, pReq->dbFName); + int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), pReq->idxName); + if (TSDB_CODE_SUCCESS == code) { + memset(&name, 0, sizeof(SName)); + code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), pReq->stbName); + } + if (TSDB_CODE_SUCCESS == code) { + memset(&name, 0, sizeof(SName)); + code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + } + if (TSDB_CODE_SUCCESS != code) return code; + (void)tNameGetFullDbName(&name, pReq->dbFName); SNode* pNode = NULL; ASSERT(LIST_LENGTH(pStmt->pCols) == 1); @@ -8959,7 +9204,8 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { SMDropSmaReq dropSmaReq = {0}; SName name; - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), dropSmaReq.name); + int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), dropSmaReq.name); + if (TSDB_CODE_SUCCESS != code) return code; dropSmaReq.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_DROP_SMA, (FSerializeFunc)tSerializeSMDropSmaReq, &dropSmaReq); } @@ -9033,23 +9279,28 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS SName name; if ('\0' != pStmt->subSTbName[0]) { pReq->subType = TOPIC_SUB_TYPE__TABLE; - toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); - tNameGetFullDbName(&name, pReq->subDbName); - tNameExtractFullName(&name, pReq->subStbName); - if (pStmt->pQuery != NULL) { - code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); + (void)toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); + (void)tNameGetFullDbName(&name, pReq->subDbName); + if (TSDB_CODE_SUCCESS == code) { + (void)tNameExtractFullName(&name, pReq->subStbName); + if (pStmt->pQuery != NULL) { + code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); + } } } else if ('\0' != pStmt->subDbName[0]) { pReq->subType = TOPIC_SUB_TYPE__DB; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->subDbName, strlen(pStmt->subDbName)); - tNameGetFullDbName(&name, pReq->subDbName); + code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->subDbName, strlen(pStmt->subDbName)); + if (TSDB_CODE_SUCCESS == code) + (void)tNameGetFullDbName(&name, pReq->subDbName); } else { pReq->subType = TOPIC_SUB_TYPE__COLUMN; char* dbName = ((SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable))->table.dbName; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, dbName, strlen(dbName)); - tNameGetFullDbName(&name, pReq->subDbName); - pCxt->pParseCxt->topicQuery = true; - code = translateQuery(pCxt, pStmt->pQuery); + code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, dbName, strlen(dbName)); + if (TSDB_CODE_SUCCESS == code) { + (void)tNameGetFullDbName(&name, pReq->subDbName); + pCxt->pParseCxt->topicQuery = true; + code = translateQuery(pCxt, pStmt->pQuery); + } if (TSDB_CODE_SUCCESS == code) { code = checkTopicQuery(pCxt, (SSelectStmt*)pStmt->pQuery); } @@ -9061,14 +9312,20 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS return code; } -static int32_t addTagList(SNodeList** ppList, SNode* pNode) { +static int32_t addTagList(SNodeList** ppList, SNode** pNode) { if (NULL == *ppList) { *ppList = nodesMakeList(); + if (!*ppList) return TSDB_CODE_OUT_OF_MEMORY; } - nodesListStrictAppend(*ppList, pNode); + int32_t code = nodesListStrictAppend(*ppList, *pNode); + *pNode = NULL; + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(*ppList); + *ppList = NULL; + } - return TSDB_CODE_SUCCESS; + return code; } static EDealRes checkColumnTagsInCond(SNode* pNode, void* pContext) { @@ -9080,7 +9337,10 @@ static EDealRes checkColumnTagsInCond(SNode* pNode, void* pContext) { pCxt->colExists = true; return DEAL_RES_ERROR; } else if (type == TCOL_TYPE_TAG) { - addTagList(&pCxt->pTags, nodesCloneNode(pNode)); + SNode* pNodeClone = nodesCloneNode(pNode); + int32_t code = addTagList(&pCxt->pTags, &pNodeClone); + if (TSDB_CODE_SUCCESS != code) nodesDestroyNode(pNodeClone); + return DEAL_RES_ERROR; } else { pCxt->colNotFound = true; return DEAL_RES_ERROR; @@ -9088,7 +9348,11 @@ static EDealRes checkColumnTagsInCond(SNode* pNode, void* pContext) { } else if (QUERY_NODE_FUNCTION == nodeType(pNode)) { SFunctionNode* pFunc = (SFunctionNode*)pNode; if (0 == strcasecmp(pFunc->functionName, "tbname")) { - addTagList(&pCxt->pTags, nodesCloneNode(pNode)); + SNode* pNodeClone = nodesCloneNode(pNode); + if (TSDB_CODE_SUCCESS != addTagList(&pCxt->pTags, &pNodeClone)) { + nodesDestroyNode(pNodeClone); + return DEAL_RES_ERROR; + } } } @@ -9116,8 +9380,11 @@ static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* strcpy(col->colName, column->name); strcpy(col->node.aliasName, col->colName); strcpy(col->node.userAlias, col->colName); - addTagList(&colCxt.pTags, (SNode*)col); - // } + int32_t code = addTagList(&colCxt.pTags, (SNode**)&col); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode((SNode*)col); + return code; + } } *ppProjection = colCxt.pTags; @@ -9207,9 +9474,11 @@ static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pStmt) { SMDropCgroupReq dropReq = {0}; - SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName)); - tNameGetFullDbName(&name, dropReq.topic); + SName name; + int32_t code = TSDB_CODE_SUCCESS; + code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName)); + if (TSDB_CODE_SUCCESS != code) return code; + (void)tNameGetFullDbName(&name, dropReq.topic); dropReq.igNotExists = pStmt->ignoreNotExists; strcpy(dropReq.cgroup, pStmt->cgroup); @@ -9242,7 +9511,7 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { int32_t origCode = code; SName name; - toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); SViewMeta* pMeta = NULL; code = getViewMetaFromMetaCache(pCxt, &name, &pMeta); if (TSDB_CODE_SUCCESS != code) { @@ -9250,7 +9519,7 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) } else { SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA}; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(&name, dbFName); + (void)tNameGetFullDbName(&name, dbFName); code = (*pCxt->pParseCxt->parseSqlFp)(pCxt->pParseCxt->parseSqlParam, name.dbname, pMeta->querySql, false, pMeta->user, &res); if (TSDB_CODE_SUCCESS == code) { @@ -9299,9 +9568,12 @@ static int32_t translateCompactRange(STranslateContext* pCxt, SCompactDatabaseSt static int32_t translateCompact(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt) { SCompactDbReq compactReq = {0}; SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); - tNameGetFullDbName(&name, compactReq.db); - int32_t code = translateCompactRange(pCxt, pStmt, &compactReq); + int32_t code = TSDB_CODE_SUCCESS; + code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + if (TSDB_CODE_SUCCESS != code) return code; + + (void)tNameGetFullDbName(&name, compactReq.db); + code = translateCompactRange(pCxt, pStmt, &compactReq); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_COMPACT_DB, (FSerializeFunc)tSerializeSCompactDbReq, &compactReq); } @@ -9380,7 +9652,7 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt static void getSourceDatabase(SNode* pStmt, int32_t acctId, char* pDbFName) { SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; strcpy(name.dbname, ((SRealTableNode*)(((SSelectStmt*)pStmt)->pFromTable))->table.dbName); - tNameGetFullDbName(&name, pDbFName); + (void)tNameGetFullDbName(&name, pDbFName); } static void getStreamQueryFirstProjectAliasName(SHashObj* pUserAliasSet, char* aliasName, int32_t len) { @@ -9568,8 +9840,8 @@ static int32_t addNullTagsToCreateStreamQuery(STranslateContext* pCxt, STableMet return addNullTagsForExistTable(pCxt, pMeta, (SSelectStmt*)pStmt->pQuery); } -static int32_t addColDefNodeByProj(SNodeList** ppCols, SNode* pProject, int8_t flags) { - SExprNode* pExpr = (SExprNode*)pProject; +static int32_t addColDefNodeByProj(SNodeList** ppCols, const SNode* pProject, int8_t flags) { + const SExprNode* pExpr = (const SExprNode*)pProject; SColumnDefNode* pColDef = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF); strcpy(pColDef->colName, pExpr->userAlias); pColDef->dataType = pExpr->resType; @@ -9702,7 +9974,9 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm int64_t watermark = 0; if (pStmt->pOptions->pWatermark) { - translateValue(pCxt, (SValueNode*)pStmt->pOptions->pWatermark); + if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pWatermark)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Invalid watermark value."); + } watermark = ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i; } if (watermark <= 0) { @@ -9805,6 +10079,7 @@ static int32_t setFillNullCols(SArray* pProjColPos, const STableMeta* pMeta, SCM } const SSchema* pSchemas = getTableColumnSchema(pMeta); int32_t indexOfBoundCols = 0; + int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) { const SSchema* pSchema = pSchemas + i; if (indexOfBoundCols < numOfBoundCols) { @@ -9815,9 +10090,12 @@ static int32_t setFillNullCols(SArray* pProjColPos, const STableMeta* pMeta, SCM } } SColLocation colLoc = {.colId = pSchema->colId, .slotId = i, .type = pSchema->type}; - taosArrayPush(pReq->fillNullCols, &colLoc); + if (NULL == taosArrayPush(pReq->fillNullCols, &colLoc)) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } } - return TSDB_CODE_SUCCESS; + return code; } static int32_t adjustOrderOfProjections(STranslateContext* pCxt, SNodeList** ppCols, const STableMeta* pMeta, @@ -9873,13 +10151,15 @@ static int32_t adjustOrderOfProjections(STranslateContext* pCxt, SNodeList** ppC int32_t num = taosArrayGetSize(pProjColPos); pNewProjections = nodesMakeList(); pNewCols = nodesMakeList(); - if (NULL == pNewProjections) { + if (NULL == pNewProjections || !pNewCols) { code = TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < num; ++i) { SProjColPos* pPos = taosArrayGet(pProjColPos, i); code = nodesListStrictAppend(pNewProjections, pPos->pProj); - addColDefNodeByProj(&pNewCols, pPos->pProj, pPos->flags); + if (TSDB_CODE_SUCCESS == code) { + code = addColDefNodeByProj(&pNewCols, pPos->pProj, pPos->flags); + } pPos->pProj = NULL; } } @@ -9889,7 +10169,7 @@ static int32_t adjustOrderOfProjections(STranslateContext* pCxt, SNodeList** ppC } if (TSDB_CODE_SUCCESS == code) { - taosArrayDestroy(pProjColPos); + (void)taosArrayDestroy(pProjColPos); nodesDestroyList(*pProjections); nodesDestroyList(*ppCols); *pProjections = pNewProjections; @@ -9998,7 +10278,7 @@ static int32_t adjustOrderOfTags(STranslateContext* pCxt, SNodeList* pTags, cons } if (TSDB_CODE_SUCCESS == code) { - taosArrayDestroy(pTagPos); + (void)taosArrayDestroy(pTagPos); nodesDestroyList(*pTagExprs); *pTagExprs = pNewTagExprs; } else { @@ -10316,17 +10596,19 @@ static int32_t translateStreamOptions(STranslateContext* pCxt, SCreateStreamStmt static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { pReq->igExists = pStmt->ignoreExists; - SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->streamName, strlen(pStmt->streamName)); - tNameGetFullDbName(&name, pReq->name); + SName name; + int32_t code = TSDB_CODE_SUCCESS; + code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->streamName, strlen(pStmt->streamName)); + if (TSDB_CODE_SUCCESS != code) return code; + (void)tNameGetFullDbName(&name, pReq->name); if ('\0' != pStmt->targetTabName[0]) { strcpy(name.dbname, pStmt->targetDbName); strcpy(name.tname, pStmt->targetTabName); - tNameExtractFullName(&name, pReq->targetStbFullName); + (void)tNameExtractFullName(&name, pReq->targetStbFullName); } - int32_t code = buildCreateStreamQuery(pCxt, pStmt, pReq); + code = buildCreateStreamQuery(pCxt, pStmt, pReq); if (TSDB_CODE_SUCCESS == code) { pReq->sql = taosStrdup(pCxt->pParseCxt->pSql); if (NULL == pReq->sql) { @@ -10346,12 +10628,14 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt* (NULL != pStmt->pOptions->pDeleteMark ? ((SValueNode*)pStmt->pOptions->pDeleteMark)->datum.i : 0); pReq->fillHistory = pStmt->pOptions->fillHistory; pReq->igExpired = pStmt->pOptions->ignoreExpired; - if (pReq->createStb) { - tagDefNodeToField(pStmt->pTags, &pReq->pTags, true); - pReq->numOfTags = LIST_LENGTH(pStmt->pTags); - } - columnDefNodeToField(pStmt->pCols, &pReq->pCols, false); pReq->igUpdate = pStmt->pOptions->ignoreUpdate; + if (pReq->createStb) { + pReq->numOfTags = LIST_LENGTH(pStmt->pTags); + code = tagDefNodeToField(pStmt->pTags, &pReq->pTags, true); + } + if (TSDB_CODE_SUCCESS == code) { + code = columnDefNodeToField(pStmt->pCols, &pReq->pCols, false); + } } return code; @@ -10419,7 +10703,7 @@ static int32_t createStreamReqVersionInfo(SSDataBlock* pBlock, SArray** pArray, for (int32_t i = 0; i < pBlock->info.rows; ++i) { SVgroupVer v = {.vgId = *(int32_t*)colDataGetData(pCol1, i), .ver = *(int64_t*)colDataGetData(pCol2, i)}; - taosArrayPush(*pArray, &v); + (void)taosArrayPush(*pArray, &v); } } else { int32_t precision = (pInterval->interval > 0) ? pInterval->precision : TSDB_TIME_PRECISION_MILLI; @@ -10458,7 +10742,8 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, SSDa code = setQuery(&cxt, pQuery); } - setRefreshMeta(&cxt, pQuery); + int32_t tmpCode = setRefreshMeta(&cxt, pQuery); + if (TSDB_CODE_SUCCESS == code) code = tmpCode; destroyTranslateContext(&cxt); tFreeSCMCreateStreamReq(pStmt->pReq); @@ -10470,10 +10755,11 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, SSDa static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pStmt) { SMDropStreamReq dropReq = {0}; SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->streamName, strlen(pStmt->streamName)); - tNameGetFullDbName(&name, dropReq.name); + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->streamName, strlen(pStmt->streamName)); + if (TSDB_CODE_SUCCESS != code) return code; + (void)tNameGetFullDbName(&name, dropReq.name); dropReq.igNotExists = pStmt->ignoreNotExists; - int32_t code = buildCmdMsg(pCxt, TDMT_MND_DROP_STREAM, (FSerializeFunc)tSerializeSMDropStreamReq, &dropReq); + code = buildCmdMsg(pCxt, TDMT_MND_DROP_STREAM, (FSerializeFunc)tSerializeSMDropStreamReq, &dropReq); tFreeMDropStreamReq(&dropReq); return code; } @@ -10481,8 +10767,9 @@ static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pSt static int32_t translatePauseStream(STranslateContext* pCxt, SPauseStreamStmt* pStmt) { SMPauseStreamReq req = {0}; SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->streamName, strlen(pStmt->streamName)); - tNameGetFullDbName(&name, req.name); + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->streamName, strlen(pStmt->streamName)); + if (TSDB_CODE_SUCCESS != code) return code; + (void)tNameGetFullDbName(&name, req.name); req.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_PAUSE_STREAM, (FSerializeFunc)tSerializeSMPauseStreamReq, &req); } @@ -10490,8 +10777,9 @@ static int32_t translatePauseStream(STranslateContext* pCxt, SPauseStreamStmt* p static int32_t translateResumeStream(STranslateContext* pCxt, SResumeStreamStmt* pStmt) { SMResumeStreamReq req = {0}; SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->streamName, strlen(pStmt->streamName)); - tNameGetFullDbName(&name, req.name); + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->streamName, strlen(pStmt->streamName)); + if (TSDB_CODE_SUCCESS != code) return code; + (void)tNameGetFullDbName(&name, req.name); req.igNotExists = pStmt->ignoreNotExists; req.igUntreated = pStmt->ignoreUntreated; return buildCmdMsg(pCxt, TDMT_MND_RESUME_STREAM, (FSerializeFunc)tSerializeSMResumeStreamReq, &req); @@ -10523,8 +10811,8 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA}; SName name; char dbFName[TSDB_DB_FNAME_LEN]; - toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); - tNameGetFullDbName(&name, dbFName); + (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); + (void)tNameGetFullDbName(&name, dbFName); int32_t code = validateCreateView(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { @@ -10564,18 +10852,20 @@ static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) SCMDropViewReq dropReq = {0}; SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); - tNameGetFullDbName(&name, dropReq.dbFName); - strncpy(dropReq.name, pStmt->viewName, sizeof(dropReq.name) - 1); - snprintf(dropReq.fullname, sizeof(dropReq.fullname) - 1, "%s.%s", dropReq.dbFName, dropReq.name); - dropReq.sql = (char*)pCxt->pParseCxt->pSql; - if (NULL == dropReq.sql) { - return TSDB_CODE_OUT_OF_MEMORY; + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + if (TSDB_CODE_SUCCESS == code) { + (void)tNameGetFullDbName(&name, dropReq.dbFName); + strncpy(dropReq.name, pStmt->viewName, sizeof(dropReq.name) - 1); + snprintf(dropReq.fullname, sizeof(dropReq.fullname) - 1, "%s.%s", dropReq.dbFName, dropReq.name); + dropReq.sql = (char*)pCxt->pParseCxt->pSql; + if (NULL == dropReq.sql) { + return TSDB_CODE_OUT_OF_MEMORY; + } + dropReq.igNotExists = pStmt->ignoreNotExists; + (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); + code = collectUseTable(&name, pCxt->pTargetTables); } - dropReq.igNotExists = pStmt->ignoreNotExists; - toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); - int32_t code = collectUseTable(&name, pCxt->pTargetTables); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -10608,12 +10898,11 @@ static int32_t readFromFile(char* pName, int32_t* len, char** buf) { int64_t s = taosReadFile(tfile, *buf, *len); if (s != *len) { - taosCloseFile(&tfile); + (void)taosCloseFile(&tfile); taosMemoryFreeClear(*buf); return TSDB_CODE_APP_ERROR; } - taosCloseFile(&tfile); - return TSDB_CODE_SUCCESS; + return taosCloseFile(&tfile); } static int32_t translateCreateFunction(STranslateContext* pCxt, SCreateFunctionStmt* pStmt) { @@ -10792,7 +11081,7 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) { SBalanceVgroupLeaderReq req = {0}; req.vgId = pStmt->vgId; - if (pStmt->dbName != NULL) strcpy(req.db, pStmt->dbName); + strcpy(req.db, pStmt->dbName); int32_t code = buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req); tFreeSBalanceVgroupLeaderReq(&req); @@ -10861,10 +11150,11 @@ static int32_t translateShowCreateDatabase(STranslateContext* pCxt, SShowCreateD } SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); - tNameGetFullDbName(&name, pStmt->dbFName); - - return getDBCfg(pCxt, pStmt->dbName, (SDbCfgInfo*)pStmt->pCfg); + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + (void)tNameGetFullDbName(&name, pStmt->dbFName); + if (TSDB_CODE_SUCCESS == code) + return getDBCfg(pCxt, pStmt->dbName, (SDbCfgInfo*)pStmt->pCfg); + return code; } static int32_t translateShowCreateTable(STranslateContext* pCxt, SShowCreateTableStmt* pStmt) { @@ -10875,7 +11165,7 @@ static int32_t translateShowCreateTable(STranslateContext* pCxt, SShowCreateTabl int32_t code = getDBCfg(pCxt, pStmt->dbName, (SDbCfgInfo*)pStmt->pDbCfg); if (TSDB_CODE_SUCCESS == code) { SName name; - toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); code = getTableCfg(pCxt, &name, (STableCfg**)&pStmt->pTableCfg); } return code; @@ -10886,7 +11176,7 @@ static int32_t translateShowCreateView(STranslateContext* pCxt, SShowCreateViewS return TSDB_CODE_OPS_NOT_SUPPORT; #else SName name; - toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); + (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); return getViewMetaFromMetaCache(pCxt, &name, (SViewMeta**)&pStmt->pViewMeta); #endif } @@ -10926,7 +11216,7 @@ static int32_t compareTsmaFuncWithFuncAndColId(SNode* pNode1, SNode* pNode2) { } // pFuncs are already sorted by funcId and colId -static int32_t deduplicateTsmaFuncs(SNodeList* pFuncs) { +static void deduplicateTsmaFuncs(SNodeList* pFuncs) { SNode* pLast = NULL; SNode* pFunc = NULL; SNodeList* pRes = NULL; @@ -10942,7 +11232,6 @@ static int32_t deduplicateTsmaFuncs(SNodeList* pFuncs) { pLast = pFunc; } } - return TSDB_CODE_SUCCESS; } static int32_t buildTSMAAstStreamSubTable(SCreateTSMAStmt* pStmt, SMCreateSmaReq* pReq, const SNode* pTbname, @@ -11020,8 +11309,9 @@ static int32_t buildTSMAAst(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, SMC code = TSDB_CODE_OUT_OF_MEMORY; break; } - nodesListAppend(info.pPartitionByList, pTagCol); - code = nodesListMakeStrictAppend(&info.pTags, nodesCloneNode(pTagCol)); + code = nodesListAppend(info.pPartitionByList, pTagCol); + if (TSDB_CODE_SUCCESS == code) + code = nodesListMakeStrictAppend(&info.pTags, nodesCloneNode(pTagCol)); } // sub table @@ -11081,13 +11371,14 @@ static int32_t rewriteTSMAFuncs(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, pFunc = (SFunctionNode*)pNode; const SSchema* pSchema = pCols + i; code = createColumnBySchema(pSchema, &pCol); - if (code) break; - nodesListErase(pFunc->pParameterList, pFunc->pParameterList->pHead); - nodesListPushFront(pFunc->pParameterList, (SNode*)pCol); + if (TSDB_CODE_SUCCESS != code) break; + (void)nodesListErase(pFunc->pParameterList, pFunc->pParameterList->pHead); + code = nodesListPushFront(pFunc->pParameterList, (SNode*)pCol); + if (TSDB_CODE_SUCCESS != code) break; snprintf(pFunc->node.userAlias, TSDB_COL_NAME_LEN, "%s", pSchema->name); // for first or last, the second param will be pk ts col, here we should remove it if (fmIsImplicitTsFunc(pFunc->funcId) && LIST_LENGTH(pFunc->pParameterList) == 2) { - nodesListErase(pFunc->pParameterList, pFunc->pParameterList->pTail); + (void)nodesListErase(pFunc->pParameterList, pFunc->pParameterList->pTail); } ++i; } @@ -11140,13 +11431,16 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm SName name; SDbCfgInfo pDbInfo = {0}; int32_t code = TSDB_CODE_SUCCESS; - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name), pReq->name); - memset(&name, 0, sizeof(SName)); - toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, useTbName); - tNameExtractFullName(useTbName, pReq->stb); - pReq->igExists = pStmt->ignoreExists; - - code = getDBCfg(pCxt, pStmt->dbName, &pDbInfo); + code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name), pReq->name); + if (TSDB_CODE_SUCCESS == code) { + memset(&name, 0, sizeof(SName)); + (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, useTbName); + code = tNameExtractFullName(useTbName, pReq->stb); + } + if (TSDB_CODE_SUCCESS == code) { + pReq->igExists = pStmt->ignoreExists; + code = getDBCfg(pCxt, pStmt->dbName, &pDbInfo); + } if (code != TSDB_CODE_SUCCESS) { return code; } @@ -11182,12 +11476,14 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm code = getTsma(pCxt, useTbName, &pRecursiveTsma); if (code == TSDB_CODE_SUCCESS) { pReq->recursiveTsma = true; - tNameExtractFullName(useTbName, pReq->baseTsmaName); - SValueNode* pInterval = (SValueNode*)pStmt->pOptions->pInterval; - if (checkRecursiveTsmaInterval(pRecursiveTsma->interval, pRecursiveTsma->unit, pInterval->datum.i, - pInterval->unit, pDbInfo.precision, true)) { - } else { - code = TSDB_CODE_TSMA_INVALID_PARA; + code = tNameExtractFullName(useTbName, pReq->baseTsmaName); + if (TSDB_CODE_SUCCESS == code) { + SValueNode* pInterval = (SValueNode*)pStmt->pOptions->pInterval; + if (checkRecursiveTsmaInterval(pRecursiveTsma->interval, pRecursiveTsma->unit, pInterval->datum.i, + pInterval->unit, pDbInfo.precision, true)) { + } else { + code = TSDB_CODE_TSMA_INVALID_PARA; + } } } if (code == TSDB_CODE_SUCCESS) { @@ -11199,17 +11495,24 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm FOREACH(pNode, pSelect->pProjectionList) { SFunctionNode* pFuncNode = (SFunctionNode*)pNode; if (!fmIsTSMASupportedFunc(pFuncNode->funcId)) continue; - nodesListMakeStrictAppend(&pStmt->pOptions->pFuncs, nodesCloneNode(pNode)); + code = nodesListMakeStrictAppend(&pStmt->pOptions->pFuncs, nodesCloneNode(pNode)); + if (TSDB_CODE_SUCCESS != code) { + break; + } } nodesDestroyNode((SNode*)pSelect); - memset(useTbName, 0, sizeof(SName)); - memcpy(pStmt->originalTbName, pRecursiveTsma->tb, TSDB_TABLE_NAME_LEN); - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pRecursiveTsma->tb, useTbName), pReq->stb); - numOfCols = pRecursiveTsma->pUsedCols->size; - numOfTags = pRecursiveTsma->pTags ? pRecursiveTsma->pTags->size : 0; - pCols = pRecursiveTsma->pUsedCols->pData; - pTags = pRecursiveTsma->pTags ? pRecursiveTsma->pTags->pData : NULL; - code = getTableMeta(pCxt, pStmt->dbName, pRecursiveTsma->targetTb, &pTableMeta); + if (TSDB_CODE_SUCCESS == code) { + memset(useTbName, 0, sizeof(SName)); + memcpy(pStmt->originalTbName, pRecursiveTsma->tb, TSDB_TABLE_NAME_LEN); + code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pRecursiveTsma->tb, useTbName), pReq->stb); + } + if (TSDB_CODE_SUCCESS == code) { + numOfCols = pRecursiveTsma->pUsedCols->size; + numOfTags = pRecursiveTsma->pTags ? pRecursiveTsma->pTags->size : 0; + pCols = pRecursiveTsma->pUsedCols->pData; + pTags = pRecursiveTsma->pTags ? pRecursiveTsma->pTags->pData : NULL; + code = getTableMeta(pCxt, pStmt->dbName, pRecursiveTsma->targetTb, &pTableMeta); + } } } else { code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta); @@ -11322,11 +11625,12 @@ int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSData if (TSDB_CODE_SUCCESS == code) { SName name = {0}; - toName(pParseCxt->acctId, pStmt->dbName, pStmt->originalTbName, &name); + (void)toName(pParseCxt->acctId, pStmt->dbName, pStmt->originalTbName, &name); code = collectUseTable(&name, cxt.pTargetTables); } - setRefreshMeta(&cxt, pQuery); + int32_t tmpCode = setRefreshMeta(&cxt, pQuery); + if (TSDB_CODE_SUCCESS == code) code = tmpCode; destroyTranslateContext(&cxt); tFreeSMCreateSmaReq(pStmt->pReq); @@ -11336,15 +11640,17 @@ int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSData } static int32_t translateDropTSMA(STranslateContext* pCxt, SDropTSMAStmt* pStmt) { - int32_t code = TSDB_CODE_SUCCESS; - SMDropSmaReq dropReq = {0}; - SName name; - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name), dropReq.name); - dropReq.igNotExists = pStmt->ignoreNotExists; + int32_t code = TSDB_CODE_SUCCESS; + SMDropSmaReq dropReq = {0}; + SName name; STableTSMAInfo* pTsma = NULL; - code = getTsma(pCxt, &name, &pTsma); + code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name), dropReq.name); + if (TSDB_CODE_SUCCESS == code) { + dropReq.igNotExists = pStmt->ignoreNotExists; + code = getTsma(pCxt, &name, &pTsma); + } if (code == TSDB_CODE_SUCCESS) { - toName(pCxt->pParseCxt->acctId, pStmt->dbName, pTsma->tb, &name); + (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pTsma->tb, &name); code = collectUseTable(&name, pCxt->pTargetTables); } if (TSDB_CODE_SUCCESS == code) @@ -11798,20 +12104,26 @@ static SNode* createProjectCol(const char* pProjCol) { return (SNode*)pCol; } -static SNodeList* createProjectCols(int32_t ncols, const char* const pCols[]) { +static int32_t createProjectCols(int32_t ncols, const char* const pCols[], SNodeList** pResList) { SNodeList* pProjections = NULL; + int32_t code = TSDB_CODE_SUCCESS; if (0 == ncols) { - nodesListMakeStrictAppend(&pProjections, createStarCol()); - return pProjections; + code = nodesListMakeStrictAppend(&pProjections, createStarCol()); + if (TSDB_CODE_SUCCESS == code) + *pResList = pProjections; + else + nodesDestroyList(pProjections); + return code; } for (int32_t i = 0; i < ncols; ++i) { - int32_t code = nodesListMakeStrictAppend(&pProjections, createProjectCol(pCols[i])); + code = nodesListMakeStrictAppend(&pProjections, createProjectCol(pCols[i])); if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pProjections); - return NULL; + break; } } - return pProjections; + if (TSDB_CODE_SUCCESS == code) *pResList = pProjections; + return code; } static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, SNodeList* pProjectionList, @@ -11842,10 +12154,8 @@ static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTabl const char* const pProjCol[], SSelectStmt** pStmt) { SNodeList* pProjectionList = NULL; if (numOfProjs >= 0) { - pProjectionList = createProjectCols(numOfProjs, pProjCol); - if (NULL == pProjectionList) { - return TSDB_CODE_OUT_OF_MEMORY; - } + int32_t code = createProjectCols(numOfProjs, pProjCol, &pProjectionList); + if (TSDB_CODE_SUCCESS != code) return code; } return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt); @@ -11924,7 +12234,8 @@ static const char* getTbNameColName(ENodeType type) { return colName; } -static int32_t createLogicCondNode(SNode* pCond1, SNode* pCond2, SNode** pCond, ELogicConditionType logicCondType) { +static int32_t createLogicCondNode(SNode** pCond1, SNode** pCond2, SNode** pCond, ELogicConditionType logicCondType) { + int32_t code = TSDB_CODE_SUCCESS; SLogicConditionNode* pCondition = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); if (NULL == pCondition) { return TSDB_CODE_OUT_OF_MEMORY; @@ -11935,114 +12246,199 @@ static int32_t createLogicCondNode(SNode* pCond1, SNode* pCond2, SNode** pCond, nodesDestroyNode((SNode*)pCondition); return TSDB_CODE_OUT_OF_MEMORY; } - if (TSDB_CODE_SUCCESS != nodesListAppend(pCondition->pParameterList, pCond1) || - TSDB_CODE_SUCCESS != nodesListAppend(pCondition->pParameterList, pCond2)) { + code = nodesListAppend(pCondition->pParameterList, *pCond1); + if (TSDB_CODE_SUCCESS == code) { + *pCond1 = NULL; + code = nodesListAppend(pCondition->pParameterList, *pCond2); + } + if (TSDB_CODE_SUCCESS == code) { + *pCond2 = NULL; + *pCond = (SNode*)pCondition; + } else { nodesDestroyNode((SNode*)pCondition); - return TSDB_CODE_OUT_OF_MEMORY; } - *pCond = (SNode*)pCondition; - return TSDB_CODE_SUCCESS; + return code; } -static int32_t insertCondIntoSelectStmt(SSelectStmt* pSelect, SNode* pCond) { +static int32_t insertCondIntoSelectStmt(SSelectStmt* pSelect, SNode** pCond) { + int32_t code = TSDB_CODE_SUCCESS; if (pSelect->pWhere == NULL) { - pSelect->pWhere = pCond; + pSelect->pWhere = *pCond; + *pCond = NULL; } else { SNodeList* pLogicCondListWhere = NULL; SNodeList* pLogicCondList2 = NULL; if (nodeType(pSelect->pWhere) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pSelect->pWhere)->condType == LOGIC_COND_TYPE_AND) { pLogicCondListWhere = ((SLogicConditionNode*)pSelect->pWhere)->pParameterList; + ((SLogicConditionNode*)pSelect->pWhere)->pParameterList = NULL; } else { - nodesListMakeAppend(&pLogicCondListWhere, pSelect->pWhere); + code = nodesListMakeAppend(&pLogicCondListWhere, pSelect->pWhere); + if (TSDB_CODE_SUCCESS == code) { + pSelect->pWhere = NULL; + } } - if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION && - ((SLogicConditionNode*)pCond)->condType == LOGIC_COND_TYPE_AND) { - pLogicCondList2 = ((SLogicConditionNode*)pCond)->pParameterList; - } else { - nodesListMakeAppend(&pLogicCondList2, pCond); + if (TSDB_CODE_SUCCESS == code) { + if (nodeType(*pCond) == QUERY_NODE_LOGIC_CONDITION && + ((SLogicConditionNode*)(*pCond))->condType == LOGIC_COND_TYPE_AND) { + pLogicCondList2 = ((SLogicConditionNode*)(*pCond))->pParameterList; + ((SLogicConditionNode*)(*pCond))->pParameterList = NULL; + } else { + code = nodesListMakeAppend(&pLogicCondList2, (*pCond)); + if (TSDB_CODE_SUCCESS == code) { + *pCond = NULL; + } + } } - nodesListAppendList(pLogicCondListWhere, pLogicCondList2); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListAppendList(pLogicCondListWhere, pLogicCondList2); + } - SLogicConditionNode* pWhere = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); - pWhere->condType = LOGIC_COND_TYPE_AND; - pWhere->pParameterList = pLogicCondListWhere; + if (TSDB_CODE_SUCCESS == code) { + SLogicConditionNode* pWhere = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (pWhere) { + pWhere->condType = LOGIC_COND_TYPE_AND; + pWhere->pParameterList = pLogicCondListWhere; - pSelect->pWhere = (SNode*)pWhere; + pSelect->pWhere = (SNode*)pWhere; + } + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pLogicCondListWhere); + nodesDestroyList(pLogicCondList2); + } } - return TSDB_CODE_SUCCESS; + return code; } static int32_t addShowUserDatabasesCond(SSelectStmt* pSelect) { + int32_t code = TSDB_CODE_SUCCESS; SNode* pNameCond1 = NULL; SNode* pNameCond2 = NULL; - SValueNode* pValNode1 = nodesMakeValueNodeFromString(TSDB_INFORMATION_SCHEMA_DB); - SValueNode* pValNode2 = nodesMakeValueNodeFromString(TSDB_PERFORMANCE_SCHEMA_DB); - createOperatorNode(OP_TYPE_NOT_EQUAL, "name", (SNode*)pValNode1, &pNameCond1); - createOperatorNode(OP_TYPE_NOT_EQUAL, "name", (SNode*)pValNode2, &pNameCond2); + SNode* pNameCond = NULL; + SValueNode* pValNode1 = NULL, *pValNode2 = NULL; + pValNode1 = nodesMakeValueNodeFromString(TSDB_INFORMATION_SCHEMA_DB); + if (!pValNode1) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + pValNode2 = nodesMakeValueNodeFromString(TSDB_PERFORMANCE_SCHEMA_DB); + if (!pValNode2) code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + code = createOperatorNode(OP_TYPE_NOT_EQUAL, "name", (SNode*)pValNode1, &pNameCond1); + } + if (TSDB_CODE_SUCCESS == code) { + code = createOperatorNode(OP_TYPE_NOT_EQUAL, "name", (SNode*)pValNode2, &pNameCond2); + } nodesDestroyNode((SNode*)pValNode2); nodesDestroyNode((SNode*)pValNode1); - SNode* pNameCond = NULL; - createLogicCondNode(pNameCond1, pNameCond2, &pNameCond, LOGIC_COND_TYPE_AND); - insertCondIntoSelectStmt(pSelect, pNameCond); - return TSDB_CODE_SUCCESS; + if (TSDB_CODE_SUCCESS == code) + code = createLogicCondNode(&pNameCond1, &pNameCond2, &pNameCond, LOGIC_COND_TYPE_AND); + + if (TSDB_CODE_SUCCESS == code) + code = insertCondIntoSelectStmt(pSelect, &pNameCond); + + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNameCond1); + nodesDestroyNode(pNameCond2); + nodesDestroyNode(pNameCond); + } + return code; } static int32_t addShowSystemDatabasesCond(SSelectStmt* pSelect) { + int32_t code = TSDB_CODE_SUCCESS; SNode* pNameCond1 = NULL; SNode* pNameCond2 = NULL; - SValueNode* pValNode1 = nodesMakeValueNodeFromString(TSDB_INFORMATION_SCHEMA_DB); - SValueNode* pValNode2 = nodesMakeValueNodeFromString(TSDB_PERFORMANCE_SCHEMA_DB); - createOperatorNode(OP_TYPE_EQUAL, "name", (SNode*)pValNode1, &pNameCond1); - createOperatorNode(OP_TYPE_EQUAL, "name", (SNode*)pValNode2, &pNameCond2); + SValueNode* pValNode1 = NULL, * pValNode2 = NULL; + SNode* pNameCond = NULL; + pValNode1 = nodesMakeValueNodeFromString(TSDB_INFORMATION_SCHEMA_DB); + if (!pValNode1) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + pValNode2 = nodesMakeValueNodeFromString(TSDB_PERFORMANCE_SCHEMA_DB); + if (!pValNode2) code = terrno; + } + if (TSDB_CODE_SUCCESS == code) { + code = createOperatorNode(OP_TYPE_EQUAL, "name", (SNode*)pValNode1, &pNameCond1); + } + if (TSDB_CODE_SUCCESS == code) { + code = createOperatorNode(OP_TYPE_EQUAL, "name", (SNode*)pValNode2, &pNameCond2); + } nodesDestroyNode((SNode*)pValNode2); nodesDestroyNode((SNode*)pValNode1); - SNode* pNameCond = NULL; - createLogicCondNode(pNameCond1, pNameCond2, &pNameCond, LOGIC_COND_TYPE_OR); - insertCondIntoSelectStmt(pSelect, pNameCond); - return TSDB_CODE_SUCCESS; + if (TSDB_CODE_SUCCESS == code) { + code = createLogicCondNode(&pNameCond1, &pNameCond2, &pNameCond, LOGIC_COND_TYPE_OR); + } + + if (TSDB_CODE_SUCCESS == code) + code = insertCondIntoSelectStmt(pSelect, &pNameCond); + + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNameCond1); + nodesDestroyNode(pNameCond2); + nodesDestroyNode(pNameCond); + } + return code; } static int32_t addShowNormalTablesCond(SSelectStmt* pSelect) { SNode* pTypeCond = NULL; + int32_t code = TSDB_CODE_SUCCESS; SValueNode* pValNode1 = nodesMakeValueNodeFromString("NORMAL_TABLE"); - createOperatorNode(OP_TYPE_EQUAL, "type", (SNode*)pValNode1, &pTypeCond); + if (!pValNode1) code = TSDB_CODE_OUT_OF_MEMORY; + + if (TSDB_CODE_SUCCESS == code) + code = createOperatorNode(OP_TYPE_EQUAL, "type", (SNode*)pValNode1, &pTypeCond); + nodesDestroyNode((SNode*)pValNode1); - insertCondIntoSelectStmt(pSelect, pTypeCond); - return TSDB_CODE_SUCCESS; + + if (TSDB_CODE_SUCCESS == code) code = insertCondIntoSelectStmt(pSelect, &pTypeCond); + if (TSDB_CODE_SUCCESS != code) nodesDestroyNode(pTypeCond); + return code; } static int32_t addShowChildTablesCond(SSelectStmt* pSelect) { + int32_t code = TSDB_CODE_SUCCESS; SNode* pTypeCond = NULL; SValueNode* pValNode1 = nodesMakeValueNodeFromString("CHILD_TABLE"); - createOperatorNode(OP_TYPE_EQUAL, "type", (SNode*)pValNode1, &pTypeCond); + if (!pValNode1) code = TSDB_CODE_OUT_OF_MEMORY; + + if (TSDB_CODE_SUCCESS == code) + code = createOperatorNode(OP_TYPE_EQUAL, "type", (SNode*)pValNode1, &pTypeCond); + nodesDestroyNode((SNode*)pValNode1); - insertCondIntoSelectStmt(pSelect, pTypeCond); - return TSDB_CODE_SUCCESS; + + if (TSDB_CODE_SUCCESS == code) code = insertCondIntoSelectStmt(pSelect, &pTypeCond); + if (TSDB_CODE_SUCCESS != code) nodesDestroyNode(pTypeCond); + return code; } static int32_t addShowKindCond(const SShowStmt* pShow, SSelectStmt* pSelect) { + int32_t code = TSDB_CODE_SUCCESS; if (pShow->type != QUERY_NODE_SHOW_DATABASES_STMT && pShow->type != QUERY_NODE_SHOW_TABLES_STMT || pShow->showKind == SHOW_KIND_ALL) { return TSDB_CODE_SUCCESS; } if (pShow->type == QUERY_NODE_SHOW_DATABASES_STMT) { if (pShow->showKind == SHOW_KIND_DATABASES_USER) { - addShowUserDatabasesCond(pSelect); + code = addShowUserDatabasesCond(pSelect); } else if (pShow->showKind == SHOW_KIND_DATABASES_SYSTEM) { - addShowSystemDatabasesCond(pSelect); + code = addShowSystemDatabasesCond(pSelect); } } else if (pShow->type == QUERY_NODE_SHOW_TABLES_STMT) { if (pShow->showKind == SHOW_KIND_TABLES_NORMAL) { - addShowNormalTablesCond(pSelect); + code = addShowNormalTablesCond(pSelect); } else if (pShow->showKind == SHOW_KIND_TABLES_CHILD) { - addShowChildTablesCond(pSelect); + code = addShowChildTablesCond(pSelect); } } - return TSDB_CODE_SUCCESS; + return code; } static int32_t createShowCondition(const SShowStmt* pShow, SSelectStmt* pSelect) { @@ -12057,7 +12453,7 @@ static int32_t createShowCondition(const SShowStmt* pShow, SSelectStmt* pSelect) } if (NULL != pDbCond && NULL != pTbCond) { - if (TSDB_CODE_SUCCESS != createLogicCondNode(pDbCond, pTbCond, &pSelect->pWhere, LOGIC_COND_TYPE_AND)) { + if (TSDB_CODE_SUCCESS != createLogicCondNode(&pDbCond, &pTbCond, &pSelect->pWhere, LOGIC_COND_TYPE_AND)) { nodesDestroyNode(pDbCond); nodesDestroyNode(pTbCond); return TSDB_CODE_OUT_OF_MEMORY; @@ -12195,7 +12591,7 @@ static int32_t rewriteShowDnodeVariables(STranslateContext* pCxt, SQuery* pQuery } if (TSDB_CODE_SUCCESS == code) { if (NULL != pLikeCond) { - code = createLogicCondNode(pDnodeCond, pLikeCond, &pSelect->pWhere, LOGIC_COND_TYPE_AND); + code = createLogicCondNode(&pDnodeCond, &pLikeCond, &pSelect->pWhere, LOGIC_COND_TYPE_AND); } else { pSelect->pWhere = pDnodeCond; } @@ -12279,8 +12675,8 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* SVgroupCreateTableBatch* pBatch) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; - strcpy(name.dbname, pStmt->dbName); - tNameGetFullDbName(&name, dbFName); + (void)strcpy(name.dbname, pStmt->dbName); + (void)tNameGetFullDbName(&name, dbFName); SVCreateTbReq req = {0}; req.type = TD_NORMAL_TABLE; @@ -12327,13 +12723,16 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* ++index; } pBatch->info = *pVgroupInfo; - strcpy(pBatch->dbName, pStmt->dbName); + (void)strcpy(pBatch->dbName, pStmt->dbName); pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); if (NULL == pBatch->req.pArray) { tdDestroySVCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(pBatch->req.pArray, &req); + if (NULL == taosArrayPush(pBatch->req.pArray, &req)) { + tdDestroySVCreateTbReq(&req); + return TSDB_CODE_OUT_OF_MEMORY; + } return TSDB_CODE_SUCCESS; } @@ -12344,6 +12743,9 @@ static int32_t serializeVgroupCreateTableBatch(SVgroupCreateTableBatch* pTbBatch int32_t ret = 0; tEncodeSize(tEncodeSVCreateTbBatchReq, &pTbBatch->req, tlen, ret); + if (TSDB_CODE_SUCCESS != ret) { + return ret; + } tlen += sizeof(SMsgHead); void* buf = taosMemoryMalloc(tlen); if (NULL == buf) { @@ -12354,20 +12756,29 @@ static int32_t serializeVgroupCreateTableBatch(SVgroupCreateTableBatch* pTbBatch void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncoderInit(&coder, pBuf, tlen - sizeof(SMsgHead)); - tEncodeSVCreateTbBatchReq(&coder, &pTbBatch->req); + ret = tEncodeSVCreateTbBatchReq(&coder, &pTbBatch->req); tEncoderClear(&coder); + if (TSDB_CODE_SUCCESS != ret) { + taosMemoryFreeClear(buf); + return ret; + } SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == pVgData) { + taosMemoryFreeClear(buf); return TSDB_CODE_OUT_OF_MEMORY; } pVgData->vg = pTbBatch->info; pVgData->pData = buf; pVgData->size = tlen; pVgData->numOfTables = (int32_t)taosArrayGetSize(pTbBatch->req.pArray); - taosArrayPush(pBufArray, &pVgData); + if (NULL == taosArrayPush(pBufArray, &pVgData)) { + ret = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFreeClear(buf); + taosMemoryFreeClear(pVgData); + } - return TSDB_CODE_SUCCESS; + return ret; } static void destroyCreateTbReqBatch(void* data) { @@ -12378,7 +12789,7 @@ static void destroyCreateTbReqBatch(void* data) { tdDestroySVCreateTbReq(pTableReq); } - taosArrayDestroy(pTbBatch->req.pArray); + (void)taosArrayDestroy(pTbBatch->req.pArray); } int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray) { @@ -12400,7 +12811,7 @@ static void destroyCreateTbReqArray(SArray* pArray) { taosMemoryFreeClear(pVg->pData); taosMemoryFreeClear(pVg); } - taosArrayDestroy(pArray); + (void)taosArrayDestroy(pArray); } static int32_t buildCreateTableDataBlock(int32_t acctId, const SCreateTableStmt* pStmt, const SVgroupInfo* pInfo, @@ -12429,7 +12840,7 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { int32_t code = checkCreateTable(pCxt, pStmt, false); SVgroupInfo info = {0}; SName name; - toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); if (TSDB_CODE_SUCCESS == code) { code = getTableHashVgroupImpl(pCxt, &name, &info); } @@ -12478,6 +12889,7 @@ static int32_t addCreateTbReqIntoVgroup(SHashObj* pVgroupHashmap, const char* db return TSDB_CODE_OUT_OF_MEMORY; } + int32_t code = TSDB_CODE_SUCCESS; SVgroupCreateTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); if (pTableBatch == NULL) { SVgroupCreateTableBatch tBatch = {0}; @@ -12485,14 +12897,28 @@ static int32_t addCreateTbReqIntoVgroup(SHashObj* pVgroupHashmap, const char* db strcpy(tBatch.dbName, dbName); tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); - taosArrayPush(tBatch.req.pArray, &req); - - taosHashPut(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &tBatch, sizeof(tBatch)); + if (!tBatch.req.pArray) { + code = terrno; + } else if (NULL == taosArrayPush(tBatch.req.pArray, &req)) { + (void)taosArrayDestroy(tBatch.req.pArray); + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + code = taosHashPut(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &tBatch, sizeof(tBatch)); + if (TSDB_CODE_SUCCESS != code) { + (void)taosArrayDestroy(tBatch.req.pArray); + } + } } else { // add to the correct vgroup - taosArrayPush(pTableBatch->req.pArray, &req); + if (NULL == taosArrayPush(pTableBatch->req.pArray, &req)) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } - return TSDB_CODE_SUCCESS; + if (TSDB_CODE_SUCCESS != code) { + tdDestroySVCreateTbReq(&req); + } + + return code; } static int32_t createCastFuncForTag(STranslateContext* pCxt, SNode* pNode, SDataType dt, SNode** pCast) { @@ -12575,7 +13001,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla taosMemoryFreeClear(p->pData); } } - taosArrayDestroy(pTagArray); + (void)taosArrayDestroy(pTagArray); return code; } @@ -12638,7 +13064,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau taosMemoryFreeClear(p->pData); } } - taosArrayDestroy(pTagArray); + (void)taosArrayDestroy(pTagArray); return code; } @@ -12661,7 +13087,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla } if (TSDB_CODE_SUCCESS == code) { SName name; - toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); code = collectUseTable(&name, pCxt->pTargetTables); } @@ -12689,7 +13115,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla taosMemoryFree(pTag); } - taosArrayDestroy(tagName); + (void)taosArrayDestroy(tagName); taosMemoryFreeClear(pSuperTableMeta); return code; } @@ -12815,6 +13241,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) int32_t numOfTags = getNumOfTags(pParFileCxt->pStbMeta); uint8_t precision = getTableInfo(pParFileCxt->pStbMeta).precision; SSchema* pSchemas = getTableTagSchema(pParFileCxt->pStbMeta); + bool isJson = false; for (int i = 0; i < sz; i++) { const char* pSql = pParFileCxt->pSql; @@ -12832,6 +13259,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) // parse tag const SSchema* pTagSchema = &pSchemas[index]; + isJson = (pTagSchema->type == TSDB_DATA_TYPE_JSON); code = checkAndTrimValue(&token, pParFileCxt->tmpTokenBuf, pMsgBuf, pTagSchema->type); if (TSDB_CODE_SUCCESS == code && TK_NK_VARIABLE == token.type) { code = buildInvalidOperationMsg(pMsgBuf, "not expected row value"); @@ -12859,7 +13287,9 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) if (TSDB_CODE_SUCCESS == code) { pParFileCxt->tagNameFilled = true; - code = tTagNew(pParFileCxt->aTagVals, 1, false, &pParFileCxt->pTag); + if (!isJson) { + code = tTagNew(pParFileCxt->aTagVals, 1, false, &pParFileCxt->pTag); + } } return code; @@ -12900,7 +13330,7 @@ static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFi if (pLine[0] == '#') continue; // ignore comment line begins with '#' - strtolower(pLine, pLine); + (void)strtolower(pLine, pLine); pParFileCxt->pSql = pLine; code = parseOneStbRow(pMsgBuf, pParFileCxt); @@ -12927,7 +13357,10 @@ static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFi .pTag = pParFileCxt->pTag, .vg = pParFileCxt->vg}; - taosArrayPush(pParFileCxt->aCreateTbData, &data); + if (NULL == taosArrayPush(pParFileCxt->aCreateTbData, &data)) { + taosMemoryFreeClear(pParFileCxt->pTag); + code = TSDB_CODE_OUT_OF_MEMORY; + } } else { taosMemoryFreeClear(pParFileCxt->pTag); } @@ -13132,10 +13565,10 @@ static int32_t createSubTableFromFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt return code; } -SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap) { +int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut) { SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); if (NULL == pBufArray) { - return NULL; + return terrno; } int32_t code = TSDB_CODE_SUCCESS; @@ -13146,10 +13579,20 @@ SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap) { break; } - serializeVgroupCreateTableBatch(pTbBatch, pBufArray); + code = serializeVgroupCreateTableBatch(pTbBatch, pBufArray); + if (TSDB_CODE_SUCCESS != code) { + taosHashCancelIterate(pVgroupHashmap, pTbBatch); + break; + } } while (true); - return pBufArray; + if (TSDB_CODE_SUCCESS != code) { + (void)taosArrayDestroy(pBufArray); + } else { + *pOut = pBufArray; + } + + return code; } static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) { @@ -13172,10 +13615,11 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) } } - SArray* pBufArray = serializeVgroupsCreateTableBatch(pVgroupHashmap); + SArray* pBufArray = NULL; + code = serializeVgroupsCreateTableBatch(pVgroupHashmap, &pBufArray); taosHashCleanup(pVgroupHashmap); - if (NULL == pBufArray) { - return TSDB_CODE_OUT_OF_MEMORY; + if (TSDB_CODE_SUCCESS != code) { + return code; } return rewriteToVnodeModifyOpStmt(pQuery, pBufArray); @@ -13209,11 +13653,12 @@ static int32_t rewriteCreateTableFromFile(STranslateContext* pCxt, SQuery* pQuer return code; } - SArray* pBufArray = serializeVgroupsCreateTableBatch(pModifyStmt->pVgroupsHashObj); + SArray* pBufArray = NULL; + code = serializeVgroupsCreateTableBatch(pModifyStmt->pVgroupsHashObj, &pBufArray); taosHashClear(pModifyStmt->pVgroupsHashObj); - if (NULL == pBufArray) { + if (TSDB_CODE_SUCCESS != code) { taosHashCleanup(pModifyStmt->pVgroupsHashObj); - return TSDB_CODE_OUT_OF_MEMORY; + return code; } pModifyStmt->pDataBlocks = pBufArray; @@ -13231,10 +13676,10 @@ int32_t continueCreateTbFromFile(SParseContext* pParseCxt, SQuery** pQuery) { int32_t code = createSubTableFromFile(&tmpBuf, pParseCxt, pModifyStmt); if (TSDB_CODE_SUCCESS != code) goto _OUT; - SArray* pBufArray = serializeVgroupsCreateTableBatch(pModifyStmt->pVgroupsHashObj); + SArray* pBufArray = NULL; + code = serializeVgroupsCreateTableBatch(pModifyStmt->pVgroupsHashObj, &pBufArray); taosHashClear(pModifyStmt->pVgroupsHashObj); - if (NULL == pBufArray) { - code = TSDB_CODE_OUT_OF_MEMORY; + if (TSDB_CODE_SUCCESS != code) { goto _OUT; } @@ -13256,18 +13701,31 @@ typedef struct SVgroupDropTableBatch { char dbName[TSDB_DB_NAME_LEN]; } SVgroupDropTableBatch; -static void addDropTbReqIntoVgroup(SHashObj* pVgroupHashmap, SVgroupInfo* pVgInfo, SVDropTbReq* pReq) { +static int32_t addDropTbReqIntoVgroup(SHashObj* pVgroupHashmap, SVgroupInfo* pVgInfo, SVDropTbReq* pReq) { SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); + int32_t code = TSDB_CODE_SUCCESS; if (NULL == pTableBatch) { SVgroupDropTableBatch tBatch = {0}; tBatch.info = *pVgInfo; tBatch.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq)); - taosArrayPush(tBatch.req.pArray, pReq); + if (NULL == taosArrayPush(tBatch.req.pArray, pReq)) { + (void)taosArrayDestroy(tBatch.req.pArray); + tBatch.req.pArray = NULL; + return TSDB_CODE_OUT_OF_MEMORY; + } - taosHashPut(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &tBatch, sizeof(tBatch)); + code = taosHashPut(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &tBatch, sizeof(tBatch)); + if (TSDB_CODE_SUCCESS != code) { + (void)taosArrayDestroy(tBatch.req.pArray); + tBatch.req.pArray = NULL; + return code; + } } else { // add to the correct vgroup - taosArrayPush(pTableBatch->req.pArray, pReq); + if (NULL == taosArrayPush(pTableBatch->req.pArray, pReq)) { + return TSDB_CODE_OUT_OF_MEMORY; + } } + return code; } static int32_t buildDropTableVgroupHashmap(STranslateContext* pCxt, SDropTableClause* pClause, const SName* name, @@ -13295,7 +13753,7 @@ static int32_t buildDropTableVgroupHashmap(STranslateContext* pCxt, SDropTableCl if (TSDB_CODE_SUCCESS == code) { SVDropTbReq req = {.suid = pTableMeta->suid, .igNotExists = pClause->ignoreNotExists}; req.name = pClause->tableName; - addDropTbReqIntoVgroup(pVgroupHashmap, &info, &req); + code = addDropTbReqIntoVgroup(pVgroupHashmap, &info, &req); } over: @@ -13305,7 +13763,7 @@ over: static void destroyDropTbReqBatch(void* data) { SVgroupDropTableBatch* pTbBatch = (SVgroupDropTableBatch*)data; - taosArrayDestroy(pTbBatch->req.pArray); + (void)taosArrayDestroy(pTbBatch->req.pArray); } static int32_t serializeVgroupDropTableBatch(SVgroupDropTableBatch* pTbBatch, SArray* pBufArray) { @@ -13314,6 +13772,7 @@ static int32_t serializeVgroupDropTableBatch(SVgroupDropTableBatch* pTbBatch, SA int32_t ret = 0; tEncodeSize(tEncodeSVDropTbBatchReq, &pTbBatch->req, tlen, ret); + if (TSDB_CODE_SUCCESS != ret) return ret; tlen += sizeof(SMsgHead); void* buf = taosMemoryMalloc(tlen); if (NULL == buf) { @@ -13324,26 +13783,35 @@ static int32_t serializeVgroupDropTableBatch(SVgroupDropTableBatch* pTbBatch, SA void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncoderInit(&coder, pBuf, tlen - sizeof(SMsgHead)); - tEncodeSVDropTbBatchReq(&coder, &pTbBatch->req); + ret = tEncodeSVDropTbBatchReq(&coder, &pTbBatch->req); tEncoderClear(&coder); + if (TSDB_CODE_SUCCESS != ret) { + taosMemoryFreeClear(buf); + return ret; + } SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == pVgData) { + taosMemoryFreeClear(buf); return TSDB_CODE_OUT_OF_MEMORY; } pVgData->vg = pTbBatch->info; pVgData->pData = buf; pVgData->size = tlen; pVgData->numOfTables = (int32_t)taosArrayGetSize(pTbBatch->req.pArray); - taosArrayPush(pBufArray, &pVgData); + if (NULL == taosArrayPush(pBufArray, &pVgData)) { + ret = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFreeClear(pVgData); + taosMemoryFreeClear(buf); + } - return TSDB_CODE_SUCCESS; + return ret; } -SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap) { +int32_t serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap, SArray** pOut) { SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); if (NULL == pBufArray) { - return NULL; + return terrno; } int32_t code = TSDB_CODE_SUCCESS; @@ -13354,10 +13822,16 @@ SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap) { break; } - serializeVgroupDropTableBatch(pTbBatch, pBufArray); + code = serializeVgroupDropTableBatch(pTbBatch, pBufArray); + if (TSDB_CODE_SUCCESS != code) { + taosHashCancelIterate(pVgroupHashmap, pTbBatch); + (void)taosArrayDestroy(pBufArray); + break; + } } while (true); - return pBufArray; + *pOut = pBufArray; + return code; } static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { @@ -13375,7 +13849,7 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { FOREACH(pNode, pStmt->pTables) { SDropTableClause* pClause = (SDropTableClause*)pNode; SName name; - toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &name); + (void)toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &name); int32_t code = buildDropTableVgroupHashmap(pCxt, pClause, &name, &tableType, pVgroupHashmap); if (TSDB_CODE_SUCCESS != code) { taosHashCleanup(pVgroupHashmap); @@ -13420,20 +13894,26 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { SMDropTbReqsOnSingleVg reqOnVg = {0}; reqOnVg.vgInfo = pTbBatch->info; reqOnVg.pTbs = pTbBatch->req.pArray; - taosArrayPush(req.pVgReqs, &reqOnVg); + if (NULL == taosArrayPush(req.pVgReqs, &reqOnVg)) { + taosHashCancelIterate(pVgroupHashmap, pTbBatch); + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } } while (true); - - code = buildCmdMsg(pCxt, TDMT_MND_DROP_TB_WITH_TSMA, (FSerializeFunc)tSerializeSMDropTbsReq, &req); - taosArrayDestroy(req.pVgReqs); + if (TSDB_CODE_SUCCESS == code) { + code = buildCmdMsg(pCxt, TDMT_MND_DROP_TB_WITH_TSMA, (FSerializeFunc)tSerializeSMDropTbsReq, &req); + } + (void)taosArrayDestroy(req.pVgReqs); } taosHashCleanup(pVgroupHashmap); return code; } - SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap); + SArray* pBufArray = NULL; + code = serializeVgroupsDropTableBatch(pVgroupHashmap, &pBufArray); taosHashCleanup(pVgroupHashmap); - if (NULL == pBufArray) { - return TSDB_CODE_OUT_OF_MEMORY; + if (TSDB_CODE_SUCCESS != code) { + return code; } return rewriteToVnodeModifyOpStmt(pQuery, pBufArray); @@ -13445,7 +13925,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS SArray* pTsmas = NULL; int32_t code = TSDB_CODE_SUCCESS; if (pCxt->pMetaCache) { - toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); + (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); code = getTableTsmasFromCache(pCxt->pMetaCache, &tbName, &pTsmas); if (code != TSDB_CODE_SUCCESS) return code; if (pTsmas && pTsmas->size > 0) return TSDB_CODE_TSMA_MUST_BE_DROPPED; @@ -13629,7 +14109,7 @@ static int32_t buildRenameColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt SArray* pTsmas = NULL; SName tbName; int32_t code = 0; - toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); + (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); if (pCxt->pMetaCache) code = getTableTsmasFromCache(pCxt->pMetaCache, &tbName, &pTsmas); if (TSDB_CODE_SUCCESS != code) return code; if (pTsmas && pTsmas->size > 0) { @@ -13754,7 +14234,10 @@ static int32_t serializeAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pSt void* pBuf = POINTER_SHIFT(pMsg, sizeof(SMsgHead)); SEncoder coder = {0}; tEncoderInit(&coder, pBuf, tlen - sizeof(SMsgHead)); - tEncodeSVAlterTbReq(&coder, pReq); + if (TSDB_CODE_SUCCESS != tEncodeSVAlterTbReq(&coder, pReq)) { + taosMemoryFree(pMsg); + return terrno; + } tEncoderClear(&coder); SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); @@ -13766,7 +14249,9 @@ static int32_t serializeAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pSt pVgData->pData = pMsg; pVgData->size = tlen; pVgData->numOfTables = 1; - taosArrayPush(pArray, &pVgData); + if (NULL == taosArrayPush(pArray, &pVgData)) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } return code; @@ -13783,7 +14268,7 @@ static int32_t buildModifyVnodeArray(STranslateContext* pCxt, SAlterTableStmt* p if (TSDB_CODE_SUCCESS == code) { *pArray = pTmpArray; } else { - taosArrayDestroy(pTmpArray); + (void)taosArrayDestroy(pTmpArray); } return code; @@ -13801,7 +14286,7 @@ static void destoryAlterTbReq(SVAlterTbReq* pReq) { taosMemoryFreeClear(p->pData); } } - taosArrayDestroy(pReq->pTagArray); + (void)taosArrayDestroy(pReq->pTagArray); if (pReq->tagFree) tTagFree((STag*)pReq->pTagVal); } @@ -13853,6 +14338,7 @@ static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { static int32_t serializeFlushVgroup(SVgroupInfo* pVg, SArray* pBufArray) { int32_t len = sizeof(SMsgHead); void* buf = taosMemoryMalloc(len); + int32_t code = TSDB_CODE_SUCCESS; if (NULL == buf) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -13867,9 +14353,11 @@ static int32_t serializeFlushVgroup(SVgroupInfo* pVg, SArray* pBufArray) { pVgData->vg = *pVg; pVgData->pData = buf; pVgData->size = len; - taosArrayPush(pBufArray, &pVgData); + if (NULL == taosArrayPush(pBufArray, &pVgData)) { + code = TSDB_CODE_OUT_OF_MEMORY; + } - return TSDB_CODE_SUCCESS; + return code; } static int32_t serializeFlushDb(SArray* pVgs, SArray** pOutput) { @@ -13883,7 +14371,7 @@ static int32_t serializeFlushDb(SArray* pVgs, SArray** pOutput) { for (int32_t i = 0; i < numOfVgs; ++i) { int32_t code = serializeFlushVgroup((SVgroupInfo*)taosArrayGet(pVgs, i), pBufArray); if (TSDB_CODE_SUCCESS != code) { - taosArrayDestroy(pBufArray); + (void)taosArrayDestroy(pBufArray); return code; } } @@ -13905,9 +14393,9 @@ static int32_t rewriteFlushDatabase(STranslateContext* pCxt, SQuery* pQuery) { code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); } if (TSDB_CODE_SUCCESS != code) { - taosArrayDestroy(pBufArray); + (void)taosArrayDestroy(pBufArray); } - taosArrayDestroy(pVgs); + (void)taosArrayDestroy(pVgs); return code; } @@ -14026,9 +14514,9 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) { SNode* pTemp1 = NULL; SNode* pTemp2 = NULL; SNode* pFullCond = NULL; - CHECK_RES_OUT_OF_MEM(createLogicCondNode(pCond1, pCond2, &pTemp1, LOGIC_COND_TYPE_OR)); - CHECK_RES_OUT_OF_MEM(createLogicCondNode(pTemp1, pCond3, &pTemp2, LOGIC_COND_TYPE_OR)); - CHECK_RES_OUT_OF_MEM(createLogicCondNode(pTemp2, pCond4, &pFullCond, LOGIC_COND_TYPE_OR)); + CHECK_RES_OUT_OF_MEM(createLogicCondNode(&pCond1, &pCond2, &pTemp1, LOGIC_COND_TYPE_OR)); + CHECK_RES_OUT_OF_MEM(createLogicCondNode(&pTemp1, &pCond3, &pTemp2, LOGIC_COND_TYPE_OR)); + CHECK_RES_OUT_OF_MEM(createLogicCondNode(&pTemp2, &pCond4, &pFullCond, LOGIC_COND_TYPE_OR)); SNode* pThen = nodesMakeValueNodeFromInt32(1); CHECK_POINTER_OUT_OF_MEM(pThen); @@ -14093,7 +14581,7 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) { CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_GREATER_THAN, pSumColAlias, pTempVal, &pCond2)); // leader_col = count_col and leader_col > 0 pTemp1 = NULL; - CHECK_RES_OUT_OF_MEM(createLogicCondNode(pCond1, pCond2, &pTemp1, LOGIC_COND_TYPE_AND)); + CHECK_RES_OUT_OF_MEM(createLogicCondNode(&pCond1, &pCond2, &pTemp1, LOGIC_COND_TYPE_AND)); pThen = nodesMakeValueNodeFromInt32(1); CHECK_POINTER_OUT_OF_MEM(pThen); @@ -14108,7 +14596,7 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) { CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_GREATER_THAN, pSumColAlias, pTempVal, &pCond2)); // leader_col < count_col and leader_col > 0 pTemp2 = NULL; - CHECK_RES_OUT_OF_MEM(createLogicCondNode(pCond1, pCond2, &pTemp2, LOGIC_COND_TYPE_AND)); + CHECK_RES_OUT_OF_MEM(createLogicCondNode(&pCond1, &pCond2, &pTemp2, LOGIC_COND_TYPE_AND)); nodesDestroyNode((SNode*)pTempVal); pThen = nodesMakeValueNodeFromInt32(2); @@ -14247,40 +14735,49 @@ static int32_t toMsgType(ENodeType type) { static int32_t setRefreshMeta(STranslateContext* pCxt, SQuery* pQuery) { if (NULL != pCxt->pDbs) { - taosArrayDestroy(pQuery->pDbList); + (void)taosArrayDestroy(pQuery->pDbList); pQuery->pDbList = taosArrayInit(taosHashGetSize(pCxt->pDbs), TSDB_DB_FNAME_LEN); if (NULL == pQuery->pDbList) { return TSDB_CODE_OUT_OF_MEMORY; } SFullDatabaseName* pDb = taosHashIterate(pCxt->pDbs, NULL); while (NULL != pDb) { - taosArrayPush(pQuery->pDbList, pDb->fullDbName); + if (NULL == taosArrayPush(pQuery->pDbList, pDb->fullDbName)) { + taosHashCancelIterate(pCxt->pDbs, pDb); + return TSDB_CODE_OUT_OF_MEMORY; + } pDb = taosHashIterate(pCxt->pDbs, pDb); } } if (NULL != pCxt->pTables) { - taosArrayDestroy(pQuery->pTableList); + (void)taosArrayDestroy(pQuery->pTableList); pQuery->pTableList = taosArrayInit(taosHashGetSize(pCxt->pTables), sizeof(SName)); if (NULL == pQuery->pTableList) { return TSDB_CODE_OUT_OF_MEMORY; } SName* pTable = taosHashIterate(pCxt->pTables, NULL); while (NULL != pTable) { - taosArrayPush(pQuery->pTableList, pTable); + if (NULL == taosArrayPush(pQuery->pTableList, pTable)) { + taosHashCancelIterate(pCxt->pTables, pTable); + return TSDB_CODE_OUT_OF_MEMORY; + } pTable = taosHashIterate(pCxt->pTables, pTable); } } if (NULL != pCxt->pTargetTables) { - taosArrayDestroy(pQuery->pTargetTableList); + (void)taosArrayDestroy(pQuery->pTargetTableList); pQuery->pTargetTableList = taosArrayInit(taosHashGetSize(pCxt->pTargetTables), sizeof(SName)); if (NULL == pQuery->pTargetTableList) { return TSDB_CODE_OUT_OF_MEMORY; } SName* pTable = taosHashIterate(pCxt->pTargetTables, NULL); while (NULL != pTable) { - taosArrayPush(pQuery->pTargetTableList, pTable); + if (NULL == taosArrayPush(pQuery->pTargetTableList, pTable)) { + taosHashCancelIterate(pCxt->pTargetTables, pTable); + return TSDB_CODE_OUT_OF_MEMORY; + } pTable = taosHashIterate(pCxt->pTargetTables, pTable); } } @@ -14381,7 +14878,8 @@ int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMe if (TSDB_CODE_SUCCESS == code) { code = setQuery(&cxt, pQuery); } - setRefreshMeta(&cxt, pQuery); + int32_t tmpCode = setRefreshMeta(&cxt, pQuery); + if (TSDB_CODE_SUCCESS == code) code = tmpCode; destroyTranslateContext(&cxt); return code; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 9ff6fc3e49..8502856754 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -158,7 +158,9 @@ int32_t cleanupTaskQueue() { } int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) { - SSchedMsg* pSchedMsg = taosAllocateQitem(sizeof(SSchedMsg), DEF_QITEM, 0); + SSchedMsg* pSchedMsg; + int32_t rc = taosAllocateQitem(sizeof(SSchedMsg), DEF_QITEM, 0, (void **)&pSchedMsg); + if (rc) return rc; pSchedMsg->fp = NULL; pSchedMsg->ahandle = execFn; pSchedMsg->thandle = execParam; diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index dfd00dc5a7..93bbc44a25 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -218,8 +218,8 @@ typedef struct SQWorkerMgmt { #define QW_IDS() sId, qId, tId, rId, eId #define QW_FPARAMS() mgmt, QW_IDS() -#define QW_STAT_INC(_item, _n) atomic_add_fetch_64(&(_item), _n) -#define QW_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n) +#define QW_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n) +#define QW_STAT_DEC(_item, _n) (void)atomic_sub_fetch_64(&(_item), _n) #define QW_STAT_GET(_item) atomic_load_64(&(_item)) #define QW_GET_EVENT(ctx, event) atomic_load_8(&(ctx)->events[event]) @@ -250,7 +250,7 @@ typedef struct SQWorkerMgmt { } while (0) #define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code) -#define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) +#define QW_UPDATE_RSP_CODE(ctx, code) (void)atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) #define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY) #define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch) diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index b7a4b718e2..d3b8d36b25 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -221,11 +221,11 @@ void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) { SEpSet epSet = {0}; epSet.inUse = 1; epSet.numOfEps = 3; - strcpy(epSet.eps[0].fqdn, "localhost"); + TAOS_STRCPY(epSet.eps[0].fqdn, "localhost"); epSet.eps[0].port = 7100; - strcpy(epSet.eps[1].fqdn, "localhost"); + TAOS_STRCPY(epSet.eps[1].fqdn, "localhost"); epSet.eps[1].port = 7200; - strcpy(epSet.eps[2].fqdn, "localhost"); + TAOS_STRCPY(epSet.eps[2].fqdn, "localhost"); epSet.eps[2].port = 7300; ctx->phase = QW_PHASE_POST_QUERY; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 9f4a540f3c..f84e5e463e 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -21,7 +21,7 @@ int32_t qwMallocFetchRsp(int8_t rpcMalloc, int32_t length, SRetrieveTableRsp **r } if (NULL == *rsp) { - memset(pRsp, 0, sizeof(SRetrieveTableRsp)); + TAOS_MEMSET(pRsp, 0, sizeof(SRetrieveTableRsp)); } *rsp = pRsp; @@ -169,7 +169,7 @@ int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieve if (NULL == pRsp) { QW_RET(terrno); } - memset(pRsp, 0, sizeof(SRetrieveTableRsp)); + TAOS_MEMSET(pRsp, 0, sizeof(SRetrieveTableRsp)); dataLength = 0; } diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 0451532cbe..0b404ec176 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -342,7 +342,7 @@ int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { if (NULL == pExec) { QW_ERR_JRET(terrno); } - memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo)); + (void)memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo)); localRsp.rsp.subplanInfo = pExec; localRsp.qId = qId; localRsp.tId = tId; @@ -537,7 +537,7 @@ int32_t qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) { } if (dbFName[0] && tbName[0]) { - sprintf(tbInfo.tbFName, "%s.%s", dbFName, tbName); + (void)sprintf(tbInfo.tbFName, "%s.%s", dbFName, tbName); } else { tbInfo.tbFName[0] = 0; } @@ -613,7 +613,7 @@ void qwDestroyImpl(void *pMgmt) { taosMemoryFree(mgmt); - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); qwCloseRef(); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 5840cc0245..6e684545f7 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -452,8 +452,8 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes pRes->skey = pDelRes->skey; pRes->ekey = pDelRes->ekey; pRes->affectedRows = pDelRes->affectedRows; - strcpy(pRes->tableFName, pDelRes->tableName); - strcpy(pRes->tsColName, pDelRes->tsColName); + TAOS_STRCPY(pRes->tableFName, pDelRes->tableName); + TAOS_STRCPY(pRes->tsColName, pDelRes->tsColName); taosMemoryFree(output.pData); return TSDB_CODE_SUCCESS; @@ -1126,8 +1126,8 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { sch->hbConnInfo.handle = NULL; } - memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo)); - memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId)); + (void)memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo)); + (void)memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId)); QW_UNLOCK(QW_WRITE, &sch->hbConnLock); @@ -1138,7 +1138,7 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { _return: - memcpy(&rsp.epId, &req->epId, sizeof(req->epId)); + (void)memcpy(&rsp.epId, &req->epId, sizeof(req->epId)); code = qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code); if (code) { @@ -1299,19 +1299,19 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S int32_t qwNum = atomic_add_fetch_32(&gQwMgmt.qwNum, 1); if (1 == qwNum) { - memset(gQwMgmt.param, 0, sizeof(gQwMgmt.param)); + TAOS_MEMSET(gQwMgmt.param, 0, sizeof(gQwMgmt.param)); } int32_t code = qwOpenRef(); if (code) { - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); QW_RET(code); } SQWorker *mgmt = taosMemoryCalloc(1, sizeof(SQWorker)); if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorker)); - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); QW_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -1345,7 +1345,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S if (pMsgCb) { mgmt->msgCb = *pMsgCb; } else { - memset(&mgmt->msgCb, 0, sizeof(mgmt->msgCb)); + TAOS_MEMSET(&mgmt->msgCb, 0, sizeof(mgmt->msgCb)); } mgmt->refId = taosAddRef(gQwMgmt.qwRef, mgmt); @@ -1379,7 +1379,7 @@ _return: taosTmrCleanUp(mgmt->timer); taosMemoryFreeClear(mgmt); - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); } QW_RET(code); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index d292b271c5..91e1dda2ac 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -170,7 +170,7 @@ int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { printf("malloc failed"); assert(0); } - memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); + (void)memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = newMsg; if (qwtTestFetchQueueWIdx >= qwtTestFetchQueueSize) { qwtTestFetchQueueWIdx = 0; @@ -199,7 +199,7 @@ int32_t qwtPutReqToQueue(void *node, EQueueType qtype, struct SRpcMsg *pMsg) { printf("malloc failed"); assert(0); } - memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); + (void)memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = newMsg; if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) { qwtTestQueryQueueWIdx = 0; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index c7f6c20b7d..5ea79c6ae9 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -336,7 +336,7 @@ extern SSchedulerMgmt schMgmt; ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && \ (!SCH_IS_DATA_BIND_QRY_TASK(_task))) -#define SCH_UPDATE_REDIRECT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) +#define SCH_UPDATE_REDIRECT_CODE(job, _code) (void)atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) #define SCH_GET_REDIRECT_CODE(job, _code) \ (((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode) @@ -413,7 +413,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOG_TASK_START_TS(_task) \ do { \ int64_t us = taosGetTimestampUs(); \ - taosArrayPush((_task)->profile.execTime, &us); \ + (void)taosArrayPush((_task)->profile.execTime, &us); \ if (0 == (_task)->execId) { \ (_task)->profile.startTs = us; \ } \ @@ -422,7 +422,10 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOG_TASK_WAIT_TS(_task) \ do { \ int64_t us = taosGetTimestampUs(); \ - (_task)->profile.waitTime += us - *(int64_t *)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ + int64_t* startus = (int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ + if (NULL != startus) { \ + (_task)->profile.waitTime += us - *startus; \ + } \ } while (0) #define SCH_LOG_TASK_END_TS(_task) \ @@ -430,7 +433,9 @@ extern SSchedulerMgmt schMgmt; int64_t us = taosGetTimestampUs(); \ int32_t idx = (_task)->execId % (_task)->maxExecTimes; \ int64_t *startts = taosArrayGet((_task)->profile.execTime, (_task)->execId); \ - *startts = us - *startts; \ + if (NULL != startts) { \ + *startts = us - *startts; \ + } \ (_task)->profile.endTs = us; \ } while (0) @@ -538,7 +543,7 @@ void schCleanClusterHb(void *pTrans); int32_t schLaunchTask(SSchJob *job, SSchTask *task); int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType, void *param); -SSchJob *schAcquireJob(int64_t refId); +int32_t schAcquireJob(int64_t refId, SSchJob **ppJob); int32_t schReleaseJob(int64_t refId); void schFreeFlowCtrl(SSchJob *pJob); int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); @@ -578,12 +583,12 @@ int32_t schJobFetchRows(SSchJob *pJob); int32_t schJobFetchRowsA(SSchJob *pJob); int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList); -char *schDumpEpSet(SEpSet *pEpSet); +int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes); char *schGetOpStr(SCH_OP_TYPE type); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq); -int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes); +void schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes); int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet); int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode); void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode); @@ -606,7 +611,7 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask); void schDropTaskInHashList(SSchJob *pJob, SHashObj *list); int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type, SSchTask *pTask); int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); -int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); +void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel); int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode); diff --git a/source/libs/scheduler/src/schDbg.c b/source/libs/scheduler/src/schDbg.c index d6c2b638b8..deaaaec7c1 100644 --- a/source/libs/scheduler/src/schDbg.c +++ b/source/libs/scheduler/src/schDbg.c @@ -16,17 +16,5 @@ #include "query.h" #include "schInt.h" -tsem_t schdRspSem; SSchDebug gSCHDebug = {0}; -void schdExecCallback(SExecResult* pResult, void* param, int32_t code) { - if (code) { - pResult->code = code; - } - - *(SExecResult*)param = *pResult; - - taosMemoryFree(pResult); - - tsem_post(&schdRspSem); -} diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 8c2b65e125..20e1600737 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -50,6 +50,10 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks); for (int32_t i = 0; i < taskNum; ++i) { SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i); + if (NULL == pTask) { + SCH_JOB_DLOG("fail to get the %dth task", i); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } sum += pTask->plan->execNodeStat.tableNum; } @@ -214,6 +218,10 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { for (int32_t i = 0; i < taskNum; ++i) { pTask = *(SSchTask **)taosArrayGet(ctrl->taskList, i); + if (NULL == pTask) { + SCH_JOB_ELOG("fail to get the %dth task", i); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); if (pTask->plan->execNodeStat.tableNum > remainNum && ctrl->execTaskNum > 0) { @@ -243,6 +251,11 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { if (i < (taskNum - 1)) { SSchTask *pLastTask = *(SSchTask **)taosArrayGetLast(ctrl->taskList); + if (NULL == pLastTask) { + SCH_JOB_ELOG("fail to get the last task, num:%d", (int32_t)taosArrayGetSize(ctrl->taskList)); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + if (remainNum < pLastTask->plan->execNodeStat.tableNum) { SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d, smallestInList:%d", ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 278768981a..58a0706223 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -47,6 +47,7 @@ void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) { return; _return: + SCH_JOB_DLOG("job errCode updated to %s", tstrerror(errCode)); } @@ -166,9 +167,18 @@ _return: int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { for (int32_t i = 0; i < pJob->levelNum; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); - + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, pJob->levelNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + for (int32_t m = 0; m < pLevel->taskNum; ++m) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); + if (NULL == pTask) { + SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum: %d", m, pLevel->level, pLevel->taskNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SSubplan *pPlan = pTask->plan; int32_t childNum = pPlan->pChildren ? (int32_t)LIST_LENGTH(pPlan->pChildren) : 0; int32_t parentNum = pPlan->pParents ? (int32_t)LIST_LENGTH(pPlan->pParents) : 0; @@ -188,6 +198,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { for (int32_t n = 0; n < childNum; ++n) { SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); + if (NULL == child) { + SCH_JOB_ELOG("fail to get the %dth child subplan, childNum: %d", n, childNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES); if (NULL == childTask || NULL == *childTask) { SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -222,6 +237,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { for (int32_t n = 0; n < parentNum; ++n) { SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); + if (NULL == parent) { + SCH_JOB_ELOG("fail to get the %dth parent subplan, parentNum: %d", n, parentNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); if (NULL == parentTask || NULL == *parentTask) { SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -241,6 +261,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } SSchLevel *pLevel = taosArrayGet(pJob->levels, 0); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get level 0 level, levelNum:%d", (int32_t)taosArrayGetSize(pJob->levels)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + if (SCH_IS_QUERY_JOB(pJob)) { if (pLevel->taskNum > 1) { SCH_JOB_ELOG("invalid query plan, level:0, taskNum:%d", pLevel->taskNum); @@ -248,6 +273,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } SSchTask *pTask = taosArrayGet(pLevel->subTasks, 0); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the first task in level 0, taskNum:%d", (int32_t)taosArrayGetSize(pLevel->subTasks)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + if (SUBPLAN_TYPE_MODIFY != pTask->plan->subplanType || EXPLAIN_MODE_DISABLE != pJob->attr.explainMode) { pJob->attr.needFetch = true; } @@ -261,7 +291,9 @@ int32_t schAppendJobDataSrc(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - taosArrayPush(pJob->dataSrcTasks, &pTask); + if (NULL == taosArrayPush(pJob->dataSrcTasks, &pTask)) { + SCH_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -318,6 +350,11 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { } pLevel = taosArrayGet(pJob->levels, i); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, levelNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + pLevel->level = i; plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); @@ -342,6 +379,10 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { for (int32_t n = 0; n < taskNum; ++n) { SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); + if (NULL == plan) { + SCH_JOB_ELOG("fail to get the %dth subplan, taskNum: %d", n, taskNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } SCH_SET_JOB_TYPE(pJob, plan->subplanType); @@ -383,7 +424,7 @@ _return: SCH_RET(code); } -int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) { +void schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) { pRes->code = atomic_load_32(&pJob->errCode); pRes->numOfRows = pJob->resNumOfRows; @@ -395,8 +436,6 @@ int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) { SCH_UNLOCK(SCH_WRITE, &pJob->resLock); SCH_JOB_DLOG("execRes dumped, code: %s", tstrerror(pRes->code)); - - return TSDB_CODE_SUCCESS; } int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) { @@ -421,10 +460,13 @@ int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) { if (NULL == *pData) { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); - if (rsp) { - rsp->completed = 1; + if (NULL == rsp) { + SCH_JOB_ELOG("malloc SRetrieveTableRsp %d failed, code:%x", (int32_t)sizeof(SRetrieveTableRsp), terrno); + SCH_ERR_JRET(terrno); } + rsp->completed = 1; + *pData = rsp; SCH_JOB_DLOG("empty res and set query complete, code:%x", code); } @@ -440,10 +482,13 @@ _return: int32_t schNotifyUserExecRes(SSchJob *pJob) { SExecResult *pRes = taosMemoryCalloc(1, sizeof(SExecResult)); - if (pRes) { - schDumpJobExecRes(pJob, pRes); + if (NULL == pRes) { + qError("malloc execResult %d failed, error: %x", (int32_t)sizeof(SExecResult), terrno); + SCH_RET(terrno); } + schDumpJobExecRes(pJob, pRes); + SCH_JOB_DLOG("sch start to invoke exec cb, code: %s", tstrerror(pJob->errCode)); (*pJob->userRes.execFp)(pRes, pJob->userRes.cbParam, atomic_load_32(&pJob->errCode)); SCH_JOB_DLOG("sch end from exec cb, code: %s", tstrerror(pJob->errCode)); @@ -454,7 +499,10 @@ int32_t schNotifyUserExecRes(SSchJob *pJob) { int32_t schNotifyUserFetchRes(SSchJob *pJob) { void *pRes = NULL; - schDumpJobFetchRes(pJob, &pRes); + int32_t code = schDumpJobFetchRes(pJob, &pRes); + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_SUCCESS == atomic_load_32(&pJob->errCode)) { + atomic_store_32(&pJob->errCode, code); + } SCH_JOB_DLOG("sch start to invoke fetch cb, code: %s", tstrerror(pJob->errCode)); (*pJob->userRes.fetchFp)(pRes, pJob->userRes.cbParam, atomic_load_32(&pJob->errCode)); @@ -478,13 +526,13 @@ void schPostJobRes(SSchJob *pJob, SCH_OP_TYPE op) { if (SCH_JOB_IN_SYNC_OP(pJob)) { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - tsem_post(&pJob->rspSem); + (void)tsem_post(&pJob->rspSem); // ignore error } else if (SCH_JOB_IN_ASYNC_EXEC_OP(pJob)) { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - schNotifyUserExecRes(pJob); + (void)schNotifyUserExecRes(pJob); // ignore error } else if (SCH_JOB_IN_ASYNC_FETCH_OP(pJob)) { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - schNotifyUserFetchRes(pJob); + (void)schNotifyUserFetchRes(pJob); // ignore error } else { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); SCH_JOB_ELOG("job not in any operation, status:%s", jobTaskStatusStr(pJob->status)); @@ -519,7 +567,8 @@ int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } - schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode); + (void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode); // ignore error + return TSDB_CODE_SCH_IGNORE_ERROR; } @@ -530,7 +579,8 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } - schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode); + (void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode); // ignore error + return TSDB_CODE_SCH_IGNORE_ERROR; } @@ -569,12 +619,21 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { SSchLevel *pLevel = pTask->level; int32_t doneNum = atomic_load_32(&pLevel->taskExecDoneNum); if (doneNum == pLevel->taskNum) { - atomic_sub_fetch_32(&pJob->levelIdx, 1); + (void)atomic_sub_fetch_32(&pJob->levelIdx, 1); pLevel = taosArrayGet(pJob->levels, pJob->levelIdx); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, (int32_t)taosArrayGetSize(pJob->levels)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + for (int32_t i = 0; i < pLevel->taskNum; ++i) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, i); - + if (NULL == pTask) { + SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum:%d", i, pLevel->level, pLevel->taskNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + if (pTask->children && taosArrayGetSize(pTask->children) > 0) { continue; } @@ -602,7 +661,11 @@ int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp) { } } - taosArrayAddBatch((SArray *)pJob->execRes.res, taosArrayGet(rsp->tbVerInfo, 0), taosArrayGetSize(rsp->tbVerInfo)); + if (NULL == taosArrayAddBatch((SArray *)pJob->execRes.res, taosArrayGet(rsp->tbVerInfo, 0), taosArrayGetSize(rsp->tbVerInfo))) { + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + SCH_ERR_RET(terrno); + } + taosArrayDestroy(rsp->tbVerInfo); pJob->execRes.msgType = TDMT_SCH_QUERY; @@ -629,6 +692,11 @@ int32_t schLaunchJob(SSchJob *pJob) { SCH_ERR_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL)); } else { SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx); + if (NULL == level) { + SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, (int32_t)taosArrayGetSize(pJob->levels)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_ERR_RET(schLaunchLevelTasks(pJob, level)); } @@ -661,10 +729,19 @@ void schFreeJobImpl(void *job) { int32_t numOfLevels = taosArrayGetSize(pJob->levels); for (int32_t i = 0; i < numOfLevels; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", i, numOfLevels); + continue; + } int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); for (int32_t j = 0; j < numOfTasks; ++j) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the %dth task, taskNum:%d", j, numOfTasks); + continue; + } + schFreeTask(pJob, pTask); } @@ -687,12 +764,12 @@ void schFreeJobImpl(void *job) { destroyQueryExecRes(&pJob->execRes); qDestroyQueryPlan(pJob->pDag); - nodesReleaseAllocatorWeakRef(pJob->allocatorRefId); + (void)nodesReleaseAllocatorWeakRef(pJob->allocatorRefId); // ignore error taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); taosMemoryFreeClear(pJob->sql); - tsem_destroy(&pJob->rspSem); + (void)tsem_destroy(&pJob->rspSem); // ignore error taosMemoryFree(pJob); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); @@ -711,7 +788,7 @@ int32_t schJobFetchRows(SSchJob *pJob) { if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - tsem_wait(&pJob->rspSem); + (void)tsem_wait(&pJob->rspSem); // ignore error SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); } } else { @@ -739,9 +816,19 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->conn = *pReq->pConn; if (pReq->sql) { pJob->sql = taosStrdup(pReq->sql); + if (NULL == pJob->sql) { + qError("QID:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } pJob->pDag = pReq->pDag; - pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); + if (pReq->allocatorRefId > 0) { + pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); + if (pJob->allocatorRefId <= 0) { + qError("QID:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId); + SCH_ERR_JRET(terrno); + } + } pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillParam = pReq->chkKillParam; pJob->userRes.execFp = pReq->execFp; @@ -752,6 +839,10 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); } else { pJob->nodeList = taosArrayDup(pReq->pNodeList, NULL); + if (NULL == pJob->nodeList) { + qError("QID:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, (int32_t)taosArrayGetSize(pReq->pNodeList)); + SCH_ERR_JRET(terrno); + } } pJob->taskList = taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, @@ -785,7 +876,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { SCH_ERR_JRET(terrno); } - atomic_add_fetch_32(&schMgmt.jobNum, 1); + (void)atomic_add_fetch_32(&schMgmt.jobNum, 1); *pJobId = pJob->refId; @@ -800,7 +891,7 @@ _return: } else if (pJob->refId < 0) { schFreeJobImpl(pJob); } else { - taosRemoveRef(schMgmt.jobRef, pJob->refId); + (void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error } SCH_RET(code); @@ -814,7 +905,7 @@ int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { if (pReq->syncReq) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - tsem_wait(&pJob->rspSem); + (void)tsem_wait(&pJob->rspSem); // ignore error } SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); @@ -845,7 +936,7 @@ int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode) { } SCH_UNLOCK(SCH_WRITE, &pJob->resLock); - schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC); + SCH_ERR_RET(schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC)); } return TSDB_CODE_SUCCESS; @@ -866,6 +957,10 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) { int32_t numOfLevels = taosArrayGetSize(pJob->levels); for (int32_t i = 0; i < numOfLevels; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", i, numOfLevels); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } pLevel->taskExecDoneNum = 0; pLevel->taskLaunchedNum = 0; @@ -873,6 +968,11 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) { int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); for (int32_t j = 0; j < numOfTasks; ++j) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); + if (NULL == pTask) { + SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum:%d", j, i, numOfTasks); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_LOCK_TASK(pTask); code = schChkUpdateRedirectCtx(pJob, pTask, NULL, rspCode); if (TSDB_CODE_SUCCESS != code) { @@ -973,7 +1073,7 @@ void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int } if (errCode) { - schHandleJobFailure(pJob, errCode); + (void)schHandleJobFailure(pJob, errCode); // handle internal } SCH_JOB_DLOG("job end %s operation with code %s", schGetOpStr(type), tstrerror(errCode)); @@ -1052,11 +1152,11 @@ void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { } if (errCode) { - schHandleJobFailure(pJob, errCode); + (void)schHandleJobFailure(pJob, errCode); // ignore error } if (pJob) { - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); // ignore error } } @@ -1065,7 +1165,9 @@ int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_ int8_t status = 0; SSchTask *pTask = NULL; - SSchJob *pJob = schAcquireJob(rId); + SSchJob *pJob = NULL; + + (void)schAcquireJob(rId, &pJob); if (NULL == pJob) { qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); SCH_ERR_RET(TSDB_CODE_QRY_JOB_NOT_EXIST); @@ -1091,7 +1193,7 @@ _return: SCH_UNLOCK_TASK(pTask); } if (pJob) { - schReleaseJob(rId); + (void)schReleaseJob(rId); // ignore error } SCH_RET(code); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index d0dc04d6b4..08a8f684f5 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -113,7 +113,7 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs } atomic_store_ptr(&pJob->fetchRes, rsp); - atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows)); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows)); if (rsp->completed) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); @@ -166,13 +166,27 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SCH_LOCK(SCH_WRITE, &pJob->resLock); if (NULL == pJob->execRes.res) { pJob->execRes.res = (void*)taosArrayInit(batchRsp.nRsps, POINTER_BYTES); + if (NULL == pJob->execRes.res) { + code = terrno; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + tDecoderClear(&coder); + SCH_ERR_JRET(code); + } + pJob->execRes.msgType = TDMT_VND_CREATE_TABLE; } for (int32_t i = 0; i < batchRsp.nRsps; ++i) { SVCreateTbRsp *rsp = batchRsp.pRsps + i; if (rsp->pMeta) { - taosArrayPush((SArray*)pJob->execRes.res, &rsp->pMeta); + if (NULL == taosArrayPush((SArray*)pJob->execRes.res, &rsp->pMeta)) { + code = terrno; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + tDecoderClear(&coder); + SCH_ERR_JRET(code); + } } if (TSDB_CODE_SUCCESS != rsp->code) { @@ -264,7 +278,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SCH_ERR_JRET(code); } - atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); int32_t createTbRspNum = taosArrayGetSize(rsp->aCreateTbRsp); SCH_TASK_DLOG("submit succeed, affectedRows:%d, createTbRspNum:%d", rsp->affectedRows, createTbRspNum); @@ -275,7 +289,12 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SSubmitRsp2 *sum = pJob->execRes.res; sum->affectedRows += rsp->affectedRows; if (sum->aCreateTbRsp) { - taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp); + if (NULL == taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp)) { + code = terrno; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + SCH_ERR_JRET(code); + } + taosArrayDestroy(rsp->aCreateTbRsp); } else { TSWAP(sum->aCreateTbRsp, rsp->aCreateTbRsp); @@ -313,10 +332,14 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SDecoder coder = {0}; SVDeleteRsp rsp = {0}; tDecoderInit(&coder, msg, msgSize); - tDecodeSVDeleteRsp(&coder, &rsp); + if (tDecodeSVDeleteRsp(&coder, &rsp) < 0) { + code = terrno; + tDecoderClear(&coder); + SCH_ERR_JRET(code); + } tDecoderClear(&coder); - atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); SCH_TASK_DLOG("delete succeed, affectedRows:%" PRId64, rsp.affectedRows); } @@ -351,7 +374,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SCH_ERR_JRET(schSaveJobExecRes(pJob, &rsp)); - atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); taosMemoryFreeClear(msg); @@ -479,7 +502,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); // called if drop task rsp received code - rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error if (pMsg) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -501,7 +524,7 @@ int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchCallbackParamHeader *head = (SSchCallbackParamHeader *)param; - rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error qDebug("handle %p is broken", pMsg->handle); @@ -531,7 +554,7 @@ int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) { if (code) { qError("hb rsp error:%s", tstrerror(code)); - rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error SCH_ERR_JRET(code); } @@ -585,9 +608,14 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo param->head.isHbParam = true; SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + taosMemoryFree(param); + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum: %d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } param->nodeEpId.nodeId = addr->nodeId; SEp *pEp = SCH_GET_CUR_EP(addr); - strcpy(param->nodeEpId.ep.fqdn, pEp->fqdn); + TAOS_STRCPY(param->nodeEpId.ep.fqdn, pEp->fqdn); param->nodeEpId.ep.port = pEp->port; param->pTrans = trans->pTrans; *pParam = param; @@ -712,7 +740,7 @@ int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { int32_t code = 0; - memcpy(pDst, pSrc, sizeof(SRpcCtx)); + TAOS_MEMCPY(pDst, pSrc, sizeof(SRpcCtx)); pDst->brokenVal.val = NULL; pDst->args = NULL; @@ -760,7 +788,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; - memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); + TAOS_MEMCPY(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { @@ -877,7 +905,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(dst, pSrc, sizeof(*dst)); + TAOS_MEMCPY(dst, pSrc, sizeof(*dst)); *pDst = (SSchCallbackParamHeader *)dst; return TSDB_CODE_SUCCESS; @@ -889,7 +917,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(dst, pSrc, sizeof(*dst)); + TAOS_MEMCPY(dst, pSrc, sizeof(*dst)); *pDst = (SSchCallbackParamHeader *)dst; return TSDB_CODE_SUCCESS; @@ -904,7 +932,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pDst, pSrc, sizeof(*pSrc)); + TAOS_MEMCPY(pDst, pSrc, sizeof(*pSrc)); pDst->param = NULL; SCH_ERR_JRET(schCloneCallbackParam(pSrc->param, (SSchCallbackParamHeader **)&pDst->param)); @@ -948,6 +976,10 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery if (isHb && persistHandle && trans->pHandle == 0) { trans->pHandle = rpcAllocHandle(); + if (NULL == trans->pHandle) { + SCH_TASK_ELOG("rpcAllocHandle failed, code:%x", terrno); + SCH_ERR_JRET(terrno); + } } if (pJob && pTask) { @@ -1000,7 +1032,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { req.header.vgId = nodeEpId->nodeId; req.sId = schMgmt.sId; - memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); + TAOS_MEMCPY(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); SCH_LOCK(SCH_READ, &schMgmt.hbLock); SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId)); @@ -1013,7 +1045,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { SCH_LOCK(SCH_WRITE, &hb->lock); code = schCloneHbRpcCtx(&hb->rpcCtx, &rpcCtx); - memcpy(&trans, &hb->trans, sizeof(trans)); + TAOS_MEMCPY(&trans, &hb->trans, sizeof(trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); @@ -1039,7 +1071,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { SQueryNodeAddr addr = {.nodeId = nodeEpId->nodeId}; addr.epSet.inUse = 0; addr.epSet.numOfEps = 1; - memcpy(&addr.epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); + TAOS_MEMCPY(&addr.epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); code = schAsyncSendMsg(NULL, NULL, &trans, &addr, msgType, msg, msgSize, true, &rpcCtx); msg = NULL; @@ -1064,6 +1096,11 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get condidateAddr, candidateIdx %d, totalNum: %d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_JRET(terrno); + } + isCandidateAddr = true; SCH_TASK_DLOG("target candidateIdx %d, epInUse %d/%d", pTask->candidateIdx, addr->epSet.inUse, addr->epSet.numOfEps); @@ -1082,7 +1119,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(msg, pTask->msg, msgSize); + TAOS_MEMCPY(msg, pTask->msg, msgSize); break; } @@ -1098,13 +1135,21 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.msg = pTask->msg; req.source = pJob->source; msgSize = tSerializeSVDeleteReq(NULL, 0, &req); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSVDeleteReq failed, code:%x", terrno); + SCH_ERR_JRET(terrno); + } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - tSerializeSVDeleteReq(msg, msgSize, &req); + msgSize = tSerializeSVDeleteReq(msg, msgSize, &req); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSVDeleteReq second failed, code:%x", terrno); + SCH_ERR_JRET(terrno); + } break; } case TDMT_SCH_QUERY: @@ -1221,7 +1266,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.sId = schMgmt.sId; req.header.vgId = addr->nodeId; req.epId.nodeId = addr->nodeId; - memcpy(&req.epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); + TAOS_MEMCPY(&req.epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index d37393137f..f24ee74101 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -43,7 +43,7 @@ int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) { SCH_RET(schProcessOnJobFailure(pJob, (param ? *(int32_t*)param : 0))); break; case JOB_TASK_STATUS_DROP: - schProcessOnJobDropped(pJob, *(int32_t*)param); + (void)schProcessOnJobDropped(pJob, *(int32_t*)param); // ignore error if (taosRemoveRef(schMgmt.jobRef, pJob->refId)) { SCH_JOB_ELOG("remove job from job list failed, refId:0x%" PRIx64, pJob->refId); @@ -65,7 +65,8 @@ _return: } int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) { - SSchJob* pJob = schAcquireJob(jobId); + SSchJob* pJob = NULL; + (void)schAcquireJob(jobId, &pJob); if (NULL == pJob) { qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); SCH_ERR_RET(TSDB_CODE_SCH_JOB_NOT_EXISTS); @@ -90,7 +91,7 @@ int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq code = pJob->errCode; } - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); // ignore error return code; } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index c317a63ce1..6dd6aa9aae 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -165,7 +165,7 @@ int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, v SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execId)); } - schUpdateTaskExecNode(pJob, pTask, handle, execId); + SCH_ERR_RET(schUpdateTaskExecNode(pJob, pTask, handle, execId)); if ((execId != pTask->execId || execId <= pTask->failedExecId) || pTask->waitRetry) { // ignore it SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, @@ -297,7 +297,11 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { for (int32_t i = 0; i < parentNum; ++i) { SSchTask *parent = *(SSchTask **)taosArrayGet(pTask->parents, i); - + if (NULL == parent) { + SCH_TASK_ELOG("fail to get task %d parent, parentNum: %d", i, parentNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_LOCK(SCH_WRITE, &parent->planLock); SDownstreamSourceNode source = { .type = QUERY_NODE_DOWNSTREAM_SOURCE, @@ -308,9 +312,14 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { .fetchMsgType = SCH_FETCH_TYPE(pTask), .localExec = SCH_IS_LOCAL_EXEC_TASK(pJob, pTask), }; - qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); + code = qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); + if (TSDB_CODE_SUCCESS != code) { + SCH_TASK_ELOG("qSetSubplanExecutionNode failed, groupId: %d", pTask->plan->id.groupId); + } SCH_UNLOCK(SCH_WRITE, &parent->planLock); + SCH_ERR_RET(code); + int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1); if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { @@ -413,16 +422,16 @@ void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask) { schDropTaskOnExecNode(pJob, pTask); if (pTask->delayTimer) { - taosTmrStopA(&pTask->delayTimer); + (void)taosTmrStopA(&pTask->delayTimer); // ignore error } taosHashClear(pTask->execNodes); - schRemoveTaskFromExecList(pJob, pTask); + (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error schDeregisterTaskHb(pJob, pTask); taosMemoryFreeClear(pTask->msg); pTask->msgLen = 0; pTask->lastMsgType = 0; pTask->childReady = 0; - memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); + TAOS_MEMSET(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); } int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { @@ -443,11 +452,21 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_ERR_JRET(schUpdateTaskCandidateAddr(pJob, pTask, pData->pEpSet)); } else if (SYNC_SELF_LEADER_REDIRECT_ERROR(rspCode)) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d, code:%s", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps, pEp->fqdn, pEp->port, tstrerror(rspCode)); } else { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_SWITCH_EPSET(addr); SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); } @@ -476,7 +495,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 for (int32_t i = 0; i < childrenNum; ++i) { SSchTask *pChild = taosArrayGetP(pTask->children, i); SCH_LOCK_TASK(pChild); - schDoTaskRedirect(pJob, pChild, NULL, rspCode); + (void)schDoTaskRedirect(pJob, pChild, NULL, rspCode); // error handled internal SCH_UNLOCK_TASK(pChild); } @@ -494,18 +513,23 @@ int32_t schResetTaskSetLevelInfo(SSchJob *pJob, SSchTask *pTask) { atomic_load_32(&pLevel->taskExecDoneNum), atomic_load_32(&pLevel->taskLaunchedNum)); if (SCH_GET_TASK_STATUS(pTask) >= JOB_TASK_STATUS_PART_SUCC) { - atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); } - atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); int32_t childrenNum = taosArrayGetSize(pTask->children); for (int32_t i = 0; i < childrenNum; ++i) { SSchTask *pChild = taosArrayGetP(pTask->children, i); + if (NULL == pChild) { + SCH_TASK_ELOG("fail to get the %dth child, childrenNum:%d", i, childrenNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_LOCK_TASK(pChild); pLevel = pChild->level; - atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); - atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); SCH_UNLOCK_TASK(pChild); } @@ -711,9 +735,9 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo } int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { - atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); + (void)atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); - schRemoveTaskFromExecList(pJob, pTask); + (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -724,6 +748,11 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_DATA_BIND_TASK(pTask)) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_SWITCH_EPSET(addr); } else { SCH_ERR_RET(schSwitchTaskCandidateAddr(pJob, pTask)); @@ -743,6 +772,11 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { for (int32_t i = 0; i < nodeNum; ++i) { SQueryNodeLoad *nload = taosArrayGet(pJob->nodeList, i); + if (NULL == nload) { + SCH_TASK_ELOG("fail to get the %dth node in nodeList, nodeNum:%d", i, nodeNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SQueryNodeAddr *naddr = &nload->addr; if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { @@ -810,6 +844,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { } int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) { + int32_t code = TSDB_CODE_SUCCESS; if (NULL == pTask->candidateAddrs || 1 != taosArrayGetSize(pTask->candidateAddrs)) { SCH_TASK_ELOG("not able to update cndidate addr, addr num %d", (int32_t)(pTask->candidateAddrs ? taosArrayGetSize(pTask->candidateAddrs) : 0)); @@ -817,18 +852,27 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSe } SQueryNodeAddr *pAddr = taosArrayGet(pTask->candidateAddrs, 0); + if (NULL == pAddr) { + SCH_TASK_ELOG("fail to get task 0th condidataAddr, totalNum:%d", (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } - char *origEpset = schDumpEpSet(&pAddr->epSet); - char *newEpset = schDumpEpSet(pEpSet); + char *origEpset = NULL; + char *newEpset = NULL; + + SCH_ERR_RET(schDumpEpSet(&pAddr->epSet, &origEpset)); + SCH_ERR_JRET(schDumpEpSet(pEpSet, &newEpset)); SCH_TASK_DLOG("update task target node %d epset from %s to %s", pAddr->nodeId, origEpset, newEpset); + TAOS_MEMCPY(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet)); + +_return: + taosMemoryFree(origEpset); taosMemoryFree(newEpset); - memcpy(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet)); - - return TSDB_CODE_SUCCESS; + return code; } int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) { @@ -877,7 +921,6 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { } int32_t size = (int32_t)taosHashGetSize(pTask->execNodes); - if (size <= 0) { SCH_TASK_DLOG("task has no execNodes, no need to drop it, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); return; @@ -889,7 +932,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { if (nodeInfo->handle) { SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); void *pExecId = taosHashGetKey(nodeInfo, NULL); - schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, pExecId); + (void)schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, pExecId); // ignore error and continue SCH_TASK_DLOG("start to drop task's %dth execNode", i); } else { @@ -939,6 +982,11 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { for (int32_t i = 0; i < taskNum; ++i) { STaskStatus *pStatus = taosArrayGet(pStatusList, i); + if (NULL == pStatus) { + qError("fail to get the %dth task status in hb rsp, taskNum:%d", i, taskNum); + continue; + } + int32_t code = 0; qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, @@ -983,10 +1031,15 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { for (int32_t i = 0; i < resNum; ++i) { SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i); + if (NULL == localRsp) { + qError("fail to get the %dth LOCAL explain rsp msg, total:%d", i, resNum); + continue; + } qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); - pJob = schAcquireJob(localRsp->rId); + pJob = NULL; + (void)schAcquireJob(localRsp->rId, &pJob); if (NULL == pJob) { qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, localRsp->tId, localRsp->rId); @@ -996,7 +1049,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { int8_t status = 0; if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("will not do further processing cause of job status %s", jobTaskStatusStr(status)); - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } @@ -1006,7 +1059,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { code = schProcessExplainRsp(pJob, pTask, &localRsp->rsp); } - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, localRsp->tId, code); @@ -1022,6 +1075,11 @@ _return: for (int32_t i = 0; i < resNum; ++i) { SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i); + if (NULL == localRsp) { + qError("in _return fail to get the %dth LOCAL explain rsp msg, total:%d", i, resNum); + continue; + } + tFreeSExplainRsp(&localRsp->rsp); } @@ -1076,6 +1134,9 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); + if (NULL == explainRes) { + SCH_ERR_RET(terrno); + } } SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, @@ -1097,7 +1158,9 @@ _return: int32_t schLaunchTaskImpl(void *param) { SSchTaskCtx *pCtx = (SSchTaskCtx *)param; - SSchJob *pJob = schAcquireJob(pCtx->jobRid); + SSchJob *pJob = NULL; + + (void)schAcquireJob(pCtx->jobRid, &pJob); if (NULL == pJob) { qDebug("job refId 0x%" PRIx64 " already not exist", pCtx->jobRid); taosMemoryFree(param); @@ -1113,7 +1176,7 @@ int32_t schLaunchTaskImpl(void *param) { int8_t status = 0; int32_t code = 0; - atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); + (void)atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); pTask->execId++; pTask->retryTimes++; pTask->waitRetry = false; @@ -1160,7 +1223,7 @@ _return: SCH_UNLOCK_TASK(pTask); } - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); taosMemoryFree(param); @@ -1178,7 +1241,7 @@ int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { param->asyncLaunch = true; - taosAsyncExec(schLaunchTaskImpl, param, NULL); + SCH_ERR_RET(taosAsyncExec(schLaunchTaskImpl, param, NULL)); } else { SCH_ERR_RET(schLaunchTaskImpl(param)); } @@ -1252,7 +1315,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer); + (void)taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer); return TSDB_CODE_SUCCESS; } @@ -1283,7 +1346,7 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { SCH_LOCK_TASK(pTask); if (pTask->delayTimer) { - taosTmrStopA(&pTask->delayTimer); + (void)taosTmrStopA(&pTask->delayTimer); } schDropTaskOnExecNode(pJob, pTask); SCH_UNLOCK_TASK(pTask); @@ -1327,6 +1390,9 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); + if (NULL == explainRes) { + SCH_ERR_RET(terrno); + } } SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 82b2e021af..811890dde5 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -22,9 +22,14 @@ #include "tref.h" #include "trpc.h" -FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { +FORCE_INLINE int32_t schAcquireJob(int64_t refId, SSchJob** ppJob) { qDebug("sch acquire jobId:0x%" PRIx64, refId); - return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); + *ppJob = (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); + if (NULL == *ppJob) { + return terrno; + } + + return TSDB_CODE_SUCCESS; } FORCE_INLINE int32_t schReleaseJob(int64_t refId) { @@ -36,15 +41,16 @@ FORCE_INLINE int32_t schReleaseJob(int64_t refId) { return taosReleaseRef(schMgmt.jobRef, refId); } -char *schDumpEpSet(SEpSet *pEpSet) { +int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes) { + *ppRes = NULL; if (NULL == pEpSet) { - return NULL; + return TSDB_CODE_SUCCESS; } int32_t maxSize = 1024; char *str = taosMemoryMalloc(maxSize); if (NULL == str) { - return NULL; + return terrno; } int32_t n = 0; @@ -54,7 +60,8 @@ char *schDumpEpSet(SEpSet *pEpSet) { n += snprintf(str + n, maxSize - n, "[%s:%d]", pEp->fqdn, pEp->port); } - return str; + *ppRes = str; + return TSDB_CODE_SUCCESS; } char *schGetOpStr(SCH_OP_TYPE type) { @@ -73,7 +80,7 @@ char *schGetOpStr(SCH_OP_TYPE type) { } void schFreeHbTrans(SSchHbTrans *pTrans) { - rpcReleaseHandle((void *)pTrans->trans.pHandleId, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle((void *)pTrans->trans.pHandleId, TAOS_CONN_CLIENT); schFreeRpcCtx(&pTrans->rpcCtx); } @@ -86,7 +93,7 @@ void schCleanClusterHb(void *pTrans) { if (hb->trans.pTrans == pTrans) { SQueryNodeEpId *pEpId = taosHashGetKey(hb, NULL); schFreeHbTrans(hb); - taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); + (void)taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); } hb = taosHashIterate(schMgmt.hbConnections, hb); @@ -109,7 +116,7 @@ int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *ep int64_t taskNum = atomic_load_64(&hb->taskNum); if (taskNum <= 0) { schFreeHbTrans(hb); - taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + (void)taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); } SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); @@ -165,7 +172,7 @@ int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId * break; } - atomic_add_fetch_64(&hb->taskNum, 1); + (void)atomic_add_fetch_64(&hb->taskNum, 1); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); @@ -178,12 +185,17 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { } SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + return; + } + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; SEp *pEp = SCH_GET_CUR_EP(addr); - strcpy(epId.ep.fqdn, pEp->fqdn); + TAOS_STRCPY(epId.ep.fqdn, pEp->fqdn); epId.ep.port = pEp->port; SCH_LOCK(SCH_READ, &schMgmt.hbLock); @@ -197,7 +209,7 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { int64_t taskNum = atomic_sub_fetch_64(&hb->taskNum, 1); if (0 == taskNum) { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); - schRemoveHbConnection(pJob, pTask, &epId); + (void)schRemoveHbConnection(pJob, pTask, &epId); } else { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); } @@ -211,12 +223,17 @@ int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { } SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + return TSDB_CODE_SCH_INTERNAL_ERROR; + } + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; SEp *pEp = SCH_GET_CUR_EP(addr); - strcpy(epId.ep.fqdn, pEp->fqdn); + TAOS_STRCPY(epId.ep.fqdn, pEp->fqdn); epId.ep.port = pEp->port; SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId)); @@ -240,7 +257,7 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { } SCH_LOCK(SCH_WRITE, &hb->lock); - memcpy(&hb->trans, trans, sizeof(*trans)); + TAOS_MEMCPY(&hb->trans, trans, sizeof(*trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); @@ -256,7 +273,7 @@ void schCloseJobRef(void) { } if (schMgmt.jobRef >= 0) { - taosCloseRef(schMgmt.jobRef); + (void)taosCloseRef(schMgmt.jobRef); schMgmt.jobRef = -1; } } @@ -316,18 +333,18 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { } } -int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) { +void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) { + *pTask = NULL; + int32_t s = taosHashGetSize(pTaskList); if (s <= 0) { - return TSDB_CODE_SUCCESS; + return; } SSchTask **task = taosHashGet(pTaskList, &taskId, sizeof(taskId)); if (NULL == task || NULL == (*task)) { - return TSDB_CODE_SUCCESS; + return; } *pTask = *task; - - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index fc92be8214..45e0ecf738 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -108,14 +108,26 @@ int32_t schedulerGetTasksStatus(int64_t jobId, SArray *pSub) { for (int32_t i = pJob->levelNum - 1; i >= 0; --i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + if (NULL == pLevel) { + qError("failed to get level %d", i); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } for (int32_t m = 0; m < pLevel->taskNum; ++m) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); + if (NULL == pTask) { + qError("failed to get task %d, total: %d", m, pLevel->taskNum); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SQuerySubDesc subDesc = {0}; subDesc.tid = pTask->taskId; - strcpy(subDesc.status, jobTaskStatusStr(pTask->status)); + TAOS_STRCPY(subDesc.status, jobTaskStatusStr(pTask->status)); - taosArrayPush(pSub, &subDesc); + if (NULL == taosArrayPush(pSub, &subDesc)) { + qError("taosArrayPush task %d failed, error: %x, ", m, terrno); + SCH_ERR_JRET(terrno); + } } } @@ -141,7 +153,7 @@ int32_t schedulerUpdatePolicy(int32_t policy) { qDebug("schedule policy updated to %d", schMgmt.cfg.schPolicy); break; default: - return TSDB_CODE_TSC_INVALID_INPUT; + SCH_RET(TSDB_CODE_TSC_INVALID_INPUT); } return TSDB_CODE_SUCCESS; @@ -157,16 +169,17 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) { return; } - SSchJob *pJob = schAcquireJob(*jobId); + SSchJob *pJob = NULL; + (void)schAcquireJob(*jobId, &pJob); if (NULL == pJob) { - qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); + qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); return; } SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode)); - schHandleJobDrop(pJob, errCode); + (void)schHandleJobDrop(pJob, errCode); // ignore any error - schReleaseJob(*jobId); + (void)schReleaseJob(*jobId); // ignore error *jobId = 0; } @@ -182,7 +195,7 @@ void schedulerDestroy(void) { if (refId == 0) { break; } - taosRemoveRef(schMgmt.jobRef, pJob->refId); + (void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error pJob = taosIterateRef(schMgmt.jobRef, refId); } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 78e876f82c..f906e3ec3e 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -75,9 +75,7 @@ int32_t schtStartFetch = 0; void schtInitLogFile() { const char *defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; - rpcInit(); tsAsyncLog = 0; - rpcInit(); qDebugFlag = 159; strcpy(tsLogDir, TD_LOG_DIR_PATH); @@ -136,8 +134,13 @@ int32_t schtBuildSubmitRspMsg(uint32_t *msize, void **rspMsg) { tEncodeSize(tEncodeSSubmitRsp2, &submitRsp, msgSize, ret); void *msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + return terrno; + } tEncoderInit(&ec, (uint8_t *)msg, msgSize); - tEncodeSSubmitRsp2(&ec, &submitRsp); + if (tEncodeSSubmitRsp2(&ec, &submitRsp) < 0) { + return -1; + } tEncoderClear(&ec); *rspMsg = msg; @@ -152,11 +155,26 @@ void schtBuildQueryDag(SQueryPlan *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); + if (NULL == dag->pSubplans) { + return; + } SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == scan) { + return; + } SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == merge) { + return; + } SSubplan *scanPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == scanPlan) { + return; + } SSubplan *mergePlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == mergePlan) { + return; + } scanPlan->id.queryId = qId; scanPlan->id.groupId = 0x0000000000000002; @@ -170,7 +188,13 @@ void schtBuildQueryDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); + if (NULL == scanPlan->pParents) { + return; + } scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + if (NULL == scanPlan->pNode) { + return; + } scanPlan->msgType = TDMT_SCH_QUERY; mergePlan->id.queryId = qId; @@ -181,21 +205,33 @@ void schtBuildQueryDag(SQueryPlan *dag) { mergePlan->execNode.epSet.numOfEps = 0; mergePlan->pChildren = nodesMakeList(); + if (NULL == mergePlan->pChildren) { + return; + } mergePlan->pParents = NULL; mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE); + if (NULL == mergePlan->pNode) { + return; + } mergePlan->msgType = TDMT_SCH_QUERY; merge->pNodeList = nodesMakeList(); + if (NULL == merge->pNodeList) { + return; + } scan->pNodeList = nodesMakeList(); + if (NULL == scan->pNodeList) { + return; + } - nodesListAppend(merge->pNodeList, (SNode *)mergePlan); - nodesListAppend(scan->pNodeList, (SNode *)scanPlan); + (void)nodesListAppend(merge->pNodeList, (SNode *)mergePlan); + (void)nodesListAppend(scan->pNodeList, (SNode *)scanPlan); - nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); - nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); + (void)nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); + (void)nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); - nodesListAppend(dag->pSubplans, (SNode *)merge); - nodesListAppend(dag->pSubplans, (SNode *)scan); + (void)nodesListAppend(dag->pSubplans, (SNode *)merge); + (void)nodesListAppend(dag->pSubplans, (SNode *)scan); } void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { @@ -205,18 +241,42 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); + if (NULL == dag->pSubplans) { + return; + } SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == scan) { + return; + } SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == merge) { + return; + } SSubplan *mergePlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == mergePlan) { + return; + } merge->pNodeList = nodesMakeList(); + if (NULL == merge->pNodeList) { + return; + } scan->pNodeList = nodesMakeList(); + if (NULL == scan->pNodeList) { + return; + } mergePlan->pChildren = nodesMakeList(); + if (NULL == mergePlan->pChildren) { + return; + } for (int32_t i = 0; i < scanPlanNum; ++i) { SSubplan *scanPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == scanPlan) { + return; + } scanPlan->id.queryId = qId; scanPlan->id.groupId = 0x0000000000000002; scanPlan->id.subplanId = 0x0000000000000003 + i; @@ -233,13 +293,19 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); + if (NULL == scanPlan->pParents) { + return; + } scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + if (NULL == scanPlan->pNode) { + return; + } scanPlan->msgType = TDMT_SCH_QUERY; - nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); - nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); + (void)nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); + (void)nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); - nodesListAppend(scan->pNodeList, (SNode *)scanPlan); + (void)nodesListAppend(scan->pNodeList, (SNode *)scanPlan); } mergePlan->id.queryId = qId; @@ -251,12 +317,15 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { mergePlan->pParents = NULL; mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE); + if (NULL == mergePlan->pNode) { + return; + } mergePlan->msgType = TDMT_SCH_QUERY; - nodesListAppend(merge->pNodeList, (SNode *)mergePlan); + (void)nodesListAppend(merge->pNodeList, (SNode *)mergePlan); - nodesListAppend(dag->pSubplans, (SNode *)merge); - nodesListAppend(dag->pSubplans, (SNode *)scan); + (void)nodesListAppend(dag->pSubplans, (SNode *)merge); + (void)nodesListAppend(dag->pSubplans, (SNode *)scan); } void schtFreeQueryDag(SQueryPlan *dag) {} @@ -267,10 +336,22 @@ void schtBuildInsertDag(SQueryPlan *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); + if (NULL == dag->pSubplans) { + return; + } SNodeListNode *inserta = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == inserta) { + return; + } inserta->pNodeList = nodesMakeList(); + if (NULL == inserta->pNodeList) { + return; + } SSubplan *insertPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == insertPlan) { + return; + } insertPlan->id.queryId = qId; insertPlan->id.groupId = 0x0000000000000003; @@ -286,13 +367,22 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan->pParents = NULL; insertPlan->pNode = NULL; insertPlan->pDataSink = (SDataSinkNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT); + if (NULL == insertPlan->pDataSink) { + return; + } ((SDataInserterNode *)insertPlan->pDataSink)->size = 1; ((SDataInserterNode *)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1); + if (NULL == ((SDataInserterNode *)insertPlan->pDataSink)->pData) { + return; + } insertPlan->msgType = TDMT_VND_SUBMIT; - nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); + (void)nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); insertPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == insertPlan) { + return; + } insertPlan->id.queryId = qId; insertPlan->id.groupId = 0x0000000000000003; @@ -308,22 +398,31 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan->pParents = NULL; insertPlan->pNode = NULL; insertPlan->pDataSink = (SDataSinkNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT); + if (NULL == insertPlan->pDataSink) { + return; + } ((SDataInserterNode *)insertPlan->pDataSink)->size = 1; ((SDataInserterNode *)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1); + if (NULL == ((SDataInserterNode *)insertPlan->pDataSink)->pData) { + return; + } insertPlan->msgType = TDMT_VND_SUBMIT; - nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); + (void)nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); - nodesListAppend(dag->pSubplans, (SNode *)inserta); + (void)nodesListAppend(dag->pSubplans, (SNode *)inserta); } int32_t schtPlanToString(const SSubplan *subplan, char **str, int32_t *len) { *str = (char *)taosMemoryCalloc(1, 20); + if (NULL == *str) { + return -1; + } *len = 20; return 0; } -void schtExecNode(SSubplan *subplan, uint64_t groupId, SQueryNodeAddr *ep) {} +int32_t schtExecNode(SSubplan *subplan, uint64_t groupId, SQueryNodeAddr *ep) { return 0; } void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) {} @@ -431,24 +530,27 @@ void *schtSendRsp(void *param) { taosMsleep(1); } - pJob = schAcquireJob(job); - + code = schAcquireJob(job, &pJob); + if (code) { + return NULL; + } + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildSubmitRspMsg(&msg.len, &rmsg); + (void)schtBuildSubmitRspMsg(&msg.len, &rmsg); msg.msgType = TDMT_VND_SUBMIT_RSP; msg.pData = rmsg; - schHandleResponseMsg(pJob, task, task->execId, &msg, 0); + (void)schHandleResponseMsg(pJob, task, task->execId, &msg, 0); pIter = taosHashIterate(pJob->execTasks, pIter); } - schReleaseJob(job); + (void)schReleaseJob(job); schtJobDone = true; @@ -457,20 +559,25 @@ void *schtSendRsp(void *param) { void *schtCreateFetchRspThread(void *param) { int64_t job = *(int64_t *)param; - SSchJob *pJob = schAcquireJob(job); + SSchJob *pJob = NULL; + (void)schAcquireJob(job, &pJob); + if (NULL == pJob) { + return NULL; + } + taosSsleep(1); int32_t code = 0; SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildFetchRspMsg(&msg.len, &rmsg); + (void)schtBuildFetchRspMsg(&msg.len, &rmsg); msg.msgType = TDMT_SCH_MERGE_FETCH_RSP; msg.pData = rmsg; code = schHandleResponseMsg(pJob, pJob->fetchTask, pJob->fetchTask->execId, &msg, 0); - schReleaseJob(job); + (void)schReleaseJob(job); assert(code == 0); return NULL; @@ -488,12 +595,17 @@ void *schtFetchRspThread(void *aa) { taosUsleep(100); param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); - + if (NULL == param) { + return NULL; + } param->queryId = schtQueryId; param->taskId = schtFetchTaskId; int32_t code = 0; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); + if (NULL == rsp) { + return NULL; + } rsp->completed = 1; rsp->numOfRows = 10; @@ -549,12 +661,17 @@ void *schtRunJobThread(void *aa) { schtBuildQueryDag(dag); SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad)); - + if (NULL == qnodeList) { + assert(0); + } + SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + if (NULL == taosArrayPush(qnodeList, &load)) { + assert(0); + } queryDone = 0; @@ -572,7 +689,9 @@ void *schtRunJobThread(void *aa) { code = schedulerExecJob(&req, &queryJobRefId); assert(code == 0); - pJob = schAcquireJob(queryJobRefId); + pJob = NULL; + code = schAcquireJob(queryJobRefId, &pJob); + if (NULL == pJob) { taosArrayDestroy(qnodeList); schtFreeQueryDag(dag); @@ -580,16 +699,24 @@ void *schtRunJobThread(void *aa) { } execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == execTasks) { + assert(0); + } void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; schtFetchTaskId = task->taskId - 1; - taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task)); + if (taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task))) { + assert(0); + } pIter = taosHashIterate(pJob->execTasks, pIter); } param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); + if (NULL == param) { + assert(0); + } param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -601,7 +728,9 @@ void *schtRunJobThread(void *aa) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + if (schtBuildQueryRspMsg(&msg.len, &rmsg)) { + assert(0); + } msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -612,6 +741,9 @@ void *schtRunJobThread(void *aa) { } param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); + if (NULL == param) { + assert(0); + } param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -622,7 +754,9 @@ void *schtRunJobThread(void *aa) { param->taskId = task->taskId - 1; SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + if (schtBuildQueryRspMsg(&msg.len, &rmsg)) { + assert(0); + } msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -728,7 +862,9 @@ TEST(queryTest, normalCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); + SSchJob *pJob = NULL; + code = schAcquireJob(job, &pJob); + ASSERT_EQ(code, 0); void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { @@ -839,8 +975,10 @@ TEST(queryTest, readyFirstCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); - + SSchJob *pJob = NULL; + code = schAcquireJob(job, &pJob); + ASSERT_EQ(code, 0); + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; @@ -956,7 +1094,9 @@ TEST(queryTest, flowCtrlCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); + SSchJob *pJob = NULL; + code = schAcquireJob(job, &pJob); + ASSERT_EQ(code, 0); while (!queryDone) { void *pIter = taosHashIterate(pJob->execTasks, NULL); @@ -1094,13 +1234,17 @@ TEST(otherTest, otherCase) { schReleaseJob(0); schFreeRpcCtx(NULL); - ASSERT_EQ(schDumpEpSet(NULL), (char *)NULL); + char* ep = NULL; + ASSERT_EQ(schDumpEpSet(NULL, &ep), TSDB_CODE_SUCCESS); ASSERT_EQ(strcmp(schGetOpStr(SCH_OP_NULL), "NULL"), 0); ASSERT_EQ(strcmp(schGetOpStr((SCH_OP_TYPE)100), "UNKNOWN"), 0); } int main(int argc, char **argv) { schtInitLogFile(); + if (rpcInit()) { + assert(0); + } taosSeedRand(taosGetTimestampSec()); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 3b15e5122d..d0ad38ea79 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -29,11 +29,13 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint static int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList); static void checkpointTriggerMonitorFn(void* param, void* tmrId); -int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId, - int32_t srcTaskId, SStreamDataBlock** pRes) { - SStreamDataBlock* pChkpoint = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock)); - if (pChkpoint == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; +int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, + int32_t transId, int32_t srcTaskId) { + SStreamDataBlock* pChkpoint; + + int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock), (void**)&pChkpoint); + if (code) { + return code; } pChkpoint->type = checkpointType; @@ -436,7 +438,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream } void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { - pTask->chkInfo.startTs = 0; // clear the recorded start time + pTask->chkInfo.startTs = 0; // clear the recorded start time streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks (void) taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock); diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index e54794d75f..bf499293b4 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -16,8 +16,11 @@ #include "streamInt.h" SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg) { - SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, pReq->totalLen); - if (pData == NULL) { + SStreamDataBlock* pData; + + int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, pReq->totalLen, (void**)&pData); + if (code) { + terrno = code; return NULL; } @@ -34,7 +37,7 @@ SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pRe ASSERT((pReq->blockNum == taosArrayGetSize(pReq->data)) && (pReq->blockNum == taosArrayGetSize(pReq->dataLen))); for (int32_t i = 0; i < blockNum; i++) { - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*) taosArrayGetP(pReq->data, i); + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pReq->data, i); SSDataBlock* pDataBlock = taosArrayGet(pArray, i); int32_t compLen = *(int32_t*)pRetrieve->data; @@ -42,7 +45,7 @@ SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pRe char* pInput = pRetrieve->data + PAYLOAD_PREFIX_LEN; if (pRetrieve->compressed && compLen < fullLen) { - char* p = taosMemoryMalloc(fullLen); + char* p = taosMemoryMalloc(fullLen); int32_t len = tsDecompressString(pInput, compLen, 1, p, fullLen, ONE_STAGE_COMP, NULL, 0); ASSERT(len == fullLen); pInput = p; @@ -69,10 +72,14 @@ SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pRe return pData; } -SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, SArray* pRes) { - SStreamDataBlock* pStreamBlocks = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, resultSize); - if (pStreamBlocks == NULL) { +SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, + SArray* pRes) { + SStreamDataBlock* pStreamBlocks; + + int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, resultSize, (void**)&pStreamBlocks); + if (code) { taosArrayClearEx(pRes, (FDelete)blockDataFreeRes); + terrno = code; return NULL; } @@ -132,11 +139,10 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock } int32_t streamDataSubmitNew(SPackedData* pData, int32_t type, SStreamDataSubmit** pSubmit) { - *pSubmit = NULL; - - SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, pData->msgLen); - if (pDataSubmit == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + SStreamDataSubmit* pDataSubmit = NULL; + int32_t code = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, pData->msgLen, (void**)&pDataSubmit); + if (code) { + return code; } pDataSubmit->ver = pData->ver; @@ -154,8 +160,11 @@ void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit) { } SStreamMergedSubmit* streamMergedSubmitNew() { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0); - if (pMerged == NULL) { + SStreamMergedSubmit* pMerged; + + int32_t code = taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0, (void**)&pMerged); + if (code) { + terrno = code; return NULL; } @@ -181,7 +190,7 @@ int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubm // todo handle memory error SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem) { terrno = 0; - + if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)pElem; @@ -215,7 +224,8 @@ SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueI taosFreeQitem(pElem); return (SStreamQueueItem*)pMerged; } else { - stDebug("block type:%s not merged with existed blocks list, type:%d", streamQueueItemGetTypeStr(pElem->type), dst->type); + stDebug("block type:%s not merged with existed blocks list, type:%d", streamQueueItemGetTypeStr(pElem->type), + dst->type); return NULL; } } @@ -248,8 +258,9 @@ void streamFreeQitem(SStreamQueueItem* data) { SStreamRefDataBlock* pRefBlock = (SStreamRefDataBlock*)data; blockDataDestroy(pRefBlock->pBlock); taosFreeQitem(pRefBlock); - } else if (type == STREAM_INPUT__CHECKPOINT || type == STREAM_INPUT__CHECKPOINT_TRIGGER || type == STREAM_INPUT__TRANS_STATE) { - SStreamDataBlock* pBlock = (SStreamDataBlock*) data; + } else if (type == STREAM_INPUT__CHECKPOINT || type == STREAM_INPUT__CHECKPOINT_TRIGGER || + type == STREAM_INPUT__TRANS_STATE) { + SStreamDataBlock* pBlock = (SStreamDataBlock*)data; taosArrayDestroyEx(pBlock->blocks, freeItems); taosFreeQitem(pBlock); } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index d6929c5f00..2023da43ad 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1166,10 +1166,10 @@ void streamClearChkptReadyMsg(SActiveCheckpointInfo* pActiveInfo) { static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId, int32_t downstreamNodeId) { stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); - bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); - clearBufferedDispatchMsg(pTask); - int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs; + bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); + + clearBufferedDispatchMsg(pTask); // put data into inputQ of current task is also allowed if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { @@ -1193,13 +1193,24 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId return 0; } -static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, const char* id) { +static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, int32_t* pNotRsp, const char* id) { int32_t numOfRsp = 0; bool alreadySet = false; bool updated = false; + bool allRsp = false; + *pNotRsp = 0; taosThreadMutexLock(&pMsgInfo->lock); - for (int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) { + int32_t numOfDispatchBranch = taosArrayGetSize(pMsgInfo->pSendInfo); + + for(int32_t i = 0; i < numOfDispatchBranch; ++i) { + SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, i); + if (pEntry->rspTs != -1) { + numOfRsp += 1; + } + } + + for (int32_t j = 0; j < numOfDispatchBranch; ++j) { SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j); if (pEntry->nodeId == vgId) { ASSERT(!alreadySet); @@ -1207,18 +1218,20 @@ static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int3 pEntry->status = code; alreadySet = true; updated = true; - stDebug("s-task:%s record the rsp recv, ts:%" PRId64 " code:%d, idx:%d", id, now, code, j); - } - - if (pEntry->rspTs != -1) { numOfRsp += 1; + + stDebug("s-task:%s record the rsp recv, ts:%" PRId64 " code:%d, idx:%d, total recv:%d/%d", id, now, code, j, + numOfRsp, numOfDispatchBranch); } } - taosThreadMutexUnlock(&pMsgInfo->lock); - ASSERT(updated); + *pNotRsp = numOfDispatchBranch - numOfRsp; + allRsp = (numOfRsp == numOfDispatchBranch); - return numOfRsp; + taosThreadMutexUnlock(&pMsgInfo->lock); + + ASSERT(updated); + return allRsp; } bool isDispatchRspTimeout(SDispatchEntry* pEntry, int64_t now) { @@ -1244,7 +1257,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int32_t vgId = pTask->pMeta->vgId; SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo; int64_t now = taosGetTimestampMs(); - int32_t totalRsp = 0; + bool allRsp = false; + int32_t notRsp = 0; taosThreadMutexLock(&pMsgInfo->lock); int32_t msgId = pMsgInfo->msgId; @@ -1273,18 +1287,18 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (code == TSDB_CODE_STREAM_TASK_NOT_EXIST) { // destination task does not exist, not retry anymore stError("s-task:%s failed to dispatch msg to task:0x%x(vgId:%d), msgId:%d no retry, since task destroyed already", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId, msgId); - totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, id); + allRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, ¬Rsp, id); } else { stError("s-task:%s failed to dispatch msgId:%d to task:0x%x(vgId:%d), code:%s, add to retry list", id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, tstrerror(code)); - totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, code, now, id); + allRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, code, now, ¬Rsp, id); } } else { // code == 0 if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { pTask->inputq.status = TASK_INPUT_STATUS__BLOCKED; // block the input of current task, to push pressure to upstream - totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, pRsp->inputStatus, now, id); + allRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, pRsp->inputStatus, now, ¬Rsp, id); stTrace("s-task:%s inputQ of downstream task:0x%x(vgId:%d) is full, wait for retry dispatch", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); } else { @@ -1296,7 +1310,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); } - totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, id); + allRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, ¬Rsp, id); { bool delayDispatch = (pMsgInfo->dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); @@ -1321,13 +1335,11 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } } - int32_t notRsp = taosArrayGetSize(pMsgInfo->pSendInfo) - totalRsp; if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { - if (notRsp > 0) { + if (!allRsp) { stDebug( "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, " - "waiting " - "for %d rsp", + "waiting for %d rsp", id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code), notRsp); } else { stDebug( @@ -1341,7 +1353,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } // all msg rsp already, continue - if (notRsp == 0) { + if (allRsp) { ASSERT(pTask->outputq.status == TASK_OUTPUT_STATUS__WAIT); // we need to re-try send dispatch msg to downstream tasks diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 9c5c230a3d..7bc50417bd 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -15,8 +15,8 @@ #include "streamInt.h" -#define MAX_STREAM_EXEC_BATCH_NUM 32 -#define MAX_SMOOTH_BURST_RATIO 5 // 5 sec +#define MAX_STREAM_EXEC_BATCH_NUM 32 +#define MAX_SMOOTH_BURST_RATIO 5 // 5 sec // todo refactor: // read data from input queue @@ -42,18 +42,26 @@ static void streamQueueCleanup(SStreamQueue* pQueue) { static void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; } SStreamQueue* streamQueueOpen(int64_t cap) { + int32_t code; + SStreamQueue* pQueue = taosMemoryCalloc(1, sizeof(SStreamQueue)); if (pQueue == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pQueue->pQueue = taosOpenQueue(); - pQueue->qall = taosAllocateQall(); + code = taosOpenQueue(&pQueue->pQueue); + if (code) { + taosMemoryFreeClear(pQueue); + terrno = code; + return NULL; + } - if (pQueue->pQueue == NULL || pQueue->qall == NULL) { - if (pQueue->pQueue) taosCloseQueue(pQueue->pQueue); - if (pQueue->qall) taosFreeQall(pQueue->qall); + code = taosAllocateQall(&pQueue->qall); + if (code) { + taosCloseQueue(pQueue->pQueue); taosMemoryFree(pQueue); + terrno = code; return NULL; } @@ -64,7 +72,8 @@ SStreamQueue* streamQueueOpen(int64_t cap) { } void streamQueueClose(SStreamQueue* pQueue, int32_t taskId) { - stDebug("s-task:0x%x free the queue:%p, items in queue:%d", taskId, pQueue->pQueue, taosQueueItemSize(pQueue->pQueue)); + stDebug("s-task:0x%x free the queue:%p, items in queue:%d", taskId, pQueue->pQueue, + taosQueueItemSize(pQueue->pQueue)); streamQueueCleanup(pQueue); taosFreeQall(pQueue->qall); @@ -129,12 +138,12 @@ int32_t streamQueueGetItemSize(const SStreamQueue* pQueue) { } int32_t streamQueueItemGetSize(const SStreamQueueItem* pItem) { - STaosQnode* p = (STaosQnode*)((char*) pItem - sizeof(STaosQnode)); + STaosQnode* p = (STaosQnode*)((char*)pItem - sizeof(STaosQnode)); return p->dataSize; } void streamQueueItemIncSize(const SStreamQueueItem* pItem, int32_t size) { - STaosQnode* p = (STaosQnode*)((char*) pItem - sizeof(STaosQnode)); + STaosQnode* p = (STaosQnode*)((char*)pItem - sizeof(STaosQnode)); p->dataSize += size; } @@ -152,7 +161,7 @@ const char* streamQueueItemGetTypeStr(int32_t type) { } EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks, - int32_t* blockSize) { + int32_t* blockSize) { const char* id = pTask->id.idStr; int32_t taskLevel = pTask->info.taskLevel; @@ -174,7 +183,6 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputq.queue); if (qItem == NULL) { - // restore the token to bucket if (*numOfBlocks > 0) { *blockSize = streamQueueItemGetSize(*pInput); @@ -203,7 +211,7 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte *numOfBlocks = 1; *pInput = qItem; return EXEC_CONTINUE; - } else { // previous existed blocks needs to be handle, before handle the checkpoint msg block + } else { // previous existed blocks needs to be handle, before handle the checkpoint msg block stDebug("s-task:%s %s msg extracted, handle previous blocks, numOfBlocks:%d", id, p, *numOfBlocks); *blockSize = streamQueueItemGetSize(*pInput); if (taskLevel == TASK_LEVEL__SINK) { @@ -223,7 +231,7 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte if (newRet == NULL) { if (terrno != 0) { stError("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d, code:%s", id, *numOfBlocks, - tstrerror(terrno)); + tstrerror(terrno)); } *blockSize = streamQueueItemGetSize(*pInput); @@ -284,14 +292,14 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) // use the local variable to avoid the pItem be freed by other threads, since it has been put into queue already. stDebug("s-task:%s submit enqueue msgLen:%d ver:%" PRId64 ", total in queue:%d, size:%.2fMiB", pTask->id.idStr, - msgLen, ver, total, size + SIZE_IN_MiB(msgLen)); + msgLen, ver, total, size + SIZE_IN_MiB(msgLen)); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { if (streamQueueIsFull(pTask->inputq.queue)) { double size = SIZE_IN_MiB(taosQueueMemorySize(pQueue)); stTrace("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", - pTask->id.idStr, STREAM_TASK_QUEUE_CAPACITY, STREAM_TASK_QUEUE_CAPACITY_IN_SIZE, total, size); + pTask->id.idStr, STREAM_TASK_QUEUE_CAPACITY, STREAM_TASK_QUEUE_CAPACITY_IN_SIZE, total, size); streamFreeQitem(pItem); return -1; } @@ -314,7 +322,7 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) double size = SIZE_IN_MiB(taosQueueMemorySize(pQueue)); stDebug("s-task:%s level:%d %s blockdata enqueue, total in queue:%d, size:%.2fMiB", pTask->id.idStr, - pTask->info.taskLevel, streamQueueItemGetTypeStr(type), total, size); + pTask->info.taskLevel, streamQueueItemGetTypeStr(type), total, size); } else if (type == STREAM_INPUT__GET_RES) { // use the default memory limit, refactor later. int32_t code = taosWriteQitem(pQueue, pItem); @@ -332,16 +340,20 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && type != STREAM_INPUT__CHECKPOINT_TRIGGER && (pTask->info.delaySchedParam != 0)) { atomic_val_compare_exchange_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); - stDebug("s-task:%s new data arrived, active the sched-trigger, triggerStatus:%d", pTask->id.idStr, pTask->schedInfo.status); + stDebug("s-task:%s new data arrived, active the sched-trigger, triggerStatus:%d", pTask->id.idStr, + pTask->schedInfo.status); } return 0; } int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask) { - SStreamDataBlock* pTranstate = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock)); - if (pTranstate == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + int32_t code; + SStreamDataBlock* pTranstate; + + code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock), (void**)&pTranstate); + if (code) { + return code; } SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); @@ -371,13 +383,13 @@ int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask) { // the result should be put into the outputQ in any cases, the result may be lost otherwise. int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBlock) { STaosQueue* pQueue = pTask->outputq.queue->pQueue; - int32_t code = taosWriteQitem(pQueue, pBlock); + int32_t code = taosWriteQitem(pQueue, pBlock); int32_t total = streamQueueGetNumOfItems(pTask->outputq.queue); double size = SIZE_IN_MiB(taosQueueMemorySize(pQueue)); if (code != 0) { stError("s-task:%s failed to put res into outputQ, outputQ items:%d, size:%.2fMiB code:%s, result lost", - pTask->id.idStr, total + 1, size, tstrerror(code)); + pTask->id.idStr, total + 1, size, tstrerror(code)); } else { if (streamQueueIsFull(pTask->outputq.queue)) { stWarn( @@ -427,7 +439,7 @@ static void fillTokenBucket(STokenBucket* pBucket, const char* id) { // increase the new available quota as time goes on int64_t deltaQuota = now - pBucket->quotaFillTimestamp; - double incSize = (deltaQuota / 1000.0) * pBucket->quotaRate; + double incSize = (deltaQuota / 1000.0) * pBucket->quotaRate; if (incSize > 0) { pBucket->quotaRemain = TMIN(pBucket->quotaRemain + incSize, pBucket->quotaCapacity); pBucket->quotaFillTimestamp = now; @@ -447,7 +459,7 @@ bool streamTaskExtractAvailableToken(STokenBucket* pBucket, const char* id) { if (pBucket->quotaRemain > 0) { pBucket->numOfToken -= 1; return true; - } else { // no available size quota now + } else { // no available size quota now return false; } } else { @@ -460,8 +472,6 @@ void streamTaskPutbackToken(STokenBucket* pBucket) { } // size in KB -void streamTaskConsumeQuota(STokenBucket* pBucket, int32_t bytes) { - pBucket->quotaRemain -= SIZE_IN_MiB(bytes); -} +void streamTaskConsumeQuota(STokenBucket* pBucket, int32_t bytes) { pBucket->quotaRemain -= SIZE_IN_MiB(bytes); } void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputq.status, TASK_INPUT_STATUS__FAILED); } \ No newline at end of file diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 4cb87820c6..f11b135168 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -134,11 +134,14 @@ void streamTaskSchedHelper(void* param, void* tmrId) { stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, nextTrigger); } else { if (status == TASK_TRIGGER_STATUS__ACTIVE) { - SStreamTrigger* pTrigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM, 0); - if (pTrigger == NULL) { + SStreamTrigger* pTrigger; + + int32_t code = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM, 0, (void**)&pTrigger); + if (code) { stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", nextTrigger); taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); + terrno = code; return; } @@ -156,7 +159,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) { atomic_store_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE); pTrigger->pBlock->info.type = STREAM_GET_ALL; - int32_t code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger); + code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger); if (code != TSDB_CODE_SUCCESS) { taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); return; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 892ed6f5bf..11eea98e98 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -266,6 +266,7 @@ void tFreeStreamTask(SStreamTask* pTask) { tDeleteSchemaWrapper(pTask->outputInfo.tbSink.pSchemaWrapper); taosMemoryFree(pTask->outputInfo.tbSink.pTSchema); tSimpleHashCleanup(pTask->outputInfo.tbSink.pTblInfo); + tDeleteSchemaWrapper(pTask->outputInfo.tbSink.pTagSchema); } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { taosArrayDestroy(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos); } @@ -1010,10 +1011,13 @@ int32_t createStreamTaskIdStr(int64_t streamId, int32_t taskId, const char** pId } static int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq) { - SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SStreamDataBlock)); - if (pData == NULL) { + int32_t code; + SStreamDataBlock* pData; + + code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SStreamDataBlock), (void**)&pData); + if (code) { stError("s-task:%s failed to allocated retrieve-block", pTask->id.idStr); - return terrno; + return terrno = code; } // enqueue @@ -1023,7 +1027,7 @@ static int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pData->type = STREAM_INPUT__DATA_RETRIEVE; pData->srcVgId = 0; - int32_t code = streamRetrieveReqToData(pReq, pData, pTask->id.idStr); + code = streamRetrieveReqToData(pReq, pData, pTask->id.idStr); if (code != TSDB_CODE_SUCCESS) { taosFreeQitem(pData); return code; diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 0adb7050c6..61b7e02a4c 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -18,6 +18,7 @@ #include "tencode.h" #include "tstreamUpdate.h" #include "ttime.h" +#include "tutil.h" #define DEFAULT_FALSE_POSITIVE 0.01 #define DEFAULT_BUCKET_SIZE 131072 @@ -34,7 +35,8 @@ static int64_t adjustExpEntries(int64_t entries) { return TMIN(DEFAULT_EXPECTED_ENTRIES, entries); } int compareKeyTs(void* pTs1, void* pTs2, void* pPkVal, __compar_fn_t cmpPkFn) { - return compareInt64Val(pTs1, pTs2);; + return compareInt64Val(pTs1, pTs2); + ; } int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpPkFn) { @@ -49,7 +51,7 @@ int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpP int32_t getKeyBuff(TSKEY ts, int64_t tbUid, void* pVal, int32_t len, char* buff) { *(TSKEY*)buff = ts; - memcpy(buff+ sizeof(TSKEY), &tbUid, sizeof(int64_t)); + memcpy(buff + sizeof(TSKEY), &tbUid, sizeof(int64_t)); if (len == 0) { return sizeof(TSKEY) + sizeof(int64_t); } @@ -66,26 +68,40 @@ int32_t getValueBuff(TSKEY ts, char* pVal, int32_t len, char* buff) { return sizeof(TSKEY) + len; } -void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { +int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pInfo->numSBFs < count) { count = pInfo->numSBFs; } for (uint64_t i = 0; i < count; ++i) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - SScalableBf *tsSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); - taosArrayPush(pInfo->pTsSBFs, &tsSBF); + SScalableBf* tsSBF = NULL; + code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &tsSBF); + TSDB_CHECK_CODE(code, lino, _error); + void* res = taosArrayPush(pInfo->pTsSBFs, &tsSBF); + if (!res) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } } + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static void clearItemHelper(void *p) { - SScalableBf **pBf = p; +static void clearItemHelper(void* p) { + SScalableBf** pBf = p; tScalableBfDestroy(*pBf); } -void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) { +void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count) { if (count < pInfo->numSBFs) { for (uint64_t i = 0; i < count; ++i) { - SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0); + SScalableBf* pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0); tScalableBfDestroy(pTsSBFs); taosArrayRemove(pInfo->pTsSBFs, 0); } @@ -124,14 +140,19 @@ static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t w return watermark; } -SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen) { - return updateInfoInit(pInterval->interval, pInterval->precision, watermark, igUp, pkType, pkLen); +int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, + SUpdateInfo** ppInfo) { + return updateInfoInit(pInterval->interval, pInterval->precision, watermark, igUp, pkType, pkLen, ppInfo); } -SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen) { - SUpdateInfo *pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); +int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, + SUpdateInfo** ppInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SUpdateInfo* pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); if (pInfo == NULL) { - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } pInfo->pTsBuckets = NULL; pInfo->pTsSBFs = NULL; @@ -145,17 +166,19 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma bfSize = (uint64_t)(pInfo->watermark / pInfo->interval); pInfo->numSBFs = bfSize; - pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void *)); + pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void*)); if (pInfo->pTsSBFs == NULL) { updateInfoDestroy(pInfo); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } windowSBfAdd(pInfo, bfSize); pInfo->pTsBuckets = taosArrayInit(DEFAULT_BUCKET_SIZE, sizeof(TSKEY)); if (pInfo->pTsBuckets == NULL) { updateInfoDestroy(pInfo); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } TSKEY dumy = 0; @@ -167,31 +190,54 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK); + if (!pInfo->pMap) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } pInfo->maxDataVersion = 0; pInfo->pkColLen = pkLen; pInfo->pkColType = pkType; pInfo->pKeyBuff = taosMemoryCalloc(1, sizeof(TSKEY) + sizeof(int64_t) + pkLen); + if (!pInfo->pKeyBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } pInfo->pValueBuff = taosMemoryCalloc(1, sizeof(TSKEY) + pkLen); + if (!pInfo->pValueBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } if (pkLen != 0) { pInfo->comparePkRowFn = compareKeyTsAndPk; - pInfo->comparePkCol = getKeyComparFunc(pkType, TSDB_ORDER_ASC);; + pInfo->comparePkCol = getKeyComparFunc(pkType, TSDB_ORDER_ASC); + ; } else { pInfo->comparePkRowFn = compareKeyTs; pInfo->comparePkCol = NULL; } - return pInfo; + (*ppInfo) = pInfo; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { +static int32_t getSBf(SUpdateInfo* pInfo, TSKEY ts, SScalableBf** ppSBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (ts <= 0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->minTS < 0) { pInfo->minTS = (TSKEY)(ts / pInfo->interval * pInfo->interval); } int64_t index = (int64_t)((ts - pInfo->minTS) / pInfo->interval); if (index < 0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } if (index >= pInfo->numSBFs) { uint64_t count = index + 1 - pInfo->numSBFs; @@ -199,22 +245,31 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { windowSBfAdd(pInfo, count); index = pInfo->numSBFs - 1; } - SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index); + SScalableBf* res = taosArrayGetP(pInfo->pTsSBFs, index); if (res == NULL) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - res = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); + code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res); + TSDB_CHECK_CODE(code, lino, _end); + taosArrayPush(pInfo->pTsSBFs, &res); } - return res; + (*ppSBf) = res; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid) { - void *pVal = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); +bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) { + void* pVal = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); if (pVal || taosHashGetSize(pInfo->pMap) >= DEFAULT_MAP_SIZE) return true; return false; } -TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) { +TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) { + int32_t code = TSDB_CODE_SUCCESS; if (pBlock == NULL || pBlock->info.rows == 0) return INT64_MIN; TSKEY maxTs = INT64_MIN; void* pPkVal = NULL; @@ -223,14 +278,14 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p int32_t len = 0; int64_t tbUid = pBlock->info.id.uid; - SColumnInfoData *pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol); - SColumnInfoData *pPkDataInfo = NULL; + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol); + SColumnInfoData* pPkDataInfo = NULL; if (primaryKeyCol >= 0) { pPkDataInfo = taosArrayGet(pBlock->pDataBlock, primaryKeyCol); } for (int32_t i = 0; i < pBlock->info.rows; i++) { - TSKEY ts = ((TSKEY *)pColDataInfo->pData)[i]; + TSKEY ts = ((TSKEY*)pColDataInfo->pData)[i]; if (maxTs < ts) { maxTs = ts; if (primaryKeyCol >= 0) { @@ -238,17 +293,22 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p maxLen = colDataGetRowLength(pPkDataInfo, i); } } - SScalableBf *pSBf = getSBf(pInfo, ts); + SScalableBf* pSBf = NULL; + int32_t code = getSBf(pInfo, ts, &pSBf); + if (code != TSDB_CODE_SUCCESS) { + uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } if (pSBf) { if (primaryKeyCol >= 0) { pPkVal = colDataGetData(pPkDataInfo, i); len = colDataGetRowLength(pPkDataInfo, i); } int32_t buffLen = getKeyBuff(ts, tbUid, pPkVal, len, pInfo->pKeyBuff); + // we don't care whether the data is updated or not tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen); } } - void *pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); + void* pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); if (pMaxTs == NULL || pInfo->comparePkRowFn(pMaxTs, &maxTs, pMaxPkVal, pInfo->comparePkCol) == -1) { int32_t valueLen = getValueBuff(maxTs, pMaxPkVal, maxLen, pInfo->pValueBuff); taosHashPut(pInfo->pMap, &tbUid, sizeof(int64_t), pInfo->pValueBuff, valueLen); @@ -256,14 +316,14 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p return maxTs; } -bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { +bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { int32_t res = TSDB_CODE_FAILED; int32_t buffLen = 0; buffLen = getKeyBuff(ts, tableId, pPkVal, len, pInfo->pKeyBuff); - void* *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); + void** pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; - TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index); + TSKEY maxTs = *(TSKEY*)taosArrayGet(pInfo->pTsBuckets, index); if (ts < maxTs - pInfo->watermark) { // this window has been closed. if (pInfo->pCloseWinSBF) { @@ -277,10 +337,15 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* p return true; } - SScalableBf *pSBf = getSBf(pInfo, ts); + SScalableBf* pSBf = NULL; + int32_t code = getSBf(pInfo, ts, &pSBf); + if (code != TSDB_CODE_SUCCESS) { + uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } int32_t size = taosHashGetSize(pInfo->pMap); - if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1 )) { + if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || + (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1)) { int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff); taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen); // pSBf may be a null pointer @@ -309,7 +374,7 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* p return true; } -void updateInfoDestroy(SUpdateInfo *pInfo) { +void updateInfoDestroy(SUpdateInfo* pInfo) { if (pInfo == NULL) { return; } @@ -317,7 +382,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { uint64_t size = taosArrayGetSize(pInfo->pTsSBFs); for (uint64_t i = 0; i < size; i++) { - SScalableBf *pSBF = taosArrayGetP(pInfo->pTsSBFs, i); + SScalableBf* pSBF = taosArrayGetP(pInfo->pTsSBFs, i); tScalableBfDestroy(pSBF); } @@ -329,15 +394,19 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { taosMemoryFree(pInfo); } -void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) { +void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo) { if (pInfo->pCloseWinSBF) { return; } int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - pInfo->pCloseWinSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); + int32_t code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &pInfo->pCloseWinSBF); + if (code != TSDB_CODE_SUCCESS) { + pInfo->pCloseWinSBF = NULL; + uError("%s failed to add close window SBF since %s", __func__, tstrerror(code)); + } } -void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { +void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo) { if (!pInfo || !pInfo->pCloseWinSBF) { return; } @@ -345,62 +414,124 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { pInfo->pCloseWinSBF = NULL; } -int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) { +int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (!pInfo) { - return 0; + return TSDB_CODE_SUCCESS; } SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; + if (tStartEncode(&encoder) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t size = taosArrayGetSize(pInfo->pTsBuckets); - if (tEncodeI32(&encoder, size) < 0) return -1; - for (int32_t i = 0; i < size; i++) { - TSKEY *pTs = (TSKEY *)taosArrayGet(pInfo->pTsBuckets, i); - if (tEncodeI64(&encoder, *pTs) < 0) return -1; + if (tEncodeI32(&encoder, size) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } - if (tEncodeU64(&encoder, pInfo->numBuckets) < 0) return -1; + for (int32_t i = 0; i < size; i++) { + TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->pTsBuckets, i); + if (tEncodeI64(&encoder, *pTs) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + } + + if (tEncodeU64(&encoder, pInfo->numBuckets) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t sBfSize = taosArrayGetSize(pInfo->pTsSBFs); - if (tEncodeI32(&encoder, sBfSize) < 0) return -1; + if (tEncodeI32(&encoder, sBfSize) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } for (int32_t i = 0; i < sBfSize; i++) { - SScalableBf *pSBf = taosArrayGetP(pInfo->pTsSBFs, i); - if (tScalableBfEncode(pSBf, &encoder) < 0) return -1; + SScalableBf* pSBf = taosArrayGetP(pInfo->pTsSBFs, i); + if (tScalableBfEncode(pSBf, &encoder) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } } - if (tEncodeU64(&encoder, pInfo->numSBFs) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->interval) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->watermark) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->minTS) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->numSBFs) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI64(&encoder, pInfo->interval) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI64(&encoder, pInfo->watermark) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI64(&encoder, pInfo->minTS) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } - if (tScalableBfEncode(pInfo->pCloseWinSBF, &encoder) < 0) return -1; + if (tScalableBfEncode(pInfo->pCloseWinSBF, &encoder) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t mapSize = taosHashGetSize(pInfo->pMap); - if (tEncodeI32(&encoder, mapSize) < 0) return -1; - void *pIte = NULL; + if (tEncodeI32(&encoder, mapSize) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pInfo->pMap, pIte)) != NULL) { - void *key = taosHashGetKey(pIte, &keyLen); - if (tEncodeU64(&encoder, *(uint64_t *)key) < 0) return -1; + void* key = taosHashGetKey(pIte, &keyLen); + if (tEncodeU64(&encoder, *(uint64_t*)key) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t valueSize = taosHashGetValueSize(pIte); - if (tEncodeBinary(&encoder, (const uint8_t *)pIte, valueSize) < 0) return -1; + if (tEncodeBinary(&encoder, (const uint8_t*)pIte, valueSize) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } } - if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } - if (tEncodeI32(&encoder, pInfo->pkColLen) < 0) return -1; - if (tEncodeI8(&encoder, pInfo->pkColType) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->pkColLen) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI8(&encoder, pInfo->pkColType) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } tEndEncode(&encoder); int32_t tlen = encoder.pos; tEncoderClear(&encoder); - return tlen; + *pLen = tlen; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { +int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; ASSERT(pInfo); SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -419,10 +550,12 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { int32_t sBfSize = 0; if (tDecodeI32(&decoder, &sBfSize) < 0) return -1; - pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void *)); + pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void*)); for (int32_t i = 0; i < sBfSize; i++) { - SScalableBf *pSBf = tScalableBfDecode(&decoder); - if (!pSBf) return -1; + SScalableBf* pSBf = NULL; + code = tScalableBfDecode(&decoder, &pSBf); + TSDB_CHECK_CODE(code, lino, _error); + taosArrayPush(pInfo->pTsSBFs, &pSBf); } @@ -430,15 +563,20 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { if (tDecodeI64(&decoder, &pInfo->interval) < 0) return -1; if (tDecodeI64(&decoder, &pInfo->watermark) < 0) return -1; if (tDecodeI64(&decoder, &pInfo->minTS) < 0) return -1; - pInfo->pCloseWinSBF = tScalableBfDecode(&decoder); + + code = tScalableBfDecode(&decoder, &pInfo->pCloseWinSBF); + if (code != TSDB_CODE_SUCCESS) { + pInfo->pCloseWinSBF = NULL; + code = TSDB_CODE_SUCCESS; + } int32_t mapSize = 0; if (tDecodeI32(&decoder, &mapSize) < 0) return -1; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); pInfo->pMap = taosHashInit(mapSize, hashFn, true, HASH_NO_LOCK); uint64_t uid = 0; - void* pVal = NULL; - int32_t valSize = 0; + void* pVal = NULL; + int32_t valSize = 0; for (int32_t i = 0; i < mapSize; i++) { if (tDecodeU64(&decoder, &uid) < 0) return -1; if (tDecodeBinary(&decoder, (uint8_t**)&pVal, &valSize) < 0) return -1; @@ -454,7 +592,8 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { pInfo->pValueBuff = taosMemoryCalloc(1, sizeof(TSKEY) + pInfo->pkColLen); if (pInfo->pkColLen != 0) { pInfo->comparePkRowFn = compareKeyTsAndPk; - pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC);; + pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC); + ; } else { pInfo->comparePkRowFn = compareKeyTs; pInfo->comparePkCol = NULL; @@ -463,11 +602,16 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { tEndDecode(&decoder); tDecoderClear(&decoder); - return 0; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { - TSKEY *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); +bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { + TSKEY* pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); bool res = true; if (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == 1) { res = false; diff --git a/source/libs/transport/test/svrBench.c b/source/libs/transport/test/svrBench.c index a3fa81662c..dff3efef12 100644 --- a/source/libs/transport/test/svrBench.c +++ b/source/libs/transport/test/svrBench.c @@ -67,7 +67,7 @@ void *processShellMsg(void *arg) { int type; SQueueInfo qinfo = {0}; - qall = taosAllocateQall(); + taosAllocateQall(&qall); while (1) { int numOfMsgs = taosReadAllQitemsFromQset(multiQ->qset[idx], qall, &qinfo); @@ -129,7 +129,7 @@ void *processShellMsg(void *arg) { void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0); + taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0, (void **)&pTemp); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); int32_t idx = balance % multiQ->numOfThread; @@ -212,8 +212,8 @@ int main(int argc, char *argv[]) { multiQ->qset = (STaosQset **)taosMemoryMalloc(sizeof(STaosQset *) * numOfAthread); for (int i = 0; i < numOfAthread; i++) { - multiQ->qhandle[i] = taosOpenQueue(); - multiQ->qset[i] = taosOpenQset(); + taosOpenQueue(&multiQ->qhandle[i]); + taosOpenQset(&multiQ->qset[i]); taosAddIntoQset(multiQ->qset[i], multiQ->qhandle[i], NULL); } TThread *threads = taosMemoryMalloc(sizeof(TThread) * numOfAthread); diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index 3e5a86588d..d7cd0348f4 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -147,10 +147,14 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, int64_t size, cons } } -void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) { +int32_t taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) { char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer + if (NULL == buf) { + return TSDB_CODE_OUT_OF_MEMORY; + } tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf); taosMemoryFreeClear(buf); + return 0; } #define DOSWAP(a, b, size) \ @@ -324,14 +328,17 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size } } -void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, - __ext_compar_fn_t compar, char *buf, bool maxroot) { +int32_t taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, + __ext_compar_fn_t compar, char *buf, bool maxroot) { int32_t parent; int32_t child; char *tmp = NULL; if (buf == NULL) { tmp = taosMemoryMalloc(size); + if (NULL == tmp) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { tmp = buf; } @@ -378,24 +385,31 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const if (buf == NULL) { taosMemoryFree(tmp); } + + return 0; } -void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, - bool maxroot) { +int32_t taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, + bool maxroot) { int32_t i; char *buf = taosMemoryCalloc(1, size); if (buf == NULL) { - return; + return TSDB_CODE_OUT_OF_MEMORY; } if (base && size > 0) { for (i = len / 2 - 1; i >= 0; i--) { - taosheapadjust(base, size, i, len - 1, parcompar, compar, buf, maxroot); + int32_t code = taosheapadjust(base, size, i, len - 1, parcompar, compar, buf, maxroot); + if (code) { + taosMemoryFree(buf); + return code; + } } } taosMemoryFree(buf); + return 0; } static void taosMerge(void *src, int32_t start, int32_t leftend, int32_t end, int64_t size, const void *param, @@ -441,7 +455,10 @@ static int32_t taosMergeSortHelper(void *src, int64_t numOfElem, int64_t size, c // short array sort, instead of merge sort process const int32_t THRESHOLD_SIZE = 6; char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer - if (buf == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t start = 0; start < numOfElem - 1; start += THRESHOLD_SIZE) { int32_t end = (start + THRESHOLD_SIZE - 1) <= numOfElem - 1 ? (start + THRESHOLD_SIZE - 1) : numOfElem - 1; tInsertSort(src, size, start, end, param, comparFn, buf); @@ -451,7 +468,9 @@ static int32_t taosMergeSortHelper(void *src, int64_t numOfElem, int64_t size, c if (numOfElem > THRESHOLD_SIZE) { int32_t currSize; void *tmp = taosMemoryMalloc(numOfElem * size); - if (tmp == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } for (currSize = THRESHOLD_SIZE; currSize <= numOfElem - 1; currSize = 2 * currSize) { int32_t leftStart; diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 4c056178c2..ec5e41f256 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -518,9 +518,8 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t // todo remove it // order array -void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { - taosqsort(pArray->pData, pArray->size, pArray->elemSize, param, fn); - // taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); +int32_t taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { + return taosqsort(pArray->pData, pArray->size, pArray->elemSize, param, fn); } void taosArraySwap(SArray* a, SArray* b) { diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 150faab571..1cf4ecf72e 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -22,26 +22,30 @@ #define UNIT_NUM_BITS 64ULL #define UNIT_ADDR_NUM_BITS 6ULL -static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { +static FORCE_INLINE bool setBit(uint64_t* buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; uint64_t old = buf[unitIndex]; buf[unitIndex] |= (1ULL << (index % UNIT_NUM_BITS)); return buf[unitIndex] != old; } -static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) { +static FORCE_INLINE bool getBit(uint64_t* buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); return buf[unitIndex] & mask; } -SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) { +int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF) { + int32_t code = 0; + int32_t lino = 0; if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); + SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); if (pBF == NULL) { - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pBF->expectedEntries = expectedEntries; pBF->errorRate = errorRate; @@ -61,12 +65,19 @@ SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) { pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); if (pBF->buffer == NULL) { tBloomFilterDestroy(pBF); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } - return pBF; + (*ppBF) = pBF; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { +int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) { ASSERT(!tBloomFilterIsFull(pBF)); bool hasChange = false; const register uint64_t size = pBF->numBits; @@ -82,7 +93,7 @@ int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { return TSDB_CODE_FAILED; } -int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { +int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len) { uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len); bool hasChange = false; @@ -99,7 +110,7 @@ int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { return TSDB_CODE_FAILED; } -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { +int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) { const register uint64_t size = pBF->numBits; uint64_t cbHash = hash1; for (uint32_t i = 0; i < pBF->hashFunctions; ++i) { @@ -111,7 +122,7 @@ int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t return TSDB_CODE_FAILED; } -void tBloomFilterDestroy(SBloomFilter *pBF) { +void tBloomFilterDestroy(SBloomFilter* pBF) { if (pBF == NULL) { return; } @@ -119,41 +130,70 @@ void tBloomFilterDestroy(SBloomFilter *pBF) { taosMemoryFree(pBF); } -int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder) { +int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder) { if (tEncodeU32(pEncoder, pBF->hashFunctions) < 0) return -1; if (tEncodeU64(pEncoder, pBF->expectedEntries) < 0) return -1; if (tEncodeU64(pEncoder, pBF->numUnits) < 0) return -1; if (tEncodeU64(pEncoder, pBF->numBits) < 0) return -1; if (tEncodeU64(pEncoder, pBF->size) < 0) return -1; for (uint64_t i = 0; i < pBF->numUnits; i++) { - uint64_t *pUnits = (uint64_t *)pBF->buffer; + uint64_t* pUnits = (uint64_t*)pBF->buffer; if (tEncodeU64(pEncoder, pUnits[i]) < 0) return -1; } if (tEncodeDouble(pEncoder, pBF->errorRate) < 0) return -1; return 0; } -SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder) { - SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); +int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF) { + int32_t code = 0; + int32_t lino = 0; + SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); + if (!pBF) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } pBF->buffer = NULL; - if (tDecodeU32(pDecoder, &pBF->hashFunctions) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->expectedEntries) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->numUnits) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->numBits) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->size) < 0) goto _error; + if (tDecodeU32(pDecoder, &pBF->hashFunctions) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->expectedEntries) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->numUnits) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->numBits) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->size) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); for (int32_t i = 0; i < pBF->numUnits; i++) { - uint64_t *pUnits = (uint64_t *)pBF->buffer; - if (tDecodeU64(pDecoder, pUnits + i) < 0) goto _error; + uint64_t* pUnits = (uint64_t*)pBF->buffer; + if (tDecodeU64(pDecoder, pUnits + i) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + } + if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) goto _error; pBF->hashFn1 = HASH_FUNCTION_1; pBF->hashFn2 = HASH_FUNCTION_2; - return pBF; + (*ppBF) = pBF; + return TSDB_CODE_SUCCESS; _error: tBloomFilterDestroy(pBF); - return NULL; + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + return code; } -bool tBloomFilterIsFull(const SBloomFilter *pBF) { return pBF->size >= pBF->expectedEntries; } +bool tBloomFilterIsFull(const SBloomFilter* pBF) { return pBF->size >= pBF->expectedEntries; } diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index 7f8a0edfdd..aa9f157f3a 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -72,10 +72,14 @@ int32_t tStartEncode(SEncoder* pCoder) { SEncoderNode* pNode; if (pCoder->data) { - if (pCoder->size - pCoder->pos < sizeof(int32_t)) return -1; + if (pCoder->size - pCoder->pos < sizeof(int32_t)) { + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + } pNode = tEncoderMalloc(pCoder, sizeof(*pNode)); - if (pNode == NULL) return -1; + if (pNode == NULL) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } pNode->data = pCoder->data; pNode->pos = pCoder->pos; @@ -111,7 +115,7 @@ void tEndEncode(SEncoder* pCoder) { (void)tEncodeI32(pCoder, len); - TD_CODER_MOVE_POS(pCoder, len); + pCoder->pos += len; } } @@ -119,10 +123,12 @@ int32_t tStartDecode(SDecoder* pCoder) { SDecoderNode* pNode; int32_t len; - if (tDecodeI32(pCoder, &len) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI32(pCoder, &len)); pNode = tDecoderMalloc(pCoder, sizeof(*pNode)); - if (pNode == NULL) return -1; + if (pNode == NULL) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } pNode->data = pCoder->data; pNode->pos = pCoder->pos; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 347f0be4ff..ab50b83937 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -789,6 +789,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_DIR, "Invalid TDLite open TAOS_DEFINE_ERROR(TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY, "Queue out of memory") +//AUDIT +TAOS_DEFINE_ERROR(TSDB_CODE_AUDIT_NOT_FORMAT_TO_JSON, "can't format to json") +TAOS_DEFINE_ERROR(TSDB_CODE_AUDIT_FAIL_SEND_AUDIT_RECORD, "Failed to send out audit record") +TAOS_DEFINE_ERROR(TSDB_CODE_AUDIT_FAIL_GENERATE_JSON, "Failed to generate json") #ifdef TAOS_ERROR_C }; #endif diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 45a8a462fb..b2d2622407 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -17,6 +17,7 @@ #include "tqueue.h" #include "taoserror.h" #include "tlog.h" +#include "tutil.h" int64_t tsRpcQueueMemoryAllowed = 0; int64_t tsRpcQueueMemoryUsed = 0; @@ -58,20 +59,20 @@ struct STaosQall { void taosSetQueueMemoryCapacity(STaosQueue *queue, int64_t cap) { queue->memLimit = cap; } void taosSetQueueCapacity(STaosQueue *queue, int64_t size) { queue->itemLimit = size; } -STaosQueue *taosOpenQueue() { - STaosQueue *queue = taosMemoryCalloc(1, sizeof(STaosQueue)); - if (queue == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; +int32_t taosOpenQueue(STaosQueue **queue) { + *queue = taosMemoryCalloc(1, sizeof(STaosQueue)); + if (*queue == NULL) { + return (terrno = TSDB_CODE_OUT_OF_MEMORY); } - if (taosThreadMutexInit(&queue->mutex, NULL) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + int32_t code = taosThreadMutexInit(&(*queue)->mutex, NULL); + if (code) { + taosMemoryFreeClear(*queue); + return (terrno = TAOS_SYSTEM_ERROR(code)); } uDebug("queue:%p is opened", queue); - return queue; + return 0; } void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp) { @@ -146,11 +147,12 @@ int64_t taosQueueMemorySize(STaosQueue *queue) { return memOfItems; } -void *taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize) { +int32_t taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize, void **item) { + *item = NULL; + STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size); if (pNode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } pNode->dataSize = dataSize; @@ -165,15 +167,15 @@ void *taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize) { tsRpcQueueMemoryAllowed); atomic_sub_fetch_64(&tsRpcQueueMemoryUsed, size + dataSize); taosMemoryFree(pNode); - terrno = TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE; - return NULL; + return (terrno = TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE); } uTrace("item:%p, node:%p is allocated, alloc:%" PRId64, pNode->item, pNode, alloced); } else { uTrace("item:%p, node:%p is allocated", pNode->item, pNode); } - return pNode->item; + *item = pNode->item; + return 0; } void taosFreeQitem(void *pItem) { @@ -221,13 +223,17 @@ int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { } queue->numOfItems++; queue->memOfItems += (pNode->size + pNode->dataSize); - if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1); + if (queue->qset) { + atomic_add_fetch_32(&queue->qset->numOfItems, 1); + } uTrace("item:%p is put into queue:%p, items:%d mem:%" PRId64, pItem, queue, queue->numOfItems, queue->memOfItems); taosThreadMutexUnlock(&queue->mutex); - if (queue->qset) tsem_post(&queue->qset->sem); + if (queue->qset) { + tsem_post(&queue->qset->sem); + } return code; } @@ -241,10 +247,14 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { pNode = queue->head; *ppItem = pNode->item; queue->head = pNode->next; - if (queue->head == NULL) queue->tail = NULL; + if (queue->head == NULL) { + queue->tail = NULL; + } queue->numOfItems--; queue->memOfItems -= (pNode->size + pNode->dataSize); - if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, 1); + if (queue->qset) { + atomic_sub_fetch_32(&queue->qset->numOfItems, 1); + } code = 1; uTrace("item:%p is read out from queue:%p, items:%d mem:%" PRId64, *ppItem, queue, queue->numOfItems, queue->memOfItems); @@ -255,12 +265,12 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { return code; } -STaosQall *taosAllocateQall() { - STaosQall *qall = taosMemoryCalloc(1, sizeof(STaosQall)); - if (qall != NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; +int32_t taosAllocateQall(STaosQall **qall) { + *qall = taosMemoryCalloc(1, sizeof(STaosQall)); + if (*qall == NULL) { + return terrno = TSDB_CODE_OUT_OF_MEMORY; } - return qall; + return 0; } void taosFreeQall(STaosQall *qall) { taosMemoryFree(qall); } @@ -290,7 +300,9 @@ int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { queue->memOfItems = 0; uTrace("read %d items from queue:%p, items:%d mem:%" PRId64, numOfItems, queue, queue->numOfItems, queue->memOfItems); - if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, qall->numOfItems); + if (queue->qset) { + atomic_sub_fetch_32(&queue->qset->numOfItems, qall->numOfItems); + } } taosThreadMutexUnlock(&queue->mutex); @@ -326,18 +338,17 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) { return num; } -STaosQset *taosOpenQset() { - STaosQset *qset = taosMemoryCalloc(sizeof(STaosQset), 1); - if (qset == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; +int32_t taosOpenQset(STaosQset **qset) { + *qset = taosMemoryCalloc(sizeof(STaosQset), 1); + if (*qset == NULL) { + return terrno = TSDB_CODE_OUT_OF_MEMORY; } - taosThreadMutexInit(&qset->mutex, NULL); - tsem_init(&qset->sem, 0, 0); + taosThreadMutexInit(&(*qset)->mutex, NULL); + tsem_init(&(*qset)->sem, 0, 0); uDebug("qset:%p is opened", qset); - return qset; + return 0; } void taosCloseQset(STaosQset *qset) { @@ -533,9 +544,7 @@ int64_t taosQallUnAccessedMemSize(STaosQall *qall) { return qall->unAccessMemOfI void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; } -void taosQueueSetThreadId(STaosQueue* pQueue, int64_t threadId) { - pQueue->threadId = threadId; -} +void taosQueueSetThreadId(STaosQueue *pQueue, int64_t threadId) { pQueue->threadId = threadId; } int64_t taosQueueGetThreadId(STaosQueue *pQueue) { return pQueue->threadId; } diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 7af794546b..4a4dfd2653 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -17,6 +17,7 @@ #include "tscalablebf.h" #include "taoserror.h" +#include "tutil.h" #define DEFAULT_GROWTH 2 #define DEFAULT_TIGHTENING_RATIO 0.5 @@ -24,82 +25,120 @@ #define SBF_INVALID -1 #define SBF_VALID 0 -static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate); +static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate, + SBloomFilter** ppNormalBf); -SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) { +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; const uint32_t defaultSize = 8; if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (pSBf == NULL) { - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; pSBf->status = SBF_VALID; pSBf->numBits = 0; - pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *)); - if (tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO) == NULL) { + pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void*)); + if (!pSBf->bfArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } + + SBloomFilter* pNormalBf = NULL; + code = tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + if (code != TSDB_CODE_SUCCESS) { tScalableBfDestroy(pSBf); - return NULL; + TSDB_CHECK_CODE(code, lino, _error); } pSBf->growth = DEFAULT_GROWTH; pSBf->hashFn1 = HASH_FUNCTION_1; pSBf->hashFn2 = HASH_FUNCTION_2; - return pSBf; + (*ppSBf) = pSBf; + return TSDB_CODE_SUCCESS; + +_error: + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + return code; } -int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { +int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pSBf->status == SBF_INVALID) { - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + int32_t size = taosArrayGetSize(pSBf->bfArray); + SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); + if (!pNormalBf) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } - int32_t size = taosArrayGetSize(pSBf->bfArray); - SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); - ASSERT(pNormalBf); if (tBloomFilterIsFull(pNormalBf)) { - pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, - pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); - if (pNormalBf == NULL) { + code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, + pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + if (code != TSDB_CODE_SUCCESS) { pSBf->status = SBF_INVALID; - return TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } } return tBloomFilterPut(pNormalBf, keyBuf, len); + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { +int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pSBf->status == SBF_INVALID) { - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); - int32_t size = taosArrayGetSize(pSBf->bfArray); + int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 2; i >= 0; --i) { if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } } - SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); + SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); ASSERT(pNormalBf); if (tBloomFilterIsFull(pNormalBf)) { - pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, - pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); - if (pNormalBf == NULL) { + code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, + pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + if (code != TSDB_CODE_SUCCESS) { pSBf->status = SBF_INVALID; - return TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } } return tBloomFilterPutHash(pNormalBf, h1, h2); + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) { +int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len) { if (pSBf->status == SBF_INVALID) { return TSDB_CODE_FAILED; } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); - int32_t size = taosArrayGetSize(pSBf->bfArray); + int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 1; i >= 0; --i) { if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; @@ -108,24 +147,35 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 return TSDB_CODE_SUCCESS; } -static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate) { +static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate, + SBloomFilter** ppNormalBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - SBloomFilter *pNormalBf = tBloomFilterInit(expectedEntries, errorRate); - if (pNormalBf == NULL) { - return NULL; - } + SBloomFilter* pNormalBf = NULL; + code = tBloomFilterInit(expectedEntries, errorRate, &pNormalBf); + TSDB_CHECK_CODE(code, lino, _error); + if (taosArrayPush(pSBf->bfArray, &pNormalBf) == NULL) { tBloomFilterDestroy(pNormalBf); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pSBf->numBits += pNormalBf->numBits; - return pNormalBf; + (*ppNormalBf) = pNormalBf; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -void tScalableBfDestroy(SScalableBf *pSBf) { +void tScalableBfDestroy(SScalableBf* pSBf) { if (pSBf == NULL) { return; } @@ -135,7 +185,7 @@ void tScalableBfDestroy(SScalableBf *pSBf) { taosMemoryFree(pSBf); } -int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { +int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder) { if (!pSBf) { if (tEncodeI32(pEncoder, 0) < 0) return -1; return 0; @@ -143,7 +193,7 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { int32_t size = taosArrayGetSize(pSBf->bfArray); if (tEncodeI32(pEncoder, size) < 0) return -1; for (int32_t i = 0; i < size; i++) { - SBloomFilter *pBF = taosArrayGetP(pSBf->bfArray, i); + SBloomFilter* pBF = taosArrayGetP(pSBf->bfArray, i); if (tBloomFilterEncode(pBF, pEncoder) < 0) return -1; } if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1; @@ -153,30 +203,61 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { return 0; } -SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { - SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); +int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + if (!pSBf) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } pSBf->hashFn1 = HASH_FUNCTION_1; pSBf->hashFn2 = HASH_FUNCTION_2; pSBf->bfArray = NULL; int32_t size = 0; - if (tDecodeI32(pDecoder, &size) < 0) goto _error; - if (size == 0) { - tScalableBfDestroy(pSBf); - return NULL; + if (tDecodeI32(pDecoder, &size) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - pSBf->bfArray = taosArrayInit(size * 2, sizeof(void *)); + if (size == 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + pSBf->bfArray = taosArrayInit(size * 2, POINTER_BYTES); + if (!pSBf->bfArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } + for (int32_t i = 0; i < size; i++) { - SBloomFilter *pBF = tBloomFilterDecode(pDecoder); - if (!pBF) goto _error; + SBloomFilter* pBF = NULL; + code = tBloomFilterDecode(pDecoder, &pBF); + TSDB_CHECK_CODE(code, lino, _error); taosArrayPush(pSBf->bfArray, &pBF); } - if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error; - if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) goto _error; - if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) goto _error; - if (tDecodeI8(pDecoder, &pSBf->status) < 0) goto _error; - return pSBf; + if (tDecodeU32(pDecoder, &pSBf->growth) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeI8(pDecoder, &pSBf->status) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + (*ppSBf) = pSBf; _error: - tScalableBfDestroy(pSBf); - return NULL; + + if (code != TSDB_CODE_SUCCESS) { + tScalableBfDestroy(pSBf); + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 7a97dc3527..0960d37817 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -16,20 +16,22 @@ #define _DEFAULT_SOURCE #include "tworker.h" #include "taoserror.h" +#include "tcompare.h" #include "tgeosctx.h" #include "tlog.h" -#include "tcompare.h" #define QUEUE_THRESHOLD (1000 * 1000) typedef void *(*ThreadFp)(void *param); int32_t tQWorkerInit(SQWorkerPool *pool) { - pool->qset = taosOpenQset(); + int32_t code = taosOpenQset(&pool->qset); + if (code) return code; + pool->workers = taosMemoryCalloc(pool->max, sizeof(SQueueWorker)); if (pool->workers == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + taosCloseQset(pool->qset); + return terrno = TSDB_CODE_OUT_OF_MEMORY; } (void)taosThreadMutexInit(&pool->mutex, NULL); @@ -109,8 +111,14 @@ static void *tQWorkerThreadFp(SQueueWorker *worker) { } STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { - STaosQueue *queue = taosOpenQueue(); - if (queue == NULL) return NULL; + int32_t code; + STaosQueue *queue; + + code = taosOpenQueue(&queue); + if (code) { + terrno = code; + return NULL; + } taosThreadMutexLock(&pool->mutex); taosSetQueueFp(queue, fp, NULL); @@ -150,11 +158,17 @@ void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) { } int32_t tAutoQWorkerInit(SAutoQWorkerPool *pool) { - pool->qset = taosOpenQset(); + int32_t code; + + code = taosOpenQset(&pool->qset); + if (code) { + return terrno = code; + } + pool->workers = taosArrayInit(2, sizeof(SQueueWorker *)); if (pool->workers == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + taosCloseQset(pool->qset); + return terrno = TSDB_CODE_OUT_OF_MEMORY; } (void)taosThreadMutexInit(&pool->mutex, NULL); @@ -228,8 +242,14 @@ static void *tAutoQWorkerThreadFp(SQueueWorker *worker) { } STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem fp) { - STaosQueue *queue = taosOpenQueue(); - if (queue == NULL) return NULL; + int32_t code; + STaosQueue *queue; + + code = taosOpenQueue(&queue); + if (code) { + terrno = code; + return NULL; + } taosThreadMutexLock(&pool->mutex); taosSetQueueFp(queue, fp, NULL); @@ -375,20 +395,21 @@ static void *tWWorkerThreadFp(SWWorker *worker) { STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { taosThreadMutexLock(&pool->mutex); - SWWorker *worker = pool->workers + pool->nextId; - int32_t code = -1; + SWWorker *worker = pool->workers + pool->nextId; + int32_t code = -1; + STaosQueue *queue; - STaosQueue *queue = taosOpenQueue(); - if (queue == NULL) goto _OVER; + code = taosOpenQueue(&queue); + if (code) goto _OVER; taosSetQueueFp(queue, NULL, fp); if (worker->qset == NULL) { - worker->qset = taosOpenQset(); - if (worker->qset == NULL) goto _OVER; + code = taosOpenQset(&worker->qset); + if (code) goto _OVER; taosAddIntoQset(worker->qset, queue, ahandle); - worker->qall = taosAllocateQall(); - if (worker->qall == NULL) goto _OVER; + code = taosAllocateQall(&worker->qall); + if (code) goto _OVER; TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); @@ -406,12 +427,10 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { pool->nextId = (pool->nextId + 1) % pool->max; } - code = 0; - _OVER: taosThreadMutexUnlock(&pool->mutex); - if (code == -1) { + if (code) { if (queue != NULL) taosCloseQueue(queue); if (worker->qset != NULL) taosCloseQset(worker->qset); if (worker->qall != NULL) taosFreeQall(worker->qall); @@ -517,13 +536,13 @@ void tMultiWorkerCleanup(SMultiWorker *pWorker) { tWWorkerFreeQueue(&pWorker->pool, pWorker->queue); } -static int32_t tQueryAutoQWorkerAddWorker(SQueryAutoQWorkerPool* pool); +static int32_t tQueryAutoQWorkerAddWorker(SQueryAutoQWorkerPool *pool); static int32_t tQueryAutoQWorkerBeforeBlocking(void *p); static int32_t tQueryAutoQWorkerRecoverFromBlocking(void *p); -static int32_t tQueryAutoQWorkerWaitingCheck(SQueryAutoQWorkerPool* pPool); -static bool tQueryAutoQWorkerTryRecycleWorker(SQueryAutoQWorkerPool* pPool, SQueryAutoQWorker* pWorker); +static int32_t tQueryAutoQWorkerWaitingCheck(SQueryAutoQWorkerPool *pPool); +static bool tQueryAutoQWorkerTryRecycleWorker(SQueryAutoQWorkerPool *pPool, SQueryAutoQWorker *pWorker); -#define GET_ACTIVE_N(int64_val) (int32_t)((int64_val) >> 32) +#define GET_ACTIVE_N(int64_val) (int32_t)((int64_val) >> 32) #define GET_RUNNING_N(int64_val) (int32_t)(int64_val & 0xFFFFFFFF) static int32_t atomicFetchSubActive(int64_t *ptr, int32_t val) { @@ -544,7 +563,7 @@ static int32_t atomicFetchAddActive(int64_t *ptr, int32_t val) { static int32_t atomicFetchAddRunning(int64_t *ptr, int32_t val) { return GET_RUNNING_N(atomic_fetch_add_64(ptr, val)); } -static bool atomicCompareExchangeActive(int64_t* ptr, int32_t* expectedVal, int32_t newVal) { +static bool atomicCompareExchangeActive(int64_t *ptr, int32_t *expectedVal, int32_t newVal) { int64_t oldVal64 = *expectedVal, newVal64 = newVal; int32_t running = GET_RUNNING_N(*ptr); oldVal64 <<= 32; @@ -560,7 +579,7 @@ static bool atomicCompareExchangeActive(int64_t* ptr, int32_t* expectedVal, int3 } } -static int64_t atomicCompareExchangeRunning(int64_t* ptr, int32_t* expectedVal, int32_t newVal) { +static int64_t atomicCompareExchangeRunning(int64_t *ptr, int32_t *expectedVal, int32_t newVal) { int64_t oldVal64 = *expectedVal, newVal64 = newVal; int64_t activeShifted = GET_ACTIVE_N(*ptr); activeShifted <<= 32; @@ -576,7 +595,7 @@ static int64_t atomicCompareExchangeRunning(int64_t* ptr, int32_t* expectedVal, } static int64_t atomicCompareExchangeActiveAndRunning(int64_t *ptr, int32_t *expectedActive, int32_t newActive, - int32_t *expectedRunning, int32_t newRunning) { + int32_t *expectedRunning, int32_t newRunning) { int64_t oldVal64 = *expectedActive, newVal64 = newActive; oldVal64 <<= 32; oldVal64 |= *expectedRunning; @@ -641,7 +660,7 @@ static void *tQueryAutoQWorkerThreadFp(SQueryAutoQWorker *worker) { static bool tQueryAutoQWorkerTrySignalWaitingAfterBlock(void *p) { SQueryAutoQWorkerPool *pPool = p; bool ret = false; - int32_t waiting = pPool->waitingAfterBlockN; + int32_t waiting = pPool->waitingAfterBlockN; while (waiting > 0) { int32_t waitingNew = atomic_val_compare_exchange_32(&pPool->waitingAfterBlockN, waiting, waiting - 1); if (waitingNew == waiting) { @@ -656,10 +675,10 @@ static bool tQueryAutoQWorkerTrySignalWaitingAfterBlock(void *p) { return ret; } -static bool tQueryAutoQWorkerTrySignalWaitingBeforeProcess(void* p) { +static bool tQueryAutoQWorkerTrySignalWaitingBeforeProcess(void *p) { SQueryAutoQWorkerPool *pPool = p; bool ret = false; - int32_t waiting = pPool->waitingBeforeProcessMsgN; + int32_t waiting = pPool->waitingBeforeProcessMsgN; while (waiting > 0) { int32_t waitingNew = atomic_val_compare_exchange_32(&pPool->waitingBeforeProcessMsgN, waiting, waiting - 1); if (waitingNew == waiting) { @@ -674,10 +693,10 @@ static bool tQueryAutoQWorkerTrySignalWaitingBeforeProcess(void* p) { return ret; } -static bool tQueryAutoQWorkerTryDecActive(void* p, int32_t minActive) { +static bool tQueryAutoQWorkerTryDecActive(void *p, int32_t minActive) { SQueryAutoQWorkerPool *pPool = p; bool ret = false; - int64_t val64 = pPool->activeRunningN; + int64_t val64 = pPool->activeRunningN; int32_t active = GET_ACTIVE_N(val64), running = GET_RUNNING_N(val64); while (active > minActive) { if (atomicCompareExchangeActiveAndRunning(&pPool->activeRunningN, &active, active - 1, &running, running - 1)) @@ -687,7 +706,7 @@ static bool tQueryAutoQWorkerTryDecActive(void* p, int32_t minActive) { return false; } -static int32_t tQueryAutoQWorkerWaitingCheck(SQueryAutoQWorkerPool* pPool) { +static int32_t tQueryAutoQWorkerWaitingCheck(SQueryAutoQWorkerPool *pPool) { while (1) { int64_t val64 = pPool->activeRunningN; int32_t running = GET_RUNNING_N(val64), active = GET_ACTIVE_N(val64); @@ -709,17 +728,17 @@ static int32_t tQueryAutoQWorkerWaitingCheck(SQueryAutoQWorkerPool* pPool) { return TSDB_CODE_SUCCESS; } -bool tQueryAutoQWorkerTryRecycleWorker(SQueryAutoQWorkerPool* pPool, SQueryAutoQWorker* pWorker) { +bool tQueryAutoQWorkerTryRecycleWorker(SQueryAutoQWorkerPool *pPool, SQueryAutoQWorker *pWorker) { if (tQueryAutoQWorkerTrySignalWaitingAfterBlock(pPool) || tQueryAutoQWorkerTrySignalWaitingBeforeProcess(pPool) || tQueryAutoQWorkerTryDecActive(pPool, pPool->num)) { taosThreadMutexLock(&pPool->poolLock); - SListNode* pNode = listNode(pWorker); + SListNode *pNode = listNode(pWorker); tdListPopNode(pPool->workers, pNode); // reclaim some workers if (pWorker->id >= pPool->maxInUse) { while (listNEles(pPool->exitedWorkers) > pPool->maxInUse - pPool->num) { - SListNode* head = tdListPopHead(pPool->exitedWorkers); - SQueryAutoQWorker* pWorker = (SQueryAutoQWorker*)head->data; + SListNode *head = tdListPopHead(pPool->exitedWorkers); + SQueryAutoQWorker *pWorker = (SQueryAutoQWorker *)head->data; if (pWorker && taosCheckPthreadValid(pWorker->thread)) { taosThreadJoin(pWorker->thread, NULL); taosThreadClear(&pWorker->thread); @@ -758,8 +777,9 @@ bool tQueryAutoQWorkerTryRecycleWorker(SQueryAutoQWorkerPool* pPool, SQueryAutoQ } int32_t tQueryAutoQWorkerInit(SQueryAutoQWorkerPool *pool) { - pool->qset = taosOpenQset(); - if (!pool->qset) return terrno; + int32_t code; + code = taosOpenQset(&pool->qset); + if (code) return terrno = code; pool->workers = tdListNew(sizeof(SQueryAutoQWorker)); if (!pool->workers) return TSDB_CODE_OUT_OF_MEMORY; pool->backupWorkers = tdListNew(sizeof(SQueryAutoQWorker)); @@ -812,16 +832,16 @@ void tQueryAutoQWorkerCleanup(SQueryAutoQWorkerPool *pPool) { taosThreadCondBroadcast(&pPool->waitingBeforeProcessMsgCond); taosThreadMutexUnlock(&pPool->waitingBeforeProcessMsgLock); - int32_t idx = 0; - SQueryAutoQWorker* worker = NULL; + int32_t idx = 0; + SQueryAutoQWorker *worker = NULL; while (true) { taosThreadMutexLock(&pPool->poolLock); if (listNEles(pPool->workers) == 0) { taosThreadMutexUnlock(&pPool->poolLock); break; } - SListNode* pNode = tdListPopHead(pPool->workers); - worker = (SQueryAutoQWorker*)pNode->data; + SListNode *pNode = tdListPopHead(pPool->workers); + worker = (SQueryAutoQWorker *)pNode->data; taosThreadMutexUnlock(&pPool->poolLock); if (worker && taosCheckPthreadValid(worker->thread)) { taosThreadJoin(worker->thread, NULL); @@ -831,8 +851,8 @@ void tQueryAutoQWorkerCleanup(SQueryAutoQWorkerPool *pPool) { } while (listNEles(pPool->backupWorkers) > 0) { - SListNode* pNode = tdListPopHead(pPool->backupWorkers); - worker = (SQueryAutoQWorker*)pNode->data; + SListNode *pNode = tdListPopHead(pPool->backupWorkers); + worker = (SQueryAutoQWorker *)pNode->data; if (worker && taosCheckPthreadValid(worker->thread)) { taosThreadJoin(worker->thread, NULL); taosThreadClear(&worker->thread); @@ -841,8 +861,8 @@ void tQueryAutoQWorkerCleanup(SQueryAutoQWorkerPool *pPool) { } while (listNEles(pPool->exitedWorkers) > 0) { - SListNode* pNode = tdListPopHead(pPool->exitedWorkers); - worker = (SQueryAutoQWorker*)pNode->data; + SListNode *pNode = tdListPopHead(pPool->exitedWorkers); + worker = (SQueryAutoQWorker *)pNode->data; if (worker && taosCheckPthreadValid(worker->thread)) { taosThreadJoin(worker->thread, NULL); taosThreadClear(&worker->thread); @@ -867,14 +887,18 @@ void tQueryAutoQWorkerCleanup(SQueryAutoQWorkerPool *pPool) { } STaosQueue *tQueryAutoQWorkerAllocQueue(SQueryAutoQWorkerPool *pool, void *ahandle, FItem fp) { - STaosQueue *queue = taosOpenQueue(); - if (queue == NULL) return NULL; + STaosQueue *queue; + int32_t code = taosOpenQueue(&queue); + if (code) { + terrno = code; + return NULL; + } taosThreadMutexLock(&pool->poolLock); taosSetQueueFp(queue, fp, NULL); taosAddIntoQset(pool->qset, queue, ahandle); - SQueryAutoQWorker worker = {0}; - SQueryAutoQWorker* pWorker = NULL; + SQueryAutoQWorker worker = {0}; + SQueryAutoQWorker *pWorker = NULL; // spawn a thread to process queue if (pool->num < pool->max) { @@ -882,14 +906,14 @@ STaosQueue *tQueryAutoQWorkerAllocQueue(SQueryAutoQWorkerPool *pool, void *ahand worker.id = listNEles(pool->workers); worker.backupIdx = -1; worker.pool = pool; - SListNode* pNode = tdListAdd(pool->workers, &worker); + SListNode *pNode = tdListAdd(pool->workers, &worker); if (!pNode) { taosCloseQueue(queue); queue = NULL; terrno = TSDB_CODE_OUT_OF_MEMORY; break; } - pWorker = (SQueryAutoQWorker*)pNode->data; + pWorker = (SQueryAutoQWorker *)pNode->data; TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); @@ -915,11 +939,9 @@ STaosQueue *tQueryAutoQWorkerAllocQueue(SQueryAutoQWorkerPool *pool, void *ahand return queue; } -void tQueryAutoQWorkerFreeQueue(SQueryAutoQWorkerPool *pPool, STaosQueue *pQ) { - taosCloseQueue(pQ); -} +void tQueryAutoQWorkerFreeQueue(SQueryAutoQWorkerPool *pPool, STaosQueue *pQ) { taosCloseQueue(pQ); } -static int32_t tQueryAutoQWorkerAddWorker(SQueryAutoQWorkerPool* pool) { +static int32_t tQueryAutoQWorkerAddWorker(SQueryAutoQWorkerPool *pool) { // try backup pool int32_t backup = pool->backupNum; while (backup > 0) { @@ -931,20 +953,20 @@ static int32_t tQueryAutoQWorkerAddWorker(SQueryAutoQWorkerPool* pool) { backup = backupNew; } // backup pool is empty, create new - SQueryAutoQWorker* pWorker = NULL; - SQueryAutoQWorker worker = {0}; + SQueryAutoQWorker *pWorker = NULL; + SQueryAutoQWorker worker = {0}; worker.pool = pool; worker.backupIdx = -1; taosThreadMutexLock(&pool->poolLock); worker.id = listNEles(pool->workers); - SListNode* pNode = tdListAdd(pool->workers, &worker); + SListNode *pNode = tdListAdd(pool->workers, &worker); if (!pNode) { taosThreadMutexUnlock(&pool->poolLock); terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } taosThreadMutexUnlock(&pool->poolLock); - pWorker = (SQueryAutoQWorker*)pNode->data; + pWorker = (SQueryAutoQWorker *)pNode->data; TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); @@ -960,7 +982,7 @@ static int32_t tQueryAutoQWorkerAddWorker(SQueryAutoQWorkerPool* pool) { } static int32_t tQueryAutoQWorkerBeforeBlocking(void *p) { - SQueryAutoQWorkerPool* pPool = p; + SQueryAutoQWorkerPool *pPool = p; if (tQueryAutoQWorkerTrySignalWaitingAfterBlock(p) || tQueryAutoQWorkerTrySignalWaitingBeforeProcess(p) || tQueryAutoQWorkerTryDecActive(p, 1)) { } else { @@ -974,9 +996,9 @@ static int32_t tQueryAutoQWorkerBeforeBlocking(void *p) { } static int32_t tQueryAutoQWorkerRecoverFromBlocking(void *p) { - SQueryAutoQWorkerPool* pPool = p; - int64_t val64 = pPool->activeRunningN; - int32_t running = GET_RUNNING_N(val64), active = GET_ACTIVE_N(val64); + SQueryAutoQWorkerPool *pPool = p; + int64_t val64 = pPool->activeRunningN; + int32_t running = GET_RUNNING_N(val64), active = GET_ACTIVE_N(val64); while (running < pPool->num) { if (atomicCompareExchangeActiveAndRunning(&pPool->activeRunningN, &active, active + 1, &running, running + 1)) { return TSDB_CODE_SUCCESS; diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index c51de3c8a4..9d83cbf086 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -8,12 +8,15 @@ using namespace std; TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { int64_t ts1 = 1650803518000; - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 0)); - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 1)); - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, -0.1)); - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(0, 0.01)); + SBloomFilter* pBFTmp = NULL; + GTEST_ASSERT_NE(0, tBloomFilterInit(100, 0, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(100, 1, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(100, -0.1, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(0, 0.01, &pBFTmp)); - SBloomFilter *pBF1 = tBloomFilterInit(100, 0.005); + SBloomFilter *pBF1 = NULL; + int32_t code = tBloomFilterInit(100, 0.005, &pBF1); + GTEST_ASSERT_EQ(0, code); GTEST_ASSERT_EQ(pBF1->numBits, 1152); GTEST_ASSERT_EQ(pBF1->numUnits, 1152 / 64); int64_t count = 0; @@ -25,16 +28,19 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { } ASSERT_TRUE(tBloomFilterIsFull(pBF1)); - SBloomFilter *pBF2 = tBloomFilterInit(1000 * 10000, 0.1); + SBloomFilter* pBF2 = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(1000 * 10000, 0.1, &pBF2)); GTEST_ASSERT_EQ(pBF2->numBits, 47925312); GTEST_ASSERT_EQ(pBF2->numUnits, 47925312 / 64); - SBloomFilter *pBF3 = tBloomFilterInit(10000 * 10000, 0.001); + SBloomFilter* pBF3 = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(10000 * 10000, 0.001, &pBF3)); GTEST_ASSERT_EQ(pBF3->numBits, 1437758784); GTEST_ASSERT_EQ(pBF3->numUnits, 1437758784 / 64); int64_t size = 10000; - SBloomFilter *pBF4 = tBloomFilterInit(size, 0.001); + SBloomFilter* pBF4 = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(size, 0.001, &pBF4)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; GTEST_ASSERT_EQ(tBloomFilterPut(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); @@ -42,17 +48,17 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { ASSERT_TRUE(!tBloomFilterIsFull(pBF4)); for (int64_t i = 0; i < 1000; i++) { - int64_t ts = i + ts1; - uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); - uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); + int64_t ts = i + ts1; + uint64_t h1 = (uint64_t)pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); + uint64_t h2 = (uint64_t)pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_FAILED); } for (int64_t i = 2000; i < 3000; i++) { - int64_t ts = i + ts1; - uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); - uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); - GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS); + int64_t ts = i + ts1; + uint64_t h1 = (uint64_t)pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); + uint64_t h2 = (uint64_t)pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); + GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS); } tBloomFilterDestroy(pBF1); @@ -64,12 +70,14 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int64_t ts1 = 1650803518000; - GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 0)); - GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 1)); - GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, -0.1)); - GTEST_ASSERT_EQ(NULL, tScalableBfInit(0, 0.01)); + SScalableBf* tsSBF = NULL; + GTEST_ASSERT_NE(0, tScalableBfInit(100, 0, &tsSBF)); + GTEST_ASSERT_NE(0, tScalableBfInit(100, 1, &tsSBF)); + GTEST_ASSERT_NE(0, tScalableBfInit(100, -0.1, &tsSBF)); + GTEST_ASSERT_NE(0, tScalableBfInit(0, 0.01, &tsSBF)); - SScalableBf *pSBF1 = tScalableBfInit(100, 0.01); + SScalableBf* pSBF1 = NULL; + GTEST_ASSERT_EQ(0, tScalableBfInit(100, 0.01, &pSBF1)); GTEST_ASSERT_EQ(pSBF1->numBits, 1152); int64_t count = 0; int64_t index = 0; @@ -108,7 +116,7 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int32_t aSize = taosArrayGetSize(pSBF1->bfArray); int64_t totalBits = 0; for (int64_t i = 0; i < aSize; i++) { - SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF1->bfArray, i); + SBloomFilter* pBF = (SBloomFilter*)taosArrayGetP(pSBF1->bfArray, i); ASSERT_TRUE(tBloomFilterIsFull(pBF)); totalBits += pBF->numBits; } @@ -120,7 +128,8 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { } int64_t size = 10000; - SScalableBf *pSBF4 = tScalableBfInit(size, 0.001); + SScalableBf* pSBF4 = NULL; + GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, &pSBF4)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; GTEST_ASSERT_EQ(tScalableBfPut(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); diff --git a/tests/army/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py index 77a22b9256..6fc8e858a3 100644 --- a/tests/army/cmdline/fullopt.py +++ b/tests/army/cmdline/fullopt.py @@ -28,13 +28,13 @@ from frame import * class TDTestCase(TBase): updatecfgDict = { - 'queryMaxConcurrentTables': '2K', - 'streamMax': '1M', + 'queryMaxConcurrentTables': '2K', + 'streamMax': '1M', 'totalMemoryKB': '1G', 'streamMax': '1P', 'streamBufferSize':'1T', 'slowLogScope':"query" - } + } def insertData(self): tdLog.info(f"insert data.") @@ -53,7 +53,7 @@ class TDTestCase(TBase): def doTaos(self): tdLog.info(f"check taos command options...") - + # local command options = [ "DebugFlag 143", @@ -71,7 +71,6 @@ class TDTestCase(TBase): "smlTsDefaultName tsdef", "serverPort 6030", "timezone tz", - "tempDir /var/tmp" ] # exec for option in options: @@ -83,7 +82,7 @@ class TDTestCase(TBase): # help rets = etool.runBinFile("taos", "--help") self.checkListNotEmpty(rets) - # b r w s + # b r w s sql = f"select * from {self.db}.{self.stb} limit 10" rets = etool.runBinFile("taos", f'-B -r -w 100 -s "{sql}" ') self.checkListNotEmpty(rets) @@ -124,16 +123,16 @@ class TDTestCase(TBase): tdLog.info(f"check taosd command options...") idx = 1 # dnode1 cfg = sc.dnodeCfgPath(idx) - + # -s sdb = "./sdb.json" eos.delFile(sdb) etool.exeBinFile("taosd", f"-s -c {cfg}") - + # -C etool.exeBinFile("taosd", "-C") - # -k + # -k etool.exeBinFile("taosd", "-k", False) # -V rets = etool.runBinFile("taosd", "-V") @@ -153,7 +152,7 @@ class TDTestCase(TBase): sc.dnodeStop(idx) etool.exeBinFile("taosd", "-a http://192.168.1.10") - #exe + #exe etool.exeBinFile("taosd", f"-E abc -c {cfg}", False) sc.dnodeStop(idx) etool.exeBinFile("taosd", f"-e def -c {cfg}", False) @@ -182,7 +181,7 @@ class TDTestCase(TBase): # do action self.doTaosd() - + tdLog.success(f"{__file__} successfully executed") diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 8368837cc4..1eb61d2394 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -311,7 +311,7 @@ static void shellInitArgs(int argc, char *argv[]) { if (strlen(argv[i]) == 2) { printf("Enter password: "); taosSetConsoleEcho(false); - if (scanf("%20s", shell.args.password) > 1) { + if (scanf("%128s", shell.args.password) > 1) { fprintf(stderr, "password reading error\n"); } taosSetConsoleEcho(true); diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 2338674ae3..a3830d2518 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -121,7 +121,8 @@ int smlProcess_json1_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -149,7 +150,8 @@ int smlProcess_json1_Test() { char *sql3[1] = {0}; for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); - strncpy(sql3[i], sql2[i], 1023); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, @@ -176,7 +178,8 @@ int smlProcess_json1_Test() { char *sql5[1] = {0}; for (int i = 0; i < 1; i++) { sql5[i] = taosMemoryCalloc(1, 1024); - strncpy(sql5[i], sql4[i], 1023); + ASSERT(sql5[i] != NULL); + (void)strncpy(sql5[i], sql4[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, @@ -215,7 +218,8 @@ int smlProcess_json2_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -249,7 +253,8 @@ int smlProcess_json3_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -286,7 +291,8 @@ int smlProcess_json_tag_not_same_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -312,7 +318,8 @@ int smlProcess_json_tag_not_same_Test() { char *sql3[1] = {0}; for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); - strncpy(sql3[i], sql2[i], 1023); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, @@ -338,7 +345,8 @@ int smlProcess_json_tag_not_same_Test() { char *sql5[1] = {0}; for (int i = 0; i < 1; i++) { sql5[i] = taosMemoryCalloc(1, 1024); - strncpy(sql5[i], sql4[i], 1023); + ASSERT(sql5[i] != NULL); + (void)strncpy(sql5[i], sql4[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, @@ -1147,7 +1155,8 @@ int sml_19221_Test() { taos_free_result(pRes); char *tmp = (char *)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql[0], strlen(sql[0])); + ASSERT(tmp != NULL); + (void)memcpy(tmp, sql[0], strlen(sql[0])); *(char *)(tmp + 44) = 0; int32_t totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, @@ -1218,7 +1227,8 @@ int sml_ts3116_Test() { taos_free_result(pRes); int32_t totalRows = 0; char *tmp = (char *)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql, strlen(sql)); + ASSERT(tmp != NULL); + (void)memcpy(tmp, sql, strlen(sql)); totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); @@ -1235,7 +1245,8 @@ int sml_ts3116_Test() { taos_free_result(pRes); tmp = (char *)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql1, strlen(sql1)); + ASSERT(tmp != NULL); + (void)memcpy(tmp, sql1, strlen(sql1)); totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); @@ -2049,7 +2060,8 @@ int sml_td29373_Test() { char *sql3[1] = {0}; for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); - strncpy(sql3[i], sql2[i], 1023); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL,