xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/TD-31202

This commit is contained in:
dapan1121 2024-08-22 16:43:54 +08:00 committed by GitHub
parent 3ce05d79b8 adfa742ac6
commit e3f2b8e091
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
197 changed files with 3429 additions and 2569 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -134,3 +134,4 @@ tags
.clangd
*CMakeCache*
*CMakeFiles*
.history/

10
docs/examples/c/with_reqid_demo.c
View File

@ -33,7 +33,8 @@ static int DemoWithReqId() {
// connect
TAOS *taos = taos_connect(host, user, password, NULL, port);
if (taos == NULL) {
fprintf(stderr, "Failed to connect to %s:%hu, ErrCode: 0x%x, ErrMessage: %s.\n", host, port, taos_errno(NULL), taos_errstr(NULL));
fprintf(stderr, "Failed to connect to %s:%hu, ErrCode: 0x%x, ErrMessage: %s.\n", host, port, taos_errno(NULL),
taos_errstr(NULL));
taos_cleanup();
return -1;
}
@ -44,7 +45,8 @@ static int DemoWithReqId() {
TAOS_RES *result = taos_query_with_reqid(taos, sql, reqid);
code = taos_errno(result);
if (code != 0) {
fprintf(stderr, "Failed to execute sql with reqId: %ld, ErrCode: 0x%x, ErrMessage: %s\n.", reqid, code, taos_errstr(result));
fprintf(stderr, "Failed to execute sql with QID: %ld, ErrCode: 0x%x, ErrMessage: %s\n.", reqid, code,
taos_errstr(result));
taos_close(taos);
taos_cleanup();
return -1;
@ -73,6 +75,4 @@ static int DemoWithReqId() {
// ANCHOR_END: with_reqid
}
int main(int argc, char *argv[]) {
return DemoWithReqId();
}
int main(int argc, char *argv[]) { return DemoWithReqId(); }

0
docs/zh/14-reference/03-taos-sql/25-grant.md → docs/zh/14-reference/03-taos-sql/25-user.md
View File

168
docs/zh/14-reference/03-taos-sql/26-grant.md Normal file
View File

@ -0,0 +1,168 @@
---
toc_max_heading_level: 4
title: 权限管理
---
TDengine 中的权限管理分为[用户管理](../user)、数据库授权管理以及消息订阅授权管理，本节重点说明数据库授权和订阅授权。
## 数据库访问授权
系统管理员可以根据业务需要对系统中的每个用户针对每个数据库进行特定的授权，以防止业务数据被不恰当的用户读取或修改。对某个用户进行数据库访问授权的语法如下：
```sql
GRANT privileges ON priv_level TO user_name
privileges : {
ALL
| priv_type [, priv_type] ...
}
priv_type : {
READ
| WRITE
}
priv_level : {
dbname.tbname
| dbname.*
| *.*
}
```
对数据库的访问权限包含读和写两种权限，它们可以被分别授予，也可以被同时授予。
说明
- priv_level 格式中 "." 之前为数据库名称， "." 之后为表名称，意思为表级别的授权控制。如果 "." 之后为 "\*" ，意为 "." 前所指定的数据库中的所有表
- "dbname.\*" 意思是名为 "dbname" 的数据库中的所有表
- "\*.\*" 意思是所有数据库名中的所有表
### 数据库权限说明
对 root 用户和普通用户的权限的说明如下表
| 用户 | 描述 | 权限说明 |
| -------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| 超级用户 | 只有 root 是超级用户 | DB 外部 所有操作权限，例如user、dnode、udf、qnode等的CRUD DB 权限，包括 创建 删除 更新，例如修改 Option，移动 Vgruop等 读 写 Enable/Disable 用户 |
| 普通用户 | 除 root 以外的其它用户均为普通用户 | 在可读的 DB 中，普通用户可以进行读操作 select describe show subscribe 在可写 DB 的内部，用户可以进行写操作： 创建、删除、修改 超级表 创建、删除、修改 子表 创建、删除、修改 topic 写入数据 被限制系统信息时，不可进行如下操作 show dnode、mnode、vgroups、qnode、snode 修改用户包括自身密码 show db时只能看到自己的db，并且不能看到vgroups、副本、cache等信息 无论是否被限制系统信息，都可以 管理 udf 可以创建 DB 自己创建的 DB 具备所有权限 非自己创建的 DB ，参照读、写列表中的权限 |
## 消息订阅授权
任意用户都可以在自己拥有读权限的数据库上创建 topic。超级用户 root 可以在任意数据库上创建 topic。每个 topic 的订阅权限都可以被独立授权给任何用户，不管该用户是否拥有该数据库的访问权限。删除 topic 只能由 root 用户或者该 topic 的创建者进行。topic 只能由超级用户、topic的创建者或者被显式授予 subscribe 权限的用户订阅。
具体的 SQL 语法如下：
```sql
GRANT SUBSCRIBE ON topic_name TO user_name
REVOKE SUBSCRIBE ON topic_name FROM user_name
```
## 基于标签的授权（表级授权）
从 TDengine 3.0.5.0 开始，我们支持按标签授权某个超级表中部分特定的子表。具体的 SQL 语法如下。
```sql
GRANT privileges ON priv_level [WITH tag_condition] TO user_name
privileges : {
ALL
| priv_type [, priv_type] ...
}
priv_type : {
READ
| WRITE
}
priv_level : {
dbname.tbname
| dbname.*
| *.*
}
REVOKE privileges ON priv_level [WITH tag_condition] FROM user_name
privileges : {
ALL
| priv_type [, priv_type] ...
}
priv_type : {
READ
| WRITE
}
priv_level : {
dbname.tbname
| dbname.*
| *.*
}
```
上面 SQL 的语义为：
- 用户可以通过 dbname.tbname 来为指定的表（包括超级表和普通表）授予或回收其读写权限，不支持直接对子表授予或回收权限。
- 用户可以通过 dbname.tbname 和 WITH 子句来为符合条件的所有子表授予或回收其读写权限。使用 WITH 子句时，权限级别必须为超级表。
## 表级权限和数据库权限的关系
下表列出了在不同的数据库授权和表级授权的组合下产生的实际权限。
| | **表无授权** | **表读授权** | **表读授权有标签条件** | **表写授权** | **表写授权有标签条件** |
| ---------------- | ---------------- | ---------------------------------------- | ------------------------------------------------------------ | ---------------------------------------- | ---------------------------------------------------------- |
| **数据库无授权** | 无授权 | 对此表有读权限，对数据库下的其他表无权限 | 对此表符合标签权限的子表有读权限，对数据库下的其他表无权限 | 对此表有写权限，对数据库下的其他表无权限 | 对此表符合标签权限的子表有写权限，对数据库下的其他表无权限 |
| **数据库读授权** | 对所有表有读权限 | 对所有表有读权限 | 对此表符合标签权限的子表有读权限，对数据库下的其他表有读权限 | 对此表有写权限，对所有表有读权限 | 对此表符合标签权限的子表有写权限，所有表有读权限 |
| **数据库写授权** | 对所有表有写权限 | 对此表有读权限，对所有表有写权限 | 对此表符合标签权限的子表有读权限，对所有表有写权限 | 对所有表有写权限 | 对此表符合标签权限的子表有写权限，数据库下的其他表有写权限 |
## 查看用户授权
使用下面的命令可以显示一个用户所拥有的授权：
```sql
show user privileges
```
## 撤销授权
1. 撤销数据库访问的授权
```sql
REVOKE privileges ON priv_level FROM user_name
privileges : {
ALL
| priv_type [, priv_type] ...
}
priv_type : {
READ
| WRITE
}
priv_level : {
dbname.tbname
| dbname.*
| *.*
}
```
2. 撤销数据订阅的授权
```sql
REVOKE privileges ON priv_level FROM user_name
privileges : {
ALL
| priv_type [, priv_type] ...
}
priv_type : {
SUBSCRIBE
}
priv_level : {
topic_name
}
```

0
docs/zh/14-reference/03-taos-sql/26-udf.md → docs/zh/14-reference/03-taos-sql/27-udf.md
View File

0
docs/zh/14-reference/03-taos-sql/27-indexing.md → docs/zh/14-reference/03-taos-sql/28-index.md
View File

0
docs/zh/14-reference/03-taos-sql/28-recovery.md → docs/zh/14-reference/03-taos-sql/29-recovery.md
View File

0
docs/zh/14-reference/03-taos-sql/29-changes.md → docs/zh/14-reference/03-taos-sql/30-changes.md
View File

0
docs/zh/14-reference/03-taos-sql/30-join.md → docs/zh/14-reference/03-taos-sql/31-join.md
View File

0
docs/zh/14-reference/03-taos-sql/31-compress.md → docs/zh/14-reference/03-taos-sql/32-compress.md
View File

0
docs/zh/14-reference/03-taos-sql/32-view.md → docs/zh/14-reference/03-taos-sql/33-view.md
View File

40
docs/zh/20-third-party/01-collection/11-kafka.md vendored
View File

@ -27,22 +27,21 @@ TDengine Source Connector 用于把数据实时地从 TDengine 读出来发送
## 安装 Kafka
在任意目录下执行：
- 在任意目录下执行：
```shell
curl -O https://downloads.apache.org/kafka/3.4.0/kafka_2.13-3.4.0.tgz
tar xzf kafka_2.13-3.4.0.tgz -C /opt/
ln -s /opt/kafka_2.13-3.4.0 /opt/kafka
```
```shell
curl -O https://downloads.apache.org/kafka/3.4.0/kafka_2.13-3.4.0.tgz
tar xzf kafka_2.13-3.4.0.tgz -C /opt/
ln -s /opt/kafka_2.13-3.4.0 /opt/kafka
```
然后需要把 `$KAFKA_HOME/bin` 目录加入 PATH。
- 然后需要把 `$KAFKA_HOME/bin` 目录加入 PATH。
```title=".profile"
export KAFKA_HOME=/opt/kafka
export PATH=$PATH:$KAFKA_HOME/bin
```
以上脚本可以追加到当前用户的 profile 文件（~/.profile 或 ~/.bash_profile）
```title=".profile"
export KAFKA_HOME=/opt/kafka
export PATH=$PATH:$KAFKA_HOME/bin
```
以上脚本可以追加到当前用户的 profile 文件（~/.profile 或 ~/.bash_profile）
## 安装 TDengine Connector 插件
@ -325,6 +324,21 @@ curl -X DELETE http://localhost:8083/connectors/TDengineSinkConnector
curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector
```
### 性能调优
如果在从 TDengine 同步数据到 Kafka 的过程中发现性能不达预期，可以尝试使用如下参数提升 Kafka 的写入吞吐量。
1. 打开 KAFKA_HOME/config/producer.properties 配置文件。
2. 参数说明及配置建议如下：
| **参数** | **参数说明** | **设置建议** |
| --------| --------------------------------- | -------------- |
| producer.type | 此参数用于设置消息的发送方式，默认值为 `sync` 表示同步发送，`async` 表示异步发送。采用异步发送能够提升消息发送的吞吐量。 | async |
| request.required.acks | 参数用于配置生产者发送消息后需要等待的确认数量。当设置为1时，表示只要领导者副本成功写入消息就会给生产者发送确认，而无需等待集群中的其他副本写入成功。这种设置可以在一定程度上保证消息的可靠性，同时也能保证一定的吞吐量。因为不需要等待所有副本都写入成功，所以可以减少生产者的等待时间，提高发送消息的效率。|1|
| max.request.size| 该参数决定了生产者在一次请求中可以发送的最大数据量。其默认值为 1048576，也就是 1M。如果设置得太小，可能会导致频繁的网络请求，降低吞吐量。如果设置得太大，可能会导致内存占用过高，或者在网络状况不佳时增加请求失败的概率。建议设置为 100M。|104857600|
|batch.size| 此参数用于设定 batch 的大小，默认值为 16384，即 16KB。在消息发送过程中，发送到 Kafka 缓冲区中的消息会被划分成一个个的 batch。故而减小 batch 大小有助于降低消息延迟，而增大 batch 大小则有利于提升吞吐量，可根据实际的数据量大小进行合理配置。可根据实际情况进行调整，建议设置为 512K。|524288|
| buffer.memory| 此参数用于设置生产者缓冲待发送消息的内存总量。较大的缓冲区可以允许生产者积累更多的消息后批量发送，提高吞吐量，但也会增加延迟和内存使用。可根据机器资源来配置，建议配置为 1G。|1073741824|
## 配置参考
### 通用配置

2
include/common/tdataformat.h
View File

@ -193,7 +193,7 @@ int32_t tColDataDecompress(void *input, SColDataCompressInfo *info, SColData *co
// for stmt bind
int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen);
void tColDataSortMerge(SArray *colDataArr);
int32_t tColDataSortMerge(SArray *colDataArr);
// for raw block
int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap,

5
include/common/trow.h
View File

@ -213,7 +213,6 @@ static FORCE_INLINE SKvRowIdx *tdKvRowColIdxAt(STSRow *pRow, col_id_t idx) {
}
static FORCE_INLINE int16_t tdKvRowColIdAt(STSRow *pRow, col_id_t idx) {
ASSERT(idx >= 0);
if (idx == 0) {
return PRIMARYKEY_TIMESTAMP_COL_ID;
}
@ -288,9 +287,7 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT
*
*/
static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) {
pBuilder->sver = sver;
}
static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) { pBuilder->sver = sver; }
int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen);
int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen);
int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen,

4
include/common/tvariant.h
View File

@ -46,14 +46,14 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value);
/**
* non floating point integers
*/
*/
int32_t toIntegerPure(const char *z, int32_t n, int32_t base, int64_t *value);
void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type);
void taosVariantDestroy(SVariant *pV);
void taosVariantAssign(SVariant *pDst, const SVariant *pSrc);
int32_t taosVariantAssign(SVariant *pDst, const SVariant *pSrc);
int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2);

3
include/libs/monitor/monitor.h
View File

@ -30,6 +30,8 @@ extern "C" {
#define MON_VER_LEN 12
#define MON_LOG_LEN 1024
#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows"
typedef struct {
int64_t ts;
ELogLevel level;
@ -216,6 +218,7 @@ typedef struct {
} SDmNotifyHandle;
int32_t monInit(const SMonCfg *pCfg);
void monInitVnode();
void monCleanup();
void monRecordLog(int64_t ts, ELogLevel level, const char *content);
int32_t monGetLogs(SMonLogs *logs);

30
include/libs/sync/sync.h
View File

@ -156,11 +156,11 @@ typedef struct SSnapshotParam {
typedef struct SSnapshot {
int32_t type;
SSyncTLV* data;
SSyncTLV* data;
ESyncFsmState state;
SyncIndex lastApplyIndex;
SyncTerm lastApplyTerm;
SyncIndex lastConfigIndex;
SyncIndex lastApplyIndex;
SyncTerm lastApplyTerm;
SyncIndex lastConfigIndex;
} SSnapshot;
typedef struct SSnapshotMeta {
@ -263,16 +263,16 @@ typedef struct SSyncState {
int64_t startTimeMs;
} SSyncState;
int32_t syncInit();
void syncCleanUp();
int64_t syncOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion);
int32_t syncStart(int64_t rid);
void syncStop(int64_t rid);
void syncPreStop(int64_t rid);
void syncPostStop(int64_t rid);
int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq);
int32_t syncCheckMember(int64_t rid);
int32_t syncIsCatchUp(int64_t rid);
int32_t syncInit();
void syncCleanUp();
int64_t syncOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion);
int32_t syncStart(int64_t rid);
void syncStop(int64_t rid);
void syncPreStop(int64_t rid);
void syncPostStop(int64_t rid);
int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq);
int32_t syncCheckMember(int64_t rid);
int32_t syncIsCatchUp(int64_t rid);
ESyncRole syncGetRole(int64_t rid);
int64_t syncGetTerm(int64_t rid);
int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg);
@ -296,7 +296,7 @@ int32_t syncGetAssignedLogSynced(int64_t rid);
void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet);
const char* syncStr(ESyncState state);
int32_t syncNodeGetConfig(int64_t rid, SSyncCfg *cfg);
int32_t syncNodeGetConfig(int64_t rid, SSyncCfg* cfg);
// util
int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size);

10
include/util/taoserror.h
View File

@ -91,7 +91,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_RPC_NETWORK_BUSY TAOS_DEF_ERROR_CODE(0, 0x0024)
#define TSDB_CODE_HTTP_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0025)
#define TSDB_CODE_RPC_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0026)
#define TSDB_CODE_RPC_ASYNC_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0027)
#define TSDB_CODE_RPC_ASYNC_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0027)
@ -152,6 +152,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_FAILED_TO_CONNECT_S3 TAOS_DEF_ERROR_CODE(0, 0x0135)
#define TSDB_CODE_MSG_PREPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0136) // internal
#define TSDB_CODE_OUT_OF_BUFFER TAOS_DEF_ERROR_CODE(0, 0x0137)
#define TSDB_CODE_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0138)
//client
#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200)
@ -604,6 +605,9 @@ int32_t taosGetErrSize();
#define TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0732)
#define TSDB_CODE_QRY_INVALID_JOIN_CONDITION TAOS_DEF_ERROR_CODE(0, 0x0733)
#define TSDB_CODE_QRY_FILTER_NOT_SUPPORT_TYPE TAOS_DEF_ERROR_CODE(0, 0x0734)
#define TSDB_CODE_QRY_FILTER_WRONG_OPTR_TYPE TAOS_DEF_ERROR_CODE(0, 0x0735)
#define TSDB_CODE_QRY_FILTER_RANGE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0736)
#define TSDB_CODE_QRY_FILTER_INVALID_TYPE TAOS_DEF_ERROR_CODE(0, 0x0737)
// grant
#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800)
@ -878,6 +882,8 @@ int32_t taosGetErrSize();
#define TSDB_CODE_FUNC_INVALID_VALUE_RANGE TAOS_DEF_ERROR_CODE(0, 0x280C)
#define TSDB_CODE_FUNC_SETUP_ERROR TAOS_DEF_ERROR_CODE(0, 0x280D)
#define TSDB_CODE_FUNC_INVALID_RES_LENGTH TAOS_DEF_ERROR_CODE(0, 0x280E)
#define TSDB_CODE_FUNC_HISTOGRAM_ERROR TAOS_DEF_ERROR_CODE(0, 0x280F)
#define TSDB_CODE_FUNC_PERCENTILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x2810)
//udf
@ -892,6 +898,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2909)
#define TSDB_CODE_UDF_FUNC_EXEC_FAILURE TAOS_DEF_ERROR_CODE(0, 0x290A)
#define TSDB_CODE_UDF_UV_EXEC_FAILURE TAOS_DEF_ERROR_CODE(0, 0x290B)
#define TSDB_CODE_UDF_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2920)
// sml
#define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000)
@ -958,6 +965,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_STREAM_INVALID_STATETRANS TAOS_DEF_ERROR_CODE(0, 0x4103)
#define TSDB_CODE_STREAM_TASK_IVLD_STATUS TAOS_DEF_ERROR_CODE(0, 0x4104)
#define TSDB_CODE_STREAM_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x4105)
#define TSDB_CODE_STREAM_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x4106)
// TDLite
#define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x5100)

3
include/util/tcoding.h
View File

@ -213,7 +213,6 @@ static FORCE_INLINE int32_t taosEncodeVariantU16(void **buf, uint16_t value) {
if (buf != NULL) ((uint8_t *)(*buf))[i] = (uint8_t)(value | ENCODE_LIMIT);
value >>= 7;
i++;
ASSERT(i < 3);
}
if (buf != NULL) {
@ -261,7 +260,6 @@ static FORCE_INLINE int32_t taosEncodeVariantU32(void **buf, uint32_t value) {
if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT);
value >>= 7;
i++;
ASSERT(i < 5);
}
if (buf != NULL) {
@ -309,7 +307,6 @@ static FORCE_INLINE int32_t taosEncodeVariantU64(void **buf, uint64_t value) {
if (buf != NULL) ((uint8_t *)(*buf))[i] = (uint8_t)(value | ENCODE_LIMIT);
value >>= 7;
i++;
ASSERT(i < 10);
}
if (buf != NULL) {

3
include/util/tref.h
View File

@ -29,8 +29,7 @@ typedef void (*RefFp)(void *);
int32_t taosOpenRef(int32_t max, RefFp fp);
// close the reference set, refId is the return value by taosOpenRef
// return 0 if success. On error, -1 is returned, and terrno is set appropriately
int32_t taosCloseRef(int32_t rsetId);
void taosCloseRef(int32_t rsetId);
// add ref, p is the pointer to resource or pointer ID
// return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately

7
include/util/tutil.h
View File

@ -139,6 +139,13 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
#define QUERY_CHECK_CODE TSDB_CHECK_CODE
#define QUERY_CHECK_CONDITION(condition, CODE, LINO, LABEL, ERRNO) \
if (!condition) { \
(CODE) = (ERRNO); \
(LINO) = __LINE__; \
goto LABEL; \
}
#define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, ERRNO) \
if ((ptr) == NULL) { \
(CODE) = (ERRNO); \

5
packaging/cfg/taos.cfg
View File

@ -98,11 +98,6 @@
# enable/disable system monitor
# monitor 1
# enable/disable audit log
# audit 1
# enable/disable audit create table
# auditCreateTable 1
# The following parameter is used to limit the maximum number of lines in log files.
# max number of lines per log filters

152
source/client/src/clientEnv.c
View File

@ -26,6 +26,7 @@
#include "qworker.h"
#include "scheduler.h"
#include "tcache.h"
#include "tcompare.h"
#include "tglobal.h"
#include "thttp.h"
#include "tmsg.h"
@ -35,7 +36,6 @@
#include "tsched.h"
#include "ttime.h"
#include "tversion.h"
#include "tcompare.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
@ -44,16 +44,16 @@
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
#define ENV_JSON_FALSE_CHECK(c) \
do { \
if (!c) { \
tscError("faild to add item to JSON object");\
code = TSDB_CODE_TSC_FAIL_GENERATE_JSON; \
goto _end; \
} \
#define ENV_JSON_FALSE_CHECK(c) \
do { \
if (!c) { \
tscError("faild to add item to JSON object"); \
code = TSDB_CODE_TSC_FAIL_GENERATE_JSON; \
goto _end; \
} \
} while (0)
#define ENV_ERR_RET(c,info) \
#define ENV_ERR_RET(c, info) \
do { \
int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \
@ -94,109 +94,110 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) {
int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1);
int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1);
tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
", current:%d, app current:%d, total:%d, QID:0x%" PRIx64,
pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
}
return code;
}
static void concatStrings(SArray *list, char* buf, int size){
int len = 0;
for(int i = 0; i < taosArrayGetSize(list); i++){
char* db = taosArrayGet(list, i);
static void concatStrings(SArray *list, char *buf, int size) {
int len = 0;
for (int i = 0; i < taosArrayGetSize(list); i++) {
char *db = taosArrayGet(list, i);
if (NULL == db) {
tscError("get dbname failed, buf:%s", buf);
break;
}
char* dot = strchr(db, '.');
char *dot = strchr(db, '.');
if (dot != NULL) {
db = dot + 1;
}
if (i != 0){
if (i != 0) {
(void)strcat(buf, ",");
len += 1;
}
int ret = snprintf(buf + len, size - len, "%s", db);
if (ret < 0) {
if (ret < 0) {
tscError("snprintf failed, buf:%s, ret:%d", buf, ret);
break;
}
len += ret;
if (len >= size){
if (len >= size) {
tscInfo("dbList is truncated, buf:%s, len:%d", buf, len);
break;
}
}
}
static int32_t generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_t reqType, int64_t duration){
cJSON* json = cJSON_CreateObject();
static int32_t generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_t reqType, int64_t duration) {
cJSON *json = cJSON_CreateObject();
int32_t code = TSDB_CODE_SUCCESS;
if (json == NULL) {
tscError("[monitor] cJSON_CreateObject failed");
return TSDB_CODE_OUT_OF_MEMORY;
}
char clusterId[32] = {0};
if (snprintf(clusterId, sizeof(clusterId), "%" PRId64, pTscObj->pAppInfo->clusterId) < 0){
if (snprintf(clusterId, sizeof(clusterId), "%" PRId64, pTscObj->pAppInfo->clusterId) < 0) {
tscError("failed to generate clusterId:%" PRId64, pTscObj->pAppInfo->clusterId);
code = TSDB_CODE_FAILED;
goto _end;
}
char startTs[32] = {0};
if (snprintf(startTs, sizeof(startTs), "%" PRId64, pRequest->metric.start/1000) < 0){
tscError("failed to generate startTs:%" PRId64, pRequest->metric.start/1000);
if (snprintf(startTs, sizeof(startTs), "%" PRId64, pRequest->metric.start / 1000) < 0) {
tscError("failed to generate startTs:%" PRId64, pRequest->metric.start / 1000);
code = TSDB_CODE_FAILED;
goto _end;
}
char requestId[32] = {0};
if (snprintf(requestId, sizeof(requestId), "%" PRIu64, pRequest->requestId) < 0){
if (snprintf(requestId, sizeof(requestId), "%" PRIu64, pRequest->requestId) < 0) {
tscError("failed to generate requestId:%" PRIu64, pRequest->requestId);
code = TSDB_CODE_FAILED;
goto _end;
}
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "cluster_id", cJSON_CreateString(clusterId)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "start_ts", cJSON_CreateString(startTs)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "request_id", cJSON_CreateString(requestId)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "query_time", cJSON_CreateNumber(duration/1000)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "code", cJSON_CreateNumber(pRequest->code)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code))));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows)));
if(pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen){
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "cluster_id", cJSON_CreateString(clusterId)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "start_ts", cJSON_CreateString(startTs)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "request_id", cJSON_CreateString(requestId)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "query_time", cJSON_CreateNumber(duration / 1000)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "code", cJSON_CreateNumber(pRequest->code)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code))));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(
json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows)));
if (pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen) {
char tmp = pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen];
pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = '\0';
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr)));
pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = tmp;
}else{
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr)));
} else {
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr)));
}
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "user", cJSON_CreateString(pTscObj->user)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "user", cJSON_CreateString(pTscObj->user)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_name", cJSON_CreateString(appInfo.appName)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "ip", cJSON_CreateString(tsLocalFqdn)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "ip", cJSON_CreateString(tsLocalFqdn)));
char pid[32] = {0};
if (snprintf(pid, sizeof(pid), "%d", appInfo.pid) < 0){
if (snprintf(pid, sizeof(pid), "%d", appInfo.pid) < 0) {
tscError("failed to generate pid:%d", appInfo.pid);
code = TSDB_CODE_FAILED;
goto _end;
}
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_id", cJSON_CreateString(pid)));
if(pRequest->dbList != NULL){
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_id", cJSON_CreateString(pid)));
if (pRequest->dbList != NULL) {
char dbList[1024] = {0};
concatStrings(pRequest->dbList, dbList, sizeof(dbList) - 1);
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(dbList)));
}else if(pRequest->pDb != NULL){
} else if (pRequest->pDb != NULL) {
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(pRequest->pDb)));
}else{
} else {
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString("")));
}
char* value = cJSON_PrintUnformatted(json);
char *value = cJSON_PrintUnformatted(json);
MonitorSlowLogData data = {0};
data.clusterId = pTscObj->pAppInfo->clusterId;
data.type = SLOW_LOG_WRITE;
@ -212,7 +213,7 @@ _end:
return code;
}
static bool checkSlowLogExceptDb(SRequestObj *pRequest, char* exceptDb) {
static bool checkSlowLogExceptDb(SRequestObj *pRequest, char *exceptDb) {
if (pRequest->pDb != NULL) {
return strcmp(pRequest->pDb, exceptDb) != 0;
}
@ -227,7 +228,7 @@ static bool checkSlowLogExceptDb(SRequestObj *pRequest, char* exceptDb) {
if (dot != NULL) {
db = dot + 1;
}
if(strcmp(db, exceptDb) == 0){
if (strcmp(db, exceptDb) == 0) {
return false;
}
}
@ -248,15 +249,14 @@ static void deregisterRequest(SRequestObj *pRequest) {
int32_t reqType = SLOW_LOG_TYPE_OTHERS;
int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64
tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", QID:0x%" PRIx64
" elapsed:%.2f ms, "
"current:%d, app current:%d",
pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst);
if (TSDB_CODE_SUCCESS == nodesSimAcquireAllocator(pRequest->allocatorRefId)) {
if ((pRequest->pQuery && pRequest->pQuery->pRoot &&
QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type &&
(0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) ||
if ((pRequest->pQuery && pRequest->pQuery->pRoot && QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type &&
(0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) ||
QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType) {
tscDebug("insert duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64
"us, planCost:%" PRId64 "us, exec:%" PRId64 "us",
@ -279,7 +279,7 @@ static void deregisterRequest(SRequestObj *pRequest) {
}
}
if(pTscObj->pAppInfo->monitorParas.tsEnableMonitor){
if (pTscObj->pAppInfo->monitorParas.tsEnableMonitor) {
if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) {
sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT);
} else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
@ -289,14 +289,15 @@ static void deregisterRequest(SRequestObj *pRequest) {
}
}
if ((duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThreshold * 1000000UL || duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThresholdTest * 1000000UL) &&
if ((duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThreshold * 1000000UL ||
duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThresholdTest * 1000000UL) &&
checkSlowLogExceptDb(pRequest, pTscObj->pAppInfo->monitorParas.tsSlowLogExceptDb)) {
(void)atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1);
if (pTscObj->pAppInfo->monitorParas.tsSlowLogScope & reqType) {
taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 " us, Duration:%" PRId64 "us, SQL:%s",
taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration,
pRequest->sqlstr);
if(pTscObj->pAppInfo->monitorParas.tsEnableMonitor){
if (pTscObj->pAppInfo->monitorParas.tsEnableMonitor) {
slowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration);
if (TSDB_CODE_SUCCESS != generateWriteSlowLog(pTscObj, pRequest, reqType, duration)) {
tscError("failed to generate write slow log");
@ -389,7 +390,7 @@ void destroyAllRequests(SHashObj *pRequests) {
SRequestObj *pRequest = acquireRequest(*rid);
if (pRequest) {
destroyRequest(pRequest);
(void)releaseRequest(*rid); // ignore error
(void)releaseRequest(*rid); // ignore error
}
pIter = taosHashIterate(pRequests, pIter);
@ -404,7 +405,7 @@ void stopAllRequests(SHashObj *pRequests) {
SRequestObj *pRequest = acquireRequest(*rid);
if (pRequest) {
taos_stop_query(pRequest);
(void)releaseRequest(*rid); // ignore error
(void)releaseRequest(*rid); // ignore error
}
pIter = taosHashIterate(pRequests, pIter);
@ -412,7 +413,7 @@ void stopAllRequests(SHashObj *pRequests) {
}
void destroyAppInst(void *info) {
SAppInstInfo* pAppInfo = *(SAppInstInfo**)info;
SAppInstInfo *pAppInfo = *(SAppInstInfo **)info;
tscDebug("destroy app inst mgr %p", pAppInfo);
int32_t code = taosThreadMutexLock(&appInfo.mutex);
@ -597,28 +598,28 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri
int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); }
/// return the most previous req ref id
int64_t removeFromMostPrevReq(SRequestObj* pRequest) {
int64_t mostPrevReqRefId = pRequest->self;
SRequestObj* pTmp = pRequest;
int64_t removeFromMostPrevReq(SRequestObj *pRequest) {
int64_t mostPrevReqRefId = pRequest->self;
SRequestObj *pTmp = pRequest;
while (pTmp->relation.prevRefId) {
pTmp = acquireRequest(pTmp->relation.prevRefId);
if (pTmp) {
mostPrevReqRefId = pTmp->self;
(void)releaseRequest(mostPrevReqRefId); // ignore error
(void)releaseRequest(mostPrevReqRefId); // ignore error
} else {
break;
}
}
(void)removeRequest(mostPrevReqRefId); // ignore error
(void)removeRequest(mostPrevReqRefId); // ignore error
return mostPrevReqRefId;
}
void destroyNextReq(int64_t nextRefId) {
if (nextRefId) {
SRequestObj* pObj = acquireRequest(nextRefId);
SRequestObj *pObj = acquireRequest(nextRefId);
if (pObj) {
(void)releaseRequest(nextRefId); // ignore error
(void)releaseRequest(nextRefId); // ignore error
(void)releaseRequest(nextRefId); // ignore error
(void)releaseRequest(nextRefId); // ignore error
}
}
}
@ -638,7 +639,7 @@ void destroySubRequests(SRequestObj *pRequest) {
pTmp = acquireRequest(tmpRefId);
if (pTmp) {
pReqList[++reqIdx] = pTmp;
(void)releaseRequest(tmpRefId); // ignore error
(void)releaseRequest(tmpRefId); // ignore error
} else {
tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId);
break;
@ -646,7 +647,7 @@ void destroySubRequests(SRequestObj *pRequest) {
}
for (int32_t i = reqIdx; i >= 0; i--) {
(void)removeRequest(pReqList[i]->self); // ignore error
(void)removeRequest(pReqList[i]->self); // ignore error
}
tmpRefId = pRequest->relation.nextRefId;
@ -654,8 +655,8 @@ void destroySubRequests(SRequestObj *pRequest) {
pTmp = acquireRequest(tmpRefId);
if (pTmp) {
tmpRefId = pTmp->relation.nextRefId;
(void)removeRequest(pTmp->self); // ignore error
(void)releaseRequest(pTmp->self); // ignore error
(void)removeRequest(pTmp->self); // ignore error
(void)releaseRequest(pTmp->self); // ignore error
} else {
tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId);
break;
@ -758,7 +759,7 @@ void stopAllQueries(SRequestObj *pRequest) {
for (int32_t i = reqIdx; i >= 0; i--) {
taosStopQueryImpl(pReqList[i]);
(void)releaseRequest(pReqList[i]->self); // ignore error
(void)releaseRequest(pReqList[i]->self); // ignore error
}
taosStopQueryImpl(pRequest);
@ -769,7 +770,7 @@ void stopAllQueries(SRequestObj *pRequest) {
if (pTmp) {
tmpRefId = pTmp->relation.nextRefId;
taosStopQueryImpl(pTmp);
(void)releaseRequest(pTmp->self); // ignore error
(void)releaseRequest(pTmp->self); // ignore error
} else {
tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId);
break;
@ -929,7 +930,8 @@ void taos_init_imp(void) {
appInfo.pid = taosGetPId();
appInfo.startTime = taosGetTimestampMs();
appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
appInfo.pInstMapByClusterId = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
appInfo.pInstMapByClusterId =
taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
if (NULL == appInfo.pInstMap || NULL == appInfo.pInstMapByClusterId) {
tscError("failed to allocate memory when init appInfo");
tscInitRes = TSDB_CODE_OUT_OF_MEMORY;
@ -993,8 +995,8 @@ int taos_init() {
return tscInitRes;
}
const char* getCfgName(TSDB_OPTION option) {
const char* name = NULL;
const char *getCfgName(TSDB_OPTION option) {
const char *name = NULL;
switch (option) {
case TSDB_OPTION_SHELL_ACTIVITY_TIMER:
@ -1025,12 +1027,12 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
char newstr[PATH_MAX];
int len = strlen(str);
if (len > 1 && str[0] != '"' && str[0] != '\'') {
if (len + 2 >= PATH_MAX) {
if (len + 2 >= PATH_MAX) {
tscError("Too long path %s", str);
return -1;
}
newstr[0] = '"';
(void)memcpy(newstr+1, str, len);
(void)memcpy(newstr + 1, str, len);
newstr[len + 1] = '"';
newstr[len + 2] = '\0';
str = newstr;

147
source/client/src/clientImpl.c
View File

@ -15,8 +15,8 @@
#include "cJSON.h"
#include "clientInt.h"
#include "clientMonitor.h"
#include "clientLog.h"
#include "clientMonitor.h"
#include "command.h"
#include "scheduler.h"
#include "tdatablock.h"
@ -28,7 +28,7 @@
#include "tref.h"
#include "tsched.h"
#include "tversion.h"
static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInfo);
static bool stringLengthCheck(const char* str, size_t maxsize) {
@ -68,15 +68,13 @@ bool chkRequestKilled(void* param) {
return killed;
}
void cleanupAppInfo() {
taosHashCleanup(appInfo.pInstMap);
}
void cleanupAppInfo() { taosHashCleanup(appInfo.pInstMap); }
static int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj);
int32_t taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
uint16_t port, int connType, STscObj** pObj) {
uint16_t port, int connType, STscObj** pObj) {
TSC_ERR_RET(taos_init());
if (!validateUserName(user)) {
TSC_ERR_RET(TSDB_CODE_TSC_INVALID_USER_LENGTH);
@ -126,7 +124,7 @@ int32_t taos_connect_internal(const char* ip, const char* user, const char* pass
}
SAppInstInfo** pInst = NULL;
int32_t code = taosThreadMutexLock(&appInfo.mutex);
int32_t code = taosThreadMutexLock(&appInfo.mutex);
if (TSDB_CODE_SUCCESS != code) {
tscError("failed to lock app info, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code)));
TSC_ERR_RET(code);
@ -166,32 +164,39 @@ int32_t taos_connect_internal(const char* ip, const char* user, const char* pass
pInst = &p;
} else {
ASSERTS((*pInst) && (*pInst)->pAppHbMgr, "*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL);
if (NULL == *pInst || NULL == (*pInst)->pAppHbMgr) {
tscError("*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL);
TSC_ERR_JRET(TSDB_CODE_TSC_INTERNAL_ERROR);
}
// reset to 0 in case of conn with duplicated user key but its user has ever been dropped.
atomic_store_8(&(*pInst)->pAppHbMgr->connHbFlag, 0);
}
_return:
code = taosThreadMutexUnlock(&appInfo.mutex);
if (TSDB_CODE_SUCCESS != code) {
tscError("failed to unlock app info, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code)));
(void)taosThreadMutexUnlock(&appInfo.mutex);
taosMemoryFreeClear(key);
return code;
} else {
code = taosThreadMutexUnlock(&appInfo.mutex);
taosMemoryFreeClear(key);
if (TSDB_CODE_SUCCESS != code) {
tscError("failed to unlock app info, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code)));
return code;
}
return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType, pObj);
}
taosMemoryFreeClear(key);
return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType, pObj);
}
//SAppInstInfo* getAppInstInfo(const char* clusterKey) {
// SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey));
// if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) {
// return *ppAppInstInfo;
// } else {
// return NULL;
// }
//}
// SAppInstInfo* getAppInstInfo(const char* clusterKey) {
// SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey));
// if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) {
// return *ppAppInstInfo;
// } else {
// return NULL;
// }
// }
void freeQueryParam(SSyncQueryParam* param) {
if (param == NULL) return;
@ -227,7 +232,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
int32_t err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
sizeof((*pRequest)->self));
if (err) {
tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
tscError("%" PRId64 " failed to add to request container, QID:0x%" PRIx64 ", conn:%" PRId64 ", %s",
(*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
destroyRequest(*pRequest);
*pRequest = NULL;
@ -238,7 +243,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) {
if (TSDB_CODE_SUCCESS !=
nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) {
tscError("%" PRId64 " failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
tscError("%" PRId64 " failed to create node allocator, QID:0x%" PRIx64 ", conn:%" PRId64 ", %s",
(*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
destroyRequest(*pRequest);
*pRequest = NULL;
@ -246,7 +251,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
}
}
tscDebugL("0x%" PRIx64 " SQL: %s, reqId:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId);
tscDebugL("0x%" PRIx64 " SQL: %s, QID:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId);
return TSDB_CODE_SUCCESS;
}
@ -360,10 +365,10 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code),
tscError("0x%" PRIx64 " fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
} else {
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64,
pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed,
pRequest->requestId);
}
@ -429,7 +434,7 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) {
return TSDB_CODE_SUCCESS;
}
int32_t qnodeRequired(SRequestObj* pRequest, bool *required) {
int32_t qnodeRequired(SRequestObj* pRequest, bool* required) {
if (QUERY_POLICY_VNODE == tsQueryPolicy || QUERY_POLICY_CLIENT == tsQueryPolicy) {
*required = false;
return TSDB_CODE_SUCCESS;
@ -551,7 +556,7 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr
if (NULL == nodeList) {
return terrno;
}
char* policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client";
char* policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client";
int32_t dbNum = taosArrayGetSize(pDbVgList);
for (int32_t i = 0; i < dbNum; ++i) {
@ -565,7 +570,7 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr
}
for (int32_t j = 0; j < vgNum; ++j) {
SVgroupInfo* pInfo = taosArrayGet(pVg, j);
SVgroupInfo* pInfo = taosArrayGet(pVg, j);
if (NULL == pInfo) {
taosArrayDestroy(nodeList);
return TSDB_CODE_OUT_OF_RANGE;
@ -969,8 +974,8 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
break;
}
default:
tscError("0x%" PRIx64 ", invalid exec result for request type %d, reqId:0x%" PRIx64, pRequest->self,
pRequest->type, pRequest->requestId);
tscError("0x%" PRIx64 ", invalid exec result for request type %d, QID:0x%" PRIx64, pRequest->self, pRequest->type,
pRequest->requestId);
code = TSDB_CODE_APP_ERROR;
}
@ -1014,12 +1019,12 @@ void returnToUser(SRequestObj* pRequest) {
(void)releaseRequest(pRequest->relation.userRefId);
return;
} else {
tscError("0x%" PRIx64 ", user ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self,
tscError("0x%" PRIx64 ", user ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self,
pRequest->relation.userRefId, pRequest->requestId);
}
}
static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock**pBlock) {
static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock** pBlock) {
int64_t lastTs = 0;
TAOS_FIELD* pResFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
@ -1029,7 +1034,7 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock*
return code;
}
for(int32_t i = 0; i < numOfFields; ++i) {
for (int32_t i = 0; i < numOfFields; ++i) {
SColumnInfoData colInfoData = createColumnInfoData(pResFields[i].type, pResFields[i].bytes, i + 1);
code = blockDataAppendColInfo(*pBlock, &colInfoData);
if (TSDB_CODE_SUCCESS != code) {
@ -1051,7 +1056,7 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock*
lastTs = ts;
}
for(int32_t j = 0; j < numOfFields; ++j) {
for (int32_t j = 0; j < numOfFields; ++j) {
SColumnInfoData* pColInfoData = taosArrayGet((*pBlock)->pDataBlock, j);
code = colDataSetVal(pColInfoData, i, pRow[j], false);
if (TSDB_CODE_SUCCESS != code) {
@ -1066,7 +1071,7 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock*
(*pBlock)->info.window.ekey = lastTs;
(*pBlock)->info.rows = numOfRows;
tscDebug("lastKey:%"PRId64" numOfRows:%d from all vgroups", lastTs, numOfRows);
tscDebug("lastKey:%" PRId64 " numOfRows:%d from all vgroups", lastTs, numOfRows);
return TSDB_CODE_SUCCESS;
}
@ -1079,7 +1084,7 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) {
SSDataBlock* pBlock = NULL;
if (TSDB_CODE_SUCCESS != createResultBlock(res, rowNum, &pBlock)) {
tscError("0x%" PRIx64 ", create result block failed, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId);
tscError("0x%" PRIx64 ", create result block failed, QID:0x%" PRIx64, pRequest->self, pRequest->requestId);
return;
}
@ -1088,7 +1093,7 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) {
continuePostSubQuery(pNextReq, pBlock);
(void)releaseRequest(pRequest->relation.nextRefId);
} else {
tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self,
tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self,
pRequest->relation.nextRefId, pRequest->requestId);
}
@ -1107,7 +1112,7 @@ void handlePostSubQuery(SSqlCallbackWrapper* pWrapper) {
continuePostSubQuery(pNextReq, NULL);
(void)releaseRequest(pRequest->relation.nextRefId);
} else {
tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self,
tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self,
pRequest->relation.nextRefId, pRequest->requestId);
}
}
@ -1135,16 +1140,14 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
(void)atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertRows, pResult->numOfRows);
}
}
schedulerFreeJob(&pRequest->body.queryJob, 0);
}
taosMemoryFree(pResult);
tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code),
tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self,
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, QID:0x%" PRIx64, pRequest->self,
tstrerror(code), pRequest->retry, pRequest->requestId);
(void)removeMeta(pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type));
restartAsyncQuery(pRequest, code);
@ -1214,7 +1217,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
}
break;
case QUERY_EXEC_MODE_SCHEDULE: {
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
if (NULL == pMnodeList) {
code = terrno;
break;
@ -1532,7 +1535,7 @@ int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* p
}
int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj) {
SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj) {
*pTscObj = NULL;
int32_t code = createTscObj(user, auth, db, connType, pAppInfo, pTscObj);
if (TSDB_CODE_SUCCESS != code) {
@ -1559,7 +1562,8 @@ int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __ta
}
int64_t transporterId = 0;
code = asyncSendMsgToServer((*pTscObj)->pAppInfo->pTransporter, &(*pTscObj)->pAppInfo->mgmtEp.epSet, &transporterId, body);
code = asyncSendMsgToServer((*pTscObj)->pAppInfo->pTransporter, &(*pTscObj)->pAppInfo->mgmtEp.epSet, &transporterId,
body);
if (TSDB_CODE_SUCCESS != code) {
destroyTscObj(*pTscObj);
tscError("failed to send connect msg to server, code:%s", tstrerror(code));
@ -1576,7 +1580,7 @@ int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __ta
*pTscObj = NULL;
return terrno;
} else {
tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, reqId:0x%" PRIx64, (*pTscObj)->id,
tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, QID:0x%" PRIx64, (*pTscObj)->id,
(*pTscObj)->connId, (*pTscObj)->pAppInfo->pTransporter, pRequest->requestId);
destroyRequest(pRequest);
}
@ -1707,7 +1711,7 @@ int32_t doProcessMsgFromServer(void* param) {
char tbuf[40] = {0};
TRACE_TO_STR(trace, tbuf);
tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, gtid: %s", pMsg->info.handle,
tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, QID:%s", pMsg->info.handle,
TMSG_INFO(pMsg->msgType), pMsg->contLen, tstrerror(pMsg->code), tbuf);
if (pSendInfo->requestObjRefId != 0) {
@ -1791,6 +1795,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
AsyncArg* arg = taosMemoryCalloc(1, sizeof(AsyncArg));
if (NULL == arg) {
pMsg->code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(tEpSet);
rpcFreeCont(pMsg->pCont);
destroySendMsgInfo(pMsg->info.ahandle);
return;
@ -1819,7 +1824,7 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
}
STscObj* pObj = NULL;
int32_t code = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY, &pObj);
int32_t code = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY, &pObj);
if (TSDB_CODE_SUCCESS == code) {
int64_t* rid = taosMemoryCalloc(1, sizeof(int64_t));
if (NULL == rid) {
@ -1889,7 +1894,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
}
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
SSchedulerReq req = { .syncReq = true, .pFetchRes = (void**)&pResInfo->pData };
SSchedulerReq req = {.syncReq = true, .pFetchRes = (void**)&pResInfo->pData};
pRequest->code = schedulerFetchRows(pRequest->body.queryJob, &req);
if (pRequest->code != TSDB_CODE_SUCCESS) {
@ -1904,7 +1909,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
return NULL;
}
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64,
pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId);
STscObj* pTscObj = pRequest->pTscObj;
@ -2032,15 +2037,15 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) {
}
static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) {
char* p = (char*)pResultInfo->pData;
char* p = (char*)pResultInfo->pData;
int32_t blockVersion = *(int32_t*)p;
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column
// length |
int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3);
if (ASSERT(numOfCols == cols)) {
if (numOfCols != cols) {
tscError("estimateJsonLen error: numOfCols:%d != cols:%d", numOfCols, cols);
return -1;
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
int32_t len = getVersion1BlockMetaSize(p, numOfCols);
@ -2125,7 +2130,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
int32_t totalLen = 0;
int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3);
if (ASSERT(numOfCols == cols)) {
if (numOfCols != cols) {
tscError("doConvertJson error: numOfCols:%d != cols:%d", numOfCols, cols);
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
@ -2150,7 +2155,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
for (int32_t i = 0; i < numOfCols; ++i) {
int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i];
int32_t colLen1 = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength1[i]) : colLength1[i];
if (ASSERT(colLen < dataLen)) {
if (colLen >= dataLen) {
tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen);
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
@ -2238,7 +2243,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows,
bool convertUcs4) {
if (ASSERT(numOfCols > 0 && pFields != NULL && pResultInfo != NULL)) {
if (numOfCols <= 0 || pFields == NULL || pResultInfo == NULL) {
tscError("setResultDataPtr paras error");
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
@ -2271,7 +2276,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
int32_t cols = *(int32_t*)p;
p += sizeof(int32_t);
if (ASSERT(rows == numOfRows && cols == numOfCols)) {
if (rows != numOfRows || cols != numOfCols) {
tscError("setResultDataPtr paras error:rows;%d numOfRows:%d cols:%d numOfCols:%d", rows, numOfRows, cols,
numOfCols);
return TSDB_CODE_TSC_INTERNAL_ERROR;
@ -2290,8 +2295,6 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
int32_t bytes = *(int32_t*)p;
p += sizeof(int32_t);
/*ASSERT(type == pFields[i].type && bytes == pFields[i].bytes);*/
}
int32_t* colLength = (int32_t*)p;
@ -2299,7 +2302,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
char* pStart = p;
for (int32_t i = 0; i < numOfCols; ++i) {
if(blockVersion == BLOCK_VERSION_1){
if (blockVersion == BLOCK_VERSION_1) {
colLength[i] = htonl(colLength[i]);
}
if (colLength[i] >= dataLen) {
@ -2413,18 +2416,23 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR
if (pRsp->compressed && compLen < rawLen) {
int32_t len = tsDecompressString(pStart, compLen, 1, pResultInfo->decompBuf, rawLen, ONE_STAGE_COMP, NULL, 0);
ASSERT(len == rawLen);
if (len < 0) {
tscError("tsDecompressString failed");
return terrno ? terrno : TSDB_CODE_FAILED;
}
if (len != rawLen) {
tscError("tsDecompressString failed, len:%d != rawLen:%d", len, rawLen);
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
pResultInfo->pData = pResultInfo->decompBuf;
pResultInfo->payloadLen = rawLen;
} else {
pResultInfo->pData = pStart;
pResultInfo->payloadLen = htonl(pRsp->compLen);
ASSERT(pRsp->compLen == pRsp->payloadLen);
if (pRsp->compLen != pRsp->payloadLen) {
tscError("pRsp->compLen:%d != pRsp->payloadLen:%d", pRsp->compLen, pRsp->payloadLen);
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
}
}
@ -2729,7 +2737,8 @@ void syncQueryFn(void* param, void* res, int32_t code) {
(void)tsem_post(&pParam->sem);
}
void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source) {
void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly,
int8_t source) {
if (sql == NULL || NULL == fp) {
terrno = TSDB_CODE_INVALID_PARA;
if (fp) {
@ -2850,7 +2859,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code,
tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, QID:0x%" PRIx64, pRequest->self, code,
tstrerror(code), pRequest->requestId);
pResultInfo->pData = pResult;
@ -2873,10 +2882,10 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4);
if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code),
tscError("0x%" PRIx64 " fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code),
pRequest->requestId);
} else {
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64,
pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed,
pRequest->requestId);

132
source/client/src/clientMain.c
View File

@ -16,19 +16,19 @@
#include "catalog.h"
#include "clientInt.h"
#include "clientLog.h"
#include "clientStmt.h"
#include "clientMonitor.h"
#include "clientStmt.h"
#include "functionMgt.h"
#include "os.h"
#include "query.h"
#include "scheduler.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tmsg.h"
#include "tref.h"
#include "trpc.h"
#include "version.h"
#include "tcompare.h"
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
@ -74,15 +74,11 @@ void taos_cleanup(void) {
int32_t id = clientReqRefPool;
clientReqRefPool = -1;
if (TSDB_CODE_SUCCESS != taosCloseRef(id)) {
tscWarn("failed to close clientReqRefPool");
}
taosCloseRef(id);
id = clientConnRefPool;
clientConnRefPool = -1;
if (TSDB_CODE_SUCCESS != taosCloseRef(id)) {
tscWarn("failed to close clientReqRefPool");
}
taosCloseRef(id);
nodesDestroyAllocatorSet();
cleanupAppInfo();
@ -124,7 +120,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
}
STscObj *pObj = NULL;
int32_t code = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY, &pObj);
int32_t code = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY, &pObj);
if (TSDB_CODE_SUCCESS == code) {
int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t));
if (NULL == rid) {
@ -187,15 +183,15 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type)
return 0;
}
typedef struct SFetchWhiteListInfo{
int64_t connId;
typedef struct SFetchWhiteListInfo {
int64_t connId;
__taos_async_whitelist_fn_t userCbFn;
void* userParam;
void *userParam;
} SFetchWhiteListInfo;
int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) {
SFetchWhiteListInfo* pInfo = (SFetchWhiteListInfo*)param;
TAOS* taos = &pInfo->connId;
int32_t fetchWhiteListCallbackFn(void *param, SDataBuf *pMsg, int32_t code) {
SFetchWhiteListInfo *pInfo = (SFetchWhiteListInfo *)param;
TAOS *taos = &pInfo->connId;
if (code != TSDB_CODE_SUCCESS) {
pInfo->userCbFn(pInfo->userParam, code, taos, 0, NULL);
taosMemoryFree(pMsg->pData);
@ -213,7 +209,7 @@ int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) {
return terrno;
}
uint64_t* pWhiteLists = taosMemoryMalloc(wlRsp.numWhiteLists * sizeof(uint64_t));
uint64_t *pWhiteLists = taosMemoryMalloc(wlRsp.numWhiteLists * sizeof(uint64_t));
if (pWhiteLists == NULL) {
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
@ -242,7 +238,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
return;
}
int64_t connId = *(int64_t*)taos;
int64_t connId = *(int64_t *)taos;
STscObj *pTsc = acquireTscObj(connId);
if (NULL == pTsc) {
@ -259,7 +255,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
return;
}
void* pReq = taosMemoryMalloc(msgLen);
void *pReq = taosMemoryMalloc(msgLen);
if (pReq == NULL) {
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
releaseTscObj(connId);
@ -273,7 +269,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
return;
}
SFetchWhiteListInfo* pParam = taosMemoryMalloc(sizeof(SFetchWhiteListInfo));
SFetchWhiteListInfo *pParam = taosMemoryMalloc(sizeof(SFetchWhiteListInfo));
if (pParam == NULL) {
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
taosMemoryFree(pReq);
@ -284,9 +280,9 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
pParam->connId = connId;
pParam->userCbFn = fp;
pParam->userParam = param;
SMsgSendInfo* pSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
SMsgSendInfo *pSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (pSendInfo == NULL) {
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
taosMemoryFree(pParam);
taosMemoryFree(pReq);
releaseTscObj(connId);
@ -301,7 +297,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
pSendInfo->msgType = TDMT_MND_GET_USER_WHITELIST;
int64_t transportId = 0;
SEpSet epSet = getEpSet_s(&pTsc->pAppInfo->mgmtEp);
SEpSet epSet = getEpSet_s(&pTsc->pAppInfo->mgmtEp);
if (TSDB_CODE_SUCCESS != asyncSendMsgToServer(pTsc->pAppInfo->pTransporter, &epSet, &transportId, pSendInfo)) {
tscWarn("failed to async send msg to server");
}
@ -447,7 +443,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
return NULL;
}
if(pRequest->inCallback) {
if (pRequest->inCallback) {
tscError("can not call taos_fetch_row before query callback ends.");
terrno = TSDB_CODE_TSC_INVALID_OPERATION;
return NULL;
@ -458,7 +454,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
SMqRspObj *msg = ((SMqRspObj *)res);
SReqResultInfo *pResultInfo = NULL;
if (msg->common.resIter == -1) {
if(tmqGetNextResInfo(res, true, &pResultInfo) != 0){
if (tmqGetNextResInfo(res, true, &pResultInfo) != 0) {
return NULL;
}
} else {
@ -470,7 +466,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
pResultInfo->current += 1;
return pResultInfo->row;
} else {
if (tmqGetNextResInfo(res, true, &pResultInfo) != 0){
if (tmqGetNextResInfo(res, true, &pResultInfo) != 0) {
return NULL;
}
@ -481,7 +477,6 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
} else if (TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return NULL;
} else {
// assert to avoid un-initialization error
tscError("invalid result passed to taos_fetch_row");
terrno = TSDB_CODE_TSC_INTERNAL_ERROR;
return NULL;
@ -545,28 +540,31 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
len += sprintf(str + len, "%lf", dv);
} break;
case TSDB_DATA_TYPE_VARBINARY:{
void* data = NULL;
case TSDB_DATA_TYPE_VARBINARY: {
void *data = NULL;
uint32_t size = 0;
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
if(taosAscii2Hex(row[i], charLen, &data, &size) < 0){
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
if (taosAscii2Hex(row[i], charLen, &data, &size) < 0) {
break;
}
(void)memcpy(str + len, data, size);
len += size;
taosMemoryFree(data);
}break;
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY: {
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_VARBINARY || fields[i].type == TSDB_DATA_TYPE_GEOMETRY) {
if (ASSERT(charLen <= fields[i].bytes && charLen >= 0)) {
if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_VARBINARY ||
fields[i].type == TSDB_DATA_TYPE_GEOMETRY) {
if (charLen > fields[i].bytes || charLen < 0) {
tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes);
break;
}
} else {
if (ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)) {
if (charLen > fields[i].bytes * TSDB_NCHAR_SIZE || charLen < 0) {
tscError("taos_print_row error. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes);
break;
}
}
@ -667,7 +665,8 @@ const char *taos_get_client_info() { return version; }
// return int32_t
int taos_affected_rows(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@ -678,7 +677,8 @@ int taos_affected_rows(TAOS_RES *res) {
// return int64_t
int64_t taos_affected_rows64(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@ -728,7 +728,8 @@ int taos_select_db(TAOS *taos, const char *db) {
}
void taos_stop_query(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res)) {
return;
}
@ -787,7 +788,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
return pRequest->code;
} else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SReqResultInfo *pResultInfo = NULL;
int32_t code = tmqGetNextResInfo(res, true, &pResultInfo);
int32_t code = tmqGetNextResInfo(res, true, &pResultInfo);
if (code != 0) return code;
pResultInfo->current = pResultInfo->numOfRows;
@ -810,7 +811,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SReqResultInfo *pResultInfo = NULL;
int32_t code = tmqGetNextResInfo(res, false, &pResultInfo);
int32_t code = tmqGetNextResInfo(res, false, &pResultInfo);
if (code != 0) {
(*numOfRows) = 0;
return 0;
@ -940,7 +941,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t
SRequestObj *pRequest = pWrapper->pRequest;
SQuery *pQuery = pRequest->pQuery;
qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId);
qDebug("0x%" PRIx64 " start to semantic analysis, QID:0x%" PRIx64, pRequest->self, pRequest->requestId);
int64_t analyseStart = taosGetTimestampUs();
pRequest->metric.ctgCostUs = analyseStart - pRequest->metric.ctgStart;
@ -956,7 +957,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t
(void)memcpy(&pRequest->parseMeta, pResultMeta, sizeof(*pResultMeta));
(void)memset(pResultMeta, 0, sizeof(*pResultMeta));
}
handleQueryAnslyseRes(pWrapper, pResultMeta, code);
}
@ -1002,7 +1003,7 @@ void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResult
}
pNewRequest->pQuery = NULL;
code = nodesMakeNode(QUERY_NODE_QUERY, (SNode**)&pNewRequest->pQuery);
code = nodesMakeNode(QUERY_NODE_QUERY, (SNode **)&pNewRequest->pQuery);
if (pNewRequest->pQuery) {
pNewRequest->pQuery->pRoot = pRoot;
pRoot = NULL;
@ -1059,15 +1060,15 @@ void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta
pRequest->pQuery = NULL;
if (NEED_CLIENT_HANDLE_ERROR(code)) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64,
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, QID:0x%" PRIx64,
pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId);
restartAsyncQuery(pRequest, code);
return;
}
// return to app directly
tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, reqId:0x%" PRIx64, pRequest->self,
tstrerror(code), pRequest->requestId);
tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, QID:0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
pRequest->code = code;
returnToUser(pRequest);
}
@ -1116,7 +1117,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c
SQuery *pQuery = pRequest->pQuery;
pRequest->metric.ctgCostUs += taosGetTimestampUs() - pRequest->metric.ctgStart;
qDebug("0x%" PRIx64 " start to continue parse, reqId:0x%" PRIx64 ", code:%s", pRequest->self, pRequest->requestId,
qDebug("0x%" PRIx64 " start to continue parse, QID:0x%" PRIx64 ", code:%s", pRequest->self, pRequest->requestId,
tstrerror(code));
if (code == TSDB_CODE_SUCCESS) {
@ -1129,7 +1130,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c
}
if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code,
tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pWrapper->pRequest->self, code,
tstrerror(code), pWrapper->pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
pRequest->pWrapper = NULL;
@ -1146,7 +1147,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest)
}
if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code,
tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pWrapper->pRequest->self, code,
tstrerror(code), pWrapper->pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
pRequest->pWrapper = NULL;
@ -1264,7 +1265,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
}
if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code),
tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pRequest->self, code, tstrerror(code),
pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
pRequest->pWrapper = NULL;
@ -1272,9 +1273,9 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
pRequest->pQuery = NULL;
if (NEED_CLIENT_HANDLE_ERROR(code)) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64,
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, QID:0x%" PRIx64,
pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId);
(void)refreshMeta(pRequest->pTscObj, pRequest); //ignore return code,try again
(void)refreshMeta(pRequest->pTscObj, pRequest); // ignore return code,try again
pRequest->prevCode = code;
doAsyncQuery(pRequest, true);
return;
@ -1288,7 +1289,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
tscInfo("restart request: %s p: %p", pRequest->sqlstr, pRequest);
SRequestObj* pUserReq = pRequest;
SRequestObj *pUserReq = pRequest;
(void)acquireRequest(pRequest->self);
while (pUserReq) {
if (pUserReq->self == pUserReq->relation.userRefId || pUserReq->relation.userRefId == 0) {
@ -1319,11 +1320,11 @@ void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
}
void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
if (ASSERT(res != NULL && fp != NULL)) {
if (res == NULL || fp == NULL) {
tscError("taos_fetch_rows_a invalid paras");
return;
}
if (ASSERT(TD_RES_QUERY(res))) {
if (!TD_RES_QUERY(res)) {
tscError("taos_fetch_rows_a res is NULL");
return;
}
@ -1338,12 +1339,12 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
}
void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
if (ASSERT(res != NULL && fp != NULL)) {
tscError("taos_fetch_rows_a invalid paras");
if (res == NULL || fp == NULL) {
tscError("taos_fetch_raw_block_a invalid paras");
return;
}
if (ASSERT(TD_RES_QUERY(res))) {
tscError("taos_fetch_rows_a res is NULL");
if (!TD_RES_QUERY(res)) {
tscError("taos_fetch_raw_block_a res is NULL");
return;
}
SRequestObj *pRequest = res;
@ -1357,12 +1358,12 @@ void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
}
const void *taos_get_raw_block(TAOS_RES *res) {
if (ASSERT(res != NULL)) {
tscError("taos_fetch_rows_a invalid paras");
if (res == NULL) {
tscError("taos_get_raw_block invalid paras");
return NULL;
}
if (ASSERT(TD_RES_QUERY(res))) {
tscError("taos_fetch_rows_a res is NULL");
if (!TD_RES_QUERY(res)) {
tscError("taos_get_raw_block res is NULL");
return NULL;
}
SRequestObj *pRequest = res;
@ -1634,7 +1635,6 @@ TAOS_STMT *taos_stmt_init_with_options(TAOS *taos, TAOS_STMT_OPTIONS *options) {
return pStmt;
}
int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
if (stmt == NULL || sql == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);
@ -1877,7 +1877,7 @@ int taos_stmt_close(TAOS_STMT *stmt) {
return stmtClose(stmt);
}
int taos_set_conn_mode(TAOS* taos, int mode, int value) {
int taos_set_conn_mode(TAOS *taos, int mode, int value) {
if (taos == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
@ -1900,6 +1900,4 @@ int taos_set_conn_mode(TAOS* taos, int mode, int value) {
return 0;
}
char* getBuildInfo(){
return buildinfo;
}
char *getBuildInfo() { return buildinfo; }

31
source/client/src/clientMonitor.c
View File

@ -120,9 +120,13 @@ static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) {
.data = NULL};
if (monitorPutData2MonitorQueue(tmp) == 0) {
p->fileName = NULL;
} else {
if(taosCloseFile(&(p->pFile)) != 0) {
tscError("failed to close file:%p", p->pFile);
}
}
}
return code;
return TSDB_CODE_SUCCESS;
}
static int32_t sendReport(void* pTransporter, SEpSet* epSet, char* pCont, MONITOR_TYPE type, void* param) {
@ -164,7 +168,10 @@ static int32_t sendReport(void* pTransporter, SEpSet* epSet, char* pCont, MONITO
int64_t transporterId = 0;
return asyncSendMsgToServer(pTransporter, epSet, &transporterId, pInfo);
FAILED:
FAILED:
if (taosCloseFile(&(((MonitorSlowLogData*)param)->pFile)) != 0) {
tscError("failed to close file:%p", ((MonitorSlowLogData*)param)->pFile);
}
monitorFreeSlowLogDataEx(param);
return TAOS_GET_TERRNO(TSDB_CODE_TSC_INTERNAL_ERROR);
}
@ -315,7 +322,7 @@ void monitorCreateClientCounter(int64_t clusterId, const char* name, const char*
void monitorCounterInc(int64_t clusterId, const char* counterName, const char** label_values) {
taosWLockLatch(&monitorLock);
if (atomic_load_32(&monitorFlag) == 1) {
taosRUnLockLatch(&monitorLock);
taosWUnLockLatch(&monitorLock);
return;
}
@ -462,11 +469,17 @@ static int64_t getFileSize(char* path) {
static int32_t sendSlowLog(int64_t clusterId, char* data, TdFilePtr pFile, int64_t offset, SLOW_LOG_QUEUE_TYPE type,
char* fileName, void* pTransporter, SEpSet* epSet) {
if (data == NULL) {
if (taosCloseFile(&pFile) != 0) {
tscError("failed to close file:%p", pFile);
}
taosMemoryFree(fileName);
return TSDB_CODE_INVALID_PARA;
}
MonitorSlowLogData* pParam = taosMemoryMalloc(sizeof(MonitorSlowLogData));
if (pParam == NULL) {
if (taosCloseFile(&pFile) != 0) {
tscError("failed to close file:%p", pFile);
}
taosMemoryFree(data);
taosMemoryFree(fileName);
return terrno;
@ -485,6 +498,9 @@ static int32_t monitorReadSend(int64_t clusterId, TdFilePtr pFile, int64_t* offs
SAppInstInfo* pInst = getAppInstByClusterId(clusterId);
if (pInst == NULL) {
tscError("failed to get app instance by clusterId:%" PRId64, clusterId);
if (taosCloseFile(&pFile) != 0) {
tscError("failed to close file:%p", pFile);
}
taosMemoryFree(fileName);
return terrno;
}
@ -710,7 +726,7 @@ static void* monitorThreadFunc(void* param) {
MonitorSlowLogData* slowLogData = NULL;
(void)taosReadQitem(monitorQueue, (void**)&slowLogData);
if (slowLogData != NULL) {
if (slowLogData->type == SLOW_LOG_READ_BEGINNIG) {
if (slowLogData->type == SLOW_LOG_READ_BEGINNIG && quitCnt == 0) {
if (slowLogData->pFile != NULL) {
monitorSendSlowLogAtBeginning(slowLogData->clusterId, &(slowLogData->fileName), slowLogData->pFile,
slowLogData->offset);
@ -729,9 +745,9 @@ static void* monitorThreadFunc(void* param) {
break;
}
}
monitorFreeSlowLogData(slowLogData);
taosFreeQitem(slowLogData);
}
monitorFreeSlowLogData(slowLogData);
taosFreeQitem(slowLogData);
if (quitCnt == 0) {
monitorSendAllSlowLog();
@ -853,6 +869,9 @@ int32_t monitorPutData2MonitorQueue(MonitorSlowLogData data) {
if (taosWriteQitem(monitorQueue, slowLogData) == 0) {
(void)tsem2_post(&monitorSem);
} else {
if (taosCloseFile(&(slowLogData->pFile)) != 0) {
tscError("failed to close file:%p", slowLogData->pFile);
}
monitorFreeSlowLogData(slowLogData);
taosFreeQitem(slowLogData);
}

141
source/client/src/clientRawBlockWrite.c
View File

@ -23,32 +23,31 @@
#include "tglobal.h"
#include "tmsgtype.h"
#define RAW_NULL_CHECK(c) \
do { \
if (c == NULL) { \
code = TSDB_CODE_OUT_OF_MEMORY; \
goto end; \
} \
#define RAW_NULL_CHECK(c) \
do { \
if (c == NULL) { \
code = TSDB_CODE_OUT_OF_MEMORY; \
goto end; \
} \
} while (0)
#define RAW_FALSE_CHECK(c) \
do { \
if (!c) { \
code = TSDB_CODE_INVALID_PARA; \
goto end; \
} \
#define RAW_FALSE_CHECK(c) \
do { \
if (!c) { \
code = TSDB_CODE_INVALID_PARA; \
goto end; \
} \
} while (0)
#define RAW_RETURN_CHECK(c) \
do { \
code = c; \
if (code != 0) { \
goto end; \
} \
#define RAW_RETURN_CHECK(c) \
do { \
code = c; \
if (code != 0) { \
goto end; \
} \
} while (0)
#define LOG_ID_TAG "connId:0x%" PRIx64 ",reqId:0x%" PRIx64
#define LOG_ID_TAG "connId:0x%" PRIx64 ",QID:0x%" PRIx64
#define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId
#define TMQ_META_VERSION "1.0"
@ -57,10 +56,10 @@ static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen
static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
int8_t t, SColCmprWrapper* pColCmprRow, cJSON** pJson) {
static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t,
SColCmprWrapper* pColCmprRow, cJSON** pJson) {
int32_t code = TSDB_CODE_SUCCESS;
int8_t buildDefaultCompress = 0;
int8_t buildDefaultCompress = 0;
if (pColCmprRow->nCols <= 0) {
buildDefaultCompress = 1;
}
@ -82,7 +81,7 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche
cJSON* columns = cJSON_CreateArray();
RAW_NULL_CHECK(columns);
for (int i = 0; i < schemaRow->nCols; i++) {
cJSON* column = cJSON_CreateObject();
cJSON* column = cJSON_CreateObject();
RAW_NULL_CHECK(column);
SSchema* s = schemaRow->pSchema + i;
cJSON* cname = cJSON_CreateString(s->name);
@ -142,7 +141,7 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche
cJSON* tags = cJSON_CreateArray();
RAW_NULL_CHECK(tags);
for (int i = 0; schemaTag && i < schemaTag->nCols; i++) {
cJSON* tag = cJSON_CreateObject();
cJSON* tag = cJSON_CreateObject();
RAW_NULL_CHECK(tag);
SSchema* s = schemaTag->pSchema + i;
cJSON* tname = cJSON_CreateString(s->name);
@ -176,7 +175,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
if (encode != 0) {
const char* encodeStr = columnEncodeStr(encode);
RAW_NULL_CHECK(encodeStr);
cJSON* encodeJson = cJSON_CreateString(encodeStr);
cJSON* encodeJson = cJSON_CreateString(encodeStr);
RAW_NULL_CHECK(encodeJson);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "encode", encodeJson));
return code;
@ -185,7 +184,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
if (compress != 0) {
const char* compressStr = columnCompressStr(compress);
RAW_NULL_CHECK(compressStr);
cJSON* compressJson = cJSON_CreateString(compressStr);
cJSON* compressJson = cJSON_CreateString(compressStr);
RAW_NULL_CHECK(compressJson);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "compress", compressJson));
return code;
@ -194,7 +193,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
if (level != 0) {
const char* levelStr = columnLevelStr(level);
RAW_NULL_CHECK(levelStr);
cJSON* levelJson = cJSON_CreateString(levelStr);
cJSON* levelJson = cJSON_CreateString(levelStr);
RAW_NULL_CHECK(levelJson);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "level", levelJson));
return code;
@ -235,7 +234,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_ADD_COLUMN: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name);
cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
cJSON* colType = cJSON_CreateNumber(field->type);
@ -259,7 +258,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: {
SFieldWithOptions* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name);
cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
cJSON* colType = cJSON_CreateNumber(field->type);
@ -285,7 +284,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_DROP_COLUMN: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name);
cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
break;
@ -294,7 +293,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name);
cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
cJSON* colType = cJSON_CreateNumber(field->type);
@ -320,7 +319,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
RAW_NULL_CHECK(oldField);
TAOS_FIELD* newField = taosArrayGet(req.pFields, 1);
RAW_NULL_CHECK(newField);
cJSON* colName = cJSON_CreateString(oldField->name);
cJSON* colName = cJSON_CreateString(oldField->name);
RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
cJSON* colNewName = cJSON_CreateString(newField->name);
@ -331,7 +330,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name);
cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
RAW_RETURN_CHECK(setCompressOption(json, field->bytes));
@ -405,7 +404,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
RAW_NULL_CHECK(tagNumJson);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tagNum", tagNumJson));
cJSON* tags = cJSON_CreateArray();
cJSON* tags = cJSON_CreateArray();
RAW_NULL_CHECK(tags);
SArray* pTagVals = NULL;
RAW_RETURN_CHECK(tTagToValArray(pTag, &pTagVals));
@ -416,13 +415,13 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
uError("p->nTag == 0");
goto end;
}
char* pJson = NULL;
char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson);
cJSON* tag = cJSON_CreateObject();
cJSON* tag = cJSON_CreateObject();
RAW_NULL_CHECK(tag);
STagVal* pTagVal = taosArrayGet(pTagVals, 0);
RAW_NULL_CHECK(pTagVal);
char* ptname = taosArrayGet(tagName, 0);
char* ptname = taosArrayGet(tagName, 0);
RAW_NULL_CHECK(ptname);
cJSON* tname = cJSON_CreateString(ptname);
RAW_NULL_CHECK(tname);
@ -443,7 +442,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
RAW_NULL_CHECK(pTagVal);
cJSON* tag = cJSON_CreateObject();
RAW_NULL_CHECK(tag);
char* ptname = taosArrayGet(tagName, i);
char* ptname = taosArrayGet(tagName, i);
RAW_NULL_CHECK(ptname);
cJSON* tname = cJSON_CreateString(ptname);
RAW_NULL_CHECK(tname);
@ -463,7 +462,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
RAW_NULL_CHECK(buf);
if (!buf) goto end;
if(dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) {
if (dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) {
taosMemoryFree(buf);
goto end;
}
@ -489,8 +488,8 @@ end:
static void buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs, cJSON** pJson) {
int32_t code = 0;
char* string = NULL;
cJSON* json = cJSON_CreateObject();
char* string = NULL;
cJSON* json = cJSON_CreateObject();
RAW_NULL_CHECK(json);
cJSON* type = cJSON_CreateString("create");
RAW_NULL_CHECK(type);
@ -534,8 +533,8 @@ static void processCreateTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
if (pCreateReq->type == TSDB_CHILD_TABLE) {
buildCreateCTableJson(req.pReqs, req.nReqs, pJson);
} else if (pCreateReq->type == TSDB_NORMAL_TABLE) {
buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid,
TSDB_NORMAL_TABLE, &pCreateReq->colCmpr, pJson);
buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE,
&pCreateReq->colCmpr, pJson);
}
}
@ -548,7 +547,7 @@ end:
static void processAutoCreateTable(STaosxRsp* rsp, char** string) {
SDecoder* decoder = NULL;
SVCreateTbReq* pCreateReq = NULL;
int32_t code = 0;
int32_t code = 0;
uDebug("auto create table data:%p", rsp);
if (rsp->createTableNum <= 0) {
uError("processAutoCreateTable rsp->createTableNum <= 0");
@ -563,7 +562,7 @@ static void processAutoCreateTable(STaosxRsp* rsp, char** string) {
// loop to create table
for (int32_t iReq = 0; iReq < rsp->createTableNum; iReq++) {
// decode
void** data = taosArrayGet(rsp->createTableReq, iReq);
void** data = taosArrayGet(rsp->createTableReq, iReq);
RAW_NULL_CHECK(data);
int32_t* len = taosArrayGet(rsp->createTableLen, iReq);
RAW_NULL_CHECK(len);
@ -735,7 +734,8 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 3, 1);
}
RAW_NULL_CHECK(buf);
if(dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL) != TSDB_CODE_SUCCESS) {
if (dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL) !=
TSDB_CODE_SUCCESS) {
taosMemoryFree(buf);
goto end;
}
@ -823,7 +823,7 @@ static void processDeleteTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
// getTbName(req.tableFName);
char sql[256] = {0};
(void)snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName,
req.tsColName, req.skey, req.tsColName, req.ekey);
req.tsColName, req.skey, req.tsColName, req.ekey);
json = cJSON_CreateObject();
RAW_NULL_CHECK(json);
@ -865,7 +865,7 @@ static void processDropTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
RAW_NULL_CHECK(tableNameList);
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
SVDropTbReq* pDropTbReq = req.pReqs + iReq;
cJSON* tableName = cJSON_CreateString(pDropTbReq->name);
cJSON* tableName = cJSON_CreateString(pDropTbReq->name);
RAW_NULL_CHECK(tableName);
RAW_FALSE_CHECK(cJSON_AddItemToArray(tableNameList, tableName));
}
@ -910,7 +910,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
}
// build create stable
pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SFieldWithOptions));
RAW_NULL_CHECK (pReq.pColumns);
RAW_NULL_CHECK(pReq.pColumns);
for (int32_t i = 0; i < req.schemaRow.nCols; i++) {
SSchema* pSchema = req.schemaRow.pSchema + i;
SFieldWithOptions field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes};
@ -957,7 +957,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
}
pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen);
RAW_NULL_CHECK(pCmdMsg.pMsg);
if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0){
if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0) {
code = TSDB_CODE_INVALID_PARA;
goto end;
}
@ -968,7 +968,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
pQuery.msgType = pQuery.pCmdMsg->msgType;
pQuery.stableQuery = true;
(void)launchQueryImpl(pRequest, &pQuery, true, NULL); //ignore, because return value is pRequest
(void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest
if (pRequest->code == TSDB_CODE_SUCCESS) {
SCatalog* pCatalog = NULL;
@ -1021,7 +1021,8 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
SName pName = {0};
(void)toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); // ignore the return value, always return pName
(void)toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name,
&pName); // ignore the return value, always return pName
STableMeta* pTableMeta = NULL;
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
@ -1059,19 +1060,18 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
}
pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen);
RAW_NULL_CHECK(pCmdMsg.pMsg);
if (tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0){
if (tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0) {
code = TSDB_CODE_INVALID_PARA;
goto end;
}
SQuery pQuery = {0};
pQuery.execMode = QUERY_EXEC_MODE_RPC;
pQuery.pCmdMsg = &pCmdMsg;
pQuery.msgType = pQuery.pCmdMsg->msgType;
pQuery.stableQuery = true;
(void)launchQueryImpl(pRequest, &pQuery, true, NULL); //ignore, because return value is pRequest
(void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest
if (pRequest->code == TSDB_CODE_SUCCESS) {
// ignore the error code
@ -1391,9 +1391,9 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
}
(void)snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName,
req.tsColName, req.skey, req.tsColName, req.ekey);
req.tsColName, req.skey, req.tsColName, req.ekey);
TAOS_RES* res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX);
TAOS_RES* res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX);
RAW_NULL_CHECK(res);
SRequestObj* pRequest = (SRequestObj*)res;
code = pRequest->code;
@ -1500,7 +1500,6 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
if (TSDB_CODE_SUCCESS != code) goto end;
RAW_RETURN_CHECK(rewriteToVnodeModifyOpStmt(pQuery, pArray));
(void)launchQueryImpl(pRequest, pQuery, true, NULL);
pVgData = NULL;
@ -1573,7 +1572,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
RAW_RETURN_CHECK(smlInitHandle(&pQuery));
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(pVgHash);
RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0));
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
@ -1632,7 +1632,8 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
RAW_RETURN_CHECK(smlInitHandle(&pQuery));
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(pVgHash);
RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0));
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
@ -1736,7 +1737,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter);
RAW_NULL_CHECK(pSW);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
RAW_NULL_CHECK(fields);
for (int i = 0; i < pSW->nCols; i++) {
fields[i].type = pSW->pSchema[i].type;
@ -1845,7 +1846,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
// find schema data info
for (int j = 0; j < rspObj.rsp.createTableNum; j++) {
void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j);
void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j);
RAW_NULL_CHECK(dataTmp);
int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j);
RAW_NULL_CHECK(dataTmp);
@ -1897,7 +1898,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter);
RAW_NULL_CHECK(pSW);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
if (fields == NULL) {
SET_ERROR_MSG("calloc fields failed");
code = TSDB_CODE_OUT_OF_MEMORY;
@ -1980,10 +1981,10 @@ static void processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp, char** string) {
RAW_NULL_CHECK(len);
void* tmpBuf = taosArrayGetP(rsp.batchMetaReq, i);
RAW_NULL_CHECK(tmpBuf);
SDecoder metaCoder = {0};
SDecoder metaCoder = {0};
SMqMetaRsp metaRsp = {0};
tDecoderInit(&metaCoder, POINTER_SHIFT(tmpBuf, sizeof(SMqRspHead)), *len - sizeof(SMqRspHead));
if(tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0 ) {
if (tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0) {
goto end;
}
cJSON* pItem = NULL;
@ -2013,18 +2014,18 @@ char* tmq_get_json_meta(TAOS_RES* res) {
if (TD_RES_TMQ_METADATA(res)) {
SMqTaosxRspObj* pMetaDataRspObj = (SMqTaosxRspObj*)res;
char* string = NULL;
char* string = NULL;
processAutoCreateTable(&pMetaDataRspObj->rsp, &string);
return string;
} else if (TD_RES_TMQ_BATCH_META(res)) {
SMqBatchMetaRspObj* pBatchMetaRspObj = (SMqBatchMetaRspObj*)res;
char* string = NULL;
char* string = NULL;
processBatchMetaToJson(&pBatchMetaRspObj->rsp, &string);
return string;
}
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
cJSON* pJson = NULL;
cJSON* pJson = NULL;
processSimpleMeta(&pMetaRspObj->metaRsp, &pJson);
char* string = cJSON_PrintUnformatted(pJson);
cJSON_Delete(pJson);
@ -2103,7 +2104,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
uDebug("tmq get raw type meta:%p", raw);
} else if (TD_RES_TMQ(res)) {
SMqRspObj* rspObj = ((SMqRspObj*)res);
int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw);
int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw);
if (code != 0) {
uError("tmq get raw type error:%d", terrno);
return code;

237
source/client/src/clientTmq.c
View File

@ -37,7 +37,7 @@ struct SMqMgmt {
static TdThreadOnce tmqInit = PTHREAD_ONCE_INIT; // initialize only once
volatile int32_t tmqInitRes = 0; // initialize rsp code
static struct SMqMgmt tmqMgmt = {0};
static int8_t pollFlag = 0;
static int8_t pollFlag = 0;
typedef struct {
int32_t code;
@ -121,7 +121,7 @@ struct tmq_t {
typedef struct SAskEpInfo {
int32_t code;
tsem2_t sem;
tsem2_t sem;
} SAskEpInfo;
enum {
@ -191,7 +191,7 @@ typedef struct {
} SMqPollRspWrapper;
typedef struct {
tsem2_t rspSem;
tsem2_t rspSem;
int32_t rspErr;
} SMqSubscribeCbParam;
@ -219,12 +219,12 @@ typedef struct SMqVgCommon {
} SMqVgCommon;
typedef struct SMqSeekParam {
tsem2_t sem;
tsem2_t sem;
int32_t code;
} SMqSeekParam;
typedef struct SMqCommittedParam {
tsem2_t sem;
tsem2_t sem;
int32_t code;
SMqVgOffset vgOffset;
} SMqCommittedParam;
@ -242,18 +242,18 @@ typedef struct {
int32_t waitingRspNum;
int32_t code;
tmq_commit_cb* callbackFn;
void* userParam;
void* userParam;
} SMqCommitCbParamSet;
typedef struct {
SMqCommitCbParamSet* params;
char topicName[TSDB_TOPIC_FNAME_LEN];
int32_t vgId;
int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
int32_t vgId;
int64_t consumerId;
} SMqCommitCbParam;
typedef struct SSyncCommitInfo {
tsem2_t sem;
tsem2_t sem;
int32_t code;
} SSyncCommitInfo;
@ -334,7 +334,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
if (strcasecmp(key, "session.timeout.ms") == 0) {
int64_t tmp = taosStr2int64(value);
if (tmp < 6000 || tmp > 1800000){
if (tmp < 6000 || tmp > 1800000) {
return TMQ_CONF_INVALID;
}
conf->sessionTimeoutMs = tmp;
@ -343,7 +343,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
if (strcasecmp(key, "heartbeat.interval.ms") == 0) {
int64_t tmp = taosStr2int64(value);
if (tmp < 1000 || tmp >= conf->sessionTimeoutMs){
if (tmp < 1000 || tmp >= conf->sessionTimeoutMs) {
return TMQ_CONF_INVALID;
}
conf->heartBeatIntervalMs = tmp;
@ -352,7 +352,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
if (strcasecmp(key, "max.poll.interval.ms") == 0) {
int64_t tmp = taosStr2int64(value);
if (tmp < 1000 || tmp > INT32_MAX){
if (tmp < 1000 || tmp > INT32_MAX) {
return TMQ_CONF_INVALID;
}
conf->maxPollIntervalMs = tmp;
@ -515,7 +515,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse
SEncoder encoder = {0};
tEncoderInit(&encoder, abuf, len);
if(tEncodeMqVgOffset(&encoder, &pOffset) < 0) {
if (tEncodeMqVgOffset(&encoder, &pOffset) < 0) {
tEncoderClear(&encoder);
taosMemoryFree(buf);
return TSDB_CODE_INVALID_PARA;
@ -562,7 +562,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse
return code;
}
static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic **topic) {
static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic** topic) {
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
for (int32_t i = 0; i < numOfTopics; ++i) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
@ -577,8 +577,8 @@ static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic
return TSDB_CODE_TMQ_INVALID_TOPIC;
}
static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam,
int32_t rspNum, SMqCommitCbParamSet** ppParamSet) {
static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, int32_t rspNum,
SMqCommitCbParamSet** ppParamSet) {
SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
if (pParamSet == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -595,7 +595,7 @@ static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, voi
static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClientVg** pVg) {
SMqClientTopic* pTopic = NULL;
int32_t code = getTopicByName(tmq, pTopicName, &pTopic);
int32_t code = getTopicByName(tmq, pTopicName, &pTopic);
if (code != 0) {
tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName);
return code;
@ -723,7 +723,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
taosRUnLockLatch(&tmq->lock);
goto end;
}
int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName,
numOfVgroups);
for (int32_t j = 0; j < numOfVgroups; j++) {
@ -769,7 +769,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
if (pParamSet->waitingRspNum != 1) {
// count down since waiting rsp num init as 1
code = commitRspCountDown(pParamSet, tmq->consumerId, "", 0);
if (code != 0){
if (code != 0) {
tscError("consumer:0x%" PRIx64 " commit rsp count down failed, code:%s", tmq->consumerId, tstrerror(code));
pParamSet = NULL;
goto end;
@ -824,14 +824,14 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
}
int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
if (code != 0){
if (code != 0) {
goto _return;
}
if (pMsg == NULL || param == NULL) {
code = TSDB_CODE_INVALID_PARA;
goto _return;
}
SMqHbRsp rsp = {0};
code = tDeserializeSMqHbRsp(pMsg->pData, pMsg->len, &rsp);
if (code != 0) {
@ -858,7 +858,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
taosWUnLockLatch(&tmq->lock);
(void)taosReleaseRef(tmqMgmt.rsetId, refId);
}
tDestroySMqHbRsp(&rsp);
_return:
@ -881,32 +881,32 @@ void tmqSendHbReq(void* param, void* tmrId) {
req.epoch = tmq->epoch;
req.pollFlag = atomic_load_8(&pollFlag);
req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows));
if (req.topics == NULL){
if (req.topics == NULL) {
return;
}
taosRLockLatch(&tmq->lock);
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
if (pTopic == NULL) {
continue;
}
int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
TopicOffsetRows* data = taosArrayReserve(req.topics, 1);
if (data == NULL){
if (data == NULL) {
continue;
}
(void)strcpy(data->topicName, pTopic->topicName);
data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows));
if (data->offsetRows == NULL){
if (data->offsetRows == NULL) {
continue;
}
for (int j = 0; j < numOfVgroups; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
if (pVg == NULL){
if (pVg == NULL) {
continue;
}
OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1);
if (offRows == NULL){
OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1);
if (offRows == NULL) {
continue;
}
offRows->vgId = pVg->vgId;
@ -964,7 +964,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
(void)atomic_val_compare_exchange_8(&pollFlag, 1, 0);
OVER:
tDestroySMqHbReq(&req);
if(tmrId != NULL){
if (tmrId != NULL) {
(void)taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer);
}
(void)taosReleaseRef(tmqMgmt.rsetId, refId);
@ -1006,14 +1006,15 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) {
continue;
}
tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId);
(void)taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->epTimer);
(void)taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer,
&pTmq->epTimer);
} else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn;
asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam);
tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId,
pTmq->autoCommitInterval / 1000.0);
(void)taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer,
&pTmq->commitTimer);
&pTmq->commitTimer);
} else {
tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType);
}
@ -1100,16 +1101,16 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
taosRLockLatch(&tmq->lock);
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i);
if(topic == NULL) {
if (topic == NULL) {
tscError("topic is null");
continue;
}
char* tmp = strchr(topic->topicName, '.');
if(tmp == NULL) {
if (tmp == NULL) {
tscError("topic name is invalid:%s", topic->topicName);
continue;
}
if(tmq_list_append(*topics, tmp+ 1) != 0) {
if (tmq_list_append(*topics, tmp + 1) != 0) {
tscError("failed to append topic:%s", tmp + 1);
continue;
}
@ -1184,7 +1185,7 @@ void tmqMgmtClose(void) {
}
if (tmqMgmt.rsetId >= 0) {
(void)taosCloseRef(tmqMgmt.rsetId);
taosCloseRef(tmqMgmt.rsetId);
tmqMgmt.rsetId = -1;
}
}
@ -1227,27 +1228,31 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
}
code = taosOpenQueue(&pTmq->mqueue);
if (code) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId);
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code),
pTmq->groupId);
SET_ERROR_MSG_TMQ("open queue failed")
goto _failed;
}
code = taosOpenQueue(&pTmq->delayedTask);
if (code) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId);
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code),
pTmq->groupId);
SET_ERROR_MSG_TMQ("open delayed task queue failed")
goto _failed;
}
code = taosAllocateQall(&pTmq->qall);
if (code) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId);
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code),
pTmq->groupId);
SET_ERROR_MSG_TMQ("allocate qall failed")
goto _failed;
}
if (conf->groupId[0] == 0) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId);
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code),
pTmq->groupId);
SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty")
goto _failed;
}
@ -1287,8 +1292,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
// init semaphore
if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) {
tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, tstrerror(TAOS_SYSTEM_ERROR(errno)),
pTmq->groupId);
tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId,
tstrerror(TAOS_SYSTEM_ERROR(errno)), pTmq->groupId);
SET_ERROR_MSG_TMQ("init t_sem failed")
goto _failed;
}
@ -1371,7 +1376,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
SName name = {0};
code = tNameSetDbName(&name, tmq->pTscObj->acctId, topic, strlen(topic));
if (code) {
tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId, code);
tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId,
code);
goto FAIL;
}
char* topicFName = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
@ -1382,7 +1388,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
code = tNameExtractFullName(&name, topicFName);
if (code) {
tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId, code);
tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId,
code);
taosMemoryFree(topicFName);
goto FAIL;
}
@ -1459,7 +1466,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
}
tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer);
tmq->commitTimer =taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer);
tmq->commitTimer =
taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer);
if (tmq->epTimer == NULL || tmq->commitTimer == NULL) {
code = TSDB_CODE_TSC_INTERNAL_ERROR;
goto FAIL;
@ -1516,12 +1524,12 @@ static void setVgIdle(tmq_t* tmq, char* topicName, int32_t vgId) {
}
int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
tmq_t* tmq = NULL;
tmq_t* tmq = NULL;
SMqPollRspWrapper* pRspWrapper = NULL;
int8_t rspType = 0;
int32_t vgId = 0;
uint64_t requestId = 0;
SMqPollCbParam* pParam = (SMqPollCbParam*)param;
int8_t rspType = 0;
int32_t vgId = 0;
uint64_t requestId = 0;
SMqPollCbParam* pParam = (SMqPollCbParam*)param;
if (pMsg == NULL) {
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
@ -1530,7 +1538,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFreeClear(pMsg->pEpSet);
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
int64_t refId = pParam->refId;
int64_t refId = pParam->refId;
vgId = pParam->vgId;
requestId = pParam->requestId;
tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
@ -1562,7 +1570,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
if (msgEpoch < clientEpoch) {
// do not write into queue since updating epoch reset
tscWarn("consumer:0x%" PRIx64
" msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64,
" msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, QID:0x%" PRIx64,
tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId);
code = TSDB_CODE_TMQ_CONSUMER_MISMATCH;
goto END;
@ -1589,7 +1597,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset);
tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64,
tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, QID:0x%" PRIx64,
tmq->consumerId, vgId, pRspWrapper->dataRsp.common.reqOffset.version, buf, rspType, requestId);
} else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
SDecoder decoder = {0};
@ -1621,23 +1629,24 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
}
tDecoderClear(&decoder);
(void)memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead));
tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, vgId,requestId);
tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, QID:0x%" PRIx64, tmq->consumerId, vgId,
requestId);
} else { // invalid rspType
tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType);
}
END:
if (pRspWrapper){
if (pRspWrapper) {
pRspWrapper->code = code;
pRspWrapper->vgId = vgId;
(void)strcpy(pRspWrapper->topicName, pParam->topicName);
code = taosWriteQitem(tmq->mqueue, pRspWrapper);
if(code != 0){
if (code != 0) {
tscError("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code);
}
}
int32_t total = taosQueueItemSize(tmq->mqueue);
tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, reqId:0x%" PRIx64,
tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, QID:0x%" PRIx64,
tmq ? tmq->consumerId : 0, rspType, vgId, total, requestId);
if (tmq) (void)tsem2_post(&tmq->rspSem);
@ -1676,7 +1685,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
}
for (int32_t j = 0; j < vgNumGet; j++) {
SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j);
if (pVgEp == NULL){
if (pVgEp == NULL) {
continue;
}
(void)sprintf(vgKey, "%s:%d", pTopic->topicName, pVgEp->vgId);
@ -1712,7 +1721,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
clientVg.offsetInfo.committedOffset = offsetNew;
clientVg.offsetInfo.beginOffset = offsetNew;
}
if (taosArrayPush(pTopic->vgs, &clientVg) == NULL){
if (taosArrayPush(pTopic->vgs, &clientVg) == NULL) {
tscError("consumer:0x%" PRIx64 ", failed to push vg:%d into topic:%s", tmq->consumerId, pVgEp->vgId,
pTopic->topicName);
freeClientVg(&clientVg);
@ -1773,7 +1782,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
.commitOffset = pVgCur->offsetInfo.committedOffset,
.numOfRows = pVgCur->numOfRows,
.vgStatus = pVgCur->vgStatus};
if(taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0){
if (taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0) {
tscError("consumer:0x%" PRIx64 ", failed to put vg:%d into hashmap", tmq->consumerId, pVgCur->vgId);
}
}
@ -1787,7 +1796,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
continue;
}
initClientTopicFromRsp(&topic, pTopicEp, pVgOffsetHashMap, tmq);
if(taosArrayPush(newTopics, &topic) == NULL){
if (taosArrayPush(newTopics, &topic) == NULL) {
tscError("consumer:0x%" PRIx64 ", failed to push topic:%s into new topics", tmq->consumerId, topic.topicName);
freeClientTopic(&topic);
}
@ -1919,7 +1928,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
if (!pDataRsp->withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable
pDataRsp->withSchema = true;
pDataRsp->blockSchema = taosArrayInit(pDataRsp->blockNum, sizeof(void*));
if (pDataRsp->blockSchema == NULL){
if (pDataRsp->blockSchema == NULL) {
tscError("failed to allocate memory for blockSchema");
return;
}
@ -1938,7 +1947,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
if (needTransformSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable
SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
if (schema) {
if (taosArrayPush(pDataRsp->blockSchema, &schema) == NULL){
if (taosArrayPush(pDataRsp->blockSchema, &schema) == NULL) {
tscError("failed to push schema into blockSchema");
continue;
}
@ -1947,7 +1956,8 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
}
}
int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj** ppRspObj) {
int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows,
SMqRspObj** ppRspObj) {
SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
if (pRspObj == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -1959,7 +1969,8 @@ int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, in
return 0;
}
int32_t tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqTaosxRspObj** ppRspObj) {
int32_t tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows,
SMqTaosxRspObj** ppRspObj) {
SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj));
if (pRspObj == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -2030,7 +2041,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
char offsetFormatBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset);
code = asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, reqId:0x%" PRIx64,
tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, QID:0x%" PRIx64,
pTmq->consumerId, pTopic->topicName, pVg->vgId, code, pTmq->epoch, offsetFormatBuf, req.reqId);
if (code != 0) {
return code;
@ -2056,10 +2067,10 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
for (int i = 0; i < numOfTopics; i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
if (pTopic == NULL){
if (pTopic == NULL) {
continue;
}
int32_t numOfVg = taosArrayGetSize(pTopic->vgs);
int32_t numOfVg = taosArrayGetSize(pTopic->vgs);
if (pTopic->noPrivilege) {
tscDebug("consumer:0x%" PRIx64 " has no privilegr for topic:%s", tmq->consumerId, pTopic->topicName);
continue;
@ -2069,7 +2080,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
if (pVg == NULL) {
continue;
}
int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs;
int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs;
if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms
tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId,
tmq->epoch, pVg->vgId);
@ -2203,7 +2214,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset);
if (pDataRsp->blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64
", total:%" PRId64 ", reqId:0x%" PRIx64,
", total:%" PRId64 ", QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId);
pVg->emptyBlockReceiveTs = taosGetTimestampMs();
tmqFreeRspWrapper(pRspWrapper);
@ -2220,13 +2231,13 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
pVg->blockSleepForReplay = pRsp->rsp.sleepTime;
if (pVg->blockSleepForReplay > 0) {
if (taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, (void*)(tmq->refId), tmqMgmt.timer) == NULL) {
tscError("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%"PRId64, tmq->consumerId,
pVg->vgId, pVg->blockSleepForReplay);
tscError("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%" PRId64,
tmq->consumerId, pVg->vgId, pVg->blockSleepForReplay);
}
}
}
tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64,
", vg total:%" PRId64 ", total:%" PRId64 ", QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
pollRspWrapper->reqId);
taosFreeQitem(pRspWrapper);
@ -2302,7 +2313,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever,
tmq->consumerId, true);
SMqBatchMetaRspObj* pRsp = NULL;
(void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp) ;
(void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp);
taosFreeQitem(pRspWrapper);
taosWUnLockLatch(&tmq->lock);
return pRsp;
@ -2337,7 +2348,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
tmq->consumerId, pDataRsp->blockNum != 0);
if (pDataRsp->blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64,
tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId);
pVg->emptyBlockReceiveTs = taosGetTimestampMs();
tmqFreeRspWrapper(pRspWrapper);
@ -2349,9 +2360,9 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
}
// build rsp
int64_t numOfRows = 0;
SMqTaosxRspObj* pRsp = NULL;
if (tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp) !=0 ) {
int64_t numOfRows = 0;
SMqTaosxRspObj* pRsp = NULL;
if (tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp) != 0) {
tscError("consumer:0x%" PRIx64 " build taosx rsp failed, vgId:%d", tmq->consumerId, pVg->vgId);
}
tmq->totalRows += numOfRows;
@ -2359,7 +2370,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset);
tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64,
", vg total:%" PRId64 ", total:%" PRId64 ", QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
pollRspWrapper->reqId);
@ -2471,7 +2482,7 @@ static void displayConsumeStatistics(tmq_t* pTmq) {
tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId);
}
static int32_t innerClose(tmq_t* tmq){
static int32_t innerClose(tmq_t* tmq) {
if (tmq->status != TMQ_CONSUMER_STATUS__READY) {
tscInfo("consumer:0x%" PRIx64 " not in ready state, unsubscribe it directly", tmq->consumerId);
return 0;
@ -2485,7 +2496,7 @@ static int32_t innerClose(tmq_t* tmq){
tmqSendHbReq((void*)(tmq->refId), NULL);
tmq_list_t* lst = tmq_list_new();
if (lst == NULL){
if (lst == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = tmq_subscribe(tmq, lst);
@ -2499,7 +2510,7 @@ int32_t tmq_unsubscribe(tmq_t* tmq) {
int32_t code = 0;
if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) {
code = innerClose(tmq);
if(code == 0){
if (code == 0) {
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED);
}
}
@ -2514,12 +2525,12 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
int32_t code = 0;
if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) {
code = innerClose(tmq);
if(code == 0){
if (code == 0) {
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED);
}
}
if (code == 0){
if (code == 0) {
(void)taosRemoveRef(tmqMgmt.rsetId, tmq->refId);
}
return code;
@ -2562,13 +2573,13 @@ const char* tmq_get_topic_name(TAOS_RES* res) {
return NULL;
}
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
char *tmp = strchr(((SMqRspObjCommon*)res)->topic, '.');
char* tmp = strchr(((SMqRspObjCommon*)res)->topic, '.');
if (tmp == NULL) {
return NULL;
}
return tmp + 1;
} else if (TD_RES_TMQ_META(res)) {
char *tmp = strchr(((SMqMetaRspObj*)res)->topic, '.');
char* tmp = strchr(((SMqMetaRspObj*)res)->topic, '.');
if (tmp == NULL) {
return NULL;
}
@ -2584,13 +2595,13 @@ const char* tmq_get_db_name(TAOS_RES* res) {
}
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
char *tmp = strchr(((SMqRspObjCommon*)res)->db, '.');
char* tmp = strchr(((SMqRspObjCommon*)res)->db, '.');
if (tmp == NULL) {
return NULL;
}
return tmp + 1;
} else if (TD_RES_TMQ_META(res)) {
char *tmp = strchr(((SMqMetaRspObj*)res)->db, '.');
char* tmp = strchr(((SMqMetaRspObj*)res)->db, '.');
if (tmp == NULL) {
return NULL;
}
@ -2690,7 +2701,7 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) {
int32_t code = 0;
SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo));
if(pInfo == NULL) {
if (pInfo == NULL) {
tscError("failed to allocate memory for sync commit");
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -2836,7 +2847,7 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
if (param == NULL) {
goto FAIL;
}
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId);
if (tmq == NULL) {
@ -2857,7 +2868,7 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch);
if (pParam->sync) {
SMqAskEpRsp rsp = {0};
if(tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL){
if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL) {
(void)doUpdateLocalEp(tmq, head->epoch, &rsp);
}
tDeleteSMqAskEpRsp(&rsp);
@ -2871,10 +2882,10 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP;
pWrapper->epoch = head->epoch;
(void)memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) == NULL){
if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) == NULL) {
tmqFreeRspWrapper((SMqRspWrapper*)pWrapper);
taosFreeQitem(pWrapper);
}else{
} else {
(void)taosWriteQitem(tmq->mqueue, pWrapper);
}
}
@ -2972,7 +2983,7 @@ int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) {
sendInfo->msgType = TDMT_MND_TMQ_ASK_EP;
SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp);
tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, reqId:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId);
tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, QID:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId);
return asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo);
}
@ -3015,7 +3026,7 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes
if (common->withSchema) {
doFreeReqResultInfo(&pRspObj->resInfo);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(common->blockSchema, pRspObj->resIter);
if (pSW){
if (pSW) {
TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols));
}
}
@ -3032,9 +3043,9 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes
pRspObj->resInfo.precision = precision;
pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows;
int32_t code = setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols, pRspObj->resInfo.numOfRows,
convertUcs4);
if (code != 0){
int32_t code = setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols,
pRspObj->resInfo.numOfRows, convertUcs4);
if (code != 0) {
return code;
}
*pResInfo = &pRspObj->resInfo;
@ -3062,18 +3073,18 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) {
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
code = tDecodeMqDataRsp(&decoder, &rsp);
tDecoderClear(&decoder);
if (code != 0){
if (code != 0) {
goto END;
}
SMqRspHead* pHead = pMsg->pData;
SMqRspHead* pHead = pMsg->pData;
tmq_topic_assignment assignment = {.begin = pHead->walsver,
.end = pHead->walever + 1,
.currentOffset = rsp.common.rspOffset.version,
.vgId = pParam->vgId};
(void)taosThreadMutexLock(&pCommon->mutex);
if(taosArrayPush(pCommon->pList, &assignment) == NULL){
if (taosArrayPush(pCommon->pList, &assignment) == NULL) {
tscError("consumer:0x%" PRIx64 " failed to push the wal info from vgId:%d for topic:%s", pCommon->consumerId,
pParam->vgId, pCommon->pTopicName);
code = TSDB_CODE_TSC_INTERNAL_ERROR;
@ -3184,7 +3195,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep
taosMemoryFree(sendInfo);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (tsem2_init(&pParam->sem, 0, 0) != 0){
if (tsem2_init(&pParam->sem, 0, 0) != 0) {
taosMemoryFree(buf);
taosMemoryFree(sendInfo);
taosMemoryFree(pParam);
@ -3348,7 +3359,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
taosWLockLatch(&tmq->lock);
SMqClientTopic* pTopic = NULL;
int32_t code = getTopicByName(tmq, tname, &pTopic);
int32_t code = getTopicByName(tmq, tname, &pTopic);
if (code != 0) {
tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName);
goto end;
@ -3358,10 +3369,10 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
*numOfAssignment = taosArrayGetSize(pTopic->vgs);
for (int32_t j = 0; j < (*numOfAssignment); ++j) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j);
if (pClientVg == NULL){
if (pClientVg == NULL) {
continue;
}
int32_t type = pClientVg->offsetInfo.beginOffset.type;
int32_t type = pClientVg->offsetInfo.beginOffset.type;
if (isInSnapshotMode(type, tmq->useSnapshot)) {
tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type);
code = TSDB_CODE_TMQ_SNAPSHOT_ERROR;
@ -3381,7 +3392,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
for (int32_t j = 0; j < (*numOfAssignment); ++j) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j);
if (pClientVg == NULL){
if (pClientVg == NULL) {
continue;
}
if (pClientVg->offsetInfo.beginOffset.type != TMQ_OFFSET__LOG) {
@ -3410,7 +3421,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
if (tsem2_init(&pCommon->rsp, 0, 0) != 0){
if (tsem2_init(&pCommon->rsp, 0, 0) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
@ -3420,7 +3431,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
for (int32_t i = 0; i < (*numOfAssignment); ++i) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i);
if (pClientVg == NULL){
if (pClientVg == NULL) {
continue;
}
SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam));
@ -3479,8 +3490,8 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
char offsetFormatBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.beginOffset);
tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64,
tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId);
tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, QID:0x%" PRIx64, tmq->consumerId,
pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId);
code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo);
if (code != 0) {
goto end;
@ -3504,7 +3515,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
for (int32_t i = 0; i < taosArrayGetSize(pTopic->vgs); ++i) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i);
if (pClientVg == NULL){
if (pClientVg == NULL) {
continue;
}
if (pClientVg->vgId != p->vgId) {
@ -3631,7 +3642,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_
taosMemoryFree(sendInfo);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (tsem2_init(&pParam->sem, 0, 0) != 0){
if (tsem2_init(&pParam->sem, 0, 0) != 0) {
taosMemoryFree(msg);
taosMemoryFree(sendInfo);
taosMemoryFree(pParam);

42
source/common/src/cos.c
View File

@ -12,7 +12,7 @@ extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN];
extern char tsS3BucketName[TSDB_FQDN_LEN];
extern char tsS3AppId[][TSDB_FQDN_LEN];
extern char tsS3Hostname[][TSDB_FQDN_LEN];
extern int8_t tsS3Https;
extern int8_t tsS3Https[];
static int32_t s3ListBucketByEp(char const *bucketname, int8_t epIndex);
static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *object_name, int64_t offset, int64_t size,
@ -33,13 +33,13 @@ static int verifyPeerG = 0;
static const char *awsRegionG = NULL;
static int forceG = 0;
static int showResponsePropertiesG = 0;
static S3Protocol protocolG = S3ProtocolHTTPS;
static S3Protocol protocolG[TSDB_MAX_EP_NUM] = {S3ProtocolHTTPS};
// static S3Protocol protocolG = S3ProtocolHTTP;
static S3UriStyle uriStyleG = S3UriStylePath;
static S3UriStyle uriStyleG[TSDB_MAX_EP_NUM] = {S3UriStylePath};
static int retriesG = 5;
static int timeoutMsG = 0;
extern int8_t tsS3Oss;
extern int8_t tsS3Oss[];
int32_t s3Begin() {
S3Status status;
@ -55,9 +55,9 @@ int32_t s3Begin() {
TAOS_RETURN(TSDB_CODE_FAILED);
}
protocolG = !tsS3Https;
if (tsS3Oss) {
uriStyleG = S3UriStyleVirtualHost;
for (int i = 0; i < tsS3EpNum; i++) {
protocolG[i] = tsS3Https[i] ? S3ProtocolHTTPS : S3ProtocolHTTP;
uriStyleG[i] = tsS3Oss[i] ? S3UriStyleVirtualHost : S3UriStylePath;
}
TAOS_RETURN(TSDB_CODE_SUCCESS);
@ -976,8 +976,8 @@ int32_t s3PutObjectFromFile2ByEp(const char *file, const char *object_name, int8
S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName,
protocolG,
uriStyleG,
protocolG[epIndex],
uriStyleG[epIndex],
tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex],
0,
@ -1059,8 +1059,8 @@ static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *objec
S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName,
protocolG,
uriStyleG,
protocolG[epIndex],
uriStyleG[epIndex],
tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex],
0,
@ -1155,8 +1155,8 @@ static void s3FreeObjectKey(void *pItem) {
static SArray *getListByPrefixByEp(const char *prefix, int8_t epIndex) {
S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName,
protocolG,
uriStyleG,
protocolG[epIndex],
uriStyleG[epIndex],
tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex],
0,
@ -1223,8 +1223,8 @@ static int32_t s3DeleteObjectsByEp(const char *object_name[], int nobject, int8_
S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName,
protocolG,
uriStyleG,
protocolG[epIndex],
uriStyleG[epIndex],
tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex],
0,
@ -1299,8 +1299,8 @@ static int32_t s3GetObjectBlockByEp(const char *object_name, int64_t offset, int
S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName,
protocolG,
uriStyleG,
protocolG[epIndex],
uriStyleG[epIndex],
tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex],
0,
@ -1372,8 +1372,8 @@ static int32_t s3GetObjectToFileByEp(const char *object_name, const char *fileNa
S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName,
protocolG,
uriStyleG,
protocolG[epIndex],
uriStyleG[epIndex],
tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex],
0,
@ -1449,8 +1449,8 @@ static long s3SizeByEp(const char *object_name, int8_t epIndex) {
S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName,
protocolG,
uriStyleG,
protocolG[epIndex],
uriStyleG[epIndex],
tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex],
0,

26
source/common/src/tdatablock.c
View File

@ -1670,6 +1670,8 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
int32_t code = 0;
dst->info = src->info;
dst->info.pks[0].pData = NULL;
dst->info.pks[1].pData = NULL;
dst->info.rows = 0;
dst->info.capacity = 0;
@ -1707,6 +1709,8 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
uint32_t cap = dst->info.capacity;
dst->info = src->info;
dst->info.pks[0].pData = NULL;
dst->info.pks[1].pData = NULL;
dst->info.capacity = cap;
return code;
}
@ -1737,6 +1741,8 @@ int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc) {
uint32_t cap = pDst->info.capacity;
pDst->info = pSrc->info;
pDst->info.pks[0].pData = NULL;
pDst->info.pks[1].pData = NULL;
code = copyPkVal(&pDst->info, &pSrc->info);
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
@ -1854,6 +1860,8 @@ int32_t blockCopyOneRow(const SSDataBlock* pDataBlock, int32_t rowIdx, SSDataBlo
}
pBlock->info = pDataBlock->info;
pBlock->info.pks[0].pData = NULL;
pBlock->info.pks[1].pData = NULL;
pBlock->info.rows = 0;
pBlock->info.capacity = 0;
@ -1916,20 +1924,20 @@ int32_t copyPkVal(SDataBlockInfo* pDst, const SDataBlockInfo* pSrc) {
// prepare the pk buffer if needed
SValue* p = &pDst->pks[0];
p->type = pDst->pks[0].type;
p->pData = taosMemoryCalloc(1, pDst->pks[0].nData);
p->type = pSrc->pks[0].type;
p->pData = taosMemoryCalloc(1, pSrc->pks[0].nData);
QUERY_CHECK_NULL(p->pData, code, lino, _end, terrno);
p->nData = pDst->pks[0].nData;
memcpy(p->pData, pDst->pks[0].pData, p->nData);
p->nData = pSrc->pks[0].nData;
memcpy(p->pData, pSrc->pks[0].pData, p->nData);
p = &pDst->pks[1];
p->type = pDst->pks[1].type;
p->pData = taosMemoryCalloc(1, pDst->pks[1].nData);
p->type = pSrc->pks[1].type;
p->pData = taosMemoryCalloc(1, pSrc->pks[1].nData);
QUERY_CHECK_NULL(p->pData, code, lino, _end, terrno);
p->nData = pDst->pks[1].nData;
memcpy(p->pData, pDst->pks[1].pData, p->nData);
p->nData = pSrc->pks[1].nData;
memcpy(p->pData, pSrc->pks[1].pData, p->nData);
_end:
if (code != TSDB_CODE_SUCCESS) {
@ -1951,6 +1959,8 @@ int32_t createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData, SSDataB
}
pDstBlock->info = pDataBlock->info;
pDstBlock->info.pks[0].pData = NULL;
pDstBlock->info.pks[1].pData = NULL;
pDstBlock->info.rows = 0;
pDstBlock->info.capacity = 0;

147
source/common/src/tdataformat.c
View File

@ -119,7 +119,6 @@ static FORCE_INLINE void tRowBuildScanAddValue(SRowBuildScanInfo *sinfo, SColVal
bool isPK = ((pTColumn->flags & COL_IS_KEY) != 0);
if (isPK) {
ASSERTS(sinfo->numOfPKs < TD_MAX_PK_COLS, "too many primary keys");
sinfo->tupleIndices[sinfo->numOfPKs].type = colVal->value.type;
sinfo->tupleIndices[sinfo->numOfPKs].offset =
IS_VAR_DATA_TYPE(pTColumn->type) ? sinfo->tupleVarSize + sinfo->tupleFixedSize : pTColumn->offset;
@ -149,9 +148,15 @@ static int32_t tRowBuildScan(SArray *colVals, const STSchema *schema, SRowBuildS
int32_t numOfColVals = TARRAY_SIZE(colVals);
SColVal *colValArray = (SColVal *)TARRAY_DATA(colVals);
ASSERT(numOfColVals > 0);
ASSERT(colValArray[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID);
ASSERT(colValArray[0].value.type == TSDB_DATA_TYPE_TIMESTAMP);
if (!(numOfColVals > 0)) {
return TSDB_CODE_INVALID_PARA;
}
if (!(colValArray[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID)) {
return TSDB_CODE_INVALID_PARA;
}
if (!(colValArray[0].value.type == TSDB_DATA_TYPE_TIMESTAMP)) {
return TSDB_CODE_INVALID_PARA;
}
*sinfo = (SRowBuildScanInfo){
.tupleFixedSize = schema->flen,
@ -166,7 +171,10 @@ static int32_t tRowBuildScan(SArray *colVals, const STSchema *schema, SRowBuildS
}
if (colValArray[colValIndex].cid == schema->columns[i].colId) {
ASSERT(colValArray[colValIndex].value.type == schema->columns[i].type);
if (!(colValArray[colValIndex].value.type == schema->columns[i].type)) {
code = TSDB_CODE_INVALID_PARA;
goto _exit;
}
if (COL_VAL_IS_VALUE(&colValArray[colValIndex])) {
tRowBuildScanAddValue(sinfo, &colValArray[colValIndex], schema->columns + i);
@ -272,7 +280,6 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo
(*ppRow)->ts = colValArray[0].value.val;
if (sinfo->tupleFlag == HAS_NONE || sinfo->tupleFlag == HAS_NULL) {
ASSERT(sinfo->tupleRowSize == sizeof(SRow));
return 0;
}
@ -285,7 +292,6 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo
for (int32_t i = 0; i < sinfo->numOfPKs; i++) {
primaryKeys += tPutPrimaryKeyIndex(primaryKeys, sinfo->tupleIndices + i);
}
ASSERT(primaryKeys == bitmap);
// bitmap + fixed + varlen
int32_t numOfColVals = TARRAY_SIZE(aColVal);
@ -356,7 +362,9 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c
(*ppRow)->len = sinfo->kvRowSize;
(*ppRow)->ts = colValArray[0].value.val;
ASSERT(sinfo->flag != HAS_NONE && sinfo->flag != HAS_NULL);
if (!(sinfo->flag != HAS_NONE && sinfo->flag != HAS_NULL)) {
return TSDB_CODE_INVALID_PARA;
}
uint8_t *primaryKeys = (*ppRow)->data;
SKVIdx *indices = (SKVIdx *)(primaryKeys + sinfo->kvPKSize);
@ -367,7 +375,6 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c
for (int32_t i = 0; i < sinfo->numOfPKs; i++) {
primaryKeys += tPutPrimaryKeyIndex(primaryKeys, sinfo->kvIndices + i);
}
ASSERT(primaryKeys == (uint8_t *)indices);
int32_t numOfColVals = TARRAY_SIZE(aColVal);
int32_t colValIndex = 1;
@ -504,8 +511,8 @@ _exit:
}
int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
ASSERT(iCol < pTSchema->numOfCols);
ASSERT(pRow->sver == pTSchema->version);
if (!(iCol < pTSchema->numOfCols)) return TSDB_CODE_INVALID_PARA;
if (!(pRow->sver == pTSchema->version)) return TSDB_CODE_INVALID_PARA;
STColumn *pTColumn = pTSchema->columns + iCol;
@ -786,7 +793,7 @@ struct SRowIter {
};
int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) {
ASSERT(pRow->sver == pTSchema->version);
if (!(pRow->sver == pTSchema->version)) return TSDB_CODE_INVALID_PARA;
int32_t code = 0;
@ -839,7 +846,6 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) {
pIter->pv = pIter->pf + pTSchema->flen;
break;
default:
ASSERT(0);
break;
}
}
@ -952,7 +958,6 @@ SColVal *tRowIterNext(SRowIter *pIter) {
bv = GET_BIT2(pIter->pb, pIter->iTColumn - 1);
break;
default:
ASSERT(0);
break;
}
@ -1070,14 +1075,15 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *
pv = pf + pTSchema->flen;
break;
default:
ASSERTS(0, "Invalid row flag");
return TSDB_CODE_INVALID_DATA_FMT;
}
while (pColData) {
if (pTColumn) {
if (pTColumn->colId == pColData->cid) {
ASSERT(pTColumn->type == pColData->type);
if (!(pTColumn->type == pColData->type)) {
return TSDB_CODE_INVALID_PARA;
}
if (pb) {
uint8_t bv;
switch (pRow->flag) {
@ -1095,7 +1101,6 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *
bv = GET_BIT2(pb, iTColumn - 1);
break;
default:
ASSERTS(0, "Invalid row flag");
return TSDB_CODE_INVALID_DATA_FMT;
}
@ -1178,7 +1183,7 @@ static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aCo
} else if (pRow->flag & KV_FLG_BIG) {
pv = pKVIdx->idx + (pKVIdx->nCol << 2);
} else {
ASSERT(0);
return TSDB_CODE_INVALID_PARA;
}
while (pColData) {
@ -1193,7 +1198,6 @@ static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aCo
} else if (pRow->flag & KV_FLG_BIG) {
pData = pv + ((uint32_t *)pKVIdx->idx)[iCol];
} else {
ASSERTS(0, "Invalid KV row format");
return TSDB_CODE_INVALID_DATA_FMT;
}
@ -1256,8 +1260,8 @@ _exit:
* flag < 0: backward update
*/
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag) {
ASSERT(pRow->sver == pTSchema->version);
ASSERT(nColData > 0);
if (!(pRow->sver == pTSchema->version)) return TSDB_CODE_INVALID_PARA;
if (!(nColData > 0)) return TSDB_CODE_INVALID_PARA;
if (pRow->flag == HAS_NONE) {
return tRowNoneUpsertColData(aColData, nColData, flag);
@ -1277,8 +1281,6 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) {
return;
}
ASSERT(row->numOfPKs <= TD_MAX_PK_COLS);
SPrimaryKeyIndex indices[TD_MAX_PK_COLS];
uint8_t *data = row->data;
@ -1317,8 +1319,6 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) {
} while (0)
int32_t tValueCompare(const SValue *tv1, const SValue *tv2) {
ASSERT(tv1->type == tv2->type);
switch (tv1->type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
@ -1476,7 +1476,6 @@ static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const c
printf("%s:%d type:%d vlen:%d, val:%" PRIi8 "\n", tag, ln, (int32_t)type, vlen, *(int8_t *)val);
break;
default:
ASSERT(0);
break;
}
}
@ -1526,7 +1525,6 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
n += tPutCStr(p ? p + n : p, pTagVal->pKey);
} else {
n += tPutI16v(p ? p + n : p, pTagVal->cid);
ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid);
}
// type
@ -1602,8 +1600,6 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) {
isLarge = 1;
}
ASSERT(szTag <= INT16_MAX);
// build tag
(*ppTag) = (STag *)taosMemoryCalloc(szTag, 1);
if ((*ppTag) == NULL) {
@ -1806,8 +1802,14 @@ STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) {
pTSchema->version = version;
// timestamp column
ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP);
ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID);
if (!(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP)) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
if (!(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID)) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
pTSchema->columns[0].colId = aSchema[0].colId;
pTSchema->columns[0].type = aSchema[0].type;
pTSchema->columns[0].flags = aSchema[0].flags;
@ -1910,7 +1912,9 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData,
pColData->nData += nData;
}
} else {
ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal);
if (!(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal)) {
return TSDB_CODE_INVALID_PARA;
}
code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes);
if (code) goto _exit;
if (pData) {
@ -2259,7 +2263,9 @@ static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pDa
// VALUE NONE NULL
};
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->value.type);
if (!(pColData->cid == pColVal->cid && pColData->type == pColVal->value.type)) {
return TSDB_CODE_INVALID_PARA;
}
return tColDataAppendValueImpl[pColData->flag][pColVal->flag](
pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val,
pColVal->value.nData);
@ -2569,8 +2575,8 @@ static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pDa
// VALUE NONE NULL
};
int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward) {
ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->value.type);
ASSERT(pColData->nVal > 0);
if (!(pColData->cid == pColVal->cid && pColData->type == pColVal->value.type)) return TSDB_CODE_INVALID_PARA;
if (!(pColData->nVal > 0)) return TSDB_CODE_INVALID_PARA;
if (tColDataUpdateValueImpl[pColData->flag][pColVal->flag] == NULL) return 0;
@ -2687,7 +2693,6 @@ uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) {
case (HAS_VALUE | HAS_NULL | HAS_NONE):
return GET_BIT2(pColData->pBitMap, iVal);
default:
ASSERTS(0, "not possible");
return 0;
}
}
@ -3053,7 +3058,9 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32
int32_t code = 0;
if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) {
ASSERT(pColData->type == pBind->buffer_type);
if (!(pColData->type == pBind->buffer_type)) {
return TSDB_CODE_INVALID_PARA;
}
}
if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type
@ -3521,15 +3528,21 @@ static void tColDataMerge(SColData *aColData, int32_t nColData) {
}
}
void tColDataSortMerge(SArray *colDataArr) {
int32_t tColDataSortMerge(SArray *colDataArr) {
int32_t nColData = TARRAY_SIZE(colDataArr);
SColData *aColData = (SColData *)TARRAY_DATA(colDataArr);
if (aColData[0].nVal <= 1) goto _exit;
if (!(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP)) {
return TSDB_CODE_INVALID_PARA;
}
if (!(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID)) {
return TSDB_CODE_INVALID_PARA;
}
if (!(aColData[0].flag == HAS_VALUE)) {
return TSDB_CODE_INVALID_PARA;
}
ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP);
ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID);
ASSERT(aColData[0].flag == HAS_VALUE);
if (aColData[0].nVal <= 1) goto _exit;
int8_t doSort = 0;
int8_t doMerge = 0;
@ -3578,7 +3591,7 @@ void tColDataSortMerge(SArray *colDataArr) {
}
_exit:
return;
return 0;
}
static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) {
@ -3685,8 +3698,7 @@ int32_t tPutColData(uint8_t version, uint8_t *pBuf, SColData *pColData) {
} else if (version == 1) {
return tPutColDataVersion1(pBuf, pColData);
} else {
ASSERT(0);
return -1;
return TSDB_CODE_INVALID_PARA;
}
}
@ -3696,8 +3708,7 @@ int32_t tGetColData(uint8_t version, uint8_t *pBuf, SColData *pColData) {
} else if (version == 1) {
return tGetColDataVersion1(pBuf, pColData);
} else {
ASSERT(0);
return -1;
return TSDB_CODE_INVALID_PARA;
}
}
@ -3733,7 +3744,6 @@ static FORCE_INLINE void tColDataCalcSMABool(SColData *pColData, int64_t *sum, i
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
break;
default:
ASSERT(0);
break;
}
}
@ -3765,7 +3775,6 @@ static FORCE_INLINE void tColDataCalcSMATinyInt(SColData *pColData, int64_t *sum
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
break;
default:
ASSERT(0);
break;
}
}
@ -3797,7 +3806,6 @@ static FORCE_INLINE void tColDataCalcSMATinySmallInt(SColData *pColData, int64_t
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
break;
default:
ASSERT(0);
break;
}
}
@ -3829,7 +3837,6 @@ static FORCE_INLINE void tColDataCalcSMAInt(SColData *pColData, int64_t *sum, in
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
break;
default:
ASSERT(0);
break;
}
}
@ -3861,7 +3868,6 @@ static FORCE_INLINE void tColDataCalcSMABigInt(SColData *pColData, int64_t *sum,
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
break;
default:
ASSERT(0);
break;
}
}
@ -3893,7 +3899,6 @@ static FORCE_INLINE void tColDataCalcSMAFloat(SColData *pColData, int64_t *sum,
CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val);
break;
default:
ASSERT(0);
break;
}
}
@ -3925,7 +3930,6 @@ static FORCE_INLINE void tColDataCalcSMADouble(SColData *pColData, int64_t *sum,
CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val);
break;
default:
ASSERT(0);
break;
}
}
@ -3957,7 +3961,6 @@ static FORCE_INLINE void tColDataCalcSMAUTinyInt(SColData *pColData, int64_t *su
CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val);
break;
default:
ASSERT(0);
break;
}
}
@ -3989,7 +3992,6 @@ static FORCE_INLINE void tColDataCalcSMATinyUSmallInt(SColData *pColData, int64_
CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val);
break;
default:
ASSERT(0);
break;
}
}
@ -4021,7 +4023,6 @@ static FORCE_INLINE void tColDataCalcSMAUInt(SColData *pColData, int64_t *sum, i
CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val);
break;
default:
ASSERT(0);
break;
}
}
@ -4053,7 +4054,6 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum
CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val);
break;
default:
ASSERT(0);
break;
}
}
@ -4092,7 +4092,6 @@ static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum
}
break;
default:
ASSERT(0);
break;
}
}
@ -4153,7 +4152,9 @@ int32_t tValueColumnAppend(SValueColumn *valCol, const SValue *value) {
valCol->type = value->type;
}
ASSERT(value->type == valCol->type);
if (!(value->type == valCol->type)) {
return TSDB_CODE_INVALID_PARA;
}
if (IS_VAR_DATA_TYPE(value->type)) {
if ((code = tBufferPutI32(&valCol->offsets, tBufferGetSize(&valCol->data)))) {
@ -4230,7 +4231,9 @@ int32_t tValueColumnGet(SValueColumn *valCol, int32_t idx, SValue *value) {
int32_t tValueColumnCompress(SValueColumn *valCol, SValueColumnCompressInfo *info, SBuffer *output, SBuffer *assist) {
int32_t code;
ASSERT(valCol->numOfValues > 0);
if (!(valCol->numOfValues > 0)) {
return TSDB_CODE_INVALID_PARA;
}
(*info) = (SValueColumnCompressInfo){
.cmprAlg = info->cmprAlg,
@ -4344,7 +4347,7 @@ int32_t tValueColumnCompressInfoDecode(SBufferReader *reader, SValueColumnCompre
if ((code = tBufferGetI32v(reader, &info->dataOriginalSize))) return code;
if ((code = tBufferGetI32v(reader, &info->dataCompressedSize))) return code;
} else {
ASSERT(0);
return TSDB_CODE_INVALID_PARA;
}
return 0;
@ -4360,7 +4363,9 @@ int32_t tCompressData(void *input, // input
int32_t code;
extraSizeNeeded = (info->cmprAlg == NO_COMPRESSION) ? info->originalSize : info->originalSize + COMP_OVERFLOW_BYTES;
ASSERT(outputSize >= extraSizeNeeded);
if (!(outputSize >= extraSizeNeeded)) {
return TSDB_CODE_INVALID_PARA;
}
if (info->cmprAlg == NO_COMPRESSION) {
(void)memcpy(output, input, info->originalSize);
@ -4442,10 +4447,14 @@ int32_t tDecompressData(void *input, // input
) {
int32_t code;
ASSERT(outputSize >= info->originalSize);
if (!(outputSize >= info->originalSize)) {
return TSDB_CODE_INVALID_PARA;
}
if (info->cmprAlg == NO_COMPRESSION) {
ASSERT(info->compressedSize == info->originalSize);
if (!(info->compressedSize == info->originalSize)) {
return TSDB_CODE_INVALID_PARA;
}
(void)memcpy(output, input, info->compressedSize);
} else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) {
SBuffer local;
@ -4478,7 +4487,9 @@ int32_t tDecompressData(void *input, // input
return TSDB_CODE_COMPRESS_ERROR;
}
ASSERT(decompressedSize == info->originalSize);
if (!(decompressedSize == info->originalSize)) {
return TSDB_CODE_COMPRESS_ERROR;
}
tBufferDestroy(&local);
} else {
DEFINE_VAR(info->cmprAlg);
@ -4512,7 +4523,9 @@ int32_t tDecompressData(void *input, // input
return TSDB_CODE_COMPRESS_ERROR;
}
ASSERT(decompressedSize == info->originalSize);
if (!(decompressedSize == info->originalSize)) {
return TSDB_CODE_COMPRESS_ERROR;
}
tBufferDestroy(&local);
}

15
source/common/src/tglobal.c
View File

@ -303,10 +303,10 @@ char tsS3BucketName[TSDB_FQDN_LEN] = "<bucketname>";
char tsS3AppId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<appid>"};
int8_t tsS3Enabled = false;
int8_t tsS3EnabledCfg = false;
int8_t tsS3Oss = false;
int8_t tsS3Oss[TSDB_MAX_EP_NUM] = {false};
int8_t tsS3StreamEnabled = false;
int8_t tsS3Https = true;
int8_t tsS3Https[TSDB_MAX_EP_NUM] = {true};
char tsS3Hostname[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<hostname>"};
int32_t tsS3BlockSize = -1; // number of tsdb pages (4096)
@ -431,11 +431,10 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
tstrncpy(tsS3AppId[i], appid + 1, TSDB_FQDN_LEN);
}
}
tsS3Https[i] = (strstr(tsS3Endpoint[i], "https://") != NULL);
tsS3Oss[i] = (strstr(tsS3Endpoint[i], "aliyuncs.") != NULL);
}
tsS3Https = (strstr(tsS3Endpoint[0], "https://") != NULL);
tsS3Oss = (strstr(tsS3Endpoint[0], "aliyuncs.") != NULL);
if (tsS3BucketName[0] != '<') {
#if defined(USE_COS) || defined(USE_S3)
#ifdef TD_ENTERPRISE
@ -1600,6 +1599,12 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minDiskFreeSize");
tsMinDiskFreeSize = pItem->i64;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "s3MigrateIntervalSec");
tsS3MigrateIntervalSec = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "s3MigrateEnabled");
tsS3MigrateEnabled = (bool)pItem->bval;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "s3PageCacheSize");
tsS3PageCacheSize = pItem->i32;

27
source/common/src/tmsg.c
View File

@ -103,7 +103,9 @@ int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
pIter->totalLen = htonl(pMsg->length);
pIter->numOfBlocks = htonl(pMsg->numOfBlocks);
ASSERT(pIter->totalLen > 0);
if (!(pIter->totalLen > 0)) {
return terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
}
pIter->len = 0;
pIter->pMsg = pMsg;
if (pIter->totalLen <= sizeof(SSubmitReq)) {
@ -114,17 +116,21 @@ int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
}
int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
ASSERT(pIter->len >= 0);
if (!(pIter->len >= 0)) {
return terrno = TSDB_CODE_INVALID_MSG_LEN;
}
if (pIter->len == 0) {
pIter->len += sizeof(SSubmitReq);
} else {
if (pIter->len >= pIter->totalLen) {
ASSERT(0);
return terrno = TSDB_CODE_INVALID_MSG_LEN;
}
pIter->len += (sizeof(SSubmitBlk) + pIter->dataLen + pIter->schemaLen);
ASSERT(pIter->len > 0);
if (!(pIter->len > 0)) {
return terrno = TSDB_CODE_INVALID_MSG_LEN;
}
}
if (pIter->len > pIter->totalLen) {
@ -377,7 +383,9 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq)
}
}
ASSERT(desc.subPlanNum == taosArrayGetSize(desc.subDesc));
if (!(desc.subPlanNum == taosArrayGetSize(desc.subDesc))) {
return TSDB_CODE_INVALID_MSG;
}
if (!taosArrayPush(pReq->query->queryDesc, &desc)) {
return terrno;
@ -8476,7 +8484,7 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) {
} else if (pReq->type == TSDB_NORMAL_TABLE) {
if (tEncodeSSchemaWrapper(pCoder, &pReq->ntb.schemaRow) < 0) return -1;
} else {
ASSERT(0);
return TSDB_CODE_INVALID_MSG;
}
// ENCODESQL
@ -8528,7 +8536,7 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) {
} else if (pReq->type == TSDB_NORMAL_TABLE) {
if (tDecodeSSchemaWrapperEx(pCoder, &pReq->ntb.schemaRow) < 0) return -1;
} else {
ASSERT(0);
return TSDB_CODE_INVALID_MSG;
}
// DECODESQL
@ -9311,7 +9319,6 @@ bool tOffsetEqual(const STqOffsetVal *pLeft, const STqOffsetVal *pRight) {
return pLeft->uid == pRight->uid;
} else {
uError("offset type:%d", pLeft->type);
ASSERT(0);
}
}
return false;
@ -9727,7 +9734,9 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm
// auto create table
if (pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) {
ASSERT(pSubmitTbData->pCreateTbReq);
if (!(pSubmitTbData->pCreateTbReq)) {
return TSDB_CODE_INVALID_MSG;
}
if (tEncodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq) < 0) return -1;
}

6
source/common/src/tname.c
View File

@ -105,7 +105,6 @@ int32_t tNameExtractFullName(const SName* name, char* dst) {
size_t tnameLen = strlen(name->tname);
if (tnameLen > 0) {
/*ASSERT(name->type == TSDB_TABLE_NAME_T);*/
dst[len] = TS_PATH_DELIMITER[0];
memcpy(dst + len + 1, name->tname, tnameLen);
@ -160,7 +159,10 @@ int32_t tNameGetFullDbName(const SName* name, char* dst) {
bool tNameIsEmpty(const SName* name) { return name->type == 0 || name->acctId == 0; }
const char* tNameGetTableName(const SName* name) {
ASSERT(name != NULL && name->type == TSDB_TABLE_NAME_T);
if (!(name != NULL && name->type == TSDB_TABLE_NAME_T)) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
return &name->tname[0];
}

9
source/common/src/tvariant.c
View File

@ -459,8 +459,8 @@ void taosVariantDestroy(SVariant *pVar) {
}
}
void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
if (pSrc == NULL || pDst == NULL) return;
int32_t taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
if (pSrc == NULL || pDst == NULL) return 0;
pDst->nType = pSrc->nType;
if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_VARBINARY ||
@ -468,19 +468,20 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) {
int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE;
char *p = taosMemoryRealloc(pDst->pz, len);
ASSERT(p);
if (!p) return terrno;
(void)memset(p, 0, len);
pDst->pz = p;
(void)memcpy(pDst->pz, pSrc->pz, pSrc->nLen);
pDst->nLen = pSrc->nLen;
return;
return 0;
}
if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) {
pDst->i = pSrc->i;
}
return 0;
}
int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) {

14
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -83,12 +83,12 @@ extern "C" {
}\
}
#define dGFatal(param, ...) {if (dDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGError(param, ...) {if (dDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGWarn(param, ...) {if (dDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGInfo(param, ...) {if (dDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dInfo(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGDebug(param, ...) {if (dDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dDebug(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGTrace(param, ...) {if (dDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGFatal(param, ...) {if (dDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGError(param, ...) {if (dDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGWarn(param, ...) {if (dDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGInfo(param, ...) {if (dDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dInfo(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGDebug(param, ...) {if (dDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dDebug(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGTrace(param, ...) {if (dDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ", QID:%s", __VA_ARGS__, buf);}}
// clang-format on
@ -208,7 +208,7 @@ void dmGetMonitorSystemInfo(SMonSysInfo *pInfo);
int32_t dmReadFile(const char *path, const char *name, bool *pDeployed);
int32_t dmWriteFile(const char *path, const char *name, bool deployed);
int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile);
//int32_t dmCheckRunningWrapper(const char *dataDir, TdFilePtr *pFile);
// int32_t dmCheckRunningWrapper(const char *dataDir, TdFilePtr *pFile);
// dmodule.c
int32_t dmInitDndInfo(SDnodeData *pData);

14
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -41,12 +41,12 @@ extern "C" {
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
#define mGFatal(param, ...) { if (mDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mFatal(param ", gtid:%s", __VA_ARGS__, buf);}}
#define mGError(param, ...) { if (mDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mError(param ", gtid:%s", __VA_ARGS__, buf);}}
#define mGWarn(param, ...) { if (mDebugFlag & DEBUG_WARN){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mWarn (param ", gtid:%s", __VA_ARGS__, buf);}}
#define mGInfo(param, ...) { if (mDebugFlag & DEBUG_INFO){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", gtid:%s", __VA_ARGS__, buf);}}
#define mGDebug(param, ...) { if (mDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", gtid:%s", __VA_ARGS__, buf);}}
#define mGTrace(param, ...) { if (mDebugFlag & DEBUG_TRACE){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", gtid:%s", __VA_ARGS__, buf);}}
#define mGFatal(param, ...) { if (mDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mFatal(param ", QID:%s", __VA_ARGS__, buf);}}
#define mGError(param, ...) { if (mDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mError(param ", QID:%s", __VA_ARGS__, buf);}}
#define mGWarn(param, ...) { if (mDebugFlag & DEBUG_WARN){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mWarn (param ", QID:%s", __VA_ARGS__, buf);}}
#define mGInfo(param, ...) { if (mDebugFlag & DEBUG_INFO){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", QID:%s", __VA_ARGS__, buf);}}
#define mGDebug(param, ...) { if (mDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", QID:%s", __VA_ARGS__, buf);}}
#define mGTrace(param, ...) { if (mDebugFlag & DEBUG_TRACE){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", QID:%s", __VA_ARGS__, buf);}}
// clang-format on
#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
@ -54,7 +54,7 @@ extern "C" {
#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
typedef int32_t (*MndMsgFp)(SRpcMsg *pMsg);
typedef int32_t (*MndMsgFpExt)(SRpcMsg *pMsg, SQueueInfo* pInfo);
typedef int32_t (*MndMsgFpExt)(SRpcMsg *pMsg, SQueueInfo *pInfo);
typedef int32_t (*MndInitFp)(SMnode *pMnode);
typedef void (*MndCleanupFp)(SMnode *pMnode);
typedef int32_t (*ShowRetrieveFp)(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);

2
source/dnode/mnode/impl/src/mndSync.c
View File

@ -486,7 +486,7 @@ int32_t mndInitSync(SMnode *pMnode) {
int32_t code = 0;
(void)tsem_init(&pMgmt->syncSem, 0, 0);
pMgmt->sync = syncOpen(&syncInfo, true);
pMgmt->sync = syncOpen(&syncInfo, 1); // always check
if (pMgmt->sync <= 0) {
if (terrno != 0) code = terrno;
mError("failed to open sync since %s", tstrerror(code));

10
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -52,6 +52,8 @@ static void sdbResetData(SSdb *pSdb) {
SHashObj *hash = pSdb->hashObjs[i];
if (hash == NULL) continue;
sdbWriteLock(pSdb, i);
SSdbRow **ppRow = taosHashIterate(hash, NULL);
while (ppRow != NULL) {
SSdbRow *pRow = *ppRow;
@ -60,15 +62,13 @@ static void sdbResetData(SSdb *pSdb) {
sdbFreeRow(pSdb, pRow, true);
ppRow = taosHashIterate(hash, ppRow);
}
}
for (ESdbType i = 0; i < SDB_MAX; ++i) {
SHashObj *hash = pSdb->hashObjs[i];
if (hash == NULL) continue;
taosHashClear(pSdb->hashObjs[i]);
pSdb->tableVer[i] = 0;
pSdb->maxId[i] = 0;
sdbUnLock(pSdb, i);
mInfo("sdb:%s is reset", sdbTableName(i));
}

2
source/dnode/vnode/src/inc/tsdb.h
View File

@ -898,7 +898,7 @@ typedef struct SSttDataInfoForTable {
int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoForTable *pTableInfo);
void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
bool tMergeTreeNext(SMergeTree *pMTree);
int32_t tMergeTreeNext(SMergeTree *pMTree, bool* pHasNext);
void tMergeTreePinSttBlock(SMergeTree *pMTree);
void tMergeTreeUnpinSttBlock(SMergeTree *pMTree);
bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);

12
source/dnode/vnode/src/inc/vnd.h
View File

@ -32,12 +32,12 @@ extern "C" {
#define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", DEBUG_DEBUG, vDebugFlag, __VA_ARGS__); }} while(0)
#define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", DEBUG_TRACE, vDebugFlag, __VA_ARGS__); }} while(0)
#define vGTrace(param, ...) do { if (vDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vTrace(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define vGFatal(param, ...) do { if (vDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vFatal(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define vGError(param, ...) do { if (vDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vError(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define vGWarn(param, ...) do { if (vDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vWarn(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define vGInfo(param, ...) do { if (vDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vInfo(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define vGDebug(param, ...) do { if (vDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vDebug(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define vGTrace(param, ...) do { if (vDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vTrace(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGFatal(param, ...) do { if (vDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vFatal(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGError(param, ...) do { if (vDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vError(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGWarn(param, ...) do { if (vDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vWarn(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGInfo(param, ...) do { if (vDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vInfo(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGDebug(param, ...) do { if (vDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vDebug(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
// clang-format on

15
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -101,20 +101,6 @@ typedef struct SQueryNode SQueryNode;
#define VND_INFO_FNAME "vnode.json"
#define VND_INFO_FNAME_TMP "vnode_tmp.json"
#define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count"
#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type"
#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id"
#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id"
#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep"
#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id"
#define VNODE_METRIC_TAG_NAME_USERNAME "username"
#define VNODE_METRIC_TAG_NAME_RESULT "result"
#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows"
// #define VNODE_METRIC_TAG_VALUE_INSERT "insert"
// #define VNODE_METRIC_TAG_VALUE_DELETE "delete"
// vnd.h
typedef int32_t (*_query_reseek_func_t)(void* pQHandle);
struct SQueryNode {
@ -452,7 +438,6 @@ typedef struct SVMonitorObj {
char strClusterId[TSDB_CLUSTER_ID_LEN];
char strDnodeId[TSDB_NODE_ID_LEN];
char strVgId[TSDB_VGROUP_ID_LEN];
taos_counter_t* insertCounter;
} SVMonitorObj;
typedef struct {

1
source/dnode/vnode/src/meta/metaCache.c
View File

@ -503,7 +503,6 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv
}
static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) {
// ASSERT(keyLen == sizeof(int64_t) * 2);
memcpy(&pBuf[2], key, keyLen);
}

4
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -332,6 +332,10 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL
} else if (!pTagIdxKey1->isNull && !pTagIdxKey2->isNull) {
// all not NULL, compr tag vals
__compar_fn_t func = getComparFunc(pTagIdxKey1->type, 0);
if (func == NULL) {
metaError("meta/open: %s", terrstr());
return TSDB_CODE_FAILED;
}
c = func(pTagIdxKey1->data, pTagIdxKey2->data);
if (c) return c;
}

21
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -694,9 +694,13 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv
SSchemaWrapper *pSchemaWrapper = &schema;
tDecoderInit(&dc, pData, nData);
(void)tDecodeSSchemaWrapper(&dc, pSchemaWrapper);
code = tDecodeSSchemaWrapper(&dc, pSchemaWrapper);
tDecoderClear(&dc);
tdbFree(pData);
if (TSDB_CODE_SUCCESS != code) {
taosMemoryFree(pSchemaWrapper->pSchema);
goto _exit;
}
// convert
STSchema *pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version);
@ -1099,7 +1103,12 @@ int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
SBtimeIdxKey *p = entryKey;
if (count > TRY_ERROR_LIMIT) break;
terrno = TSDB_CODE_SUCCESS;
int32_t cmp = (*param->filterFunc)((void *)&p->btime, (void *)&pBtimeKey->btime, param->type);
if (terrno != TSDB_CODE_SUCCESS) {
ret = terrno;
break;
}
if (cmp == 0) {
if (taosArrayPush(pUids, &p->uid) == NULL) {
ret = terrno;
@ -1163,7 +1172,12 @@ int32_t metaFilterTableName(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
if (count > TRY_ERROR_LIMIT) break;
char *pTableKey = (char *)pEntryKey;
terrno = TSDB_CODE_SUCCESS;
cmp = (*param->filterFunc)(pTableKey, pName, pCursor->type);
if (terrno != TSDB_CODE_SUCCESS) {
ret = terrno;
goto END;
}
if (cmp == 0) {
tb_uid_t tuid = *(tb_uid_t *)pEntryVal;
if (taosArrayPush(pUids, &tuid) == NULL) {
@ -1357,7 +1371,12 @@ int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
}
}
terrno = TSDB_CODE_SUCCESS;
int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type);
if (terrno != TSDB_CODE_SUCCESS) {
TAOS_CHECK_GOTO(terrno, NULL, END);
break;
}
if (cmp == 0) {
// match
tb_uid_t tuid = 0;

4
source/dnode/vnode/src/sma/smaEnv.c
View File

@ -69,7 +69,7 @@ int32_t smaInit() {
if (!smaMgmt.refHash || !smaMgmt.tmrHandle) {
code = terrno;
(void)taosCloseRef(smaMgmt.rsetId);
taosCloseRef(smaMgmt.rsetId);
if (smaMgmt.refHash) {
taosHashCleanup(smaMgmt.refHash);
smaMgmt.refHash = NULL;
@ -103,7 +103,7 @@ void smaCleanUp() {
}
if (old == 1) {
(void)taosCloseRef(smaMgmt.rsetId);
taosCloseRef(smaMgmt.rsetId);
taosHashCleanup(smaMgmt.refHash);
smaMgmt.refHash = NULL;
taosTmrCleanUp(smaMgmt.tmrHandle);

16
source/dnode/vnode/src/tq/tq.c
View File

@ -173,7 +173,7 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) {
dataRsp.common.blockNum = 0;
char buf[TSDB_OFFSET_LEN] = {0};
(void)tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.reqOffset);
tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, reqId:0x%" PRIx64, req.consumerId, vgId, buf,
tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, QID:0x%" PRIx64, req.consumerId, vgId, buf,
req.reqId);
code = tqSendDataRsp(pHandle, pHandle->msg, &req, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
@ -193,7 +193,7 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq*
(void)tFormatOffset(buf1, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->reqOffset);
(void)tFormatOffset(buf2, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->rspOffset);
tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64,
tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, QID:0x%" PRIx64,
vgId, pReq->consumerId, pReq->epoch, ((SMqDataRspCommon*)pRsp)->blockNum, buf1, buf2, pReq->reqId);
return tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever);
@ -421,7 +421,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
char buf[TSDB_OFFSET_LEN] = {0};
(void)tFormatOffset(buf, TSDB_OFFSET_LEN, &reqOffset);
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64,
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, QID:0x%" PRIx64,
consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId);
code = tqExtractDataForMq(pTq, pHandle, &req, pMsg);
@ -1204,13 +1204,13 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
}
if (req.mndTrigger) {
qInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d, ", pTask->id.idStr,
vgId, pTask->info.taskLevel, req.checkpointId, req.transId);
tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d, ",
pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId);
} else {
const char* pPrevStatus = streamTaskGetStatusStr(streamTaskGetPrevStatus(pTask));
qInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64
", transId:%d after transfer-state, prev status:%s",
pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId, pPrevStatus);
tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64
", transId:%d after transfer-state, prev status:%s",
pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId, pPrevStatus);
}
code = streamAddCheckpointSourceRspMsg(&req, &pMsg->info, pTask);

145
source/dnode/vnode/src/tq/tqRead.c
View File

@ -74,7 +74,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) {
pCreateReq = req.pReqs + iReq;
if (pCreateReq->type == TSDB_CHILD_TABLE && pCreateReq->ctb.suid == tbSuid) {
reqNew.nReqs++;
if (taosArrayPush(reqNew.pArray, pCreateReq) == NULL){
if (taosArrayPush(reqNew.pArray, pCreateReq) == NULL) {
taosArrayDestroy(reqNew.pArray);
tDeleteSVCreateTbBatchReq(&req);
goto end;
@ -155,7 +155,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) {
pDropReq = req.pReqs + iReq;
if (pDropReq->suid == tbSuid) {
reqNew.nReqs++;
if (taosArrayPush(reqNew.pArray, pDropReq) == NULL){
if (taosArrayPush(reqNew.pArray, pDropReq) == NULL) {
taosArrayDestroy(reqNew.pArray);
goto end;
}
@ -214,12 +214,12 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t
while (offset <= appliedVer) {
if (walFetchHead(pHandle->pWalReader, offset) < 0) {
tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64
", no more log to return, reqId:0x%" PRIx64 " 0x%" PRIx64,
", no more log to return, QID:0x%" PRIx64 " 0x%" PRIx64,
pHandle->consumerId, pHandle->epoch, vgId, offset, reqId, id);
goto END;
}
tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64 " 0x%" PRIx64,
tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, QID:0x%" PRIx64 " 0x%" PRIx64,
vgId, pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId, id);
if (pHandle->pWalReader->pHead->head.msgType == TDMT_VND_SUBMIT) {
@ -261,10 +261,10 @@ END:
bool tqGetTablePrimaryKey(STqReader* pReader) { return pReader->hasPrimaryKey; }
void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid){
bool ret = false;
SSchemaWrapper *schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1);
if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY){
void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) {
bool ret = false;
SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1);
if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true;
}
tDeleteSchemaWrapper(schema);
@ -486,7 +486,7 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) {
(pReader->nextBlk + 1), numOfBlocks, idstr);
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
if (pSubmitTbData == NULL){
if (pSubmitTbData == NULL) {
return false;
}
if (pReader->tbIdHash == NULL) {
@ -639,9 +639,9 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol
int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) {
tqTrace("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk);
int32_t code = 0;
int32_t line = 0;
STSchema* pTSchema = NULL;
int32_t code = 0;
int32_t line = 0;
STSchema* pTSchema = NULL;
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++);
TSDB_CHECK_NULL(pSubmitTbData, code, line, END, terrno);
SSDataBlock* pBlock = pReader->pResBlock;
@ -712,10 +712,11 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
continue;
}
SColData* pCol = taosArrayGet(pCols, sourceIdx);
SColData* pCol = taosArrayGet(pCols, sourceIdx);
TSDB_CHECK_NULL(pCol, code, line, END, terrno);
SColVal colVal = {0};
tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual, sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId);
SColVal colVal = {0};
tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual,
sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId);
if (pCol->cid < pColData->info.colId) {
sourceIdx++;
} else if (pCol->cid == pColData->info.colId) {
@ -738,7 +739,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
TSDB_CHECK_NULL(pTSchema, code, line, END, terrno);
for (int32_t i = 0; i < numOfRows; i++) {
SRow* pRow = taosArrayGetP(pRows, i);
SRow* pRow = taosArrayGetP(pRows, i);
TSDB_CHECK_NULL(pRow, code, line, END, terrno);
int32_t sourceIdx = 0;
for (int32_t j = 0; j < colActual; j++) {
@ -765,44 +766,43 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
}
}
}
}
END:
if (code != 0){
if (code != 0) {
tqError("tqRetrieveDataBlock failed, line:%d, code:%d", line, code);
}
taosMemoryFreeClear(pTSchema);
return code;
}
#define PROCESS_VAL \
if (curRow == 0) {\
assigned[j] = !COL_VAL_IS_NONE(&colVal);\
buildNew = true;\
} else {\
bool currentRowAssigned = !COL_VAL_IS_NONE(&colVal);\
if (currentRowAssigned != assigned[j]) {\
assigned[j] = currentRowAssigned;\
buildNew = true;\
}\
#define PROCESS_VAL \
if (curRow == 0) { \
assigned[j] = !COL_VAL_IS_NONE(&colVal); \
buildNew = true; \
} else { \
bool currentRowAssigned = !COL_VAL_IS_NONE(&colVal); \
if (currentRowAssigned != assigned[j]) { \
assigned[j] = currentRowAssigned; \
buildNew = true; \
} \
}
#define SET_DATA \
if (colVal.cid < pColData->info.colId) {\
sourceIdx++;\
} else if (colVal.cid == pColData->info.colId) {\
TQ_ERR_GO_TO_END(doSetVal(pColData, curRow - lastRow, &colVal));\
sourceIdx++;\
targetIdx++;\
#define SET_DATA \
if (colVal.cid < pColData->info.colId) { \
sourceIdx++; \
} else if (colVal.cid == pColData->info.colId) { \
TQ_ERR_GO_TO_END(doSetVal(pColData, curRow - lastRow, &colVal)); \
sourceIdx++; \
targetIdx++; \
}
static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks,
SArray* schemas, SSchemaWrapper* pSchemaWrapper, char* assigned,
int32_t numOfRows, int32_t curRow, int32_t* lastRow){
int32_t code = 0;
SSchemaWrapper* pSW = NULL;
SSDataBlock* block = NULL;
static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas,
SSchemaWrapper* pSchemaWrapper, char* assigned, int32_t numOfRows, int32_t curRow,
int32_t* lastRow) {
int32_t code = 0;
SSchemaWrapper* pSW = NULL;
SSDataBlock* block = NULL;
if (taosArrayGetSize(blocks) > 0) {
SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
TQ_NULL_GO_TO_END(pLastBlock);
@ -834,33 +834,34 @@ END:
taosMemoryFree(block);
return code;
}
static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas){
int32_t code = 0;
int32_t curRow = 0;
int32_t lastRow = 0;
static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas) {
int32_t code = 0;
int32_t curRow = 0;
int32_t lastRow = 0;
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
TQ_NULL_GO_TO_END(assigned);
SArray* pCols = pSubmitTbData->aCol;
SColData* pCol = taosArrayGet(pCols, 0);
SArray* pCols = pSubmitTbData->aCol;
SColData* pCol = taosArrayGet(pCols, 0);
TQ_NULL_GO_TO_END(pCol);
int32_t numOfRows = pCol->nVal;
int32_t numOfCols = taosArrayGetSize(pCols);
int32_t numOfRows = pCol->nVal;
int32_t numOfCols = taosArrayGetSize(pCols);
for (int32_t i = 0; i < numOfRows; i++) {
bool buildNew = false;
for (int32_t j = 0; j < numOfCols; j++) {
pCol = taosArrayGet(pCols, j);
TQ_NULL_GO_TO_END(pCol);
SColVal colVal = {0};
SColVal colVal = {0};
tColDataGetValue(pCol, i, &colVal);
PROCESS_VAL
}
if (buildNew) {
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows, curRow, &lastRow));
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows,
curRow, &lastRow));
}
SSDataBlock* pBlock = taosArrayGetLast(blocks);
@ -877,7 +878,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
TQ_NULL_GO_TO_END(pCol);
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
TQ_NULL_GO_TO_END(pColData);
SColVal colVal = {0};
SColVal colVal = {0};
tColDataGetValue(pCol, i, &colVal);
SET_DATA
}
@ -885,30 +886,30 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
curRow++;
}
SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
pLastBlock->info.rows = curRow - lastRow;
pLastBlock->info.rows = curRow - lastRow;
END:
taosMemoryFree(assigned);
return code;
}
int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas){
int32_t code = 0;
STSchema* pTSchema = NULL;
int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas) {
int32_t code = 0;
STSchema* pTSchema = NULL;
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
TQ_NULL_GO_TO_END(assigned);
int32_t curRow = 0;
int32_t lastRow = 0;
SArray* pRows = pSubmitTbData->aRowP;
int32_t numOfRows = taosArrayGetSize(pRows);
pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version);
int32_t curRow = 0;
int32_t lastRow = 0;
SArray* pRows = pSubmitTbData->aRowP;
int32_t numOfRows = taosArrayGetSize(pRows);
pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version);
for (int32_t i = 0; i < numOfRows; i++) {
bool buildNew = false;
SRow* pRow = taosArrayGetP(pRows, i);
SRow* pRow = taosArrayGetP(pRows, i);
TQ_NULL_GO_TO_END(pRow);
for (int32_t j = 0; j < pTSchema->numOfCols; j++) {
@ -918,7 +919,8 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra
}
if (buildNew) {
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows, curRow, &lastRow));
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows,
curRow, &lastRow));
}
SSDataBlock* pBlock = taosArrayGetLast(blocks);
@ -932,7 +934,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra
int32_t colActual = blockDataGetNumOfCols(pBlock);
while (targetIdx < colActual) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
SColVal colVal = {0};
SColVal colVal = {0};
TQ_ERR_GO_TO_END(tRowGet(pRow, pTSchema, sourceIdx, &colVal));
SET_DATA
}
@ -940,7 +942,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra
curRow++;
}
SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
pLastBlock->info.rows = curRow - lastRow;
pLastBlock->info.rows = curRow - lastRow;
END:
taosMemoryFreeClear(pTSchema);
@ -950,10 +952,10 @@ END:
int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) {
tqDebug("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk);
SSDataBlock* block = NULL;
SSDataBlock* block = NULL;
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
if(pSubmitTbData == NULL){
if (pSubmitTbData == NULL) {
return terrno;
}
pReader->nextBlk++;
@ -1034,7 +1036,7 @@ bool tqCurrentBlockConsumed(const STqReader* pReader) { return pReader->msg.msgS
void tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) {
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
if (pKey && taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)) != 0){
if (pKey && taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)) != 0) {
tqError("failed to remove table uid:%" PRId64 " from hash", *pKey);
}
}
@ -1064,7 +1066,8 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
int32_t sz = taosArrayGetSize(tbUidList);
for (int32_t i = 0; i < sz; i++) {
int64_t* tbUid = (int64_t*)taosArrayGet(tbUidList, i);
if (tbUid && taosHashPut(pTqHandle->execHandle.execDb.pFilterOutTbUid, tbUid, sizeof(int64_t), NULL, 0) != 0){
if (tbUid &&
taosHashPut(pTqHandle->execHandle.execDb.pFilterOutTbUid, tbUid, sizeof(int64_t), NULL, 0) != 0) {
tqError("failed to add table uid:%" PRId64 " to hash", *tbUid);
continue;
}

32
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -92,7 +92,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
char formatBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(formatBuf, TSDB_OFFSET_LEN, pOffsetVal);
tqDebug("tmq poll: consumer:0x%" PRIx64
", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%" PRIx64,
", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. QID:0x%" PRIx64,
consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId);
return 0;
} else {
@ -117,7 +117,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer + 1);
code = tqInitDataRsp(&dataRsp.common, *pOffsetVal);
if (code != 0){
if (code != 0) {
return code;
}
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId,
@ -145,7 +145,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
terrno = 0;
SMqDataRsp dataRsp = {0};
int code = tqInitDataRsp(&dataRsp.common, *pOffset);
int code = tqInitDataRsp(&dataRsp.common, *pOffset);
if (code != 0) {
goto end;
}
@ -176,7 +176,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
end : {
char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.rspOffset);
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, QID:0x%" PRIx64
" code:%d",
consumerId, pHandle->subKey, vgId, dataRsp.common.blockNum, buf, pRequest->reqId, code);
tDeleteMqDataRsp(&dataRsp);
@ -206,10 +206,10 @@ static void tDeleteCommon(void* parm) {}
static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
SRpcMsg* pMsg, STqOffsetVal* offset) {
int32_t vgId = TD_VID(pTq->pVnode);
STaosxRsp taosxRsp = {0};
SMqBatchMetaRsp btMetaRsp = {0};
int32_t code = 0;
int32_t vgId = TD_VID(pTq->pVnode);
STaosxRsp taosxRsp = {0};
SMqBatchMetaRsp btMetaRsp = {0};
int32_t code = 0;
TQ_ERR_GO_TO_END(tqInitTaosxRsp(&taosxRsp.common, *offset));
if (offset->type != TMQ_OFFSET__LOG) {
@ -323,9 +323,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
continue;
}
int32_t tLen = sizeof(SMqRspHead) + len;
void* tBuf = taosMemoryCalloc(1, tLen);
if (tBuf == NULL){
int32_t tLen = sizeof(SMqRspHead) + len;
void* tBuf = taosMemoryCalloc(1, tLen);
if (tBuf == NULL) {
code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);
goto END;
}
@ -339,11 +339,11 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
continue;
}
if (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf) == NULL){
if (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf) == NULL) {
code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);
goto END;
}
if (taosArrayPush(btMetaRsp.batchMetaLen, &tLen) == NULL){
if (taosArrayPush(btMetaRsp.batchMetaLen, &tLen) == NULL) {
code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);
goto END;
}
@ -601,7 +601,7 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void*
SColumnInfoData* pUidCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX);
TSDB_CHECK_NULL(pUidCol, code, line, END, terrno)
int64_t* pUid = taosArrayGet(pRes->uidList, i);
int64_t* pUid = taosArrayGet(pRes->uidList, i);
code = colDataSetVal(pUidCol, i, (const char*)pUid, false);
TSDB_CHECK_CODE(code, line, END);
void* tmp = taosArrayGet(pDelBlock->pDataBlock, GROUPID_COLUMN_INDEX);
@ -632,7 +632,7 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void*
}
END:
if (code != 0){
if (code != 0) {
tqError("failed to extract delete data block, line:%d code:%d", line, code);
}
tDecoderClear(pCoder);
@ -662,7 +662,7 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
for (int32_t i = 0; i < numOfTasks; ++i) {
SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i);
if (pId == NULL){
if (pId == NULL) {
continue;
}
if (pId->streamId != streamId) {

73
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -207,7 +207,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
updated = streamTaskUpdateEpsetInfo(pTask, req.pNodeList);
// send the checkpoint-source-rsp for source task to end the checkpoint trans in mnode
(void) streamTaskSendCheckpointsourceRsp(pTask);
(void)streamTaskSendCheckpointsourceRsp(pTask);
streamTaskResetStatus(pTask);
streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, pTask->id.idStr);
@ -303,7 +303,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList);
return rsp.code; // always return true
return rsp.code; // always return true
}
int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -324,7 +324,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tqDebug("s-task:0x%x recv dispatch msg from 0x%x(vgId:%d)", req.taskId, req.upstreamTaskId, req.upstreamNodeId);
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask && (code == 0)) {
SRpcMsg rsp = {.info = pMsg->info, .code = 0};
if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) {
@ -384,7 +384,7 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
pRsp->downstreamTaskId, pRsp->downstreamNodeId);
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask);
if (pTask && (code == 0)) {
code = streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
streamMetaReleaseTask(pMeta, pTask);
@ -482,8 +482,8 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
}
tDecoderClear(&decoder);
tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d",
rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status);
tqDebug("tq task:0x%x (vgId:%d) recv check rsp(QID:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId,
rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status);
if (!isLeader) {
tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId,
@ -681,7 +681,8 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
code = streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
if (code) {
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId,
(int32_t)hTaskId.taskId);
}
}
@ -701,7 +702,7 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
}
streamMetaWUnLock(pMeta);
return 0; // always return success
return 0; // always return success
}
int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) {
@ -793,23 +794,23 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
int32_t vgId = pMeta->vgId;
if (type == STREAM_EXEC_T_START_ONE_TASK) {
(void) streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId);
(void)streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId);
return 0;
} else if (type == STREAM_EXEC_T_START_ALL_TASKS) {
(void) streamMetaStartAllTasks(pMeta);
(void)streamMetaStartAllTasks(pMeta);
return 0;
} else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) {
(void) restartStreamTasks(pMeta, isLeader);
(void)restartStreamTasks(pMeta, isLeader);
return 0;
} else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) {
(void) streamMetaStopAllTasks(pMeta);
(void)streamMetaStopAllTasks(pMeta);
return 0;
} else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) {
int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId);
return code;
} else if (type == STREAM_EXEC_T_RESUME_TASK) { // task resume to run after idle for a while
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL && (code == 0)) {
char* pStatus = NULL;
@ -831,13 +832,13 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
}
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if ((pTask != NULL) && (code == 0)) { // even in halt status, the data in inputQ must be processed
char* p = NULL;
if (streamTaskReadyToRun(pTask, &p)) {
tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId,
pTask->id.idStr, p, pTask->chkInfo.nextProcessVer);
(void) streamExecTask(pTask);
(void)streamExecTask(pTask);
} else {
int8_t status = streamTaskSetSchedStatusInactive(pTask);
tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId,
@ -900,7 +901,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg;
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pReq->taskId);
@ -924,9 +925,9 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
streamTaskSetStatusReady(pTask);
} else if (pState.state == TASK_STATUS__UNINIT) {
// tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr);
// ASSERT(pTask->status.downstreamReady == 0);
// tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId);
// tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr);
// ASSERT(pTask->status.downstreamReady == 0);
// tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId);
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
} else {
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
@ -942,7 +943,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont;
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64
" from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already",
@ -958,7 +959,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
pTask->id.idStr, (int32_t)pReq->downstreamTaskId);
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_STREAM_TASK_IVLD_STATUS);
TSDB_CODE_STREAM_TASK_IVLD_STATUS);
streamMetaReleaseTask(pMeta, pTask);
return code;
}
@ -996,7 +997,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
pTask->id.idStr, recv, total);
}
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS);
TSDB_CODE_ACTION_IN_PROGRESS);
}
} else { // upstream not recv the checkpoint-source/trigger till now
ASSERT(pState.state == TASK_STATUS__READY || pState.state == TASK_STATUS__HALT);
@ -1005,7 +1006,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"upstream sending checkpoint-source/trigger",
pTask->id.idStr);
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS);
TSDB_CODE_ACTION_IN_PROGRESS);
}
streamMetaReleaseTask(pMeta, pTask);
@ -1016,7 +1017,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg)
SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError(
"vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already",
@ -1038,7 +1039,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg;
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
pReq->taskId);
@ -1122,7 +1123,7 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m
SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle;
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId);
return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
@ -1169,13 +1170,15 @@ int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { ret
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) {
return doProcessDummyRspMsg(pMeta, pMsg);
}
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont;
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped",
pRsp->downstreamNodeId, pRsp->downstreamTaskId);
@ -1211,8 +1214,9 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, req.taskId);
tqError(
"vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, req.taskId);
(void)streamMetaAddFailedTask(pMeta, req.streamId, req.taskId);
return code;
}
@ -1221,7 +1225,8 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
if (req.startTs < pTask->execInfo.created) {
tqWarn("s-task:%s vgId:%d create time:%" PRId64 " recv expired consensus checkpointId:%" PRId64
" from task createTs:%" PRId64 " < task createTs:%" PRId64 ", discard",
pTask->id.idStr, pMeta->vgId, pTask->execInfo.created, req.checkpointId, req.startTs, pTask->execInfo.created);
pTask->id.idStr, pMeta->vgId, pTask->execInfo.created, req.checkpointId, req.startTs,
pTask->execInfo.created);
streamMetaAddFailedTaskSelf(pTask, now);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
@ -1234,15 +1239,15 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId);
if (pTask->chkInfo.consensusTransId >= req.transId) {
tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard",
pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId);
tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard", pTask->id.idStr, vgId,
pTask->chkInfo.consensusTransId, req.transId);
streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
}
if (pTask->chkInfo.checkpointId != req.checkpointId) {
tqDebug("s-task:%s vgId:%d update the checkpoint from %" PRId64 " to %" PRId64" transId:%d", pTask->id.idStr, vgId,
tqDebug("s-task:%s vgId:%d update the checkpoint from %" PRId64 " to %" PRId64 " transId:%d", pTask->id.idStr, vgId,
pTask->chkInfo.checkpointId, req.checkpointId, req.transId);
pTask->chkInfo.checkpointId = req.checkpointId;
tqSetRestoreVersionInfo(pTask);

33
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -1413,7 +1413,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
SIdxKey *idxKey = taosArrayGet(remainCols, 0);
if (idxKey->key.cid != PRIMARYKEY_TIMESTAMP_COL_ID) {
// ignore 'ts' loaded from cache and load it from tsdb
SLastCol* pLastCol = taosArrayGet(pLastArray, 0);
SLastCol *pLastCol = taosArrayGet(pLastArray, 0);
tsdbCacheUpdateLastColToNone(pLastCol, TSDB_LAST_CACHE_NO_CACHE);
SLastKey *key = &(SLastKey){.lflag = ltype, .uid = uid, .cid = PRIMARYKEY_TIMESTAMP_COL_ID};
@ -1745,12 +1745,12 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache
goto _exit;
}
}
if (taosArrayPush(remainCols, &(SIdxKey){i, key}) ==NULL) {
if (taosArrayPush(remainCols, &(SIdxKey){i, key}) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
bool ignoreRocks = pLastCol ? (pLastCol->cacheStatus == TSDB_LAST_CACHE_NO_CACHE) : false;
if (taosArrayPush(ignoreFromRocks, &ignoreRocks) ==NULL) {
if (taosArrayPush(ignoreFromRocks, &ignoreRocks) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
@ -1802,12 +1802,12 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache
}
_exit:
if (remainCols) {
taosArrayDestroy(remainCols);
}
if (ignoreFromRocks) {
taosArrayDestroy(ignoreFromRocks);
}
if (remainCols) {
taosArrayDestroy(remainCols);
}
if (ignoreFromRocks) {
taosArrayDestroy(ignoreFromRocks);
}
TAOS_RETURN(code);
}
@ -2245,17 +2245,20 @@ static int32_t lastIterClose(SFSLastIter **iter) {
}
static int32_t lastIterNext(SFSLastIter *iter, TSDBROW **ppRow) {
int32_t code = 0;
bool hasVal = false;
*ppRow = NULL;
int32_t code = tMergeTreeNext(iter->pMergeTree, &hasVal);
if (code != 0) {
return code;
}
bool hasVal = tMergeTreeNext(iter->pMergeTree);
if (!hasVal) {
*ppRow = NULL;
TAOS_RETURN(code);
}
*ppRow = tMergeTreeGetRow(iter->pMergeTree);
TAOS_RETURN(code);
}
@ -2365,6 +2368,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
if (!state->pIndexList) {
state->pIndexList = taosArrayInit(1, sizeof(SBrinBlk));
if (!state->pIndexList) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err);
}
} else {
taosArrayClear(state->pIndexList);
}
@ -2650,7 +2656,6 @@ static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarli
TAOS_RETURN(code);
}
default:
ASSERT(0);
break;
}

18
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -208,8 +208,6 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
int32_t code = 0;
int32_t lino = 0;
ASSERT(TARRAY2_SIZE(committer->sttReaderArray) == 0);
if (committer->ctx->info->fset == NULL //
|| committer->sttTrigger > 1 //
|| TARRAY2_SIZE(committer->ctx->info->fset->lvlArr) == 0 //
@ -264,11 +262,6 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
int32_t code = 0;
int32_t lino = 0;
ASSERT(TARRAY2_SIZE(committer->dataIterArray) == 0);
ASSERT(committer->dataIterMerger == NULL);
ASSERT(TARRAY2_SIZE(committer->tombIterArray) == 0);
ASSERT(committer->tombIterMerger == NULL);
STsdbIter *iter;
STsdbIterConfig config = {0};
@ -342,10 +335,6 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
committer->ctx->tbid->suid = 0;
committer->ctx->tbid->uid = 0;
ASSERT(TARRAY2_SIZE(committer->dataIterArray) == 0);
ASSERT(committer->dataIterMerger == NULL);
ASSERT(committer->writer == NULL);
TAOS_CHECK_GOTO(tsdbCommitOpenReader(committer), &lino, _exit);
TAOS_CHECK_GOTO(tsdbCommitOpenIter(committer), &lino, _exit);
TAOS_CHECK_GOTO(tsdbCommitOpenWriter(committer), &lino, _exit);
@ -637,13 +626,10 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) {
if (eno == 0) {
TAOS_CHECK_GOTO(tsdbFSEditBegin(committer->tsdb->pFS, committer->fopArray, TSDB_FEDIT_COMMIT), &lino, _exit);
} else {
// TODO
ASSERT(0);
tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(eno));
}
ASSERT(committer->writer == NULL);
ASSERT(committer->dataIterMerger == NULL);
ASSERT(committer->tombIterMerger == NULL);
TARRAY2_DESTROY(committer->dataIterArray, NULL);
TARRAY2_DESTROY(committer->tombIterArray, NULL);
TARRAY2_DESTROY(committer->sttReaderArray, NULL);

4
source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c
View File

@ -113,7 +113,7 @@ _exit:
}
static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) {
ASSERT(0);
tsdbError("vgId:%d %s failed since not implemented", TD_VID(writer->config->tsdb->pVnode), __func__);
return 0;
}
@ -122,8 +122,6 @@ static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return
static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) {
int32_t code = 0;
int32_t lino = 0;
ASSERT(writer->ctx->offset <= writer->file.size);
ASSERT(writer->config->fid == writer->file.fid);
STFileOp op = (STFileOp){
.optype = TSDB_FOP_CREATE,

54
source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
View File

@ -283,7 +283,9 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB
}
}
ASSERT(br.offset == br.buffer->size);
if (br.offset != br.buffer->size) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
_exit:
if (code) {
@ -313,7 +315,10 @@ int32_t tsdbDataFileReadBlockData(SDataFileReader *reader, const SBrinRecord *re
// decompress
SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer);
TAOS_CHECK_GOTO(tBlockDataDecompress(&br, bData, assist), &lino, _exit);
ASSERT(br.offset == buffer->size);
if (br.offset != buffer->size) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
_exit:
if (code) {
@ -345,7 +350,9 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
TAOS_CHECK_GOTO(tGetDiskDataHdr(&br, &hdr), &lino, _exit);
ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
if (hdr.delimiter != TSDB_FILE_DLMT) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
tBlockDataReset(bData);
bData->suid = hdr.suid;
@ -354,7 +361,9 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
// Key part
TAOS_CHECK_GOTO(tBlockDataDecompressKeyPart(&hdr, &br, bData, assist), &lino, _exit);
ASSERT(br.offset == buffer0->size);
if (br.offset != buffer0->size) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
int extraColIdx = -1;
for (int i = 0; i < ncid; i++) {
@ -526,7 +535,9 @@ int32_t tsdbDataFileReadBlockSma(SDataFileReader *reader, const SBrinRecord *rec
TAOS_CHECK_GOTO(tGetColumnDataAgg(&br, sma), &lino, _exit);
TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(columnDataAggArray, sma), &lino, _exit);
}
ASSERT(br.offset == record->smaSize);
if (br.offset != record->smaSize) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
}
_exit:
@ -661,7 +672,8 @@ struct SDataFileWriter {
};
static int32_t tsdbDataFileWriterCloseAbort(SDataFileWriter *writer) {
ASSERT(0);
tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, __LINE__,
"not implemented");
return 0;
}
@ -980,7 +992,9 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData
return 0;
}
ASSERT(bData->uid);
if (!bData->uid) {
return TSDB_CODE_INVALID_PARA;
}
int32_t code = 0;
int32_t lino = 0;
@ -1234,7 +1248,6 @@ static int32_t tsdbDataFileWriteTableDataEnd(SDataFileWriter *writer) {
if (writer->ctx->tbHasOldData) {
TAOS_CHECK_GOTO(tsdbDataFileDoWriteTableOldData(writer, NULL /* as the largest key */), &lino, _exit);
ASSERT(writer->ctx->tbHasOldData == false);
}
TAOS_CHECK_GOTO(tsdbDataFileDoWriteBlockData(writer, writer->blockData), &lino, _exit);
@ -1251,9 +1264,6 @@ static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TA
int32_t code = 0;
int32_t lino = 0;
ASSERT(writer->ctx->blockDataIdx == writer->ctx->blockData->nRow);
ASSERT(writer->blockData->nRow == 0);
SMetaInfo info;
bool drop = false;
TABLEID tbid1[1];
@ -1451,7 +1461,9 @@ int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFD
}
static int32_t tsdbDataFileDoWriteTombBlk(SDataFileWriter *writer) {
ASSERT(TARRAY2_SIZE(writer->tombBlkArray) > 0);
if (TARRAY2_SIZE(writer->tombBlkArray) <= 0) {
return TSDB_CODE_INVALID_PARA;
}
int32_t code = 0;
int32_t lino = 0;
@ -1523,7 +1535,9 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom
TAOS_CHECK_GOTO(tsdbDataFileDoWriteTombBlock(writer), &lino, _exit);
}
} else {
ASSERT(0);
tsdbError("vgId:%d duplicate tomb record, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64 ", version:%" PRId64,
TD_VID(writer->config->tsdb->pVnode), writer->config->cid, record->suid, record->uid,
record->version);
}
}
@ -1566,7 +1580,9 @@ _exit:
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char *encryptKey) {
ASSERT(TARRAY2_SIZE(brinBlkArray) > 0);
if (TARRAY2_SIZE(brinBlkArray) <= 0) {
return TSDB_CODE_INVALID_PARA;
}
ptr->offset = *fileSize;
ptr->size = TARRAY2_DATA_LEN(brinBlkArray);
@ -1852,7 +1868,9 @@ int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData) {
int32_t code = 0;
int32_t lino = 0;
ASSERT(bData->uid);
if (!bData->uid) {
return TSDB_CODE_INVALID_PARA;
}
if (!writer->ctx->opened) {
TAOS_CHECK_GOTO(tsdbDataFileWriterDoOpen(writer), &lino, _exit);
@ -1895,7 +1913,9 @@ _exit:
}
int32_t tsdbDataFileFlush(SDataFileWriter *writer) {
ASSERT(writer->ctx->opened);
if (!writer->ctx->opened) {
return TSDB_CODE_INVALID_PARA;
}
if (writer->blockData->nRow == 0) return 0;
if (writer->ctx->tbHasOldData) return 0;
@ -1910,8 +1930,6 @@ static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) {
char fname[TSDB_FILENAME_LEN];
int32_t ftype = TSDB_FTYPE_TOMB;
ASSERT(writer->files[ftype].size == 0);
int32_t flag = (TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
int32_t lcn = writer->files[ftype].lcn;

25
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -83,7 +83,10 @@ static int32_t tsdbBinaryToFS(uint8_t *pData, int64_t nData, STsdbFS *pFS) {
}
}
ASSERT(n + sizeof(TSCKSUM) == nData);
if (n + sizeof(TSCKSUM) != nData) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _exit;
}
_exit:
return code;
@ -450,8 +453,9 @@ static int32_t tsdbMergeFileSet(STsdb *pTsdb, SDFileSet *pSetOld, SDFileSet *pSe
taosMemoryFree(pHeadF);
}
} else {
ASSERT(pHeadF->offset == pSetNew->pHeadF->offset);
ASSERT(pHeadF->size == pSetNew->pHeadF->size);
if (pHeadF->offset != pSetNew->pHeadF->offset || pHeadF->size != pSetNew->pHeadF->size) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
}
// data
@ -499,7 +503,6 @@ static int32_t tsdbMergeFileSet(STsdb *pTsdb, SDFileSet *pSetOld, SDFileSet *pSe
// stt
if (sameDisk) {
if (pSetNew->nSttF > pSetOld->nSttF) {
ASSERT(pSetNew->nSttF == pSetOld->nSttF + 1);
pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
if (pSetOld->aSttF[pSetOld->nSttF] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@ -509,7 +512,6 @@ static int32_t tsdbMergeFileSet(STsdb *pTsdb, SDFileSet *pSetOld, SDFileSet *pSe
pSetOld->aSttF[pSetOld->nSttF]->nRef = 1;
pSetOld->nSttF++;
} else if (pSetNew->nSttF < pSetOld->nSttF) {
ASSERT(pSetNew->nSttF == 1);
for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) {
SSttFile *pSttFile = pSetOld->aSttF[iStt];
nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1);
@ -548,8 +550,10 @@ static int32_t tsdbMergeFileSet(STsdb *pTsdb, SDFileSet *pSetOld, SDFileSet *pSe
*pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt];
pSetOld->aSttF[iStt]->nRef = 1;
} else {
ASSERT(pSetOld->aSttF[iStt]->size == pSetOld->aSttF[iStt]->size);
ASSERT(pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset);
if (pSetOld->aSttF[iStt]->size != pSetOld->aSttF[iStt]->size ||
pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
}
}
}
@ -777,8 +781,6 @@ int32_t tsdbFSOpen(STsdb *pTsdb, int8_t rollback) {
// empty one
code = tsdbSaveFSToFile(&pTsdb->fs, current);
TSDB_CHECK_CODE(code, lino, _exit);
ASSERT(!rollback);
}
// scan and fix FS
@ -796,7 +798,6 @@ int32_t tsdbFSClose(STsdb *pTsdb) {
int32_t code = 0;
if (pTsdb->fs.pDelFile) {
ASSERT(pTsdb->fs.pDelFile->nRef == 1);
taosMemoryFree(pTsdb->fs.pDelFile);
}
@ -804,20 +805,16 @@ int32_t tsdbFSClose(STsdb *pTsdb) {
SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet);
// head
ASSERT(pSet->pHeadF->nRef == 1);
taosMemoryFree(pSet->pHeadF);
// data
ASSERT(pSet->pDataF->nRef == 1);
taosMemoryFree(pSet->pDataF);
// sma
ASSERT(pSet->pSmaF->nRef == 1);
taosMemoryFree(pSet->pSmaF);
// stt
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
ASSERT(pSet->aSttF[iStt]->nRef == 1);
taosMemoryFree(pSet->aSttF[iStt]);
}
}

8
source/dnode/vnode/src/tsdb/tsdbFS2.c
View File

@ -208,7 +208,9 @@ static int32_t load_fs(STsdb *pTsdb, const char *fname, TFileSetArray *arr) {
/* fmtv */
item1 = cJSON_GetObjectItem(json, "fmtv");
if (cJSON_IsNumber(item1)) {
ASSERT(item1->valuedouble == 1);
if (item1->valuedouble != 1) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
} else {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
@ -892,7 +894,6 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
// commit
code = commit_edit(fs);
ASSERT(code == 0);
TSDB_CHECK_CODE(code, lino, _exit);
// schedule merge
@ -1050,7 +1051,6 @@ static SHashObj *tsdbFSetRangeArrayToHash(TFileSetRangeArray *pRanges) {
STFileSetRange *u = TARRAY2_GET(pRanges, i);
int32_t fid = u->fid;
int32_t code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u));
ASSERT(code == 0);
tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever);
}
return pHash;
@ -1190,10 +1190,8 @@ int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset) {
(void)taosThreadCondWait(&(*fset)->beginTask, &tsdb->mutex);
(void)tsdbFSGetFSet(tsdb->pFS, fid, fset);
ASSERT(fset != NULL);
(*fset)->numWaitTask--;
ASSERT((*fset)->numWaitTask >= 0);
} else {
(*fset)->taskRunning = true;
break;

13
source/dnode/vnode/src/tsdb/tsdbFSet2.c
View File

@ -106,7 +106,9 @@ static void tsdbSttLvlRemove(SSttLvl **lvl) {
static int32_t tsdbSttLvlApplyEdit(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl *lvl2) {
int32_t code = 0;
ASSERT(lvl1->level == lvl2->level);
if (lvl1->level != lvl2->level) {
return TSDB_CODE_INVALID_PARA;
}
int32_t i1 = 0, i2 = 0;
while (i1 < TARRAY2_SIZE(lvl1->fobjArr) || i2 < TARRAY2_SIZE(lvl2->fobjArr)) {
@ -331,29 +333,24 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) {
code = TARRAY2_SORT_INSERT(lvl->fobjArr, fobj, tsdbTFileObjCmpr);
if (code) return code;
} else {
ASSERT(fset->farr[fobj->f->type] == NULL);
fset->farr[fobj->f->type] = fobj;
}
} else if (op->optype == TSDB_FOP_REMOVE) {
// delete a file
if (op->of.type == TSDB_FTYPE_STT) {
SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, op->of.stt->level);
ASSERT(lvl);
STFileObj tfobj = {.f[0] = {.cid = op->of.cid}};
STFileObj *tfobjp = &tfobj;
int32_t idx = TARRAY2_SEARCH_IDX(lvl->fobjArr, &tfobjp, tsdbTFileObjCmpr, TD_EQ);
ASSERT(idx >= 0);
TARRAY2_REMOVE(lvl->fobjArr, idx, tsdbSttLvlClearFObj);
} else {
ASSERT(tsdbIsSameTFile(&op->of, fset->farr[op->of.type]->f));
(void)tsdbTFileObjUnref(fset->farr[op->of.type]);
fset->farr[op->of.type] = NULL;
}
} else {
if (op->nf.type == TSDB_FTYPE_STT) {
SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, op->of.stt->level);
ASSERT(lvl);
STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}, *tfobjp = &tfobj;
STFileObj **fobjPtr = TARRAY2_SEARCH(lvl->fobjArr, &tfobjp, tsdbTFileObjCmpr, TD_EQ);
@ -374,7 +371,9 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) {
int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *fset2) {
int32_t code = 0;
ASSERT(fset1->fid == fset2->fid);
if (fset1->fid != fset2->fid) {
return TSDB_CODE_INVALID_PARA;
}
for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
if (!fset1->farr[ftype] && !fset2->farr[ftype]) continue;

7
source/dnode/vnode/src/tsdb/tsdbFSetRAW.c
View File

@ -154,13 +154,11 @@ _exit:
return code;
}
int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm,
char *encryptKey) {
int32_t code = 0;
int32_t lino = 0;
ASSERT(writer->ctx->offset >= 0 && writer->ctx->offset <= writer->ctx->file.size);
if (writer->ctx->offset == writer->ctx->file.size) {
code = tsdbFSetRAWWriteFileDataEnd(writer);
TSDB_CHECK_CODE(code, lino, _exit);
@ -173,7 +171,6 @@ int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeade
TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->offset += bHdr->dataLength;
ASSERT(writer->ctx->offset == writer->dataWriter->ctx->offset);
_exit:
if (code) {

2
source/dnode/vnode/src/tsdb/tsdbFSetRW.c
View File

@ -73,8 +73,6 @@ static int32_t tsdbFSetWriteTableDataEnd(SFSetWriter *writer) {
int32_t numRow = ((writer->blockData[pidx].nRow + writer->blockData[cidx].nRow) >> 1);
if (writer->blockData[pidx].nRow > 0 && numRow >= writer->config->minRow) {
ASSERT(writer->blockData[pidx].nRow == writer->config->maxRow);
SRowInfo row = {
.suid = writer->ctx->tbid->suid,
.uid = writer->ctx->tbid->uid,

37
source/dnode/vnode/src/tsdb/tsdbFile2.c
View File

@ -242,19 +242,30 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
int32_t tsdbTFileObjRef(STFileObj *fobj) {
int32_t nRef;
(void)(void)taosThreadMutexLock(&fobj->mutex);
ASSERT(fobj->ref > 0 && fobj->state == TSDB_FSTATE_LIVE);
(void)taosThreadMutexLock(&fobj->mutex);
if (fobj->ref <= 0 || fobj->state != TSDB_FSTATE_LIVE) {
(void)taosThreadMutexUnlock(&fobj->mutex);
tsdbError("file %s, fobj:%p ref %d", fobj->fname, fobj, fobj->ref);
return TSDB_CODE_FAILED;
}
nRef = ++fobj->ref;
(void)(void)taosThreadMutexUnlock(&fobj->mutex);
(void)taosThreadMutexUnlock(&fobj->mutex);
tsdbTrace("ref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
return 0;
}
int32_t tsdbTFileObjUnref(STFileObj *fobj) {
(void)(void)taosThreadMutexLock(&fobj->mutex);
(void)taosThreadMutexLock(&fobj->mutex);
int32_t nRef = --fobj->ref;
(void)(void)taosThreadMutexUnlock(&fobj->mutex);
ASSERT(nRef >= 0);
(void)taosThreadMutexUnlock(&fobj->mutex);
if (nRef < 0) {
tsdbError("file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
return TSDB_CODE_FAILED;
}
tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
if (nRef == 0) {
if (fobj->state == TSDB_FSTATE_DEAD) {
@ -319,7 +330,11 @@ static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) {
int32_t tsdbTFileObjRemove(STFileObj *fobj) {
(void)taosThreadMutexLock(&fobj->mutex);
ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0);
if (fobj->state != TSDB_FSTATE_LIVE || fobj->ref <= 0) {
(void)taosThreadMutexUnlock(&fobj->mutex);
tsdbError("file %s, fobj:%p ref %d", fobj->fname, fobj, fobj->ref);
return TSDB_CODE_FAILED;
}
fobj->state = TSDB_FSTATE_DEAD;
int32_t nRef = --fobj->ref;
(void)taosThreadMutexUnlock(&fobj->mutex);
@ -333,7 +348,13 @@ int32_t tsdbTFileObjRemove(STFileObj *fobj) {
int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj) {
(void)taosThreadMutexLock(&fobj->mutex);
ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0);
if (fobj->state != TSDB_FSTATE_LIVE || fobj->ref <= 0) {
(void)taosThreadMutexUnlock(&fobj->mutex);
tsdbError("file %s, fobj:%p ref %d", fobj->fname, fobj, fobj->ref);
return TSDB_CODE_FAILED;
}
fobj->state = TSDB_FSTATE_DEAD;
int32_t nRef = --fobj->ref;
(void)taosThreadMutexUnlock(&fobj->mutex);

28
source/dnode/vnode/src/tsdb/tsdbIter.c
View File

@ -547,8 +547,7 @@ int32_t tsdbIterOpen(const STsdbIterConfig *config, STsdbIter **iter) {
code = tsdbMemTombIterOpen(iter[0]);
break;
default:
code = TSDB_CODE_INVALID_PARA;
ASSERTS(false, "Not implemented");
return TSDB_CODE_INVALID_PARA;
}
if (code) {
@ -588,7 +587,7 @@ int32_t tsdbIterClose(STsdbIter **iter) {
case TSDB_ITER_TYPE_MEMT_TOMB:
break;
default:
ASSERT(false);
return TSDB_CODE_INVALID_PARA;
}
taosMemoryFree(iter[0]);
iter[0] = NULL;
@ -610,7 +609,7 @@ int32_t tsdbIterNext(STsdbIter *iter) {
case TSDB_ITER_TYPE_MEMT_TOMB:
return tsdbMemTombIterNext(iter, NULL);
default:
ASSERT(false);
return TSDB_CODE_INVALID_PARA;
}
return 0;
}
@ -630,7 +629,7 @@ static int32_t tsdbIterSkipTableData(STsdbIter *iter, const TABLEID *tbid) {
case TSDB_ITER_TYPE_MEMT_TOMB:
return tsdbMemTombIterNext(iter, tbid);
default:
ASSERT(false);
return TSDB_CODE_INVALID_PARA;
}
return 0;
}
@ -691,7 +690,10 @@ int32_t tsdbIterMergerOpen(const TTsdbIterArray *iterArray, SIterMerger **merger
TARRAY2_FOREACH(iterArray, iter) {
if (iter->noMoreData) continue;
node = tRBTreePut(merger[0]->iterTree, iter->node);
ASSERT(node);
if (node == NULL) {
taosMemoryFree(merger[0]);
return TSDB_CODE_INVALID_PARA;
}
}
return tsdbIterMergerNext(merger[0]);
@ -718,10 +720,8 @@ int32_t tsdbIterMergerNext(SIterMerger *merger) {
merger->iter = NULL;
} else if ((node = tRBTreeMin(merger->iterTree))) {
c = merger->iterTree->cmprFn(merger->iter->node, node);
ASSERT(c);
if (c > 0) {
node = tRBTreePut(merger->iterTree, merger->iter->node);
ASSERT(node);
merger->iter = NULL;
}
}
@ -734,15 +734,9 @@ int32_t tsdbIterMergerNext(SIterMerger *merger) {
return 0;
}
SRowInfo *tsdbIterMergerGetData(SIterMerger *merger) {
ASSERT(!merger->isTomb);
return merger->iter ? merger->iter->row : NULL;
}
SRowInfo *tsdbIterMergerGetData(SIterMerger *merger) { return merger->iter ? merger->iter->row : NULL; }
STombRecord *tsdbIterMergerGetTombRecord(SIterMerger *merger) {
ASSERT(merger->isTomb);
return merger->iter ? merger->iter->record : NULL;
}
STombRecord *tsdbIterMergerGetTombRecord(SIterMerger *merger) { return merger->iter ? merger->iter->record : NULL; }
int32_t tsdbIterMergerSkipTableData(SIterMerger *merger, const TABLEID *tbid) {
int32_t code;
@ -757,10 +751,8 @@ int32_t tsdbIterMergerSkipTableData(SIterMerger *merger, const TABLEID *tbid) {
merger->iter = NULL;
} else if ((node = tRBTreeMin(merger->iterTree))) {
c = merger->iterTree->cmprFn(merger->iter->node, node);
ASSERT(c);
if (c > 0) {
node = tRBTreePut(merger->iterTree, merger->iter->node);
ASSERT(node);
merger->iter = NULL;
}
}

17
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -170,7 +170,6 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
goto _err;
}
ASSERT(pPool != NULL);
// do delete
SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData));
if (pDelData == NULL) {
@ -183,7 +182,6 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
pDelData->pNext = NULL;
taosWLockLatch(&pTbData->lock);
if (pTbData->pHead == NULL) {
ASSERT(pTbData->pTail == NULL);
pTbData->pHead = pTbData->pTail = pDelData;
} else {
pTbData->pTail->pNext = pDelData;
@ -263,8 +261,6 @@ void tsdbTbDataIterOpen(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, S
bool tsdbTbDataIterNext(STbDataIter *pIter) {
pIter->pRow = NULL;
if (pIter->backward) {
ASSERT(pIter->pNode != pIter->pTbData->sl.pTail);
if (pIter->pNode == pIter->pTbData->sl.pHead) {
return false;
}
@ -274,8 +270,6 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) {
return false;
}
} else {
ASSERT(pIter->pNode != pIter->pTbData->sl.pHead);
if (pIter->pNode == pIter->pTbData->sl.pTail) {
return false;
}
@ -366,7 +360,6 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel;
ASSERT(pPool != NULL);
pTbData = vnodeBufPoolMallocAligned(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2);
if (pTbData == NULL) {
code = terrno;
@ -516,8 +509,6 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN
pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize + pRow->pTSRow->len);
} else if (pRow->type == TSDBROW_COL_FMT) {
pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize);
} else {
ASSERT(0);
}
if (pNode == NULL) {
code = terrno;
@ -582,10 +573,6 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
int32_t nColData = TARRAY_SIZE(pSubmitTbData->aCol);
SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol);
ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID);
ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP);
ASSERT(aColData[0].flag == HAS_VALUE);
// copy and construct block data
SBlockData *pBlockData = vnodeBufPoolMalloc(pPool, sizeof(*pBlockData));
if (pBlockData == NULL) {
@ -740,7 +727,9 @@ int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode) {
int32_t code = 0;
int32_t nRef = atomic_fetch_add_32(&pMemTable->nRef, 1);
ASSERT(nRef > 0);
if (nRef <= 0) {
tsdbError("vgId:%d, memtable ref count is invalid, ref:%d", TD_VID(pMemTable->pTsdb->pVnode), nRef);
}
(void)vnodeBufPoolRegisterQuery(pMemTable->pPool, pQNode);

16
source/dnode/vnode/src/tsdb/tsdbMerge.c
View File

@ -69,13 +69,6 @@ static int32_t tsdbMergerClose(SMerger *merger) {
int32_t lino = 0;
SVnode *pVnode = merger->tsdb->pVnode;
ASSERT(merger->writer == NULL);
ASSERT(merger->dataIterMerger == NULL);
ASSERT(merger->tombIterMerger == NULL);
ASSERT(TARRAY2_SIZE(merger->dataIterArr) == 0);
ASSERT(TARRAY2_SIZE(merger->tombIterArr) == 0);
ASSERT(TARRAY2_SIZE(merger->sttReaderArr) == 0);
// clear the merge
TARRAY2_DESTROY(merger->tombIterArr, NULL);
TARRAY2_DESTROY(merger->dataIterArr, NULL);
@ -156,8 +149,6 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
}
}
ASSERT(merger->ctx->level > 0);
if (merger->ctx->level <= TSDB_MAX_LEVEL) {
TARRAY2_FOREACH_REVERSE(merger->ctx->fset->lvlArr, lvl) {
if (TARRAY2_SIZE(lvl->fobjArr) == 0) {
@ -180,8 +171,6 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
numFile = numFile - TARRAY2_SIZE(lvl->fobjArr) * pow(merger->sttTrigger, lvl->level);
}
ASSERT(numFile >= 0);
// get file system operations
TARRAY2_FOREACH(merger->ctx->fset->lvlArr, lvl) {
if (lvl->level >= merger->ctx->level) {
@ -323,11 +312,6 @@ static int32_t tsdbMergeFileSetBegin(SMerger *merger) {
int32_t code = 0;
int32_t lino = 0;
ASSERT(TARRAY2_SIZE(merger->sttReaderArr) == 0);
ASSERT(TARRAY2_SIZE(merger->dataIterArr) == 0);
ASSERT(merger->dataIterMerger == NULL);
ASSERT(merger->writer == NULL);
TARRAY2_CLEAR(merger->fopArr, NULL);
merger->ctx->tbid->suid = 0;

44
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -115,16 +115,14 @@ void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoad
SArray *pList = taosArrayGetP(pLDataIterArray, i);
for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) {
SLDataIter *pIter = taosArrayGetP(pList, j);
if (pIter->pBlockLoadInfo == NULL) {
continue;
}
SSttBlockLoadCostInfo *pCost = &pIter->pBlockLoadInfo->cost;
if (pLoadCost != NULL) {
pLoadCost->loadBlocks += pCost->loadBlocks;
pLoadCost->loadStatisBlocks += pCost->loadStatisBlocks;
pLoadCost->blockElapsedTime += pCost->blockElapsedTime;
pLoadCost->statisElapsedTime += pCost->statisElapsedTime;
if (pIter->pBlockLoadInfo != NULL) {
SSttBlockLoadCostInfo *pCost = &pIter->pBlockLoadInfo->cost;
if (pLoadCost != NULL) {
pLoadCost->loadBlocks += pCost->loadBlocks;
pLoadCost->loadStatisBlocks += pCost->loadStatisBlocks;
pLoadCost->blockElapsedTime += pCost->blockElapsedTime;
pLoadCost->statisElapsedTime += pCost->statisElapsedTime;
}
}
destroyLDataIter(pIter);
@ -903,6 +901,10 @@ int32_t tLDataIterNextRow(SLDataIter *pIter, const char *idStr, bool* hasNext) {
pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow);
_exit:
if (code) {
tsdbError("failed to exec stt-file nextIter, lino:%d, code:%s, %s", lino, tstrerror(code), idStr);
}
*hasNext = (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL) && (pBlockData != NULL);
return code;
}
@ -1102,13 +1104,22 @@ void tMergeTreeUnpinSttBlock(SMergeTree *pMTree) {
tLDataIterUnpinSttBlock(pIter, pMTree->idStr);
}
bool tMergeTreeNext(SMergeTree *pMTree) {
int32_t tMergeTreeNext(SMergeTree *pMTree, bool *pHasNext) {
int32_t code = 0;
if (pHasNext == NULL) {
return TSDB_CODE_INVALID_PARA;
}
*pHasNext = false;
if (pMTree->pIter) {
SLDataIter *pIter = pMTree->pIter;
bool hasVal = false;
int32_t code = tLDataIterNextRow(pIter, pMTree->idStr, &hasVal);
bool hasVal = false;
code = tLDataIterNextRow(pIter, pMTree->idStr, &hasVal);
if (!hasVal || (code != 0)) {
if (code == TSDB_CODE_FILE_CORRUPTED) {
code = 0; // suppress the file corrupt error to enable all queries within this cluster can run without failed.
}
pMTree->pIter = NULL;
}
@ -1117,7 +1128,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) {
if (pMTree->pIter && pIter) {
int32_t c = pMTree->rbt.cmprFn(&pMTree->pIter->node, &pIter->node);
if (c > 0) {
(void) tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter);
(void)tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter);
pMTree->pIter = NULL;
} else {
ASSERT(c);
@ -1132,7 +1143,8 @@ bool tMergeTreeNext(SMergeTree *pMTree) {
}
}
return pMTree->pIter != NULL;
*pHasNext = (pMTree->pIter != NULL);
return code;
}
void tMergeTreeClose(SMergeTree *pMTree) {

92
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -1383,7 +1383,6 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro
static FORCE_INLINE STSchema* getTableSchemaImpl(STsdbReader* pReader, uint64_t uid) {
ASSERT(pReader->info.pSchema == NULL);
int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, -1, &pReader->info.pSchema);
if (code != TSDB_CODE_SUCCESS || pReader->info.pSchema == NULL) {
terrno = code;
@ -1414,7 +1413,9 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
if (pReader->info.pSchema == NULL) {
pSchema = getTableSchemaImpl(pReader, uid);
if (pSchema == NULL) {
tsdbDebug("%p table uid:%" PRIu64 " has been dropped, no data existed, %s", pReader, uid, pReader->idStr);
code = terrno;
tsdbError("%p table uid:%" PRIu64 " failed to get tableschema, code:%s, %s", pReader, uid, tstrerror(code),
pReader->idStr);
return code;
}
}
@ -1449,7 +1450,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
pReader->cost.blockLoadTime += elapsedTime;
pDumpInfo->allDumped = false;
return TSDB_CODE_SUCCESS;
return code;
}
/**
@ -1759,14 +1760,22 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB
return code;
}
static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int32_t pkSrcSlot,
SVersionRange* pVerRange) {
static int32_t nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int32_t pkSrcSlot,
SVersionRange* pVerRange) {
int32_t code = 0;
int32_t order = pSttBlockReader->order;
int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1;
SRowKey* pNextProc = &pScanInfo->sttKeyInfo.nextProcKey;
while (1) {
bool hasVal = tMergeTreeNext(&pSttBlockReader->mergeTree);
bool hasVal = false;
code = tMergeTreeNext(&pSttBlockReader->mergeTree, &hasVal);
if (code) {
tsdbError("failed to iter the next row in stt-file merge tree, code:%s, %s", tstrerror(code),
pSttBlockReader->mergeTree.idStr);
return code;
}
if (!hasVal) { // the next value will be the accessed key in stt
pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;
@ -1779,7 +1788,7 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc
memset(pNextProc->pks[0].pData, 0, pNextProc->pks[0].nData);
}
}
return false;
return code;
}
TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
@ -1798,13 +1807,15 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc
if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange,
pSttBlockReader->numOfPks > 0)) {
pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA;
return true;
return code;
}
} else {
pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA;
return true;
return code;
}
}
return code;
}
static void doPinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreePinSttBlock(&pSttBlockReader->mergeTree); }
@ -1819,9 +1830,14 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttB
// avoid the fetch next row replace the referenced stt block in buffer
doPinSttBlock(pSttBlockReader);
bool hasVal = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);
code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);
doUnpinSttBlock(pSttBlockReader);
if (hasVal) {
if (code) {
return code;
}
if (hasDataInSttBlock(pScanInfo)) {
SRowKey* pNext = getCurrentKeyInSttBlock(pSttBlockReader);
if (pkCompEx(pSttKey, pNext) != 0) {
code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow);
@ -2125,7 +2141,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
if (piRow->type == TSDBROW_ROW_FMT) {
piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
if (piSchema == NULL) {
return code;
return terrno;
}
}
@ -2380,14 +2396,13 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STable
return true;
}
static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
bool hasData = true;
static void initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
int32_t order = pReader->info.order;
bool asc = ASCENDING_TRAVERSE(order);
// the stt block reader has been initialized for this table.
if (pSttBlockReader->uid == pScanInfo->uid) {
return hasDataInSttBlock(pScanInfo);
return;
}
if (pSttBlockReader->uid != 0) {
@ -2396,9 +2411,14 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
pSttBlockReader->uid = pScanInfo->uid;
// second time init stt block reader
// second or third time init stt block reader
if (pScanInfo->cleanSttBlocks && (pReader->info.execMode == READER_EXEC_ROWS)) {
return !pScanInfo->sttBlockReturned;
// only allowed to retrieve clean stt blocks for count once
if (pScanInfo->sttBlockReturned) {
pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;
tsdbDebug("uid:%" PRIu64 " set no stt-file data after stt-block retrieved, %s", pScanInfo->uid, pReader->idStr);
}
return;
}
STimeWindow w = pSttBlockReader->window;
@ -2435,28 +2455,28 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
SSttDataInfoForTable info = {.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange))};
if (info.pKeyRangeList == NULL) {
pReader->code = terrno;
return false;
return;
}
int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(info.pKeyRangeList);
pReader->code = code;
return false;
return;
}
code = initMemDataIterator(pScanInfo, pReader);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(info.pKeyRangeList);
pReader->code = code;
return false;
return;
}
code = initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(info.pKeyRangeList);
pReader->code = code;
return false;
return;
}
if (conf.rspRows) {
@ -2484,27 +2504,26 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
SRowKey* p = asc ? &pScanInfo->sttRange.skey : &pScanInfo->sttRange.ekey;
tRowKeyAssign(&pScanInfo->sttKeyInfo.nextProcKey, p);
hasData = (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA);
} else { // not clean stt blocks
INIT_KEYRANGE(&pScanInfo->sttRange); // reset the time window
pScanInfo->sttBlockReturned = false;
hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);
code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);
}
} else {
pScanInfo->cleanSttBlocks = false;
INIT_KEYRANGE(&pScanInfo->sttRange); // reset the time window
pScanInfo->sttBlockReturned = false;
hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);
code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);
}
pScanInfo->sttBlockReturned = false;
taosArrayDestroy(info.pKeyRangeList);
int64_t el = taosGetTimestampUs() - st;
pReader->cost.initSttBlockReader += (el / 1000.0);
tsdbDebug("init stt block reader completed, elapsed time:%" PRId64 "us %s", el, pReader->idStr);
return hasData;
if (code != 0) {
pReader->code = code;
}
}
static bool hasDataInSttBlock(STableBlockScanInfo* pInfo) { return pInfo->sttKeyInfo.status == STT_FILE_HAS_DATA; }
@ -2772,7 +2791,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
}
SBlockData* pBlockData = &pReader->status.fileBlockData;
(void) initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader);
initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader);
if (pReader->code != 0) {
code = pReader->code;
goto _end;
@ -3190,12 +3209,12 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) {
continue;
}
bool hasDataInSttFile = initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
if (pReader->code != TSDB_CODE_SUCCESS) {
return pReader->code;
}
if (!hasDataInSttFile) {
if (!hasDataInSttBlock(pScanInfo)) {
bool hasNexTable = moveToNextTable(pUidList, pStatus);
if (!hasNexTable) {
return TSDB_CODE_SUCCESS;
@ -3287,7 +3306,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
}
if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) {
(void) initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
if (pReader->code != 0) {
return pReader->code;
}
@ -3331,7 +3350,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
int64_t st = taosGetTimestampUs();
// let's load data from stt files, make sure clear the cleanStt block flag before load the data from stt files
(void) initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
if (pReader->code != 0) {
return pReader->code;
}
@ -4087,7 +4106,12 @@ int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanI
SRowKey* pRowKey = &pScanInfo->lastProcKey;
int32_t code = TSDB_CODE_SUCCESS;
while (nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pkSrcSlot, pVerRange)) {
while (1) {
code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pkSrcSlot, pVerRange);
if (code || (!hasDataInSttBlock(pScanInfo))) {
return code;
}
SRowKey* pNextKey = getCurrentKeyInSttBlock(pSttBlockReader);
int32_t ret = pkCompEx(pRowKey, pNextKey);

4
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -194,7 +194,7 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
break;
}
default:
ASSERT(0);
return TSDB_CODE_INVALID_PARA;
}
} else {
switch (type) {
@ -223,7 +223,7 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
pKey->pks[0].val = UINT8_MAX;
break;
default:
ASSERT(0);
return TSDB_CODE_INVALID_PARA;
}
}
} else {

36
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -77,7 +77,9 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
}
}
ASSERT(pFD->szFile % szPage == 0);
if (pFD->szFile % szPage != 0) {
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_PARA, lino, _exit);
}
pFD->szFile = pFD->szFile / szPage;
_exit:
@ -149,7 +151,6 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e
offset -= chunkoffset;
}
ASSERT(offset >= 0);
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
if (n < 0) {
@ -202,7 +203,6 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
int32_t code = 0;
int32_t lino;
// ASSERT(pgno <= pFD->szFile);
if (!pFD->pFD) {
code = tsdbOpenFileImpl(pFD);
TSDB_CHECK_CODE(code, lino, _exit);
@ -216,7 +216,6 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
offset -= chunkoffset;
}
ASSERT(offset >= 0);
// seek
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
@ -317,8 +316,9 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6
int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage);
int64_t bOffset = fOffset % pFD->szPage;
// ASSERT(pgno && pgno <= pFD->szFile);
ASSERT(bOffset < szPgCont);
if (bOffset >= szPgCont) {
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_PARA, lino, _exit);
}
while (n < size) {
if (pFD->pgno != pgno) {
@ -417,7 +417,9 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
int64_t bOffset = fOffset % pFD->szPage;
ASSERT(bOffset < szPgCont);
if (bOffset >= szPgCont) {
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_PARA, lino, _exit);
}
// 1, find pgnoStart & pgnoEnd to fetch from s3, if all pgs are local, no need to fetch
// 2, fetch pgnoStart ~ pgnoEnd from s3
@ -690,7 +692,9 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
}
ASSERT(n == size);
if (n != size) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
_exit:
if (code) {
@ -731,7 +735,9 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
}
ASSERT(n == size);
if (n != size) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
_exit:
if (code) {
@ -760,7 +766,9 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m
int32_t n;
code = tGetMapData(pReader->aBuf[0], mDataBlk, &n);
if (code) goto _exit;
ASSERT(n == size);
if (n != size) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
_exit:
if (code) {
@ -861,7 +869,9 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa
}
}
ASSERT(n == size);
if (n != size) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
_exit:
if (code) {
@ -901,7 +911,9 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
}
}
ASSERT(n == size);
if (n != size) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
_exit:
if (code) {

2
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -403,8 +403,6 @@ static int32_t tsdbS3FidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int32_t s3Kee
nowSec = nowSec * 1000000l;
} else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) {
nowSec = nowSec * 1000000000l;
} else {
ASSERT(0);
}
nowSec = nowSec - pKeepCfg->keepTimeOffset * tsTickPerHour[pKeepCfg->precision];

23
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -67,9 +67,6 @@ static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) {
int32_t code = 0;
int32_t lino = 0;
ASSERT(reader->dataReader == NULL);
ASSERT(TARRAY2_SIZE(reader->sttReaderArr) == 0);
// data
SDataFileReaderConfig config = {
.tsdb = reader->tsdb,
@ -125,11 +122,6 @@ static int32_t tsdbSnapReadFileSetOpenIter(STsdbSnapReader* reader) {
int32_t code = 0;
int32_t lino = 0;
ASSERT(reader->dataIterMerger == NULL);
ASSERT(reader->tombIterMerger == NULL);
ASSERT(TARRAY2_SIZE(reader->dataIterArr) == 0);
ASSERT(TARRAY2_SIZE(reader->tombIterArr) == 0);
STsdbIter* iter;
STsdbIterConfig config = {
.filterByVersion = true,
@ -210,8 +202,6 @@ static int32_t tsdbSnapReadRangeBegin(STsdbSnapReader* reader) {
int32_t code = 0;
int32_t lino = 0;
ASSERT(reader->ctx->fsr == NULL);
if (reader->ctx->fsrArrIdx < TARRAY2_SIZE(reader->fsrArr)) {
reader->ctx->fsr = TARRAY2_GET(reader->fsrArr, reader->ctx->fsrArrIdx++);
reader->ctx->isDataDone = false;
@ -335,7 +325,6 @@ static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** dat
}
if (reader->blockData->nRow > 0) {
ASSERT(reader->blockData->suid || reader->blockData->uid);
code = tsdbSnapCmprData(reader, data);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -616,7 +605,6 @@ static int32_t tsdbSnapWriteTimeSeriesRow(STsdbSnapWriter* writer, SRowInfo* row
}
if (row->suid == INT64_MAX) {
ASSERT(writer->ctx->hasData == false);
goto _exit;
}
@ -634,9 +622,6 @@ static int32_t tsdbSnapWriteFileSetOpenReader(STsdbSnapWriter* writer) {
int32_t code = 0;
int32_t lino = 0;
ASSERT(writer->ctx->dataReader == NULL);
ASSERT(TARRAY2_SIZE(writer->ctx->sttReaderArr) == 0);
if (writer->ctx->fset) {
// open data reader
SDataFileReaderConfig dataFileReaderConfig = {
@ -825,8 +810,6 @@ static int32_t tsdbSnapWriteFileSetBegin(STsdbSnapWriter* writer, int32_t fid) {
int32_t code = 0;
int32_t lino = 0;
ASSERT(writer->ctx->fsetWriteBegin == false);
STFileSet* fset = &(STFileSet){.fid = fid};
writer->ctx->fid = fid;
@ -885,7 +868,6 @@ static int32_t tsdbSnapWriteTombRecord(STsdbSnapWriter* writer, const STombRecor
}
if (record->suid == INT64_MAX) {
ASSERT(writer->ctx->hasTomb == false);
goto _exit;
}
@ -996,7 +978,6 @@ static int32_t tsdbSnapWriteDecmprTombBlock(SSnapDataHdr* hdr, STombBlock* tombB
TAOS_UNUSED(tTombBlockClear(tombBlock));
int64_t size = hdr->size;
ASSERT(size % TOMB_RECORD_ELEM_NUM == 0);
size = size / TOMB_RECORD_ELEM_NUM;
tombBlock->numOfRecords = size / sizeof(int64_t);
@ -1043,8 +1024,6 @@ static int32_t tsdbSnapWriteTombData(STsdbSnapWriter* writer, SSnapDataHdr* hdr)
TSDB_CHECK_CODE(code, lino, _exit);
}
ASSERT(writer->ctx->hasData == false);
for (int32_t i = 0; i < TOMB_BLOCK_SIZE(tombBlock); ++i) {
code = tTombBlockGet(tombBlock, i, &record);
TSDB_CHECK_CODE(code, lino, _exit);
@ -1177,7 +1156,7 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* writer, SSnapDataHdr* hdr) {
code = tsdbSnapWriteTombData(writer, hdr);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
ASSERT(0);
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_PARA, lino, _exit);
}
_exit:

36
source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
View File

@ -61,7 +61,9 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
// // open each segment reader
int64_t offset = config->file->size - sizeof(SSttFooter);
ASSERT(offset >= TSDB_FHDR_SIZE);
if (offset < TSDB_FHDR_SIZE) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
int32_t encryptAlgoirthm = config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey;
@ -115,7 +117,9 @@ int32_t tsdbSttFileReaderClose(SSttFileReader **reader) {
int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray **statisBlkArray) {
if (!reader->ctx->statisBlkLoaded) {
if (reader->footer->statisBlkPtr->size > 0) {
ASSERT(reader->footer->statisBlkPtr->size % sizeof(SStatisBlk) == 0);
if (reader->footer->statisBlkPtr->size % sizeof(SStatisBlk) != 0) {
return TSDB_CODE_FILE_CORRUPTED;
}
int32_t size = reader->footer->statisBlkPtr->size / sizeof(SStatisBlk);
void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size);
@ -147,7 +151,9 @@ int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray *
int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tombBlkArray) {
if (!reader->ctx->tombBlkLoaded) {
if (reader->footer->tombBlkPtr->size > 0) {
ASSERT(reader->footer->tombBlkPtr->size % sizeof(STombBlk) == 0);
if (reader->footer->tombBlkPtr->size % sizeof(STombBlk) != 0) {
return TSDB_CODE_FILE_CORRUPTED;
}
int32_t size = reader->footer->tombBlkPtr->size / sizeof(STombBlk);
void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size);
@ -179,7 +185,9 @@ int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tom
int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBlkArray) {
if (!reader->ctx->sttBlkLoaded) {
if (reader->footer->sttBlkPtr->size > 0) {
ASSERT(reader->footer->sttBlkPtr->size % sizeof(SSttBlk) == 0);
if (reader->footer->sttBlkPtr->size % sizeof(SSttBlk) != 0) {
return TSDB_CODE_FILE_CORRUPTED;
}
int32_t size = reader->footer->sttBlkPtr->size / sizeof(SSttBlk);
void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size);
@ -256,7 +264,9 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
TAOS_CHECK_GOTO(tGetDiskDataHdr(&br, &hdr), &lino, _exit);
ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
if (hdr.delimiter != TSDB_FILE_DLMT) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
// set data container
tBlockDataReset(bData);
@ -266,7 +276,9 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
// key part
TAOS_CHECK_GOTO(tBlockDataDecompressKeyPart(&hdr, &br, bData, assist), &lino, _exit);
ASSERT(br.offset == buffer0->size);
if (br.offset != buffer0->size) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
bool loadExtra = false;
for (int i = 0; i < ncid; i++) {
@ -376,7 +388,10 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk
br.offset += tombBlk->size[i];
}
ASSERT(br.offset == tombBlk->dp->size);
if (br.offset != tombBlk->dp->size) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
_exit:
if (code) {
tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
@ -444,7 +459,9 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
}
}
ASSERT(br.offset == buffer0->size);
if (br.offset != buffer0->size) {
TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
}
_exit:
if (code) {
@ -814,8 +831,6 @@ _exit:
}
static void tsdbSttFWriterDoClose(SSttFileWriter *writer) {
ASSERT(writer->fd == NULL);
for (int32_t i = 0; i < ARRAY_SIZE(writer->local); ++i) {
tBufferDestroy(writer->local + i);
}
@ -854,7 +869,6 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o
tsdbCloseFile(&writer->fd);
ASSERT(writer->file->size > 0);
STFileOp op = (STFileOp){
.optype = TSDB_FOP_CREATE,
.fid = writer->config->fid,

107
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -106,7 +106,6 @@ _exit:
#endif
void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *)) {
ASSERT(idx >= 0 && idx < pMapData->nItem);
(void)tGetItemFn(pMapData->pData + pMapData->aOffset[idx], pItem);
}
@ -584,7 +583,8 @@ int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t nowSec) {
} else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) {
nowSec = nowSec * 1000000000l;
} else {
ASSERT(0);
tsdbError("invalid time precision:%d", pKeepCfg->precision);
return 0;
}
nowSec = nowSec - pKeepCfg->keepTimeOffset * tsTickPerHour[pKeepCfg->precision];
@ -628,8 +628,6 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *
*pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
}
}
} else {
ASSERT(0);
}
}
@ -697,8 +695,6 @@ int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema)
if (code) return code;
} else if (pRow->type == TSDBROW_COL_FMT) {
pIter->iColData = 0;
} else {
ASSERT(0);
}
return 0;
@ -730,8 +726,8 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) {
return NULL;
}
} else {
ASSERT(0);
return NULL; // suppress error report by compiler
tsdbError("invalid row type:%d", pIter->pRow->type);
return NULL;
}
}
@ -902,76 +898,6 @@ int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) {
return tRowBuild(pMerger->pArray, pMerger->pTSchema, ppRow);
}
/*
// delete skyline ======================================================
static int32_t tsdbMergeSkyline2(SArray *aSkyline1, SArray *aSkyline2, SArray *aSkyline) {
int32_t code = 0;
int32_t i1 = 0;
int32_t n1 = taosArrayGetSize(aSkyline1);
int32_t i2 = 0;
int32_t n2 = taosArrayGetSize(aSkyline2);
TSDBKEY *pSkyline1;
TSDBKEY *pSkyline2;
TSDBKEY item;
int64_t version1 = 0;
int64_t version2 = 0;
ASSERT(n1 > 0 && n2 > 0);
taosArrayClear(aSkyline);
while (i1 < n1 && i2 < n2) {
pSkyline1 = (TSDBKEY *)taosArrayGet(aSkyline1, i1);
pSkyline2 = (TSDBKEY *)taosArrayGet(aSkyline2, i2);
if (pSkyline1->ts < pSkyline2->ts) {
version1 = pSkyline1->version;
i1++;
} else if (pSkyline1->ts > pSkyline2->ts) {
version2 = pSkyline2->version;
i2++;
} else {
version1 = pSkyline1->version;
version2 = pSkyline2->version;
i1++;
i2++;
}
item.ts = TMIN(pSkyline1->ts, pSkyline2->ts);
item.version = TMAX(version1, version2);
if (taosArrayPush(aSkyline, &item) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
while (i1 < n1) {
pSkyline1 = (TSDBKEY *)taosArrayGet(aSkyline1, i1);
item.ts = pSkyline1->ts;
item.version = pSkyline1->version;
if (taosArrayPush(aSkyline, &item) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
i1++;
}
while (i2 < n2) {
pSkyline2 = (TSDBKEY *)taosArrayGet(aSkyline2, i2);
item.ts = pSkyline2->ts;
item.version = pSkyline2->version;
if (taosArrayPush(aSkyline, &item) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
i2++;
}
_exit:
return code;
}
*/
// delete skyline ======================================================
static void tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pSkyline) {
int32_t i1 = 0;
@ -983,8 +909,6 @@ static void tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pSkyl
int64_t version1 = 0;
int64_t version2 = 0;
ASSERT(n1 > 0 && n2 > 0);
taosArrayClear(pSkyline);
TSDBKEY **pItem = TARRAY_GET_ELEM(pSkyline, 0);
@ -1217,7 +1141,9 @@ _exit:
int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid) {
int32_t code = 0;
ASSERT(pId->suid || pId->uid);
if (!pId->suid && !pId->uid) {
return TSDB_CODE_INVALID_PARA;
}
pBlockData->suid = pId->suid;
pBlockData->uid = pId->uid;
@ -1277,8 +1203,6 @@ void tBlockDataReset(SBlockData *pBlockData) {
}
void tBlockDataClear(SBlockData *pBlockData) {
ASSERT(pBlockData->suid || pBlockData->uid);
pBlockData->nRow = 0;
for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
tColDataClear(tBlockDataGetColDataByIdx(pBlockData, iColData));
@ -1286,7 +1210,9 @@ void tBlockDataClear(SBlockData *pBlockData) {
}
int32_t tBlockDataAddColData(SBlockData *pBlockData, int16_t cid, int8_t type, int8_t cflag, SColData **ppColData) {
ASSERT(pBlockData->nColData == 0 || pBlockData->aColData[pBlockData->nColData - 1].cid < cid);
if (pBlockData->nColData != 0 && pBlockData->aColData[pBlockData->nColData - 1].cid >= cid) {
return TSDB_CODE_INVALID_PARA;
}
SColData *newColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SColData) * (pBlockData->nColData + 1));
if (newColData == NULL) {
@ -1350,7 +1276,9 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
// uid
if (pBlockData->uid == 0) {
ASSERT(uid);
if (!uid) {
return TSDB_CODE_INVALID_PARA;
}
code = tRealloc((uint8_t **)&pBlockData->aUid, sizeof(int64_t) * (pBlockData->nRow + 1));
if (code) goto _exit;
pBlockData->aUid[pBlockData->nRow] = uid;
@ -1384,7 +1312,9 @@ int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
// version
int64_t lversion = pBlockData->aVersion[pBlockData->nRow - 1];
int64_t rversion = TSDBROW_VERSION(pRow);
ASSERT(lversion != rversion);
if (lversion == rversion) {
return TSDB_CODE_INVALID_PARA;
}
if (rversion > lversion) {
pBlockData->aVersion[pBlockData->nRow - 1] = rversion;
}
@ -1398,7 +1328,8 @@ int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
code = tBlockDataUpsertBlockRow(pBlockData, pRow->pBlockData, pRow->iRow, (rversion > lversion) ? 1 : -1);
if (code) goto _exit;
} else {
ASSERT(0);
code = TSDB_CODE_INVALID_PARA;
goto _exit;
}
_exit:
@ -1745,8 +1676,6 @@ int32_t tBlockDataDecompressColData(const SDiskDataHdr *hdr, const SBlockCol *bl
code = tBlockDataAddColData(blockData, blockCol->cid, blockCol->type, blockCol->cflag, &colData);
TSDB_CHECK_CODE(code, lino, _exit);
// ASSERT(blockCol->flag != HAS_NONE);
SColDataCompressInfo info = {
.cmprAlg = blockCol->alg,
.columnFlag = blockCol->cflag,

6
source/dnode/vnode/src/tsdb/tsdbUtil2.c
View File

@ -171,7 +171,7 @@ static int32_t tStatisBlockUpdate(STbStatisBlock *block, SRowInfo *row) {
TAOS_CHECK_RETURN(tBufferPutAt(&block->counts, (block->numOfRecords - 1) * sizeof(record.count), &record.count,
sizeof(record.count)));
} else {
ASSERT(0);
return TSDB_CODE_INVALID_PARA;
}
return 0;
@ -276,7 +276,9 @@ int32_t tBrinBlockClear(SBrinBlock *brinBlock) {
}
int32_t tBrinBlockPut(SBrinBlock *brinBlock, const SBrinRecord *record) {
ASSERT(record->firstKey.key.numOfPKs == record->lastKey.key.numOfPKs);
if (record->firstKey.key.numOfPKs != record->lastKey.key.numOfPKs) {
return TSDB_CODE_INVALID_PARA;
}
if (brinBlock->numOfRecords == 0) { // the first row
brinBlock->numOfPKs = record->firstKey.key.numOfPKs;

15
source/dnode/vnode/src/vnd/vnodeAsync.c
View File

@ -165,9 +165,7 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) {
}
ret = vHashDrop(async->taskTable, task);
if (ret != 0) {
ASSERT(0);
}
TAOS_UNUSED(ret);
async->numTasks--;
if (task->numWait == 0) {
@ -403,7 +401,6 @@ static int32_t vnodeAsyncDestroy(SVAsync **async) {
}
(void)taosThreadJoin((*async)->workers[i].thread, NULL);
ASSERT((*async)->workers[i].state == EVA_WORKER_STATE_STOP);
(*async)->workers[i].state = EVA_WORKER_STATE_UINIT;
}
@ -413,18 +410,11 @@ static int32_t vnodeAsyncDestroy(SVAsync **async) {
channel->prev->next = channel->next;
int32_t ret = vHashDrop((*async)->channelTable, channel);
if (ret) {
ASSERT(0);
}
TAOS_UNUSED(ret);
(*async)->numChannels--;
taosMemoryFree(channel);
}
ASSERT((*async)->numLaunchWorkers == 0);
ASSERT((*async)->numIdleWorkers == 0);
ASSERT((*async)->numChannels == 0);
ASSERT((*async)->numTasks == 0);
(void)taosThreadMutexDestroy(&(*async)->mutex);
(void)taosThreadCondDestroy(&(*async)->hasTask);
@ -438,7 +428,6 @@ static int32_t vnodeAsyncDestroy(SVAsync **async) {
static int32_t vnodeAsyncLaunchWorker(SVAsync *async) {
for (int32_t i = 0; i < async->numWorkers; i++) {
ASSERT(async->workers[i].state != EVA_WORKER_STATE_IDLE);
if (async->workers[i].state == EVA_WORKER_STATE_ACTIVE) {
continue;
} else if (async->workers[i].state == EVA_WORKER_STATE_STOP) {

25
source/dnode/vnode/src/vnd/vnodeBufPool.c
View File

@ -103,17 +103,18 @@ int vnodeCloseBufPool(SVnode *pVnode) {
}
void vnodeBufPoolReset(SVBufPool *pPool) {
ASSERT(pPool->nQuery == 0);
if (pPool->nQuery != 0) {
vError("vgId:%d, buffer pool %p of id %d has %d queries, reset it may cause problem", TD_VID(pPool->pVnode), pPool,
pPool->id, pPool->nQuery);
}
for (SVBufPoolNode *pNode = pPool->pTail; pNode->prev; pNode = pPool->pTail) {
ASSERT(pNode->pnext == &pPool->pTail);
pNode->prev->pnext = &pPool->pTail;
pPool->pTail = pNode->prev;
pPool->size = pPool->size - sizeof(*pNode) - pNode->size;
taosMemoryFree(pNode);
}
ASSERT(pPool->size == pPool->ptr - pPool->node.data);
pPool->size = 0;
pPool->ptr = pPool->node.data;
}
@ -123,7 +124,11 @@ void *vnodeBufPoolMallocAligned(SVBufPool *pPool, int size) {
void *p = NULL;
uint8_t *ptr = NULL;
int paddingLen = 0;
ASSERT(pPool != NULL);
if (pPool == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
if (pPool->lock) taosThreadSpinLock(pPool->lock);
@ -162,7 +167,11 @@ void *vnodeBufPoolMallocAligned(SVBufPool *pPool, int size) {
void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) {
SVBufPoolNode *pNode;
void *p = NULL;
ASSERT(pPool != NULL);
if (pPool == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
if (pPool->lock) taosThreadSpinLock(pPool->lock);
if (pPool->node.size >= pPool->ptr - pPool->node.data + size) {
@ -207,7 +216,9 @@ void vnodeBufPoolFree(SVBufPool *pPool, void *p) {
void vnodeBufPoolRef(SVBufPool *pPool) {
int32_t nRef = atomic_fetch_add_32(&pPool->nRef, 1);
ASSERT(nRef > 0);
if (nRef <= 0) {
vError("vgId:%d, buffer pool %p of id %d is referenced by %d", TD_VID(pPool->pVnode), pPool, pPool->id, nRef);
}
}
void vnodeBufPoolAddToFreeList(SVBufPool *pPool) {

4
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -375,11 +375,11 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
}
tjsonGetNumberValue(pJson, "s3ChunkSize", pCfg->s3ChunkSize, code);
if (code < 0) {
if (code < 0 || pCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE) {
pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
}
tjsonGetNumberValue(pJson, "s3KeepLocal", pCfg->s3KeepLocal, code);
if (code < 0) {
if (code < 0 || pCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL) {
pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
}
tjsonGetNumberValue(pJson, "s3Compact", pCfg->s3Compact, code);

3
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -302,7 +302,6 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
TSDB_CHECK_CODE(code, lino, _exit);
(void)taosThreadMutexLock(&pVnode->mutex);
ASSERT(pVnode->onCommit == NULL);
pVnode->onCommit = pVnode->inUse;
pVnode->inUse = NULL;
(void)taosThreadMutexUnlock(&pVnode->mutex);
@ -339,7 +338,7 @@ static void vnodeReturnBufPool(SVnode *pVnode) {
pVnode->recycleTail = pPool;
}
} else {
ASSERT(0);
vError("vgId:%d, buffer pool %p of id %d nRef:%d", TD_VID(pVnode), pPool, pPool->id, nRef);
}
(void)taosThreadMutexUnlock(&pVnode->mutex);

1
source/dnode/vnode/src/vnd/vnodeHash.c
View File

@ -77,7 +77,6 @@ int32_t vHashDestroy(SVHashTable** ht) {
}
if (*ht) {
ASSERT((*ht)->numEntries == 0);
taosMemoryFree((*ht)->buckets);
taosMemoryFree(*ht);
(*ht) = NULL;

3
source/dnode/vnode/src/vnd/vnodeModule.c
View File

@ -14,6 +14,7 @@
*/
#include "cos.h"
#include "monitor.h"
#include "vnd.h"
static volatile int32_t VINIT = 0;
@ -26,6 +27,8 @@ int vnodeInit(int nthreads, StopDnodeFp stopDnodeFp) {
TAOS_CHECK_RETURN(vnodeAsyncOpen(nthreads));
TAOS_CHECK_RETURN(walInit(stopDnodeFp));
monInitVnode();
return 0;
}

22
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -494,24 +494,6 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%" PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId);
snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%" PRId32, pVnode->config.vgId);
if (tsEnableMonitor && pVnode->monitor.insertCounter == NULL) {
taos_counter_t *counter = NULL;
int32_t label_count = 7;
const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID,
VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP,
VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME,
VNODE_METRIC_TAG_NAME_RESULT};
counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", label_count, sample_labels);
vInfo("vgId:%d, new metric:%p", TD_VID(pVnode), counter);
if (taos_collector_registry_register_metric(counter) == 1) {
(void)taos_counter_destroy(counter);
counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
vInfo("vgId:%d, get metric from registry:%p", TD_VID(pVnode), counter);
}
pVnode->monitor.insertCounter = counter;
vInfo("vgId:%d, succeed to set metric:%p", TD_VID(pVnode), counter);
}
return pVnode;
_err:
@ -558,7 +540,9 @@ void vnodeClose(SVnode *pVnode) {
// start the sync timer after the queue is ready
int32_t vnodeStart(SVnode *pVnode) {
ASSERT(pVnode);
if (pVnode == NULL) {
return TSDB_CODE_INVALID_PARA;
}
return vnodeSyncStart(pVnode);
}

12
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -19,7 +19,6 @@
#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags) \
do { \
int##vType##_t newVal = atomic_sub_fetch_##vType(&(pVar), (oVal)); \
ASSERT(newVal >= 0); \
if (newVal < 0) { \
vWarn("vgId:%d, %s, abnormal val:%" PRIi64 ", old val:%" PRIi64, TD_VID(pVnode), tags, newVal, (oVal)); \
} \
@ -37,7 +36,10 @@ int32_t fillTableColCmpr(SMetaReader *reader, SSchemaExt *pExt, int32_t numOfCol
int8_t tblType = reader->me.type;
if (useCompress(tblType)) {
SColCmprWrapper *p = &(reader->me.colCmpr);
ASSERT(numOfCol == p->nCols);
if (numOfCol != p->nCols) {
vError("fillTableColCmpr table type:%d, col num:%d, col cmpr num:%d mismatch", tblType, numOfCol, p->nCols);
return TSDB_CODE_APP_ERROR;
}
for (int i = 0; i < p->nCols; i++) {
SColCmpr *pCmpr = &p->pColCmpr[i];
pExt[i].colId = pCmpr->id;
@ -104,7 +106,8 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
} else if (mer1.me.type == TSDB_NORMAL_TABLE) {
schema = mer1.me.ntbEntry.schemaRow;
} else {
ASSERT(0);
vError("vnodeGetTableMeta get invalid table type:%d", mer1.me.type);
return TSDB_CODE_APP_ERROR;
}
metaRsp.numOfTags = schemaTag.nCols;
@ -262,7 +265,8 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
}
}
} else {
ASSERT(0);
vError("vnodeGetTableCfg get invalid table type:%d", mer1.me.type);
return TSDB_CODE_APP_ERROR;
}
cfgRsp.numOfTags = schemaTag.nCols;

20
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -15,6 +15,7 @@
#include "audit.h"
#include "cos.h"
#include "monitor.h"
#include "tencode.h"
#include "tglobal.h"
#include "tmsg.h"
@ -23,6 +24,8 @@
#include "vnode.h"
#include "vnodeInt.h"
extern taos_counter_t *tsInsertCounter;
static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
@ -934,20 +937,23 @@ _exit:
}
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
int ret = 0;
SVDropTtlTableReq ttlReq = {0};
if (tDeserializeSVDropTtlTableReq(pReq, len, &ttlReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
ret = TSDB_CODE_INVALID_MSG;
goto end;
}
ASSERT(ttlReq.nUids == taosArrayGetSize(ttlReq.pTbUids));
if (ttlReq.nUids != taosArrayGetSize(ttlReq.pTbUids)) {
ret = TSDB_CODE_INVALID_MSG;
goto end;
}
if (ttlReq.nUids != 0) {
vInfo("vgId:%d, drop ttl table req will be processed, time:%d, ntbUids:%d", pVnode->config.vgId,
ttlReq.timestampSec, ttlReq.nUids);
}
int ret = 0;
if (ttlReq.nUids > 0) {
metaDropTables(pVnode->pMeta, ttlReq.pTbUids);
(void)tqUpdateTbUidList(pVnode->pTq, ttlReq.pTbUids, false);
@ -1787,8 +1793,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in
}
if (info.suid) {
code = metaGetInfo(pVnode->pMeta, info.suid, &info, NULL);
ASSERT(code == 0);
(void)metaGetInfo(pVnode->pMeta, info.suid, &info, NULL);
}
if (pSubmitTbData->sver != info.skmVer) {
@ -1901,7 +1906,8 @@ _exit:
(void)atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows);
(void)atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1);
if (tsEnableMonitor && pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0) {
if (tsEnableMonitor && tsMonitorFqdn[0] != 0 && tsMonitorPort != 0 && pSubmitRsp->affectedRows > 0 &&
strlen(pOriginalMsg->info.conn.user) > 0 && tsInsertCounter != NULL) {
const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS,
pVnode->monitor.strClusterId,
pVnode->monitor.strDnodeId,
@ -1909,7 +1915,7 @@ _exit:
pVnode->monitor.strVgId,
pOriginalMsg->info.conn.user,
"Success"};
(void)taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels);
(void)taos_counter_add(tsInsertCounter, pSubmitRsp->affectedRows, sample_labels);
}
if (code == 0) {

40
source/libs/catalog/inc/catalogInt.h
View File

@ -852,34 +852,58 @@ typedef struct SCtgCacheItemInfo {
#define CTG_LOCK(type, _lock) \
do { \
if (CTG_READ == (type)) { \
ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \
if (atomic_load_32((_lock)) < 0) { \
qError("invalid lock value before read lock"); \
break; \
} \
CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosRLockLatch(_lock); \
CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \
if (atomic_load_32((_lock)) <= 0) { \
qError("invalid lock value after read lock"); \
break; \
} \
} else { \
ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \
if (atomic_load_32((_lock)) < 0) { \
qError("invalid lock value before write lock"); \
break; \
} \
CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosWLockLatch(_lock); \
CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \
if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \
qError("invalid lock value after write lock"); \
break; \
} \
} \
} while (0)
#define CTG_UNLOCK(type, _lock) \
do { \
if (CTG_READ == (type)) { \
ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \
if (atomic_load_32((_lock)) <= 0) { \
qError("invalid lock value before read unlock"); \
break; \
} \
CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosRUnLockLatch(_lock); \
CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \
if (atomic_load_32((_lock)) < 0) { \
qError("invalid lock value after read unlock"); \
break; \
} \
} else { \
ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \
if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \
qError("invalid lock value before write unlock"); \
break; \
} \
CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosWUnLockLatch(_lock); \
CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \
if (atomic_load_32((_lock)) < 0) { \
qError("invalid lock value after write unlock"); \
break; \
} \
} \
} while (0)

2
source/libs/catalog/src/catalog.c
View File

@ -773,8 +773,6 @@ int32_t ctgGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaNa
}
CTG_ERR_JRET(code);
ASSERT(tsmaRsp.pTsmas && tsmaRsp.pTsmas->size == 1);
*pTsma = taosArrayGetP(tsmaRsp.pTsmas, 0);
taosArrayDestroy(tsmaRsp.pTsmas);

461
source/libs/catalog/src/ctgAsync.c
View File

File diff suppressed because it is too large Load Diff

81
source/libs/catalog/src/ctgRemote.c
View File

@ -42,7 +42,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
msgNum = taosArrayGetSize(batchRsp.pRsps);
}
if (ASSERTS(taskNum == msgNum || 0 == msgNum, "taskNum %d mis-match msgNum %d", taskNum, msgNum)) {
if (taskNum != msgNum && 0 != msgNum) {
ctgError("taskNum %d mis-match msgNum %d", taskNum, msgNum);
msgNum = 0;
}
@ -56,13 +57,13 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
}
for (int32_t i = 0; i < taskNum; ++i) {
int32_t* taskId = taosArrayGet(cbParam->taskId, i);
int32_t* taskId = taosArrayGet(cbParam->taskId, i);
if (NULL == taskId) {
ctgError("taosArrayGet %d taskId failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->taskId));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i);
int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i);
if (NULL == msgIdx) {
ctgError("taosArrayGet %d msgIdx failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->msgIdx));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
@ -77,7 +78,9 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
if (msgNum > 0) {
pRsp = taosArrayGet(batchRsp.pRsps, i);
if (ASSERTS(pRsp->msgIdx == *msgIdx, "rsp msgIdx %d mis-match msgIdx %d", pRsp->msgIdx, *msgIdx)) {
if (pRsp->msgIdx != *msgIdx) {
ctgError("rsp msgIdx %d mis-match msgIdx %d", pRsp->msgIdx, *msgIdx);
pRsp = &rsp;
pRsp->msgIdx = *msgIdx;
pRsp->reqType = -1;
@ -114,7 +117,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId,
pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs);
(void)(*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); // error handled internal
(void)(*gCtgAsyncFps[pTask->type].handleRspFp)(
&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); // error handled internal
}
CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs));
@ -417,12 +421,12 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) {
if (TDMT_VND_BATCH_META == cbParam->reqType || TDMT_MND_BATCH_META == cbParam->reqType) {
CTG_ERR_JRET(ctgHandleBatchRsp(pJob, cbParam, pMsg, rspCode));
} else {
int32_t* taskId = taosArrayGet(cbParam->taskId, 0);
int32_t* taskId = taosArrayGet(cbParam->taskId, 0);
if (NULL == taskId) {
ctgError("taosArrayGet %d taskId failed, total:%d", 0, (int32_t)taosArrayGetSize(cbParam->taskId));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId);
if (NULL == pTask) {
ctgError("taosArrayGet %d SCtgTask failed, total:%d", *taskId, (int32_t)taosArrayGetSize(pJob->pTasks));
@ -445,7 +449,7 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) {
ctgError("get task %d SCtgMsgCtx failed, taskType:%d", -1, pTask->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pMsgCtx->pBatchs = pBatchs;
#endif
@ -534,7 +538,7 @@ int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob* pJob,
CTG_ERR_JRET(code);
}
ctgDebug("ctg req msg sent, reqId:0x%" PRIx64 ", msg type:%d, %s", pJob->queryId, msgType, TMSG_INFO(msgType));
ctgDebug("ctg req msg sent, QID:0x%" PRIx64 ", msg type:%d, %s", pJob->queryId, msgType, TMSG_INFO(msgType));
return TSDB_CODE_SUCCESS;
_return:
@ -558,9 +562,9 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
ctgError("get task %d SCtgMsgCtx failed, taskType:%d", tReq->msgIdx, pTask->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SHashObj* pBatchs = pMsgCtx->pBatchs;
SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId));
SHashObj* pBatchs = pMsgCtx->pBatchs;
SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId));
if (NULL == pBatch) {
newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg));
newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t));
@ -599,7 +603,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx;
SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName));
} else if (CTG_TASK_GET_TB_TSMA == pTask->type){
} else if (CTG_TASK_GET_TB_TSMA == pTask->type) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
@ -616,10 +620,11 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
if (NULL == pFetch) {
ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches));
ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx,
(int32_t)taosArrayGetSize(pCtx->pFetches));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
if (NULL == pTbReq) {
ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
@ -675,7 +680,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx;
SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName));
} else if (CTG_TASK_GET_TB_TSMA == pTask->type){
} else if (CTG_TASK_GET_TB_TSMA == pTask->type) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
@ -689,22 +694,23 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
pName = ctx->pName;
}
} else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
if (NULL == pFetch) {
ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
if (NULL == pTbReq) {
ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
if (NULL == pName) {
ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
if (NULL == pFetch) {
ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx,
(int32_t)taosArrayGetSize(pCtx->pFetches));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
if (NULL == pTbReq) {
ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
if (NULL == pName) {
ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
} else {
ctgError("invalid vnode msgType %d", msgType);
CTG_ERR_JRET(TSDB_CODE_APP_ERROR);
@ -1629,9 +1635,9 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName*
}
int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* name, STableTSMAInfoRsp* out,
SCtgTaskReq* tReq, int32_t reqType) {
char* msg = NULL;
int32_t msgLen = 0;
SCtgTaskReq* tReq, int32_t reqType) {
char* msg = NULL;
int32_t msgLen = 0;
SCtgTask* pTask = tReq ? tReq->pTask : NULL;
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
char tbFName[TSDB_TABLE_FNAME_LEN];
@ -1720,7 +1726,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c
#if CTG_BATCH_FETCH
CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen));
#else
char dbFName[TSDB_DB_FNAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
(void)tNameGetFullDbName(pTbName, dbFName);
SArray* pTaskId = taosArrayInit(1, sizeof(int32_t));
if (NULL == pTaskId) {
@ -1731,7 +1737,8 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen));
CTG_RET(
ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen));
#endif
}

2
source/libs/catalog/src/ctgUtil.c
View File

@ -2631,7 +2631,7 @@ bool hasOutOfDateTSMACache(SArray* pTsmas) {
for (int32_t i = 0; i < pTsmas->size; ++i) {
STSMACache* pTsmaInfo = taosArrayGetP(pTsmas, i);
if (NULL == pTsmaInfo) {
ASSERT(0);
continue;
}
if (isCtgTSMACacheOutOfDate(pTsmaInfo)) {
return true;

8
source/libs/executor/inc/executil.h
View File

@ -24,10 +24,10 @@
#include "tpagedbuf.h"
#include "tsimplehash.h"
#define T_LONG_JMP(_obj, _c) \
do { \
ASSERT(1); \
longjmp((_obj), (_c)); \
#define T_LONG_JMP(_obj, _c) \
do { \
qError("error happens at %s, line:%d, code:%s", __func__, __LINE__, tstrerror((_c))); \
longjmp((_obj), (_c)); \
} while (0)
#define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \

2
source/libs/executor/inc/tsort.h
View File

@ -66,7 +66,7 @@ typedef struct SMsortComparParam {
typedef struct SSortHandle SSortHandle;
typedef struct STupleHandle STupleHandle;
typedef SSDataBlock* (*_sort_fetch_block_fn_t)(void* param);
typedef int32_t (*_sort_fetch_block_fn_t)(void* param, SSDataBlock** ppBlock);
typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void* param);
/**

6
source/libs/executor/src/cachescanoperator.c
View File

@ -298,7 +298,11 @@ int32_t doScanCacheNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t resultRows = pBufRes->info.rows;
// the results may be null, if last values are all null
ASSERT(resultRows == 0 || resultRows == taosArrayGetSize(pInfo->pUidList));
if (resultRows != 0 && resultRows != taosArrayGetSize(pInfo->pUidList)) {
pTaskInfo->code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(pTaskInfo->code));
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
pInfo->indexOfBufferedRes = 0;
}

13
source/libs/executor/src/countwindowoperator.c
View File

@ -66,14 +66,17 @@ static void clearWinStateBuff(SCountWindowResult* pBuff) { pBuff->winRows = 0; }
static SCountWindowResult* getCountWinStateInfo(SCountWindowSupp* pCountSup) {
SCountWindowResult* pBuffInfo = taosArrayGet(pCountSup->pWinStates, pCountSup->stateIndex);
if (!pBuffInfo) {
terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
return NULL;
}
int32_t size = taosArrayGetSize(pCountSup->pWinStates);
// coverity scan
ASSERTS(size > 0, "WinStates is empty");
if (size > 0) {
pCountSup->stateIndex = (pCountSup->stateIndex + 1) % size;
int32_t size = taosArrayGetSize(pCountSup->pWinStates);
if (size == 0) {
terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
return NULL;
}
pCountSup->stateIndex = (pCountSup->stateIndex + 1) % size;
return pBuffInfo;
}

10
source/libs/executor/src/dataDeleter.c
View File

@ -87,7 +87,10 @@ static int32_t toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* p
if (pRes->affectedRows) {
pRes->skey = *(int64_t*)pColSKey->pData;
pRes->ekey = *(int64_t*)pColEKey->pData;
ASSERT(pRes->skey <= pRes->ekey);
if (pRes->skey > pRes->ekey) {
qError("data delter skey:%" PRId64 " is bigger than ekey:%" PRId64, pRes->skey, pRes->ekey);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
} else {
pRes->skey = pHandle->pDeleter->deleteTimeRange.skey;
pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey;
@ -205,7 +208,10 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRaw
static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
if (NULL == pDeleter->nextOutput.pData) {
ASSERT(pDeleter->queryEnd);
if (!pDeleter->queryEnd) {
qError("empty res while query not end in data deleter");
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
pOutput->useconds = pDeleter->useconds;
pOutput->precision = pDeleter->pSchema->precision;
pOutput->bufStatus = DS_BUF_EMPTY;

10
source/libs/executor/src/dataDispatcher.c
View File

@ -242,7 +242,11 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRow
static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
if (NULL == pDispatcher->nextOutput.pData) {
ASSERT(pDispatcher->queryEnd);
if (!pDispatcher->queryEnd) {
qError("empty res while query not end in data dispatcher");
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
pOutput->useconds = pDispatcher->useconds;
pOutput->precision = pDispatcher->pSchema->precision;
pOutput->bufStatus = DS_BUF_EMPTY;
@ -318,6 +322,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
dispatcher->sink.fGetCacheSize = getCacheSize;
dispatcher->pManager = pManager;
pManager = NULL;
dispatcher->pSchema = pDataSink->pInputDataBlockDesc;
dispatcher->status = DS_BUF_EMPTY;
dispatcher->queryEnd = false;
@ -336,6 +341,9 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFree(pManager);
if (dispatcher) {
dsDestroyDataSinker(dispatcher);
}

6
source/libs/executor/src/dataInserter.c
View File

@ -234,7 +234,11 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY
ASSERT(pColInfoData->info.type == pCol->type);
if (pColInfoData->info.type != pCol->type) {
qError("column:%d type:%d in block dismatch with schema col:%d type:%d", colIdx, pColInfoData->info.type, k, pCol->type);
terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
goto _end;
}
if (colDataIsNull_s(pColInfoData, j)) {
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
if (NULL == taosArrayPush(pVals, &cv)) {

6
source/libs/executor/src/dynqueryctrloperator.c
View File

@ -320,7 +320,7 @@ static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initPara
return code;
}
(*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES);
if (NULL == *ppRes) {
if (NULL == (*ppRes)->pChildren) {
code = terrno;
freeOperatorParam(pChild0, OP_GET_PARAM);
freeOperatorParam(pChild1, OP_GET_PARAM);
@ -823,12 +823,14 @@ static void postProcessStbJoinTableHash(SOperatorInfo* pOperator) {
pStbJoin->execInfo.leftCacheNum = tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache);
qDebug("more than 1 ref build table num %" PRId64, (int64_t)tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache));
/*
// debug only
iter = 0;
uint32_t* num = NULL;
while (NULL != (num = tSimpleHashIterate(pStbJoin->ctx.prev.leftCache, num, &iter))) {
ASSERT(*num > 1);
A S S E R T(*num > 1);
}
*/
}
static void buildStbJoinTableList(SOperatorInfo* pOperator) {

6
source/libs/executor/src/exchangeoperator.c
View File

@ -225,7 +225,10 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) {
} else {
concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo);
}
if (TSDB_CODE_SUCCESS != pOperator->pTaskInfo->code) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, pOperator->pTaskInfo->code);
}
if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) {
return NULL;
} else {
@ -518,6 +521,7 @@ void doDestroyExchangeOperatorInfo(void* param) {
int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param;
taosMemoryFreeClear(pMsg->pEpSet);
SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId);
if (pExchangeInfo == NULL) {
qWarn("failed to acquire exchange operator, since it may have been released, %p", pExchangeInfo);

70
source/libs/executor/src/executor.c
View File

@ -31,7 +31,7 @@ int32_t exchangeObjRefPool = -1;
static void cleanupRefPool() {
int32_t ref = atomic_val_compare_exchange_32(&exchangeObjRefPool, exchangeObjRefPool, 0);
(void)taosCloseRef(ref);
taosCloseRef(ref);
}
static void initRefPool() {
@ -158,7 +158,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
SStreamScanInfo* pInfo = pOperator->info;
qDebug("s-task:%s in this batch, %d blocks need to be processed", id, (int32_t)numOfBlocks);
ASSERT(pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0);
QUERY_CHECK_CONDITION((pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0), code, lino, _end,
TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
if (type == STREAM_INPUT__MERGED_SUBMIT) {
for (int32_t i = 0; i < numOfBlocks; i++) {
@ -189,7 +190,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
pInfo->blockType = STREAM_INPUT__CHECKPOINT;
} else {
ASSERT(0);
code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
QUERY_CHECK_CODE(code, lino, _end);
}
return TSDB_CODE_SUCCESS;
@ -543,7 +545,9 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
int32_t* tversion, int32_t idx, bool* tbGet) {
*tbGet = false;
ASSERT(tinfo != NULL && dbName != NULL && tableName != NULL);
if (tinfo == NULL || dbName == NULL || tableName == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
if (taosArrayGetSize(pTaskInfo->schemaInfos) <= idx) {
@ -663,7 +667,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
if (isTaskKilled(pTaskInfo)) {
atomic_store_64(&pTaskInfo->owner, 0);
qDebug("%s already killed, abort", GET_TASKID(pTaskInfo));
return TSDB_CODE_SUCCESS;
return pTaskInfo->code;
}
// error occurs, record the error code and return to client
@ -722,7 +726,8 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
blockIndex += 1;
current += p->info.rows;
ASSERT(p->info.rows > 0 || p->info.type == STREAM_CHECKPOINT);
QUERY_CHECK_CONDITION((p->info.rows > 0 || p->info.type == STREAM_CHECKPOINT), code, lino, _end,
TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
void* tmp = taosArrayPush(pResList, &p);
QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
@ -785,7 +790,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) {
qDebug("%s already killed, abort", GET_TASKID(pTaskInfo));
taosRUnLockLatch(&pTaskInfo->lock);
return TSDB_CODE_SUCCESS;
return pTaskInfo->code;
}
if (pTaskInfo->owner != 0) {
@ -987,15 +992,17 @@ void qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) {
*scanner = pOperator->info;
break;
} else {
ASSERT(pOperator->numOfDownstream == 1);
pOperator = pOperator->pDownstream[0];
}
}
}
int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRange* pVerRange, STimeWindow* pWindow) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM);
QUERY_CHECK_CONDITION((pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM), code, lino, _end,
TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
SStreamTaskInfo* pStreamInfo = &pTaskInfo->streamInfo;
@ -1007,12 +1014,19 @@ int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRan
", window:%" PRId64 " - %" PRId64,
GET_TASKID(pTaskInfo), pStreamInfo->fillHistoryVer.minVer, pStreamInfo->fillHistoryVer.maxVer, pWindow->skey,
pWindow->ekey);
return 0;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRange* pVerRange, STimeWindow* pWindow) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM);
QUERY_CHECK_CONDITION((pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM), code, lino, _end,
TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
SStreamTaskInfo* pStreamInfo = &pTaskInfo->streamInfo;
@ -1024,14 +1038,26 @@ int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRan
"-%" PRId64,
GET_TASKID(pTaskInfo), pStreamInfo->fillHistoryVer.minVer, pStreamInfo->fillHistoryVer.maxVer, pWindow->skey,
pWindow->ekey);
return 0;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t qStreamRecoverFinish(qTaskInfo_t tinfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM);
QUERY_CHECK_CONDITION((pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM), code, lino, _end,
TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE;
return 0;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
@ -1046,9 +1072,6 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);
ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0);
qInfo("save stream param for interval: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark);
pSup->calTriggerSaved = pSup->calTrigger;
@ -1063,9 +1086,6 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);
ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0);
qInfo("save stream param for session: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark);
pSup->calTriggerSaved = pSup->calTrigger;
@ -1078,9 +1098,6 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);
ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0);
qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark);
pSup->calTriggerSaved = pSup->calTrigger;
@ -1093,9 +1110,6 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);
ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0);
qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark);
pSup->calTriggerSaved = pSup->calTrigger;
@ -1108,9 +1122,6 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);
ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0);
qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark);
pSup->calTriggerSaved = pSup->calTrigger;
@ -1126,7 +1137,6 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
if (pOperator->numOfDownstream != 1 || pOperator->pDownstream[0] == NULL) {
if (pOperator->numOfDownstream > 1) {
qError("unexpected stream, multiple downstream");
ASSERT(0);
return -1;
}
return 0;

23
source/libs/executor/src/executorInt.c
View File

@ -163,7 +163,11 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
return NULL;
}
ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset);
if (pResult->pageId != p1->pageId || pResult->offset != p1->offset) {
terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
pTaskInfo->code = terrno;
return NULL;
}
}
} else {
// In case of group by column query, the required SResultRow object must be existInCurrentResusltRowInfo in the
@ -176,7 +180,11 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
return NULL;
}
ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset);
if (pResult->pageId != p1->pageId || pResult->offset != p1->offset) {
terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
pTaskInfo->code = terrno;
return NULL;
}
}
}
@ -324,6 +332,7 @@ _end:
static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag,
bool createDummyCol) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SqlFunctionCtx* pCtx = pExprSup->pCtx;
for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) {
@ -363,7 +372,7 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
if (hasPk && (j == pkParamIdx)) {
pInput->pPrimaryKey = pInput->pData[j];
}
ASSERT(pInput->pData[j] != NULL);
QUERY_CHECK_CONDITION((pInput->pData[j] != NULL), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
} else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) {
// todo avoid case: top(k, 12), 12 is the value parameter.
// sum(11), 11 is also the value parameter.
@ -374,14 +383,16 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
pInput->blankFill = pBlock->info.blankFill;
code = doCreateConstantValColumnInfo(pInput, pFuncParam, j, pBlock->info.rows);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
QUERY_CHECK_CODE(code, lino, _end);
}
}
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}

52
source/libs/executor/src/filloperator.c
View File

@ -55,7 +55,6 @@ typedef struct SFillOperatorInfo {
SExprSupp noFillExprSupp;
} SFillOperatorInfo;
static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order);
static void destroyFillOperatorInfo(void* param);
static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag);
static int32_t fillResetPrevForNewGroup(SFillInfo* pFillInfo);
@ -168,56 +167,6 @@ _end:
return code;
}
// todo refactor: decide the start key according to the query time range.
static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order) {
if (order == TSDB_ORDER_ASC) {
int64_t skey = pBlock->info.window.skey;
if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the
ASSERT(taosFillNotStarted(pInfo->pFillInfo));
taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey);
} else if (pInfo->pFillInfo->start < skey) {
int64_t t = skey;
SInterval* pInterval = &pInfo->pFillInfo->interval;
while (1) {
int64_t prev = taosTimeAdd(t, -pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
if (prev <= pInfo->pFillInfo->start) {
t = prev;
break;
}
t = prev;
}
// todo time window chosen problem: t or prev value?
taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t);
}
} else {
int64_t ekey = pBlock->info.window.ekey;
if (ekey > pInfo->pFillInfo->start) {
ASSERT(taosFillNotStarted(pInfo->pFillInfo));
taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey);
} else if (ekey < pInfo->pFillInfo->start) {
int64_t t = ekey;
SInterval* pInterval = &pInfo->pFillInfo->interval;
int64_t prev = t;
while (1) {
int64_t next = taosTimeAdd(t, pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
if (next >= pInfo->pFillInfo->start) {
prev = t;
t = next;
break;
}
prev = t;
t = next;
}
// todo time window chosen problem: t or next value?
if (t > pInfo->pFillInfo->start) t = prev;
taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t);
}
}
}
static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
@ -602,7 +551,6 @@ static void reviseFillStartAndEndKey(SFillOperatorInfo* pInfo, int32_t order) {
}
pInfo->win.ekey = ekey;
} else {
assert(order == TSDB_ORDER_DESC);
skey = taosTimeTruncate(pInfo->win.skey, &pInfo->pFillInfo->interval);
next = skey;
while (next < pInfo->win.skey) {

6
source/libs/executor/src/hashjoin.c
View File

@ -60,7 +60,7 @@ int32_t hInnerJoinDo(struct SOperatorInfo* pOperator) {
/*
size_t keySize = 0;
int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize);
ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
A S S E R T(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
int64_t rows = getSingleKeyRowsNum(pGroup->rows);
pJoin->execInfo.expectRows += rows;
qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows);
@ -145,7 +145,7 @@ int32_t hLeftJoinHandleSeqProbeRows(struct SOperatorInfo* pOperator, SHJoinOpera
/*
size_t keySize = 0;
int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize);
ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
A S S E R T(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
int64_t rows = getSingleKeyRowsNum(pGroup->rows);
pJoin->execInfo.expectRows += rows;
qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows);
@ -248,7 +248,7 @@ int32_t hLeftJoinHandleProbeRows(struct SOperatorInfo* pOperator, SHJoinOperator
/*
size_t keySize = 0;
int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize);
ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
A S S E R T(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
int64_t rows = getSingleKeyRowsNum(pGroup->rows);
pJoin->execInfo.expectRows += rows;
qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows);

3
source/libs/executor/src/hashjoinoperator.c
View File

@ -38,8 +38,6 @@ bool hJoinBlkReachThreshold(SHJoinOperatorInfo* pInfo, int64_t blkRows) {
}
int32_t hJoinHandleMidRemains(SHJoinOperatorInfo* pJoin, SHJoinCtx* pCtx) {
ASSERT(0 < pJoin->midBlk->info.rows);
TSWAP(pJoin->midBlk, pJoin->finBlk);
pCtx->midRemains = false;
@ -1153,7 +1151,6 @@ int32_t hJoinInitResBlocks(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode* pJoinN
if (NULL == pJoin->finBlk) {
QRY_ERR_RET(terrno);
}
ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0);
int32_t code = blockDataEnsureCapacity(pJoin->finBlk, hJoinGetFinBlkCapacity(pJoin, pJoinNode));
if (TSDB_CODE_SUCCESS != code) {

Some files were not shown because too many files have changed in this diff Show More