Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/3.0_mhli
This commit is contained in:
commit
28377a9f37
|
@ -175,6 +175,7 @@ def pre_test_build_mac() {
|
|||
cd ${WK}/debug
|
||||
cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false
|
||||
make -j10
|
||||
ctest -j10 || exit 7
|
||||
'''
|
||||
sh '''
|
||||
date
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
# taosadapter
|
||||
ExternalProject_Add(taosadapter
|
||||
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
|
||||
GIT_TAG a11131c
|
||||
GIT_TAG 8c3d57d
|
||||
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
|
||||
BINARY_DIR ""
|
||||
#BUILD_IN_SOURCE TRUE
|
||||
|
|
|
@ -218,9 +218,9 @@ After compiling your function into a DLL, you add it to TDengine. For more infor
|
|||
|
||||
## Sample Code
|
||||
|
||||
### Sample scalar function: [bit_and](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/bit_and.c)
|
||||
### Sample scalar function: [bit_and](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/bit_and.c)
|
||||
|
||||
The bit_add function implements bitwise addition for multiple columns. If there is only one column, the column is returned. The bit_add function ignores null values.
|
||||
The bit_and function implements bitwise addition for multiple columns. If there is only one column, the column is returned. The bit_and function ignores null values.
|
||||
|
||||
<details>
|
||||
<summary>bit_and.c</summary>
|
||||
|
@ -231,7 +231,7 @@ The bit_add function implements bitwise addition for multiple columns. If there
|
|||
|
||||
</details>
|
||||
|
||||
### Sample aggregate function: [l2norm](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/l2norm.c)
|
||||
### Sample aggregate function: [l2norm](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/l2norm.c)
|
||||
|
||||
The l2norm function finds the second-order norm for all data in the input column. This squares the values, takes a cumulative sum, and finds the square root.
|
||||
|
||||
|
|
|
@ -217,7 +217,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
|
|||
The parameters listed in this section apply to all function modes.
|
||||
|
||||
- **filetype** : The function to be tested, with optional values `insert`, `query` and `subscribe`. These correspond to the insert, query, and subscribe functions, respectively. Users can specify only one of these in each configuration file.
|
||||
**cfgdir**: specify the TDengine cluster configuration file's directory. The default path is /etc/taos.
|
||||
**cfgdir**: specify the TDengine client configuration file's directory. The default path is /etc/taos.
|
||||
|
||||
- **host**: Specify the FQDN of the TDengine server to connect. The default value is `localhost`.
|
||||
|
||||
|
|
|
@ -106,7 +106,7 @@ The parameters described in this document by the effect that they have on the sy
|
|||
| Applicable | Server only |
|
||||
| Meaning | The switch for monitoring inside server. The main object of monitoring is to collect information about load on physical nodes, including CPU usage, memory usage, disk usage, and network bandwidth. Monitoring information is sent over HTTP to the taosKeeper service specified by `monitorFqdn` and `monitorProt`.
|
||||
| Value Range | 0: monitoring disabled, 1: monitoring enabled |
|
||||
| Default | 1 |
|
||||
| Default | 0 |
|
||||
|
||||
### monitorFqdn
|
||||
|
||||
|
@ -735,19 +735,17 @@ To prevent system resource from being exhausted by multiple concurrent streams,
|
|||
| 44 | numOfVnodeQueryThreads | No | Yes |
|
||||
| 45 | numOfVnodeStreamThreads | No | Yes |
|
||||
| 46 | numOfVnodeFetchThreads | No | Yes |
|
||||
| 47 | numOfVnodeWriteThreads | No | Yes |
|
||||
| 48 | numOfVnodeSyncThreads | No | Yes |
|
||||
| 49 | numOfVnodeRsmaThreads | No | Yes |
|
||||
| 50 | numOfQnodeQueryThreads | No | Yes |
|
||||
| 51 | numOfQnodeFetchThreads | No | Yes |
|
||||
| 52 | numOfSnodeSharedThreads | No | Yes |
|
||||
| 53 | numOfSnodeUniqueThreads | No | Yes |
|
||||
| 54 | rpcQueueMemoryAllowed | No | Yes |
|
||||
| 55 | logDir | Yes | Yes |
|
||||
| 56 | minimalLogDirGB | Yes | Yes |
|
||||
| 57 | numOfLogLines | Yes | Yes |
|
||||
| 58 | asyncLog | Yes | Yes |
|
||||
| 59 | logKeepDays | Yes | Yes |
|
||||
| 47 | numOfVnodeRsmaThreads | No | Yes |
|
||||
| 48 | numOfQnodeQueryThreads | No | Yes |
|
||||
| 49 | numOfQnodeFetchThreads | No | Yes |
|
||||
| 50 | numOfSnodeSharedThreads | No | Yes |
|
||||
| 51 | numOfSnodeUniqueThreads | No | Yes |
|
||||
| 52 | rpcQueueMemoryAllowed | No | Yes |
|
||||
| 53 | logDir | Yes | Yes |
|
||||
| 54 | minimalLogDirGB | Yes | Yes |
|
||||
| 55 | numOfLogLines | Yes | Yes |
|
||||
| 56 | asyncLog | Yes | Yes |
|
||||
| 57 | logKeepDays | Yes | Yes |
|
||||
| 60 | debugFlag | Yes | Yes |
|
||||
| 61 | tmrDebugFlag | Yes | Yes |
|
||||
| 62 | uDebugFlag | Yes | Yes |
|
||||
|
|
|
@ -6,6 +6,10 @@ description: TDengine release history, Release Notes and download links.
|
|||
|
||||
import Release from "/components/ReleaseV3";
|
||||
|
||||
## 3.0.1.6
|
||||
|
||||
<Release type="tdengine" version="3.0.1.6" />
|
||||
|
||||
## 3.0.1.5
|
||||
|
||||
<Release type="tdengine" version="3.0.1.5" />
|
||||
|
|
|
@ -6,6 +6,10 @@ description: taosTools release history, Release Notes, download links.
|
|||
|
||||
import Release from "/components/ReleaseV3";
|
||||
|
||||
## 2.2.7
|
||||
|
||||
<Release type="tools" version="2.2.7" />
|
||||
|
||||
## 2.2.6
|
||||
|
||||
<Release type="tools" version="2.2.6" />
|
||||
|
|
|
@ -9,9 +9,10 @@ namespace Examples
|
|||
{
|
||||
string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
|
||||
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
|
||||
|
||||
if (wsConn == IntPtr.Zero)
|
||||
{
|
||||
throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}");
|
||||
throw new Exception("get WS connection failed");
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
@ -13,7 +13,7 @@ namespace Examples
|
|||
// Assert if connection is validate
|
||||
if (wsConn == IntPtr.Zero)
|
||||
{
|
||||
throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}");
|
||||
throw new Exception("get WS connection failed");
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
@ -13,7 +13,7 @@ namespace Examples
|
|||
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
|
||||
if (wsConn == IntPtr.Zero)
|
||||
{
|
||||
throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}");
|
||||
throw new Exception("get WS connection failed");
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
@ -21,7 +21,7 @@ namespace Examples
|
|||
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
|
||||
if (wsConn == IntPtr.Zero)
|
||||
{
|
||||
throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}");
|
||||
throw new Exception($"get WS connection failed");
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
@ -190,3 +190,16 @@ DROP DNODE dnodeId;
|
|||
dnodeID 是集群自动分配的,不得人工指定。它在生成时是递增的,不会重复。
|
||||
|
||||
:::
|
||||
|
||||
## 常见问题
|
||||
|
||||
1、建立集群时使用 CREATE DNODE 增加新节点后,新节点始终显示 offline 状态?
|
||||
```sql
|
||||
1)首先要检查增加的新节点上的 taosd 服务是否已经正常启动
|
||||
|
||||
2)如果已经启动,再检查到新节点的网络是否通畅,可以使用 ping fqdn 验证下
|
||||
|
||||
3)如果前面两步都没有问题,这一步要检查新节点做为独立集群在运行了,可以使用 taos -h fqdn 连接上后,show dnodes; 命令查看.
|
||||
如果显示的列表与你主节点上显示的不一致,说明此节点自己单独成立了一个集群,解决的方法是停止新节点上的服务,然后清空新节点上
|
||||
taos.cfg 中配置的 dataDir 目录下的所有文件,重新启动新节点服务即可解决。
|
||||
```
|
||||
|
|
|
@ -217,7 +217,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
|
|||
本节所列参数适用于所有功能模式。
|
||||
|
||||
- **filetype** : 要测试的功能,可选值为 `insert`, `query` 和 `subscribe`。分别对应插入、查询和订阅功能。每个配置文件中只能指定其中之一。
|
||||
- **cfgdir** : TDengine 集群配置文件所在的目录,默认路径是 /etc/taos 。
|
||||
- **cfgdir** : TDengine 客户端配置文件所在的目录,默认路径是 /etc/taos 。
|
||||
|
||||
- **host** : 指定要连接的 TDengine 服务端的 FQDN,默认值为 localhost。
|
||||
|
||||
|
|
|
@ -106,7 +106,7 @@ taos --dump-config
|
|||
| 适用范围 | 仅服务端适用 |
|
||||
| 含义 | 服务器内部的系统监控开关。监控主要负责收集物理节点的负载状况,包括 CPU、内存、硬盘、网络带宽的监控记录,监控信息将通过 HTTP 协议发送给由 `monitorFqdn` 和 `monitorProt` 指定的 TaosKeeper 监控服务 |
|
||||
| 取值范围 | 0:关闭监控服务, 1:激活监控服务。 |
|
||||
| 缺省值 | 1 |
|
||||
| 缺省值 | 0 |
|
||||
|
||||
### monitorFqdn
|
||||
|
||||
|
@ -711,19 +711,17 @@ charset 的有效值是 UTF-8。
|
|||
| 44 | numOfVnodeQueryThreads | 否 | 是 | |
|
||||
| 45 | numOfVnodeStreamThreads | 否 | 是 | |
|
||||
| 46 | numOfVnodeFetchThreads | 否 | 是 | |
|
||||
| 47 | numOfVnodeWriteThreads | 否 | 是 | |
|
||||
| 48 | numOfVnodeSyncThreads | 否 | 是 | |
|
||||
| 49 | numOfVnodeRsmaThreads | 否 | 是 | |
|
||||
| 50 | numOfQnodeQueryThreads | 否 | 是 | |
|
||||
| 51 | numOfQnodeFetchThreads | 否 | 是 | |
|
||||
| 52 | numOfSnodeSharedThreads | 否 | 是 | |
|
||||
| 53 | numOfSnodeUniqueThreads | 否 | 是 | |
|
||||
| 54 | rpcQueueMemoryAllowed | 否 | 是 | |
|
||||
| 55 | logDir | 是 | 是 | |
|
||||
| 56 | minimalLogDirGB | 是 | 是 | |
|
||||
| 57 | numOfLogLines | 是 | 是 | |
|
||||
| 58 | asyncLog | 是 | 是 | |
|
||||
| 59 | logKeepDays | 是 | 是 | |
|
||||
| 47 | numOfVnodeRsmaThreads | 否 | 是 | |
|
||||
| 48 | numOfQnodeQueryThreads | 否 | 是 | |
|
||||
| 49 | numOfQnodeFetchThreads | 否 | 是 | |
|
||||
| 50 | numOfSnodeSharedThreads | 否 | 是 | |
|
||||
| 51 | numOfSnodeUniqueThreads | 否 | 是 | |
|
||||
| 52 | rpcQueueMemoryAllowed | 否 | 是 | |
|
||||
| 53 | logDir | 是 | 是 | |
|
||||
| 54 | minimalLogDirGB | 是 | 是 | |
|
||||
| 55 | numOfLogLines | 是 | 是 | |
|
||||
| 56 | asyncLog | 是 | 是 | |
|
||||
| 57 | logKeepDays | 是 | 是 | |
|
||||
| 60 | debugFlag | 是 | 是 | |
|
||||
| 61 | tmrDebugFlag | 是 | 是 | |
|
||||
| 62 | uDebugFlag | 是 | 是 | |
|
||||
|
|
|
@ -6,6 +6,10 @@ description: TDengine 发布历史、Release Notes 及下载链接
|
|||
|
||||
import Release from "/components/ReleaseV3";
|
||||
|
||||
## 3.0.1.6
|
||||
|
||||
<Release type="tdengine" version="3.0.1.6" />
|
||||
|
||||
## 3.0.1.5
|
||||
|
||||
<Release type="tdengine" version="3.0.1.5" />
|
||||
|
|
|
@ -6,6 +6,10 @@ description: taosTools 的发布历史、Release Notes 和下载链接
|
|||
|
||||
import Release from "/components/ReleaseV3";
|
||||
|
||||
## 2.2.7
|
||||
|
||||
<Release type="tools" version="2.2.7" />
|
||||
|
||||
## 2.2.6
|
||||
|
||||
<Release type="tools" version="2.2.6" />
|
||||
|
|
|
@ -225,13 +225,13 @@ typedef struct SVarColAttr {
|
|||
// pBlockAgg->numOfNull == info.rows, all data are null
|
||||
// pBlockAgg->numOfNull == 0, no data are null.
|
||||
typedef struct SColumnInfoData {
|
||||
SColumnInfo info; // column info
|
||||
bool hasNull; // if current column data has null value.
|
||||
char* pData; // the corresponding block data in memory
|
||||
char* pData; // the corresponding block data in memory
|
||||
union {
|
||||
char* nullbitmap; // bitmap, one bit for each item in the list
|
||||
SVarColAttr varmeta;
|
||||
};
|
||||
SColumnInfo info; // column info
|
||||
bool hasNull; // if current column data has null value.
|
||||
} SColumnInfoData;
|
||||
|
||||
typedef struct SQueryTableDataCond {
|
||||
|
|
|
@ -55,8 +55,6 @@ extern int32_t tsNumOfMnodeReadThreads;
|
|||
extern int32_t tsNumOfVnodeQueryThreads;
|
||||
extern int32_t tsNumOfVnodeStreamThreads;
|
||||
extern int32_t tsNumOfVnodeFetchThreads;
|
||||
extern int32_t tsNumOfVnodeWriteThreads;
|
||||
extern int32_t tsNumOfVnodeSyncThreads;
|
||||
extern int32_t tsNumOfVnodeRsmaThreads;
|
||||
extern int32_t tsNumOfQnodeQueryThreads;
|
||||
extern int32_t tsNumOfQnodeFetchThreads;
|
||||
|
|
|
@ -297,7 +297,6 @@ typedef struct {
|
|||
|
||||
typedef struct {
|
||||
int32_t code;
|
||||
int8_t hashMeta;
|
||||
int64_t uid;
|
||||
char* tblFName;
|
||||
int32_t numOfRows;
|
||||
|
@ -1128,6 +1127,7 @@ typedef struct {
|
|||
SQnodeLoad qload;
|
||||
SClusterCfg clusterCfg;
|
||||
SArray* pVloads; // array of SVnodeLoad
|
||||
int32_t statusSeq;
|
||||
} SStatusReq;
|
||||
|
||||
int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
|
||||
|
@ -1149,6 +1149,7 @@ typedef struct {
|
|||
int64_t dnodeVer;
|
||||
SDnodeCfg dnodeCfg;
|
||||
SArray* pDnodeEps; // Array of SDnodeEp
|
||||
int32_t statusSeq;
|
||||
} SStatusRsp;
|
||||
|
||||
int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp);
|
||||
|
|
|
@ -241,6 +241,7 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "vnode-stream-finish", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_MON_MSG)
|
||||
|
|
|
@ -203,13 +203,11 @@ int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg);
|
|||
|
||||
int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg);
|
||||
|
||||
int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta);
|
||||
int32_t catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta);
|
||||
|
||||
int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
|
||||
STableMeta** pTableMeta);
|
||||
int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta);
|
||||
|
||||
int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
|
||||
SVgroupInfo* pVgroup, bool* exists);
|
||||
int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists);
|
||||
|
||||
/**
|
||||
* Force refresh DB's local cached vgroup info.
|
||||
|
@ -309,7 +307,7 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f
|
|||
int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
|
||||
bool* pass);
|
||||
|
||||
int32_t catalogChkAuthFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
|
||||
int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type,
|
||||
bool* pass, bool* exists);
|
||||
|
||||
int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth);
|
||||
|
@ -326,6 +324,10 @@ SMetaData* catalogCloneMetaData(SMetaData* pData);
|
|||
|
||||
void catalogFreeMetaData(SMetaData* pData);
|
||||
|
||||
int32_t ctgdEnableDebug(char *option, bool enable);
|
||||
|
||||
int32_t ctgdHandleDbgCommand(char *command);
|
||||
|
||||
/**
|
||||
* Destroy catalog and relase all resources
|
||||
*/
|
||||
|
|
|
@ -248,7 +248,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
|
|||
(_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID))
|
||||
#define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
|
||||
((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
|
||||
#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)
|
||||
#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER)
|
||||
#define NEED_CLIENT_HANDLE_ERROR(_code) \
|
||||
(NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
|
||||
NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
|
||||
|
|
|
@ -46,7 +46,7 @@ enum {
|
|||
TASK_STATUS__DROPPING,
|
||||
TASK_STATUS__FAIL,
|
||||
TASK_STATUS__STOP,
|
||||
TASK_STATUS__RECOVER_DOWNSTREAM,
|
||||
TASK_STATUS__WAIT_DOWNSTREAM,
|
||||
TASK_STATUS__RECOVER_PREPARE,
|
||||
TASK_STATUS__RECOVER1,
|
||||
TASK_STATUS__RECOVER2,
|
||||
|
@ -332,7 +332,10 @@ typedef struct SStreamTask {
|
|||
SStreamState* pState;
|
||||
|
||||
// do not serialize
|
||||
int32_t recoverWaitingChild;
|
||||
int32_t recoverTryingDownstream;
|
||||
int32_t recoverWaitingUpstream;
|
||||
int64_t checkReqId;
|
||||
SArray* checkReqIds; // shuffle
|
||||
|
||||
} SStreamTask;
|
||||
|
||||
|
@ -418,7 +421,10 @@ typedef struct {
|
|||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
int32_t taskId;
|
||||
int32_t upstreamNodeId;
|
||||
int32_t upstreamTaskId;
|
||||
int32_t downstreamNodeId;
|
||||
int32_t downstreamTaskId;
|
||||
int8_t inputStatus;
|
||||
} SStreamDispatchRsp;
|
||||
|
||||
|
@ -440,6 +446,27 @@ typedef struct {
|
|||
int32_t rspToTaskId;
|
||||
} SStreamRetrieveRsp;
|
||||
|
||||
typedef struct {
|
||||
int64_t reqId;
|
||||
int64_t streamId;
|
||||
int32_t upstreamNodeId;
|
||||
int32_t upstreamTaskId;
|
||||
int32_t downstreamNodeId;
|
||||
int32_t downstreamTaskId;
|
||||
int32_t childId;
|
||||
} SStreamTaskCheckReq;
|
||||
|
||||
typedef struct {
|
||||
int64_t reqId;
|
||||
int64_t streamId;
|
||||
int32_t upstreamNodeId;
|
||||
int32_t upstreamTaskId;
|
||||
int32_t downstreamNodeId;
|
||||
int32_t downstreamTaskId;
|
||||
int32_t childId;
|
||||
int8_t status;
|
||||
} SStreamTaskCheckRsp;
|
||||
|
||||
typedef struct {
|
||||
SMsgHead msgHead;
|
||||
int64_t streamId;
|
||||
|
@ -455,47 +482,6 @@ typedef struct {
|
|||
int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq);
|
||||
int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq);
|
||||
|
||||
#if 0
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
int32_t taskId;
|
||||
int32_t upstreamTaskId;
|
||||
int32_t upstreamNodeId;
|
||||
} SStreamTaskRecoverReq;
|
||||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
int32_t rspTaskId;
|
||||
int32_t reqTaskId;
|
||||
int8_t inputStatus;
|
||||
} SStreamTaskRecoverRsp;
|
||||
|
||||
int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq);
|
||||
int32_t tDecodeStreamTaskRecoverReq(SDecoder* pDecoder, SStreamTaskRecoverReq* pReq);
|
||||
|
||||
int32_t tEncodeStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamTaskRecoverRsp* pRsp);
|
||||
int32_t tDecodeStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamTaskRecoverRsp* pRsp);
|
||||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
int32_t taskId;
|
||||
} SMStreamTaskRecoverReq;
|
||||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
int32_t taskId;
|
||||
} SMStreamTaskRecoverRsp;
|
||||
|
||||
int32_t tEncodeSMStreamTaskRecoverReq(SEncoder* pEncoder, const SMStreamTaskRecoverReq* pReq);
|
||||
int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq* pReq);
|
||||
|
||||
int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp);
|
||||
int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pRsp);
|
||||
|
||||
int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg);
|
||||
int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp);
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
int32_t downstreamTaskId;
|
||||
|
@ -509,20 +495,18 @@ typedef struct {
|
|||
SArray* checkpointVer; // SArray<SStreamCheckpointInfo>
|
||||
} SStreamRecoverDownstreamRsp;
|
||||
|
||||
int32_t tEncodeSStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq);
|
||||
int32_t tDecodeSStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq);
|
||||
|
||||
int32_t tEncodeSStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp);
|
||||
int32_t tDecodeSStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp);
|
||||
|
||||
int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq);
|
||||
int32_t tDecodeSStreamTaskRecoverReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq);
|
||||
|
||||
int32_t tEncodeSStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamRecoverDownstreamRsp* pRsp);
|
||||
int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamRecoverDownstreamRsp* pRsp);
|
||||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
int32_t taskId;
|
||||
int32_t waitingRspCnt;
|
||||
int32_t totReq;
|
||||
SArray* info; // SArray<SArray<SStreamCheckpointInfo>*>
|
||||
} SStreamRecoverStatus;
|
||||
|
||||
int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq);
|
||||
int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq);
|
||||
void tDeleteStreamRetrieveReq(SStreamRetrieveReq* pReq);
|
||||
|
@ -533,7 +517,7 @@ int32_t streamSetupTrigger(SStreamTask* pTask);
|
|||
|
||||
int32_t streamProcessRunReq(SStreamTask* pTask);
|
||||
int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg, bool exec);
|
||||
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp);
|
||||
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code);
|
||||
|
||||
int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg);
|
||||
int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp);
|
||||
|
@ -544,6 +528,10 @@ int32_t streamSchedExec(SStreamTask* pTask);
|
|||
int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz);
|
||||
|
||||
// recover and fill history
|
||||
int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version);
|
||||
int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version);
|
||||
int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq);
|
||||
int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version);
|
||||
// common
|
||||
int32_t streamSetParamForRecover(SStreamTask* pTask);
|
||||
int32_t streamRestoreParam(SStreamTask* pTask);
|
||||
|
|
|
@ -340,7 +340,7 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_TDB_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0618)
|
||||
#define TSDB_CODE_TDB_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0619)
|
||||
#define TSDB_CODE_TDB_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x061A)
|
||||
#define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x061B)
|
||||
#define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B)
|
||||
#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C)
|
||||
|
||||
// query
|
||||
|
@ -612,9 +612,12 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001)
|
||||
#define TSDB_CODE_TMQ_CONSUMER_CLOSED TAOS_DEF_ERROR_CODE(0, 0x4002)
|
||||
|
||||
// stream
|
||||
#define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100)
|
||||
|
||||
// TDLite
|
||||
#define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x4100)
|
||||
#define TSDB_CODE_TDLITE_IVLD_OPEN_DIR TAOS_DEF_ERROR_CODE(0, 0x4101)
|
||||
#define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x5100)
|
||||
#define TSDB_CODE_TDLITE_IVLD_OPEN_DIR TAOS_DEF_ERROR_CODE(0, 0x5101)
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
@ -59,6 +59,47 @@ typedef enum {
|
|||
typedef void (*FItem)(SQueueInfo *pInfo, void *pItem);
|
||||
typedef void (*FItems)(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfItems);
|
||||
|
||||
typedef struct STaosQnode STaosQnode;
|
||||
|
||||
typedef struct STaosQnode {
|
||||
STaosQnode *next;
|
||||
STaosQueue *queue;
|
||||
int64_t timestamp;
|
||||
int32_t size;
|
||||
int8_t itype;
|
||||
int8_t reserved[3];
|
||||
char item[];
|
||||
} STaosQnode;
|
||||
|
||||
typedef struct STaosQueue {
|
||||
STaosQnode *head;
|
||||
STaosQnode *tail;
|
||||
STaosQueue *next; // for queue set
|
||||
STaosQset *qset; // for queue set
|
||||
void *ahandle; // for queue set
|
||||
FItem itemFp;
|
||||
FItems itemsFp;
|
||||
TdThreadMutex mutex;
|
||||
int64_t memOfItems;
|
||||
int32_t numOfItems;
|
||||
int64_t threadId;
|
||||
} STaosQueue;
|
||||
|
||||
typedef struct STaosQset {
|
||||
STaosQueue *head;
|
||||
STaosQueue *current;
|
||||
TdThreadMutex mutex;
|
||||
tsem_t sem;
|
||||
int32_t numOfQueues;
|
||||
int32_t numOfItems;
|
||||
} STaosQset;
|
||||
|
||||
typedef struct STaosQall {
|
||||
STaosQnode *current;
|
||||
STaosQnode *start;
|
||||
int32_t numOfItems;
|
||||
} STaosQall;
|
||||
|
||||
STaosQueue *taosOpenQueue();
|
||||
void taosCloseQueue(STaosQueue *queue);
|
||||
void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp);
|
||||
|
|
|
@ -26,8 +26,9 @@ typedef struct SQWorkerPool SQWorkerPool;
|
|||
typedef struct SWWorkerPool SWWorkerPool;
|
||||
|
||||
typedef struct SQWorker {
|
||||
int32_t id; // worker ID
|
||||
TdThread thread; // thread
|
||||
int32_t id; // worker id
|
||||
int64_t pid; // thread pid
|
||||
TdThread thread; // thread id
|
||||
SQWorkerPool *pool;
|
||||
} SQWorker;
|
||||
|
||||
|
@ -43,9 +44,10 @@ typedef struct SQWorkerPool {
|
|||
|
||||
typedef struct SWWorker {
|
||||
int32_t id; // worker id
|
||||
TdThread thread; // thread
|
||||
int64_t pid; // thread pid
|
||||
TdThread thread; // thread id
|
||||
STaosQall *qall;
|
||||
STaosQset *qset; // queue set
|
||||
STaosQset *qset;
|
||||
SWWorkerPool *pool;
|
||||
} SWWorker;
|
||||
|
||||
|
|
|
@ -40,7 +40,7 @@ if not exist %work_dir%\debug\ver-%2-x86 (
|
|||
)
|
||||
cd %work_dir%\debug\ver-%2-x64
|
||||
call vcvarsall.bat x64
|
||||
cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DBUILD_HTTP=false -DBUILD_TEST=false -DVERNUMBER=%2 -DCPUTYPE=x64
|
||||
cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DWEBSOCKET=true -DBUILD_HTTP=false -DBUILD_TEST=false -DVERNUMBER=%2 -DCPUTYPE=x64
|
||||
cmake --build .
|
||||
rd /s /Q C:\TDengine
|
||||
cmake --install .
|
||||
|
@ -63,4 +63,4 @@ exit /b
|
|||
:RUNFAILED
|
||||
echo %*
|
||||
cd %package_dir%
|
||||
goto :eof
|
||||
goto :eof
|
||||
|
|
|
@ -67,6 +67,7 @@ fi
|
|||
}
|
||||
|
||||
|
||||
|
||||
function wgetFile {
|
||||
|
||||
file=$1
|
||||
|
@ -75,7 +76,10 @@ if [ ! -f ${file} ];then
|
|||
echoColor BD "wget https://www.taosdata.com/assets-download/3.0/${file}"
|
||||
wget https://www.taosdata.com/assets-download/3.0/${file}
|
||||
else
|
||||
echoColor YD "${file} already exists "
|
||||
echoColor YD "${file} already exists and use new file "
|
||||
rm -rf ${file}
|
||||
echoColor BD "wget https://www.taosdata.com/assets-download/3.0/${file}"
|
||||
wget https://www.taosdata.com/assets-download/3.0/${file}
|
||||
fi
|
||||
}
|
||||
|
||||
|
@ -211,7 +215,8 @@ elif [[ ${packgeName} =~ "tar" ]];then
|
|||
|
||||
if [ ${diffNumbers} != 0 ];then
|
||||
echoColor R "The number and names of files is different from the previous installation package"
|
||||
echoColor Y `cat ${installPath}/diffFile.log`
|
||||
diffLog=`cat ${installPath}/diffFile.log`
|
||||
echoColor Y "${diffLog}"
|
||||
exit -1
|
||||
else
|
||||
echoColor G "The number and names of files are the same as previous installation packages"
|
||||
|
|
|
@ -34,7 +34,6 @@ if exist %binary_dir%\\test\\cfg\\taosadapter.toml (
|
|||
copy %binary_dir%\\test\\cfg\\taosadapter.toml %tagert_dir%\\cfg\\taosadapter.toml > nul
|
||||
)
|
||||
)
|
||||
|
||||
copy %source_dir%\\include\\client\\taos.h %tagert_dir%\\include > nul
|
||||
copy %source_dir%\\include\\util\\taoserror.h %tagert_dir%\\include > nul
|
||||
copy %source_dir%\\include\\libs\\function\\taosudf.h %tagert_dir%\\include > nul
|
||||
|
@ -52,6 +51,7 @@ if exist %binary_dir%\\build\\lib\\taosws.dll.lib (
|
|||
)
|
||||
if exist %binary_dir%\\build\\lib\\taosws.dll (
|
||||
copy %binary_dir%\\build\\lib\\taosws.dll %tagert_dir%\\driver > nul
|
||||
copy %source_dir%\\tools\\taosws-rs\\target\\release\\taosws.h %tagert_dir%\\include > nul
|
||||
)
|
||||
if exist %binary_dir%\\build\\bin\\taosdump.exe (
|
||||
copy %binary_dir%\\build\\bin\\taosdump.exe %tagert_dir% > nul
|
||||
|
@ -67,3 +67,4 @@ if exist C:\\TDengine\\driver\\taosws.dll (
|
|||
copy /y C:\\TDengine\\driver\\taosws.dll C:\\Windows\\System32 > nul
|
||||
)
|
||||
sc query "taosd" >nul || sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND
|
||||
sc query "taosadapter" >nul || sc create "taosadapter" binPath= "C:\\TDengine\\taosadapter.exe" start= DEMAND
|
||||
|
|
|
@ -285,10 +285,14 @@ if [[ $dbName == "taos" ]]; then
|
|||
fi
|
||||
|
||||
# Add web files
|
||||
if [ -d "${web_dir}/admin" ]; then
|
||||
mkdir -p ${install_dir}/share/
|
||||
cp ${web_dir}/admin ${install_dir}/share/ -r
|
||||
cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png
|
||||
if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then
|
||||
if [ -d "${web_dir}/admin" ] ; then
|
||||
mkdir -p ${install_dir}/share/
|
||||
cp ${web_dir}/admin ${install_dir}/share/ -r
|
||||
cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png
|
||||
else
|
||||
echo "directory not found for enterprise release: ${web_dir}/admin"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
@ -375,4 +379,4 @@ if [ -n "${taostools_bin_files}" ]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
cd ${curr_dir}
|
||||
cd ${curr_dir}
|
|
@ -15,10 +15,10 @@
|
|||
#define MyAppExeName "\*.exe"
|
||||
#define MyAppTaosExeName "\taos.bat"
|
||||
#define MyAppTaosdemoExeName "\taosBenchmark.exe"
|
||||
#define MyAppDLLName "\driver\taos.dll"
|
||||
#define MyAppDLLName "\driver\*.dll"
|
||||
;#define MyAppVersion "3.0"
|
||||
;#define MyAppInstallName "TDengine"
|
||||
|
||||
;#define MyAppInstallName "TDengine"
|
||||
[Setup]
|
||||
VersionInfoVersion={#MyAppVersion}
|
||||
AppId={{A0F7A93C-79C4-485D-B2B8-F0D03DF42FAB}
|
||||
|
|
|
@ -318,6 +318,7 @@ void* createTscObj(const char* user, const char* auth, const char* db, int32_
|
|||
void destroyTscObj(void* pObj);
|
||||
STscObj* acquireTscObj(int64_t rid);
|
||||
int32_t releaseTscObj(int64_t rid);
|
||||
void destroyAppInst(SAppInstInfo *pAppInfo);
|
||||
|
||||
uint64_t generateRequestId();
|
||||
|
||||
|
|
|
@ -39,6 +39,7 @@ typedef enum {
|
|||
STMT_BIND_COL,
|
||||
STMT_ADD_BATCH,
|
||||
STMT_EXECUTE,
|
||||
STMT_MAX,
|
||||
} STMT_STATUS;
|
||||
|
||||
typedef struct SStmtTableCache {
|
||||
|
@ -94,12 +95,18 @@ typedef struct STscStmt {
|
|||
STscObj *taos;
|
||||
SCatalog *pCatalog;
|
||||
int32_t affectedRows;
|
||||
uint32_t seqId;
|
||||
uint32_t seqIds[STMT_MAX];
|
||||
|
||||
SStmtSQLInfo sql;
|
||||
SStmtExecInfo exec;
|
||||
SStmtBindInfo bInfo;
|
||||
} STscStmt;
|
||||
|
||||
extern char *gStmtStatusStr[];
|
||||
|
||||
#define STMT_LOG_SEQ(n) do { (pStmt)->seqId++; (pStmt)->seqIds[n]++; STMT_DLOG("the %dth:%d %s", (pStmt)->seqIds[n], (pStmt)->seqId, gStmtStatusStr[n]); } while (0)
|
||||
|
||||
#define STMT_STATUS_NE(S) (pStmt->sql.status != STMT_##S)
|
||||
#define STMT_STATUS_EQ(S) (pStmt->sql.status == STMT_##S)
|
||||
|
||||
|
@ -128,6 +135,12 @@ typedef struct STscStmt {
|
|||
} \
|
||||
} while (0)
|
||||
|
||||
#define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__)
|
||||
#define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__)
|
||||
|
||||
#define STMT_ELOG_E(param) qError("stmt:%p " param, pStmt)
|
||||
#define STMT_DLOG_E(param) qDebug("stmt:%p " param, pStmt)
|
||||
|
||||
TAOS_STMT *stmtInit(STscObj *taos);
|
||||
int stmtClose(TAOS_STMT *stmt);
|
||||
int stmtExec(TAOS_STMT *stmt);
|
||||
|
|
|
@ -133,6 +133,12 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
|
|||
taosThreadMutexInit(&p->qnodeMutex, NULL);
|
||||
p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores);
|
||||
p->pAppHbMgr = appHbMgrInit(p, key);
|
||||
if (NULL == p->pAppHbMgr) {
|
||||
destroyAppInst(p);
|
||||
taosThreadMutexUnlock(&appInfo.mutex);
|
||||
taosMemoryFreeClear(key);
|
||||
return NULL;
|
||||
}
|
||||
taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES);
|
||||
p->instKey = key;
|
||||
key = NULL;
|
||||
|
@ -1266,7 +1272,9 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) {
|
|||
pMsgSendInfo->requestObjRefId = pRequest->self;
|
||||
pMsgSendInfo->requestId = pRequest->requestId;
|
||||
pMsgSendInfo->fp = getMsgRspHandle(pMsgSendInfo->msgType);
|
||||
pMsgSendInfo->param = pRequest;
|
||||
pMsgSendInfo->param = taosMemoryCalloc(1, sizeof(pRequest->self));
|
||||
|
||||
*(int64_t*)pMsgSendInfo->param = pRequest->self;
|
||||
|
||||
SConnectReq connectReq = {0};
|
||||
STscObj* pObj = pRequest->pTscObj;
|
||||
|
|
|
@ -286,7 +286,8 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
|
|||
return NULL;
|
||||
} else {
|
||||
// assert to avoid un-initialization error
|
||||
ASSERT(0);
|
||||
tscError("invalid result passed to taos_fetch_row");
|
||||
return NULL;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
@ -45,8 +45,13 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
|
|||
}
|
||||
|
||||
int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SRequestObj* pRequest = param;
|
||||
|
||||
SRequestObj *pRequest = acquireRequest(*(int64_t*)param);
|
||||
if (NULL == pRequest) {
|
||||
setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED);
|
||||
tsem_post(&pRequest->body.rspSem);
|
||||
goto End;
|
||||
}
|
||||
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
setErrno(pRequest, code);
|
||||
tsem_post(&pRequest->body.rspSem);
|
||||
|
@ -55,6 +60,12 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
|||
|
||||
STscObj* pTscObj = pRequest->pTscObj;
|
||||
|
||||
if (NULL == pTscObj->pAppInfo || NULL == pTscObj->pAppInfo->pAppHbMgr) {
|
||||
setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED);
|
||||
tsem_post(&pRequest->body.rspSem);
|
||||
goto End;
|
||||
}
|
||||
|
||||
SConnectRsp connectRsp = {0};
|
||||
if (tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp) != 0) {
|
||||
code = TSDB_CODE_TSC_INVALID_VERSION;
|
||||
|
@ -115,10 +126,15 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
|||
|
||||
tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId,
|
||||
pTscObj->pAppInfo->numOfConns);
|
||||
|
||||
|
||||
tsem_post(&pRequest->body.rspSem);
|
||||
End:
|
||||
|
||||
if (pRequest) {
|
||||
releaseRequest(pRequest->self);
|
||||
}
|
||||
|
||||
taosMemoryFree(param);
|
||||
taosMemoryFree(pMsg->pEpSet);
|
||||
taosMemoryFree(pMsg->pData);
|
||||
return code;
|
||||
|
|
|
@ -5,6 +5,8 @@
|
|||
|
||||
#include "clientStmt.h"
|
||||
|
||||
char *gStmtStatusStr[] = {"unknown", "init", "prepare", "settbname", "settags", "fetchFields", "bind", "bindCol", "addBatch", "exec"};
|
||||
|
||||
static int32_t stmtCreateRequest(STscStmt* pStmt) {
|
||||
int32_t code = 0;
|
||||
|
||||
|
@ -21,6 +23,10 @@ static int32_t stmtCreateRequest(STscStmt* pStmt) {
|
|||
int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
|
||||
int32_t code = 0;
|
||||
|
||||
if (newStatus >= STMT_INIT && newStatus < STMT_MAX) {
|
||||
STMT_LOG_SEQ(newStatus);
|
||||
}
|
||||
|
||||
switch (newStatus) {
|
||||
case STMT_PREPARE:
|
||||
break;
|
||||
|
@ -528,13 +534,17 @@ TAOS_STMT* stmtInit(STscObj* taos) {
|
|||
pStmt->bInfo.needParse = true;
|
||||
pStmt->sql.status = STMT_INIT;
|
||||
|
||||
STMT_LOG_SEQ(STMT_INIT);
|
||||
|
||||
tscDebug("stmt:%p initialized", pStmt);
|
||||
|
||||
return pStmt;
|
||||
}
|
||||
|
||||
int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
tscDebug("stmt start to prepare");
|
||||
STMT_DLOG_E("start to prepare");
|
||||
|
||||
if (pStmt->sql.status >= STMT_PREPARE) {
|
||||
STMT_ERR_RET(stmtResetStmt(pStmt));
|
||||
|
@ -555,7 +565,7 @@ int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
|
|||
int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
tscDebug("stmt start to set tbName: %s", tbName);
|
||||
STMT_DLOG("start to set tbName: %s", tbName);
|
||||
|
||||
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME));
|
||||
|
||||
|
@ -587,7 +597,7 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) {
|
|||
int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
tscDebug("stmt start to set tbTags");
|
||||
STMT_DLOG_E("start to set tbTags");
|
||||
|
||||
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS));
|
||||
|
||||
|
@ -649,7 +659,7 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields
|
|||
int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
tscDebug("start to bind stmt data, colIdx: %d", colIdx);
|
||||
STMT_DLOG("start to bind stmt data, colIdx: %d", colIdx);
|
||||
|
||||
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
|
||||
|
||||
|
@ -743,7 +753,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
|
|||
int stmtAddBatch(TAOS_STMT* stmt) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
tscDebug("stmt start to add batch");
|
||||
STMT_DLOG_E("start to add batch");
|
||||
|
||||
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH));
|
||||
|
||||
|
@ -756,8 +766,7 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) {
|
|||
tscDebug("stmt start to update tbUid, blockNum: %d", pRsp->nBlocks);
|
||||
|
||||
if (pRsp->nBlocks <= 0) {
|
||||
tscError("invalid submit resp block number %d", pRsp->nBlocks);
|
||||
STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
size_t keyLen = 0;
|
||||
|
@ -810,7 +819,7 @@ int stmtExec(TAOS_STMT* stmt) {
|
|||
SSubmitRsp* pRsp = NULL;
|
||||
bool autoCreateTbl = pStmt->exec.autoCreateTbl;
|
||||
|
||||
tscDebug("stmt start to exec");
|
||||
STMT_DLOG_E("start to exec");
|
||||
|
||||
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
|
||||
|
||||
|
@ -885,6 +894,8 @@ int stmtAffectedRowsOnce(TAOS_STMT* stmt) { return ((STscStmt*)stmt)->exec.affec
|
|||
int stmtIsInsert(TAOS_STMT* stmt, int* insert) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
STMT_DLOG_E("start is insert");
|
||||
|
||||
if (pStmt->sql.type) {
|
||||
*insert = (STMT_TYPE_INSERT == pStmt->sql.type || STMT_TYPE_MULTI_INSERT == pStmt->sql.type);
|
||||
} else {
|
||||
|
@ -897,6 +908,8 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) {
|
|||
int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
STMT_DLOG_E("start to get tag fields");
|
||||
|
||||
if (STMT_TYPE_QUERY == pStmt->sql.type) {
|
||||
STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
|
||||
}
|
||||
|
@ -927,6 +940,8 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
|
|||
int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
STMT_DLOG_E("start to get col fields");
|
||||
|
||||
if (STMT_TYPE_QUERY == pStmt->sql.type) {
|
||||
STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
|
||||
}
|
||||
|
@ -957,6 +972,8 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
|
|||
int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
STMT_DLOG_E("start to get param num");
|
||||
|
||||
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
|
||||
|
||||
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
|
||||
|
@ -986,6 +1003,8 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
|
|||
int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
STMT_DLOG_E("start to get param");
|
||||
|
||||
if (STMT_TYPE_QUERY == pStmt->sql.type) {
|
||||
STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
|
||||
}
|
||||
|
@ -1028,6 +1047,8 @@ int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
|
|||
TAOS_RES* stmtUseResult(TAOS_STMT* stmt) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
STMT_DLOG_E("start to use result");
|
||||
|
||||
if (STMT_TYPE_QUERY != pStmt->sql.type) {
|
||||
tscError("useResult only for query statement");
|
||||
return NULL;
|
||||
|
|
|
@ -1756,7 +1756,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
|
|||
|
||||
rspObj = tmqHandleAllRsp(tmq, timeout, false);
|
||||
if (rspObj) {
|
||||
tscDebug("consumer:%" PRId64 ", return rsp", tmq->consumerId);
|
||||
tscDebug("consumer:%" PRId64 ", return rsp %p", tmq->consumerId, rspObj);
|
||||
return (TAOS_RES*)rspObj;
|
||||
} else if (terrno == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) {
|
||||
tscDebug("consumer:%" PRId64 ", return null since no committed offset", tmq->consumerId);
|
||||
|
|
|
@ -112,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) {
|
|||
}
|
||||
taos_free_result(pRes);
|
||||
|
||||
for(int32_t i = 0; i < 10000; i += 20) {
|
||||
for(int32_t i = 0; i < 20; i += 20) {
|
||||
char sql[1024] = {0};
|
||||
sprintf(sql,
|
||||
"insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
|
||||
|
@ -692,6 +692,7 @@ TEST(testCase, insert_test) {
|
|||
taos_free_result(pRes);
|
||||
taos_close(pConn);
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST(testCase, projection_query_tables) {
|
||||
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
|
||||
|
@ -703,7 +704,7 @@ TEST(testCase, projection_query_tables) {
|
|||
// }
|
||||
// taos_free_result(pRes);
|
||||
|
||||
TAOS_RES* pRes = taos_query(pConn, "use benchmarkcpu");
|
||||
TAOS_RES* pRes = taos_query(pConn, "use abc2");
|
||||
taos_free_result(pRes);
|
||||
|
||||
pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
|
||||
|
@ -725,7 +726,7 @@ TEST(testCase, projection_query_tables) {
|
|||
}
|
||||
taos_free_result(pRes);
|
||||
|
||||
for (int32_t i = 0; i < 2; ++i) {
|
||||
for (int32_t i = 0; i < 200000; ++i) {
|
||||
printf("create table :%d\n", i);
|
||||
createNewTable(pConn, i);
|
||||
}
|
||||
|
@ -750,7 +751,9 @@ TEST(testCase, projection_query_tables) {
|
|||
taos_free_result(pRes);
|
||||
taos_close(pConn);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#if 0
|
||||
|
||||
TEST(testCase, tsbs_perf_test) {
|
||||
TdThread qid[20] = {0};
|
||||
|
@ -761,7 +764,7 @@ TEST(testCase, tsbs_perf_test) {
|
|||
getchar();
|
||||
}
|
||||
|
||||
#if 0
|
||||
|
||||
TEST(testCase, projection_query_stables) {
|
||||
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
|
||||
ASSERT_NE(pConn, nullptr);
|
||||
|
|
|
@ -50,8 +50,6 @@ int32_t tsNumOfMnodeReadThreads = 1;
|
|||
int32_t tsNumOfVnodeQueryThreads = 4;
|
||||
int32_t tsNumOfVnodeStreamThreads = 2;
|
||||
int32_t tsNumOfVnodeFetchThreads = 4;
|
||||
int32_t tsNumOfVnodeWriteThreads = 2;
|
||||
int32_t tsNumOfVnodeSyncThreads = 2;
|
||||
int32_t tsNumOfVnodeRsmaThreads = 2;
|
||||
int32_t tsNumOfQnodeQueryThreads = 4;
|
||||
int32_t tsNumOfQnodeFetchThreads = 1;
|
||||
|
@ -374,14 +372,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4);
|
||||
if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfVnodeWriteThreads = tsNumOfCores;
|
||||
tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1);
|
||||
if (cfgAddInt32(pCfg, "numOfVnodeWriteThreads", tsNumOfVnodeWriteThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfVnodeSyncThreads = tsNumOfCores * 2;
|
||||
tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 16);
|
||||
if (cfgAddInt32(pCfg, "numOfVnodeSyncThreads", tsNumOfVnodeSyncThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfVnodeRsmaThreads = tsNumOfCores;
|
||||
tsNumOfVnodeRsmaThreads = TMAX(tsNumOfVnodeRsmaThreads, 4);
|
||||
if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
@ -506,22 +496,6 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
|
|||
pItem->stype = stype;
|
||||
}
|
||||
|
||||
pItem = cfgGetItem(tsCfg, "numOfVnodeWriteThreads");
|
||||
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
|
||||
tsNumOfVnodeWriteThreads = numOfCores;
|
||||
tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1);
|
||||
pItem->i32 = tsNumOfVnodeWriteThreads;
|
||||
pItem->stype = stype;
|
||||
}
|
||||
|
||||
pItem = cfgGetItem(tsCfg, "numOfVnodeSyncThreads");
|
||||
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
|
||||
tsNumOfVnodeSyncThreads = numOfCores * 2;
|
||||
tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 16);
|
||||
pItem->i32 = tsNumOfVnodeSyncThreads;
|
||||
pItem->stype = stype;
|
||||
}
|
||||
|
||||
pItem = cfgGetItem(tsCfg, "numOfVnodeRsmaThreads");
|
||||
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
|
||||
tsNumOfVnodeRsmaThreads = numOfCores;
|
||||
|
@ -699,8 +673,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
|
|||
tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
|
||||
tsNumOfVnodeStreamThreads = cfgGetItem(pCfg, "numOfVnodeStreamThreads")->i32;
|
||||
tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
|
||||
tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
|
||||
tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
|
||||
tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32;
|
||||
tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
|
||||
// tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
|
||||
|
@ -943,10 +915,6 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
|
|||
} else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) {
|
||||
tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
|
||||
*/
|
||||
} else if (strcasecmp("numOfVnodeWriteThreads", name) == 0) {
|
||||
tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
|
||||
} else if (strcasecmp("numOfVnodeSyncThreads", name) == 0) {
|
||||
tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
|
||||
} else if (strcasecmp("numOfVnodeRsmaThreads", name) == 0) {
|
||||
tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32;
|
||||
} else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) {
|
||||
|
|
|
@ -1020,6 +1020,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
|
|||
if (tEncodeI64(&encoder, pReq->qload.timeInQueryQueue) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pReq->qload.timeInFetchQueue) < 0) return -1;
|
||||
|
||||
if (tEncodeI32(&encoder, pReq->statusSeq) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
|
@ -1095,6 +1096,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
|
|||
if (tDecodeI64(&decoder, &pReq->qload.timeInQueryQueue) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &pReq->qload.timeInFetchQueue) < 0) return -1;
|
||||
|
||||
if (tDecodeI32(&decoder, &pReq->statusSeq) < 0) return -1;
|
||||
tEndDecode(&decoder);
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
|
@ -1126,6 +1128,7 @@ int32_t tSerializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
|
|||
if (tEncodeU16(&encoder, pDnodeEp->ep.port) < 0) return -1;
|
||||
}
|
||||
|
||||
if (tEncodeI32(&encoder, pRsp->statusSeq) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
|
@ -1167,6 +1170,7 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
|
|||
}
|
||||
}
|
||||
|
||||
if (tDecodeI32(&decoder, &pRsp->statusSeq) < 0) return -1;
|
||||
tEndDecode(&decoder);
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
|
@ -5432,9 +5436,12 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl
|
|||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
|
||||
if (tEncodeI32(pEncoder, pBlock->code) < 0) return -1;
|
||||
if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1;
|
||||
if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1;
|
||||
if (pBlock->tblFName) {
|
||||
if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1;
|
||||
} else {
|
||||
if (tEncodeCStr(pEncoder, "") < 0) return -1;
|
||||
}
|
||||
if (tEncodeI32v(pEncoder, pBlock->numOfRows) < 0) return -1;
|
||||
if (tEncodeI32v(pEncoder, pBlock->affectedRows) < 0) return -1;
|
||||
if (tEncodeI64v(pEncoder, pBlock->sver) < 0) return -1;
|
||||
|
@ -5451,7 +5458,6 @@ static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
|
|||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
|
||||
if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1;
|
||||
pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
|
||||
if (NULL == pBlock->tblFName) return -1;
|
||||
|
|
|
@ -36,6 +36,7 @@ typedef struct SDnodeMgmt {
|
|||
GetVnodeLoadsFp getVnodeLoadsFp;
|
||||
GetMnodeLoadsFp getMnodeLoadsFp;
|
||||
GetQnodeLoadsFp getQnodeLoadsFp;
|
||||
int32_t statusSeq;
|
||||
} SDnodeMgmt;
|
||||
|
||||
// dmHandle.c
|
||||
|
|
|
@ -32,9 +32,13 @@ static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) {
|
|||
}
|
||||
|
||||
static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
|
||||
const STraceId *trace = &pRsp->info.traceId;
|
||||
dGTrace("status rsp received from mnode, statusSeq:%d code:0x%x", pMgmt->statusSeq, pRsp->code);
|
||||
|
||||
if (pRsp->code != 0) {
|
||||
if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->pData->dropped && pMgmt->pData->dnodeId > 0) {
|
||||
dInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->pData->dnodeId);
|
||||
dGInfo("dnode:%d, set to dropped since not exist in mnode, statusSeq:%d", pMgmt->pData->dnodeId,
|
||||
pMgmt->statusSeq);
|
||||
pMgmt->pData->dropped = 1;
|
||||
dmWriteEps(pMgmt->pData);
|
||||
}
|
||||
|
@ -42,9 +46,9 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
|
|||
SStatusRsp statusRsp = {0};
|
||||
if (pRsp->pCont != NULL && pRsp->contLen > 0 &&
|
||||
tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) {
|
||||
dTrace("status msg received from mnode, dnodeVer:%" PRId64 " saved:%" PRId64, statusRsp.dnodeVer,
|
||||
pMgmt->pData->dnodeVer);
|
||||
if (pMgmt->pData->dnodeVer != statusRsp.dnodeVer) {
|
||||
dGInfo("status rsp received from mnode, statusSeq:%d:%d dnodeVer:%" PRId64 ":%" PRId64, pMgmt->statusSeq,
|
||||
statusRsp.statusSeq, pMgmt->pData->dnodeVer, statusRsp.dnodeVer);
|
||||
pMgmt->pData->dnodeVer = statusRsp.dnodeVer;
|
||||
dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg);
|
||||
dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps);
|
||||
|
@ -91,6 +95,9 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
|
|||
|
||||
(*pMgmt->getQnodeLoadsFp)(&req.qload);
|
||||
|
||||
pMgmt->statusSeq++;
|
||||
req.statusSeq = pMgmt->statusSeq;
|
||||
|
||||
int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
|
||||
void *pHead = rpcMallocCont(contLen);
|
||||
tSerializeSStatusReq(pHead, contLen, &req);
|
||||
|
@ -99,13 +106,13 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
|
|||
SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527};
|
||||
SRpcMsg rpcRsp = {0};
|
||||
|
||||
dTrace("send status msg to mnode, dnodeVer:%" PRId64, req.dnodeVer);
|
||||
dTrace("send status req to mnode, dnodeVer:%" PRId64 " statusSeq:%d", req.dnodeVer, req.statusSeq);
|
||||
|
||||
SEpSet epSet = {0};
|
||||
dmGetMnodeEpSet(pMgmt->pData, &epSet);
|
||||
rpcSendRecv(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp);
|
||||
if (rpcRsp.code != 0) {
|
||||
dError("failed to send status msg since %s, numOfEps:%d inUse:%d", tstrerror(rpcRsp.code), epSet.numOfEps,
|
||||
dError("failed to send status req since %s, numOfEps:%d inUse:%d", tstrerror(rpcRsp.code), epSet.numOfEps,
|
||||
epSet.inUse);
|
||||
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
|
||||
dDebug("index:%d, mnode ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
|
||||
|
|
|
@ -58,7 +58,11 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
|||
dTrace("msg:%p, get from snode-stream queue", pMsg);
|
||||
int32_t code = sndProcessStreamMsg(pMgmt->pSnode, pMsg);
|
||||
if (code < 0) {
|
||||
dGError("snd, msg:%p failed to process stream since %s", pMsg, terrstr(code));
|
||||
if (pMsg) {
|
||||
dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code));
|
||||
} else {
|
||||
dGError("snd, msg:%p failed to process stream empty msg since %s", pMsg, terrstr(code));
|
||||
}
|
||||
smSendRsp(pMsg, terrno);
|
||||
}
|
||||
|
||||
|
|
|
@ -33,10 +33,6 @@ typedef struct SVnodeMgmt {
|
|||
SQWorkerPool queryPool;
|
||||
SQWorkerPool streamPool;
|
||||
SWWorkerPool fetchPool;
|
||||
SWWorkerPool syncPool;
|
||||
SWWorkerPool syncCtrlPool;
|
||||
SWWorkerPool writePool;
|
||||
SWWorkerPool applyPool;
|
||||
SSingleWorker mgmtWorker;
|
||||
SHashObj *hash;
|
||||
TdThreadRwlock lock;
|
||||
|
@ -52,19 +48,19 @@ typedef struct {
|
|||
} SWrapperCfg;
|
||||
|
||||
typedef struct {
|
||||
int32_t vgId;
|
||||
int32_t vgVersion;
|
||||
int32_t refCount;
|
||||
int8_t dropped;
|
||||
char *path;
|
||||
SVnode *pImpl;
|
||||
STaosQueue *pWriteQ;
|
||||
STaosQueue *pSyncQ;
|
||||
STaosQueue *pSyncCtrlQ;
|
||||
STaosQueue *pApplyQ;
|
||||
STaosQueue *pQueryQ;
|
||||
STaosQueue *pStreamQ;
|
||||
STaosQueue *pFetchQ;
|
||||
int32_t vgId;
|
||||
int32_t vgVersion;
|
||||
int32_t refCount;
|
||||
int8_t dropped;
|
||||
char *path;
|
||||
SVnode *pImpl;
|
||||
SMultiWorker pWriteW;
|
||||
SMultiWorker pSyncW;
|
||||
SMultiWorker pSyncCtrlW;
|
||||
SMultiWorker pApplyW;
|
||||
STaosQueue *pQueryQ;
|
||||
STaosQueue *pStreamQ;
|
||||
STaosQueue *pFetchQ;
|
||||
} SVnodeObj;
|
||||
|
||||
typedef struct {
|
||||
|
|
|
@ -403,7 +403,6 @@ SArray *vmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
|
||||
|
@ -412,6 +411,9 @@ SArray *vmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RECOVER_FINISH, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK_RSP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
|
|
@ -79,29 +79,49 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
|
|||
void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
|
||||
char path[TSDB_FILENAME_LEN] = {0};
|
||||
|
||||
vnodePreClose(pVnode->pImpl);
|
||||
|
||||
taosThreadRwlockWrlock(&pMgmt->lock);
|
||||
taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t));
|
||||
taosThreadRwlockUnlock(&pMgmt->lock);
|
||||
vmReleaseVnode(pMgmt, pVnode);
|
||||
|
||||
dTrace("vgId:%d, wait for vnode ref become 0", pVnode->vgId);
|
||||
while (pVnode->refCount > 0) taosMsleep(10);
|
||||
dTrace("vgId:%d, wait for vnode queue is empty", pVnode->vgId);
|
||||
dInfo("vgId:%d, pre close", pVnode->vgId);
|
||||
vnodePreClose(pVnode->pImpl);
|
||||
|
||||
while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10);
|
||||
while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10);
|
||||
while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
|
||||
dInfo("vgId:%d, wait for vnode ref become 0", pVnode->vgId);
|
||||
while (pVnode->refCount > 0) taosMsleep(10);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode write queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
|
||||
pVnode->pWriteW.queue->threadId);
|
||||
tMultiWorkerCleanup(&pVnode->pWriteW);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode sync queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue,
|
||||
pVnode->pSyncW.queue->threadId);
|
||||
tMultiWorkerCleanup(&pVnode->pSyncW);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode sync ctrl queue:%p is empty, thread:%08" PRId64, pVnode->vgId,
|
||||
pVnode->pSyncCtrlW.queue, pVnode->pSyncCtrlW.queue->threadId);
|
||||
tMultiWorkerCleanup(&pVnode->pSyncCtrlW);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
|
||||
pVnode->pApplyW.queue->threadId);
|
||||
tMultiWorkerCleanup(&pVnode->pApplyW);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode query queue:%p is empty", pVnode->vgId, pVnode->pQueryQ);
|
||||
while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode fetch queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ,
|
||||
pVnode->pFetchQ->threadId);
|
||||
while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ);
|
||||
while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10);
|
||||
dTrace("vgId:%d, vnode queue is empty", pVnode->vgId);
|
||||
|
||||
dInfo("vgId:%d, all vnode queues is empty", pVnode->vgId);
|
||||
|
||||
vmFreeQueue(pMgmt, pVnode);
|
||||
vnodeClose(pVnode->pImpl);
|
||||
pVnode->pImpl = NULL;
|
||||
dDebug("vgId:%d, vnode is closed", pVnode->vgId);
|
||||
dInfo("vgId:%d, vnode is closed", pVnode->vgId);
|
||||
|
||||
if (pVnode->dropped) {
|
||||
dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped);
|
||||
|
|
|
@ -86,7 +86,12 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
|||
int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
|
||||
if (code != 0) {
|
||||
if (terrno != 0) code = terrno;
|
||||
dGError("vgId:%d, msg:%p failed to process stream since %s", pVnode->vgId, pMsg, terrstr(code));
|
||||
if (pMsg) {
|
||||
dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType),
|
||||
terrstr(code));
|
||||
} else {
|
||||
dGError("vgId:%d, msg:%p failed to process stream empty msg since %s", pVnode->vgId, pMsg, terrstr(code));
|
||||
}
|
||||
vmSendRsp(pMsg, code);
|
||||
}
|
||||
|
||||
|
@ -183,30 +188,20 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
|
|||
dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code));
|
||||
} else {
|
||||
dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg);
|
||||
taosWriteQitem(pVnode->pWriteQ, pMsg);
|
||||
#if 0 // tests for batch writes
|
||||
if (pMsg->msgType == TDMT_VND_CREATE_TABLE) {
|
||||
SRpcMsg *pDup = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
|
||||
memcpy(pDup, pMsg, sizeof(SRpcMsg));
|
||||
pDup->pCont = rpcMallocCont(pMsg->contLen);
|
||||
memcpy(pDup->pCont, pMsg->pCont, pMsg->contLen);
|
||||
pDup->info.handle = NULL;
|
||||
taosWriteQitem(pVnode->pWriteQ, pDup);
|
||||
}
|
||||
#endif
|
||||
taosWriteQitem(pVnode->pWriteW.queue, pMsg);
|
||||
}
|
||||
break;
|
||||
case SYNC_QUEUE:
|
||||
dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg);
|
||||
taosWriteQitem(pVnode->pSyncQ, pMsg);
|
||||
taosWriteQitem(pVnode->pSyncW.queue, pMsg);
|
||||
break;
|
||||
case SYNC_CTRL_QUEUE:
|
||||
dGTrace("vgId:%d, msg:%p put into vnode-sync-ctrl queue", pVnode->vgId, pMsg);
|
||||
taosWriteQitem(pVnode->pSyncCtrlQ, pMsg);
|
||||
taosWriteQitem(pVnode->pSyncCtrlW.queue, pMsg);
|
||||
break;
|
||||
case APPLY_QUEUE:
|
||||
dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg);
|
||||
taosWriteQitem(pVnode->pApplyQ, pMsg);
|
||||
taosWriteQitem(pVnode->pApplyW.queue, pMsg);
|
||||
break;
|
||||
default:
|
||||
code = -1;
|
||||
|
@ -218,7 +213,9 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_CTRL_QUEUE); }
|
||||
int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
||||
return vmPutMsgToQueue(pMgmt, pMsg, SYNC_CTRL_QUEUE);
|
||||
}
|
||||
|
||||
int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); }
|
||||
|
||||
|
@ -269,13 +266,13 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
|
|||
if (pVnode != NULL) {
|
||||
switch (qtype) {
|
||||
case WRITE_QUEUE:
|
||||
size = taosQueueItemSize(pVnode->pWriteQ);
|
||||
size = taosQueueItemSize(pVnode->pWriteW.queue);
|
||||
break;
|
||||
case SYNC_QUEUE:
|
||||
size = taosQueueItemSize(pVnode->pSyncQ);
|
||||
size = taosQueueItemSize(pVnode->pSyncW.queue);
|
||||
break;
|
||||
case APPLY_QUEUE:
|
||||
size = taosQueueItemSize(pVnode->pApplyQ);
|
||||
size = taosQueueItemSize(pVnode->pApplyW.queue);
|
||||
break;
|
||||
case QUERY_QUEUE:
|
||||
size = taosQueueItemSize(pVnode->pQueryQ);
|
||||
|
@ -299,40 +296,44 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
|
|||
}
|
||||
|
||||
int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
|
||||
pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeProposeWriteMsg);
|
||||
pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue);
|
||||
pVnode->pSyncCtrlQ = tWWorkerAllocQueue(&pMgmt->syncCtrlPool, pVnode, (FItems)vmProcessSyncQueue);
|
||||
pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->applyPool, pVnode->pImpl, (FItems)vnodeApplyWriteMsg);
|
||||
SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl};
|
||||
SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
|
||||
SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-ctrl", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
|
||||
SMultiWorkerCfg acfg = {.max = 1, .name = "vnode-apply", .fp = (FItems)vnodeApplyWriteMsg, .param = pVnode->pImpl};
|
||||
(void)tMultiWorkerInit(&pVnode->pWriteW, &wcfg);
|
||||
(void)tMultiWorkerInit(&pVnode->pSyncW, &scfg);
|
||||
(void)tMultiWorkerInit(&pVnode->pSyncCtrlW, &sccfg);
|
||||
(void)tMultiWorkerInit(&pVnode->pApplyW, &acfg);
|
||||
|
||||
pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue);
|
||||
pVnode->pStreamQ = tQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue);
|
||||
pVnode->pFetchQ = tWWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItems)vmProcessFetchQueue);
|
||||
|
||||
if (pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pApplyQ == NULL || pVnode->pQueryQ == NULL ||
|
||||
pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) {
|
||||
if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncCtrlW.queue == NULL ||
|
||||
pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
dDebug("vgId:%d, write-queue:%p is alloced", pVnode->vgId, pVnode->pWriteQ);
|
||||
dDebug("vgId:%d, sync-queue:%p is alloced", pVnode->vgId, pVnode->pSyncQ);
|
||||
dDebug("vgId:%d, apply-queue:%p is alloced", pVnode->vgId, pVnode->pApplyQ);
|
||||
dDebug("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ);
|
||||
dDebug("vgId:%d, stream-queue:%p is alloced", pVnode->vgId, pVnode->pStreamQ);
|
||||
dDebug("vgId:%d, fetch-queue:%p is alloced", pVnode->vgId, pVnode->pFetchQ);
|
||||
dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
|
||||
pVnode->pWriteW.queue->threadId);
|
||||
dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue,
|
||||
pVnode->pSyncW.queue->threadId);
|
||||
dInfo("vgId:%d, sync-ctrl-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncCtrlW.queue,
|
||||
pVnode->pSyncCtrlW.queue->threadId);
|
||||
dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
|
||||
pVnode->pApplyW.queue->threadId);
|
||||
dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ);
|
||||
dInfo("vgId:%d, fetch-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ,
|
||||
pVnode->pFetchQ->threadId);
|
||||
dInfo("vgId:%d, stream-queue:%p is alloced", pVnode->vgId, pVnode->pStreamQ);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
|
||||
tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
|
||||
tWWorkerFreeQueue(&pMgmt->applyPool, pVnode->pApplyQ);
|
||||
tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
|
||||
tWWorkerFreeQueue(&pMgmt->syncCtrlPool, pVnode->pSyncCtrlQ);
|
||||
tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
|
||||
tQWorkerFreeQueue(&pMgmt->streamPool, pVnode->pStreamQ);
|
||||
tWWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
|
||||
pVnode->pWriteQ = NULL;
|
||||
pVnode->pSyncQ = NULL;
|
||||
pVnode->pApplyQ = NULL;
|
||||
pVnode->pQueryQ = NULL;
|
||||
pVnode->pStreamQ = NULL;
|
||||
pVnode->pFetchQ = NULL;
|
||||
|
@ -357,26 +358,6 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
|
|||
pFPool->max = tsNumOfVnodeFetchThreads;
|
||||
if (tWWorkerInit(pFPool) != 0) return -1;
|
||||
|
||||
SWWorkerPool *pWPool = &pMgmt->writePool;
|
||||
pWPool->name = "vnode-write";
|
||||
pWPool->max = tsNumOfVnodeWriteThreads;
|
||||
if (tWWorkerInit(pWPool) != 0) return -1;
|
||||
|
||||
SWWorkerPool *pAPool = &pMgmt->applyPool;
|
||||
pAPool->name = "vnode-apply";
|
||||
pAPool->max = tsNumOfVnodeWriteThreads;
|
||||
if (tWWorkerInit(pAPool) != 0) return -1;
|
||||
|
||||
SWWorkerPool *pSPool = &pMgmt->syncPool;
|
||||
pSPool->name = "vnode-sync";
|
||||
pSPool->max = tsNumOfVnodeSyncThreads;
|
||||
if (tWWorkerInit(pSPool) != 0) return -1;
|
||||
|
||||
SWWorkerPool *pSCPool = &pMgmt->syncCtrlPool;
|
||||
pSCPool->name = "vnode-sync-ctrl";
|
||||
pSCPool->max = tsNumOfVnodeSyncThreads;
|
||||
if (tWWorkerInit(pSCPool) != 0) return -1;
|
||||
|
||||
SSingleWorkerCfg mgmtCfg = {
|
||||
.min = 1,
|
||||
.max = 1,
|
||||
|
@ -391,10 +372,6 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
|
|||
}
|
||||
|
||||
void vmStopWorker(SVnodeMgmt *pMgmt) {
|
||||
tWWorkerCleanup(&pMgmt->writePool);
|
||||
tWWorkerCleanup(&pMgmt->applyPool);
|
||||
tWWorkerCleanup(&pMgmt->syncPool);
|
||||
tWWorkerCleanup(&pMgmt->syncCtrlPool);
|
||||
tQWorkerCleanup(&pMgmt->queryPool);
|
||||
tQWorkerCleanup(&pMgmt->streamPool);
|
||||
tWWorkerCleanup(&pMgmt->fetchPool);
|
||||
|
|
|
@ -157,7 +157,11 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
|
|||
_OVER:
|
||||
if (code != 0) {
|
||||
if (terrno != 0) code = terrno;
|
||||
dGTrace("msg:%p, failed to process since %s", pMsg, terrstr());
|
||||
if (pMsg) {
|
||||
dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr());
|
||||
} else {
|
||||
dGTrace("msg:%p, failed to process empty msg since %s", pMsg, terrstr());
|
||||
}
|
||||
|
||||
if (IsReq(pRpc)) {
|
||||
SRpcMsg rsp = {.code = code, .info = pRpc->info};
|
||||
|
|
|
@ -345,6 +345,19 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
|
|||
}
|
||||
}
|
||||
|
||||
int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE);
|
||||
int64_t curMs = taosGetTimestampMs();
|
||||
bool online = mndIsDnodeOnline(pDnode, curMs);
|
||||
bool dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer);
|
||||
bool reboot = (pDnode->rebootTime != statusReq.rebootTime);
|
||||
bool needCheck = !online || dnodeChanged || reboot;
|
||||
|
||||
pDnode->accessTimes++;
|
||||
pDnode->lastAccessTime = curMs;
|
||||
const STraceId *trace = &pReq->info.traceId;
|
||||
mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id,
|
||||
pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq);
|
||||
|
||||
for (int32_t v = 0; v < taosArrayGetSize(statusReq.pVloads); ++v) {
|
||||
SVnodeLoad *pVload = taosArrayGet(statusReq.pVloads, v);
|
||||
|
||||
|
@ -396,17 +409,6 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
|
|||
mndReleaseQnode(pMnode, pQnode);
|
||||
}
|
||||
|
||||
int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE);
|
||||
int64_t curMs = taosGetTimestampMs();
|
||||
bool online = mndIsDnodeOnline(pDnode, curMs);
|
||||
bool dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer);
|
||||
bool reboot = (pDnode->rebootTime != statusReq.rebootTime);
|
||||
bool needCheck = !online || dnodeChanged || reboot;
|
||||
|
||||
pDnode->accessTimes++;
|
||||
mTrace("dnode:%d, status received, access times:%d check:%d online:%d reboot:%d changed:%d", pDnode->id,
|
||||
pDnode->accessTimes, needCheck, online, reboot, dnodeChanged);
|
||||
|
||||
if (needCheck) {
|
||||
if (statusReq.sver != tsVersion) {
|
||||
if (pDnode != NULL) {
|
||||
|
@ -454,6 +456,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
|
|||
pDnode->memTotal = statusReq.memTotal;
|
||||
|
||||
SStatusRsp statusRsp = {0};
|
||||
statusRsp.statusSeq++;
|
||||
statusRsp.dnodeVer = dnodeVer;
|
||||
statusRsp.dnodeCfg.dnodeId = pDnode->id;
|
||||
statusRsp.dnodeCfg.clusterId = pMnode->clusterId;
|
||||
|
@ -474,8 +477,6 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
|
|||
pReq->info.rsp = pHead;
|
||||
}
|
||||
|
||||
pDnode->lastAccessTime = curMs;
|
||||
pDnode->accessTimes++;
|
||||
code = 0;
|
||||
|
||||
_OVER:
|
||||
|
|
|
@ -705,7 +705,8 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-stream");
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "drop-stream");
|
||||
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb);
|
||||
if (pTrans == NULL) {
|
||||
mError("stream:%s, failed to drop since %s", dropReq.name, terrstr());
|
||||
sdbRelease(pMnode->pSdb, pStream);
|
||||
|
|
|
@ -612,14 +612,14 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
|
|||
mndReleaseSubscribe(pMnode, pSub);
|
||||
}
|
||||
|
||||
// TODO replace assert with error check
|
||||
ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0);
|
||||
if (mndDoRebalance(pMnode, &rebInput, &rebOutput) < 0) {
|
||||
mError("mq rebalance internal error");
|
||||
}
|
||||
|
||||
// if add more consumer to balanced subscribe,
|
||||
// possibly no vg is changed
|
||||
/*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/
|
||||
|
||||
// TODO replace assert with error check
|
||||
if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) {
|
||||
mError("mq rebalance persist rebalance output error, possibly vnode splitted or dropped");
|
||||
}
|
||||
|
|
|
@ -231,10 +231,10 @@ int32_t sndProcessTaskRetrieveReq(SSnode *pSnode, SRpcMsg *pMsg) {
|
|||
|
||||
int32_t sndProcessTaskDispatchRsp(SSnode *pSnode, SRpcMsg *pMsg) {
|
||||
SStreamDispatchRsp *pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
|
||||
int32_t taskId = pRsp->taskId;
|
||||
int32_t taskId = ntohl(pRsp->upstreamTaskId);
|
||||
SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId);
|
||||
if (pTask) {
|
||||
streamProcessDispatchRsp(pTask, pRsp);
|
||||
streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
|
||||
return 0;
|
||||
} else {
|
||||
return -1;
|
||||
|
|
|
@ -158,7 +158,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableL
|
|||
void tsdbReaderClose(STsdbReader *pReader);
|
||||
bool tsdbNextDataBlock(STsdbReader *pReader);
|
||||
bool tsdbTableNextDataBlock(STsdbReader *pReader, uint64_t uid);
|
||||
void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow);
|
||||
void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow);
|
||||
int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
|
||||
SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
|
||||
int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
|
||||
|
@ -240,7 +240,7 @@ bool tqNextDataBlock(STqReader *pReader);
|
|||
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
|
||||
int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader);
|
||||
|
||||
void vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg);
|
||||
int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg);
|
||||
|
||||
// sma
|
||||
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
|
||||
|
|
|
@ -643,6 +643,7 @@ typedef struct SSttBlockLoadInfo {
|
|||
STSchema *pSchema;
|
||||
int16_t *colIds;
|
||||
int32_t numOfCols;
|
||||
bool sttBlockLoaded;
|
||||
} SSttBlockLoadInfo;
|
||||
|
||||
typedef struct SMergeTree {
|
||||
|
|
|
@ -97,6 +97,7 @@ bool vnodeShouldRollback(SVnode* pVnode);
|
|||
// vnodeSync.c
|
||||
int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
|
||||
void vnodeSyncStart(SVnode* pVnode);
|
||||
void vnodeSyncPreClose(SVnode* pVnode);
|
||||
void vnodeSyncClose(SVnode* pVnode);
|
||||
void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg);
|
||||
bool vnodeIsLeader(SVnode* pVnode);
|
||||
|
|
|
@ -150,19 +150,19 @@ typedef struct {
|
|||
int32_t metaGetStbStats(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo);
|
||||
|
||||
// tsdb
|
||||
int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback);
|
||||
int tsdbClose(STsdb** pTsdb);
|
||||
int32_t tsdbBegin(STsdb* pTsdb);
|
||||
int32_t tsdbCommit(STsdb* pTsdb);
|
||||
int32_t tsdbFinishCommit(STsdb* pTsdb);
|
||||
int32_t tsdbRollbackCommit(STsdb* pTsdb);
|
||||
int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now);
|
||||
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
|
||||
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
|
||||
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock,
|
||||
SSubmitBlkRsp* pRsp);
|
||||
int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
|
||||
int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
|
||||
int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback);
|
||||
int tsdbClose(STsdb** pTsdb);
|
||||
int32_t tsdbBegin(STsdb* pTsdb);
|
||||
int32_t tsdbCommit(STsdb* pTsdb);
|
||||
int32_t tsdbFinishCommit(STsdb* pTsdb);
|
||||
int32_t tsdbRollbackCommit(STsdb* pTsdb);
|
||||
int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now);
|
||||
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
|
||||
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
|
||||
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock,
|
||||
SSubmitBlkRsp* pRsp);
|
||||
int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
|
||||
int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
|
||||
|
||||
// tq
|
||||
int tqInit();
|
||||
|
@ -183,13 +183,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg);
|
|||
// tq-stream
|
||||
int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen);
|
||||
int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen);
|
||||
int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg);
|
||||
int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen);
|
||||
int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver);
|
||||
int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver);
|
||||
int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg);
|
||||
int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec);
|
||||
int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg);
|
||||
// int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg);
|
||||
// int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg);
|
||||
int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
|
||||
int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg);
|
||||
int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg);
|
||||
|
|
|
@ -896,6 +896,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
|
|||
pTask->startVer = ver;
|
||||
|
||||
// expand executor
|
||||
if (pTask->fillHistory) {
|
||||
pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM;
|
||||
}
|
||||
|
||||
if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
|
||||
pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1);
|
||||
if (pTask->pState == NULL) {
|
||||
|
@ -911,9 +915,6 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
|
|||
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
|
||||
ASSERT(pTask->exec.executor);
|
||||
|
||||
if (pTask->fillHistory) {
|
||||
pTask->taskStatus = TASK_STATUS__RECOVER_PREPARE;
|
||||
}
|
||||
} else if (pTask->taskLevel == TASK_LEVEL__AGG) {
|
||||
pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1);
|
||||
if (pTask->pState == NULL) {
|
||||
|
@ -947,11 +948,90 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
|
|||
|
||||
streamSetupTrigger(pTask);
|
||||
|
||||
tqInfo("expand stream task on vg %d, task id %d, child id %d", TD_VID(pTq->pVnode), pTask->taskId,
|
||||
pTask->selfChildId);
|
||||
tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", TD_VID(pTq->pVnode), pTask->taskId,
|
||||
pTask->selfChildId, pTask->taskLevel);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
|
||||
char* msgStr = pMsg->pCont;
|
||||
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
|
||||
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
|
||||
SStreamTaskCheckReq req;
|
||||
SDecoder decoder;
|
||||
tDecoderInit(&decoder, msgBody, msgLen);
|
||||
tDecodeSStreamTaskCheckReq(&decoder, &req);
|
||||
tDecoderClear(&decoder);
|
||||
int32_t taskId = req.downstreamTaskId;
|
||||
SStreamTaskCheckRsp rsp = {
|
||||
.reqId = req.reqId,
|
||||
.streamId = req.streamId,
|
||||
.childId = req.childId,
|
||||
.downstreamNodeId = req.downstreamNodeId,
|
||||
.downstreamTaskId = req.downstreamTaskId,
|
||||
.upstreamNodeId = req.upstreamNodeId,
|
||||
.upstreamTaskId = req.upstreamTaskId,
|
||||
};
|
||||
SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId);
|
||||
if (pTask && atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL) {
|
||||
rsp.status = 1;
|
||||
} else {
|
||||
rsp.status = 0;
|
||||
}
|
||||
|
||||
tqDebug("tq recv task check req(reqId: %" PRId64 ") %d at node %d check req from task %d at node %d, status %d",
|
||||
rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
|
||||
|
||||
SEncoder encoder;
|
||||
int32_t code;
|
||||
int32_t len;
|
||||
tEncodeSize(tEncodeSStreamTaskCheckRsp, &rsp, len, code);
|
||||
if (code < 0) {
|
||||
ASSERT(0);
|
||||
}
|
||||
void* buf = rpcMallocCont(sizeof(SMsgHead) + len);
|
||||
((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId);
|
||||
|
||||
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
|
||||
tEncoderInit(&encoder, (uint8_t*)abuf, len);
|
||||
tEncodeSStreamTaskCheckRsp(&encoder, &rsp);
|
||||
tEncoderClear(&encoder);
|
||||
|
||||
SRpcMsg rspMsg = {
|
||||
.code = 0,
|
||||
.pCont = buf,
|
||||
.contLen = sizeof(SMsgHead) + len,
|
||||
.info = pMsg->info,
|
||||
};
|
||||
|
||||
tmsgSendRsp(&rspMsg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen) {
|
||||
int32_t code;
|
||||
SStreamTaskCheckRsp rsp;
|
||||
|
||||
SDecoder decoder;
|
||||
tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
|
||||
code = tDecodeSStreamTaskCheckRsp(&decoder, &rsp);
|
||||
if (code < 0) {
|
||||
tDecoderClear(&decoder);
|
||||
return -1;
|
||||
}
|
||||
tDecoderClear(&decoder);
|
||||
|
||||
tqDebug("tq recv task check rsp(reqId: %" PRId64 ") %d at node %d check req from task %d at node %d, status %d",
|
||||
rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
|
||||
|
||||
SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, rsp.upstreamTaskId);
|
||||
if (pTask == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
return streamProcessTaskCheckRsp(pTask, &rsp, version);
|
||||
}
|
||||
|
||||
int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) {
|
||||
int32_t code;
|
||||
#if 0
|
||||
|
@ -982,37 +1062,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg
|
|||
|
||||
// 3.go through recover steps to fill history
|
||||
if (pTask->fillHistory) {
|
||||
if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
|
||||
streamSetParamForRecover(pTask);
|
||||
streamSourceRecoverPrepareStep1(pTask, version);
|
||||
|
||||
SStreamRecoverStep1Req req;
|
||||
streamBuildSourceRecover1Req(pTask, &req);
|
||||
int32_t len = sizeof(SStreamRecoverStep1Req);
|
||||
|
||||
void* serializedReq = rpcMallocCont(len);
|
||||
if (serializedReq == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
memcpy(serializedReq, &req, len);
|
||||
|
||||
SRpcMsg rpcMsg = {
|
||||
.contLen = len,
|
||||
.pCont = serializedReq,
|
||||
.msgType = TDMT_VND_STREAM_RECOVER_STEP1,
|
||||
};
|
||||
|
||||
if (tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &rpcMsg) < 0) {
|
||||
/*ASSERT(0);*/
|
||||
}
|
||||
|
||||
} else if (pTask->taskLevel == TASK_LEVEL__AGG) {
|
||||
streamSetParamForRecover(pTask);
|
||||
streamAggRecoverPrepare(pTask);
|
||||
} else if (pTask->taskLevel == TASK_LEVEL__SINK) {
|
||||
// do nothing
|
||||
}
|
||||
streamTaskCheckDownstream(pTask, version);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
@ -1268,7 +1318,7 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) {
|
|||
if (pIter == NULL) break;
|
||||
SStreamTask* pTask = *(SStreamTask**)pIter;
|
||||
if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue;
|
||||
if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE) {
|
||||
if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
|
||||
tqDebug("skip push task %d, task status %d", pTask->taskId, pTask->taskStatus);
|
||||
continue;
|
||||
}
|
||||
|
@ -1335,10 +1385,11 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) {
|
|||
|
||||
int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
|
||||
SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
|
||||
int32_t taskId = pRsp->taskId;
|
||||
int32_t taskId = ntohl(pRsp->upstreamTaskId);
|
||||
SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId);
|
||||
tqDebug("recv dispatch rsp, code: %x", pMsg->code);
|
||||
if (pTask) {
|
||||
streamProcessDispatchRsp(pTask, pRsp);
|
||||
streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
|
||||
return 0;
|
||||
} else {
|
||||
return -1;
|
||||
|
@ -1379,12 +1430,12 @@ int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
void vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) {
|
||||
STQ* pTq = pVnode->pTq;
|
||||
char* msgStr = pMsg->pCont;
|
||||
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
|
||||
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
|
||||
int32_t code = 0;
|
||||
int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) {
|
||||
STQ* pTq = pVnode->pTq;
|
||||
SMsgHead* msgStr = pMsg->pCont;
|
||||
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
|
||||
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
|
||||
int32_t code = 0;
|
||||
|
||||
SStreamDispatchReq req;
|
||||
SDecoder decoder;
|
||||
|
@ -1407,16 +1458,45 @@ void vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) {
|
|||
streamProcessDispatchReq(pTask, &req, &rsp, false);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
|
||||
code = TSDB_CODE_STREAM_TASK_NOT_EXIST;
|
||||
|
||||
FAIL:
|
||||
if (pMsg->info.handle == NULL) return;
|
||||
if (pMsg->info.handle == NULL) return -1;
|
||||
|
||||
SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp));
|
||||
if (pRspHead == NULL) {
|
||||
SRpcMsg rsp = {
|
||||
.code = TSDB_CODE_OUT_OF_MEMORY,
|
||||
.info = pMsg->info,
|
||||
};
|
||||
tqDebug("send dispatch error rsp, code: %x", code);
|
||||
tmsgSendRsp(&rsp);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
return -1;
|
||||
}
|
||||
|
||||
pRspHead->vgId = htonl(req.upstreamNodeId);
|
||||
SStreamDispatchRsp* pRsp = POINTER_SHIFT(pRspHead, sizeof(SMsgHead));
|
||||
pRsp->streamId = htobe64(req.streamId);
|
||||
pRsp->upstreamTaskId = htonl(req.upstreamTaskId);
|
||||
pRsp->upstreamNodeId = htonl(req.upstreamNodeId);
|
||||
pRsp->downstreamNodeId = htonl(pVnode->config.vgId);
|
||||
pRsp->downstreamTaskId = htonl(req.taskId);
|
||||
pRsp->inputStatus = TASK_OUTPUT_STATUS__NORMAL;
|
||||
|
||||
SRpcMsg rsp = {
|
||||
.code = code,
|
||||
.info = pMsg->info,
|
||||
.contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp),
|
||||
.pCont = pRspHead,
|
||||
};
|
||||
tqDebug("send dispatch error rsp, code: %x", code);
|
||||
tmsgSendRsp(&rsp);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
return -1;
|
||||
}
|
||||
|
|
|
@ -333,11 +333,12 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
|
|||
code = TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
|
||||
_end:
|
||||
_end:
|
||||
tsdbDataFReaderClose(&pr->pDataFReaderLast);
|
||||
tsdbDataFReaderClose(&pr->pDataFReader);
|
||||
|
||||
tsdbUntakeReadSnap(pr->pVnode->pTsdb, pr->pReadSnap, "cache-l");
|
||||
resetLastBlockLoadInfo(pr->pLoadInfo);
|
||||
|
||||
for (int32_t j = 0; j < pr->numOfCols; ++j) {
|
||||
taosMemoryFree(pRes[j]);
|
||||
|
|
|
@ -116,6 +116,13 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
|
|||
if (info.suid) {
|
||||
metaGetInfo(pTsdb->pVnode->pMeta, info.suid, &info);
|
||||
}
|
||||
if (pMsgIter->sversion != info.skmVer) {
|
||||
tsdbError("vgId:%d, req sver:%d, skmVer:%d suid:%" PRId64 " uid:%" PRId64,
|
||||
TD_VID(pTsdb->pVnode), pMsgIter->sversion, info.skmVer, suid, uid);
|
||||
code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
|
||||
goto _err;
|
||||
}
|
||||
|
||||
pRsp->sver = info.skmVer;
|
||||
|
||||
// create/get STbData to op
|
||||
|
@ -133,6 +140,7 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
|
|||
return code;
|
||||
|
||||
_err:
|
||||
terrno = code;
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
|
@ -72,6 +72,7 @@ void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
|
|||
|
||||
pLoadInfo[i].elapsedTime = 0;
|
||||
pLoadInfo[i].loadBlocks = 0;
|
||||
pLoadInfo[i].sttBlockLoaded = false;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -278,9 +279,9 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
|
|||
|
||||
(*pIter)->pBlockLoadInfo = pBlockLoadInfo;
|
||||
|
||||
size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
|
||||
if (size == 0) {
|
||||
if (!pBlockLoadInfo->sttBlockLoaded) {
|
||||
int64_t st = taosGetTimestampUs();
|
||||
pBlockLoadInfo->sttBlockLoaded = true;
|
||||
|
||||
code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
|
||||
if (code) {
|
||||
|
@ -288,7 +289,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
|
|||
}
|
||||
|
||||
// only apply to the child tables, ordinary tables will not incur this filter procedure.
|
||||
size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
|
||||
size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
|
||||
|
||||
if (size >= 1) {
|
||||
SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0);
|
||||
|
@ -296,10 +297,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
|
|||
|
||||
// all identical
|
||||
if (pStart->suid == pEnd->suid) {
|
||||
if (pStart->suid == suid) {
|
||||
// do nothing
|
||||
} else if (pStart->suid != suid) {
|
||||
if (pStart->suid != suid) {
|
||||
// no qualified stt block existed
|
||||
taosArrayClear(pBlockLoadInfo->aSttBlk);
|
||||
|
||||
(*pIter)->iSttBlk = -1;
|
||||
double el = (taosGetTimestampUs() - st) / 1000.0;
|
||||
tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
|
||||
|
@ -330,7 +331,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
|
|||
tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
|
||||
}
|
||||
|
||||
size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
|
||||
size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
|
||||
|
||||
// find the start block
|
||||
(*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);
|
||||
|
|
|
@ -185,11 +185,11 @@ static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STabl
|
|||
SRowMerger* pMerger, SVersionRange* pVerRange);
|
||||
static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger,
|
||||
STsdbReader* pReader);
|
||||
static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid);
|
||||
static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pInfo);
|
||||
static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData,
|
||||
int32_t rowIndex);
|
||||
static void setComposedBlockFlag(STsdbReader* pReader, bool composed);
|
||||
static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pRange);
|
||||
static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pVerRange);
|
||||
|
||||
static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList,
|
||||
STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow);
|
||||
|
@ -208,7 +208,6 @@ static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader);
|
|||
static int32_t doBuildDataBlock(STsdbReader* pReader);
|
||||
static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader);
|
||||
static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo);
|
||||
static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader);
|
||||
|
||||
static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
|
||||
|
||||
|
@ -534,7 +533,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity)
|
|||
}
|
||||
|
||||
for (int32_t i = 0; i < pCond->numOfCols; ++i) {
|
||||
SColumnInfoData colInfo = {{0}, 0};
|
||||
SColumnInfoData colInfo = {0, {0}};
|
||||
colInfo.info = pCond->colList[i];
|
||||
blockDataAppendColInfo(pResBlock, &colInfo);
|
||||
}
|
||||
|
@ -1529,8 +1528,8 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB
|
|||
// opt version
|
||||
// 1. it is not a border point
|
||||
// 2. the direct next point is not an duplicated timestamp
|
||||
if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && pReader->order == TSDB_ORDER_ASC) ||
|
||||
(pDumpInfo->rowIndex > 0 && pReader->order == TSDB_ORDER_DESC)) {
|
||||
bool asc = (pReader->order == TSDB_ORDER_ASC);
|
||||
if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) {
|
||||
int32_t step = pReader->order == TSDB_ORDER_ASC ? 1 : -1;
|
||||
|
||||
int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step];
|
||||
|
@ -1749,7 +1748,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
|
|||
return code;
|
||||
}
|
||||
|
||||
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
|
||||
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
|
||||
|
||||
taosMemoryFree(pTSRow);
|
||||
tRowMergerClear(&merge);
|
||||
|
@ -1770,6 +1769,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
|
|||
// only last block exists
|
||||
if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) {
|
||||
if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) {
|
||||
pBlockScanInfo->lastKey = tsLastBlock;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else {
|
||||
int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema);
|
||||
|
@ -1786,7 +1786,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
|
|||
return code;
|
||||
}
|
||||
|
||||
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
|
||||
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
|
||||
|
||||
taosMemoryFree(pTSRow);
|
||||
tRowMergerClear(&merge);
|
||||
|
@ -1810,7 +1810,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
|
|||
return code;
|
||||
}
|
||||
|
||||
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
|
||||
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
|
||||
|
||||
taosMemoryFree(pTSRow);
|
||||
tRowMergerClear(&merge);
|
||||
|
@ -1858,7 +1858,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader
|
|||
return code;
|
||||
}
|
||||
|
||||
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
|
||||
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
|
||||
|
||||
taosMemoryFree(pTSRow);
|
||||
tRowMergerClear(&merge);
|
||||
|
@ -2082,7 +2082,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
|
|||
return code;
|
||||
}
|
||||
|
||||
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
|
||||
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
|
||||
|
||||
taosMemoryFree(pTSRow);
|
||||
tRowMergerClear(&merge);
|
||||
|
@ -2186,7 +2186,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum
|
|||
static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
|
||||
// the last block reader has been initialized for this table.
|
||||
if (pLBlockReader->uid == pScanInfo->uid) {
|
||||
return true;
|
||||
return hasDataInLastBlock(pLBlockReader);
|
||||
}
|
||||
|
||||
if (pLBlockReader->uid != 0) {
|
||||
|
@ -2233,6 +2233,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc
|
|||
STsdbReader* pReader) {
|
||||
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
|
||||
if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) {
|
||||
pBlockScanInfo->lastKey = key;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else {
|
||||
TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
|
||||
|
@ -2251,7 +2252,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc
|
|||
return code;
|
||||
}
|
||||
|
||||
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
|
||||
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
|
||||
|
||||
taosMemoryFree(pTSRow);
|
||||
tRowMergerClear(&merge);
|
||||
|
@ -2299,26 +2300,32 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
|
|||
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
|
||||
SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader;
|
||||
|
||||
bool asc = ASCENDING_TRAVERSE(pReader->order);
|
||||
int64_t st = taosGetTimestampUs();
|
||||
int32_t step = asc ? 1 : -1;
|
||||
|
||||
STableBlockScanInfo* pBlockScanInfo = NULL;
|
||||
if (pBlockInfo != NULL) {
|
||||
pBlockScanInfo = *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
|
||||
if (pBlockScanInfo == NULL) {
|
||||
void* p = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
|
||||
if (p == NULL) {
|
||||
code = TSDB_CODE_INVALID_PARA;
|
||||
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid,
|
||||
taosHashGetSize(pReader->status.pTableMap), pReader->idStr);
|
||||
goto _end;
|
||||
}
|
||||
|
||||
pBlockScanInfo = *(STableBlockScanInfo**) p;
|
||||
|
||||
SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
|
||||
TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader);
|
||||
|
||||
// it is a clean block, load it directly
|
||||
if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) {
|
||||
if (pReader->order == TSDB_ORDER_ASC ||
|
||||
(pReader->order == TSDB_ORDER_DESC && (!hasDataInLastBlock(pLastBlockReader)))) {
|
||||
if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) {
|
||||
copyBlockDataToSDataBlock(pReader, pBlockScanInfo);
|
||||
|
||||
// record the last key value
|
||||
pBlockScanInfo->lastKey = asc? pBlock->maxKey.ts:pBlock->minKey.ts;
|
||||
goto _end;
|
||||
}
|
||||
}
|
||||
|
@ -2328,7 +2335,6 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
|
|||
|
||||
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
|
||||
SBlockData* pBlockData = &pReader->status.fileBlockData;
|
||||
int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1;
|
||||
|
||||
while (1) {
|
||||
bool hasBlockData = false;
|
||||
|
@ -2710,6 +2716,9 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
|
|||
pInfo->window = (STimeWindow){.skey = pBlock->minKey.ts, .ekey = pBlock->maxKey.ts};
|
||||
setComposedBlockFlag(pReader, false);
|
||||
setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order);
|
||||
|
||||
// update the last key for the corresponding table
|
||||
pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order)? pInfo->window.ekey:pInfo->window.skey;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -3214,7 +3223,6 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
|
|||
|
||||
int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
|
||||
SRowMerger* pMerger, SVersionRange* pVerRange) {
|
||||
pScanInfo->lastKey = ts;
|
||||
while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) {
|
||||
int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
|
||||
if (next1 == ts) {
|
||||
|
@ -3407,9 +3415,10 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid) {
|
||||
int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pScanInfo) {
|
||||
int32_t numOfRows = pBlock->info.rows;
|
||||
int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
|
||||
int64_t uid = pScanInfo->uid;
|
||||
|
||||
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
|
||||
STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid);
|
||||
|
@ -3448,6 +3457,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
|
|||
}
|
||||
|
||||
pBlock->info.rows += 1;
|
||||
pScanInfo->lastKey = pTSRow->ts;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
@ -3511,7 +3521,8 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
|
|||
break;
|
||||
}
|
||||
|
||||
doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo->uid);
|
||||
doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo);
|
||||
|
||||
if (freeTSRow) {
|
||||
taosMemoryFree(pTSRow);
|
||||
}
|
||||
|
|
|
@ -242,12 +242,7 @@ _err:
|
|||
return NULL;
|
||||
}
|
||||
|
||||
void vnodePreClose(SVnode *pVnode) {
|
||||
if (pVnode) {
|
||||
syncLeaderTransfer(pVnode->sync);
|
||||
syncPreStop(pVnode->sync);
|
||||
}
|
||||
}
|
||||
void vnodePreClose(SVnode *pVnode) { vnodeSyncPreClose(pVnode); }
|
||||
|
||||
void vnodeClose(SVnode *pVnode) {
|
||||
if (pVnode) {
|
||||
|
|
|
@ -229,8 +229,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
|
|||
break;
|
||||
/* TQ */
|
||||
case TDMT_VND_TMQ_SUBSCRIBE:
|
||||
if (tqProcessSubscribeReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
|
||||
pMsg->contLen - sizeof(SMsgHead)) < 0) {
|
||||
if (tqProcessSubscribeReq(pVnode->pTq, version, pReq, len) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
break;
|
||||
|
@ -240,26 +239,22 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
|
|||
}
|
||||
break;
|
||||
case TDMT_VND_TMQ_COMMIT_OFFSET:
|
||||
if (tqProcessOffsetCommitReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
|
||||
pMsg->contLen - sizeof(SMsgHead)) < 0) {
|
||||
if (tqProcessOffsetCommitReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
break;
|
||||
case TDMT_VND_TMQ_ADD_CHECKINFO:
|
||||
if (tqProcessAddCheckInfoReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
|
||||
pMsg->contLen - sizeof(SMsgHead)) < 0) {
|
||||
if (tqProcessAddCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
break;
|
||||
case TDMT_VND_TMQ_DEL_CHECKINFO:
|
||||
if (tqProcessDelCheckInfoReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
|
||||
pMsg->contLen - sizeof(SMsgHead)) < 0) {
|
||||
if (tqProcessDelCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
break;
|
||||
case TDMT_STREAM_TASK_DEPLOY: {
|
||||
if (tqProcessTaskDeployReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
|
||||
pMsg->contLen - sizeof(SMsgHead)) < 0) {
|
||||
if (tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
} break;
|
||||
|
@ -273,6 +268,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
|
|||
goto _err;
|
||||
}
|
||||
} break;
|
||||
case TDMT_STREAM_TASK_CHECK_RSP: {
|
||||
if (tqProcessStreamTaskCheckRsp(pVnode->pTq, version, pReq, len) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
} break;
|
||||
case TDMT_VND_ALTER_CONFIRM:
|
||||
vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp);
|
||||
break;
|
||||
|
@ -390,10 +390,12 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
|
|||
return tqProcessPollReq(pVnode->pTq, pMsg);
|
||||
case TDMT_STREAM_TASK_RUN:
|
||||
return tqProcessTaskRunReq(pVnode->pTq, pMsg);
|
||||
#if 0
|
||||
#if 1
|
||||
case TDMT_STREAM_TASK_DISPATCH:
|
||||
return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true);
|
||||
#endif
|
||||
case TDMT_STREAM_TASK_CHECK:
|
||||
return tqProcessStreamTaskCheckReq(pVnode->pTq, pMsg);
|
||||
case TDMT_STREAM_TASK_DISPATCH_RSP:
|
||||
return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg);
|
||||
case TDMT_STREAM_RETRIEVE:
|
||||
|
@ -863,6 +865,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
|
|||
SEncoder encoder = {0};
|
||||
SArray *newTbUids = NULL;
|
||||
SVStatis statis = {0};
|
||||
bool tbCreated = false;
|
||||
terrno = TSDB_CODE_SUCCESS;
|
||||
|
||||
pRsp->code = 0;
|
||||
|
@ -896,11 +899,10 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
|
|||
if (pBlock == NULL) break;
|
||||
|
||||
SSubmitBlkRsp submitBlkRsp = {0};
|
||||
tbCreated = false;
|
||||
|
||||
// create table for auto create table mode
|
||||
if (msgIter.schemaLen > 0) {
|
||||
submitBlkRsp.hashMeta = 1;
|
||||
|
||||
tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen);
|
||||
if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) {
|
||||
pRsp->code = TSDB_CODE_INVALID_MSG;
|
||||
|
@ -937,12 +939,13 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
|
|||
}
|
||||
|
||||
taosArrayPush(newTbUids, &createTbReq.uid);
|
||||
|
||||
submitBlkRsp.uid = createTbReq.uid;
|
||||
submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2);
|
||||
sprintf(submitBlkRsp.tblFName, "%s.%s", pVnode->config.dbname, createTbReq.name);
|
||||
tbCreated = true;
|
||||
}
|
||||
|
||||
submitBlkRsp.uid = createTbReq.uid;
|
||||
submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2);
|
||||
sprintf(submitBlkRsp.tblFName, "%s.%s", pVnode->config.dbname, createTbReq.name);
|
||||
|
||||
|
||||
msgIter.uid = createTbReq.uid;
|
||||
if (createTbReq.type == TSDB_CHILD_TABLE) {
|
||||
msgIter.suid = createTbReq.ctb.suid;
|
||||
|
@ -955,10 +958,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
|
|||
#endif
|
||||
tDecoderClear(&decoder);
|
||||
taosArrayDestroy(createTbReq.ctb.tagName);
|
||||
} else {
|
||||
submitBlkRsp.tblFName = taosMemoryMalloc(TSDB_TABLE_FNAME_LEN);
|
||||
sprintf(submitBlkRsp.tblFName, "%s.", pVnode->config.dbname);
|
||||
}
|
||||
}
|
||||
|
||||
if (tsdbInsertTableData(pVnode->pTsdb, version, &msgIter, pBlock, &submitBlkRsp) < 0) {
|
||||
submitBlkRsp.code = terrno;
|
||||
|
@ -966,7 +966,9 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
|
|||
|
||||
submitRsp.numOfRows += submitBlkRsp.numOfRows;
|
||||
submitRsp.affectedRows += submitBlkRsp.affectedRows;
|
||||
taosArrayPush(submitRsp.pArray, &submitBlkRsp);
|
||||
if (tbCreated || submitBlkRsp.code) {
|
||||
taosArrayPush(submitRsp.pArray, &submitBlkRsp);
|
||||
}
|
||||
}
|
||||
|
||||
if (taosArrayGetSize(newTbUids) > 0) {
|
||||
|
|
|
@ -342,52 +342,26 @@ static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, cons
|
|||
TMSG_INFO(pMsg->msgType));
|
||||
}
|
||||
|
||||
#define USE_TSDB_SNAPSHOT
|
||||
|
||||
static int32_t vnodeSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) {
|
||||
#ifdef USE_TSDB_SNAPSHOT
|
||||
SVnode *pVnode = pFsm->data;
|
||||
SSnapshotParam *pSnapshotParam = pParam;
|
||||
int32_t code = vnodeSnapReaderOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapReader **)ppReader);
|
||||
return code;
|
||||
#else
|
||||
*ppReader = taosMemoryMalloc(32);
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static int32_t vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) {
|
||||
#ifdef USE_TSDB_SNAPSHOT
|
||||
SVnode *pVnode = pFsm->data;
|
||||
int32_t code = vnodeSnapReaderClose(pReader);
|
||||
return code;
|
||||
#else
|
||||
taosMemoryFree(pReader);
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static int32_t vnodeSnapshotDoRead(const SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) {
|
||||
#ifdef USE_TSDB_SNAPSHOT
|
||||
SVnode *pVnode = pFsm->data;
|
||||
int32_t code = vnodeSnapRead(pReader, (uint8_t **)ppBuf, len);
|
||||
return code;
|
||||
#else
|
||||
static int32_t times = 0;
|
||||
if (times++ < 5) {
|
||||
*len = 64;
|
||||
*ppBuf = taosMemoryMalloc(*len);
|
||||
snprintf(*ppBuf, *len, "snapshot block %d", times);
|
||||
} else {
|
||||
*len = 0;
|
||||
*ppBuf = NULL;
|
||||
}
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void **ppWriter) {
|
||||
#ifdef USE_TSDB_SNAPSHOT
|
||||
SVnode *pVnode = pFsm->data;
|
||||
SSnapshotParam *pSnapshotParam = pParam;
|
||||
|
||||
|
@ -404,14 +378,9 @@ static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void
|
|||
|
||||
int32_t code = vnodeSnapWriterOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapWriter **)ppWriter);
|
||||
return code;
|
||||
#else
|
||||
*ppWriter = taosMemoryMalloc(32);
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) {
|
||||
#ifdef USE_TSDB_SNAPSHOT
|
||||
SVnode *pVnode = pFsm->data;
|
||||
vInfo("vgId:%d, stop write vnode snapshot, apply:%d, index:%" PRId64 " term:%" PRIu64 " config:%" PRId64,
|
||||
pVnode->config.vgId, isApply, pSnapshot->lastApplyIndex, pSnapshot->lastApplyTerm, pSnapshot->lastConfigIndex);
|
||||
|
@ -419,22 +388,14 @@ static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool
|
|||
int32_t code = vnodeSnapWriterClose(pWriter, !isApply, pSnapshot);
|
||||
vInfo("vgId:%d, apply vnode snapshot finished, code:0x%x", pVnode->config.vgId, code);
|
||||
return code;
|
||||
#else
|
||||
taosMemoryFree(pWriter);
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static int32_t vnodeSnapshotDoWrite(const SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) {
|
||||
#ifdef USE_TSDB_SNAPSHOT
|
||||
SVnode *pVnode = pFsm->data;
|
||||
vDebug("vgId:%d, continue write vnode snapshot, len:%d", pVnode->config.vgId, len);
|
||||
int32_t code = vnodeSnapWrite(pWriter, pBuf, len);
|
||||
vDebug("vgId:%d, continue write vnode snapshot finished, len:%d", pVnode->config.vgId, len);
|
||||
return code;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static void vnodeRestoreFinish(const SSyncFSM *pFsm) {
|
||||
|
@ -461,7 +422,6 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) {
|
|||
SVnode *pVnode = pFsm->data;
|
||||
vDebug("vgId:%d, become follower", pVnode->config.vgId);
|
||||
|
||||
// clear old leader resource
|
||||
taosThreadMutexLock(&pVnode->lock);
|
||||
if (pVnode->blocked) {
|
||||
pVnode->blocked = false;
|
||||
|
@ -474,15 +434,6 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) {
|
|||
static void vnodeBecomeLeader(const SSyncFSM *pFsm) {
|
||||
SVnode *pVnode = pFsm->data;
|
||||
vDebug("vgId:%d, become leader", pVnode->config.vgId);
|
||||
|
||||
#if 0
|
||||
taosThreadMutexLock(&pVnode->lock);
|
||||
if (pVnode->blocked) {
|
||||
pVnode->blocked = false;
|
||||
tsem_post(&pVnode->syncSem);
|
||||
}
|
||||
taosThreadMutexUnlock(&pVnode->lock);
|
||||
#endif
|
||||
}
|
||||
|
||||
static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
|
||||
|
@ -543,12 +494,25 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
|
|||
}
|
||||
|
||||
void vnodeSyncStart(SVnode *pVnode) {
|
||||
vDebug("vgId:%d, start sync", pVnode->config.vgId);
|
||||
vInfo("vgId:%d, start sync", pVnode->config.vgId);
|
||||
syncStart(pVnode->sync);
|
||||
}
|
||||
|
||||
void vnodeSyncPreClose(SVnode *pVnode) {
|
||||
vInfo("vgId:%d, pre close sync", pVnode->config.vgId);
|
||||
syncLeaderTransfer(pVnode->sync);
|
||||
syncPreStop(pVnode->sync);
|
||||
taosThreadMutexLock(&pVnode->lock);
|
||||
if (pVnode->blocked) {
|
||||
vInfo("vgId:%d, post block after close sync", pVnode->config.vgId);
|
||||
pVnode->blocked = false;
|
||||
tsem_post(&pVnode->syncSem);
|
||||
}
|
||||
taosThreadMutexUnlock(&pVnode->lock);
|
||||
}
|
||||
|
||||
void vnodeSyncClose(SVnode *pVnode) {
|
||||
vDebug("vgId:%d, close sync", pVnode->config.vgId);
|
||||
vInfo("vgId:%d, close sync", pVnode->config.vgId);
|
||||
syncStop(pVnode->sync);
|
||||
}
|
||||
|
||||
|
|
|
@ -33,6 +33,7 @@ extern "C" {
|
|||
#define CTG_DEFAULT_MAX_RETRY_TIMES 3
|
||||
#define CTG_DEFAULT_BATCH_NUM 64
|
||||
#define CTG_DEFAULT_FETCH_NUM 8
|
||||
#define CTG_MAX_COMMAND_LEN 512
|
||||
|
||||
#define CTG_RENT_SLOT_SECOND 1.5
|
||||
|
||||
|
@ -223,6 +224,7 @@ typedef struct SCtgUserAuth {
|
|||
|
||||
typedef struct SCatalog {
|
||||
uint64_t clusterId;
|
||||
bool stopUpdate;
|
||||
SHashObj* userCache; // key:user, value:SCtgUserAuth
|
||||
SHashObj* dbCache; // key:dbname, value:SCtgDBCache
|
||||
SCtgRentMgmt dbRent;
|
||||
|
@ -671,7 +673,7 @@ void ctgdShowClusterCache(SCatalog* pCtg);
|
|||
int32_t ctgdShowCacheInfo(void);
|
||||
|
||||
int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq);
|
||||
int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
|
||||
int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
|
||||
int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx,
|
||||
int32_t* fetchIdx, int32_t baseResIdx, SArray* pList);
|
||||
|
||||
|
@ -786,6 +788,7 @@ void ctgFreeTbCacheImpl(SCtgTbCache* pCache);
|
|||
int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName);
|
||||
int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists);
|
||||
SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch);
|
||||
int32_t ctgdGetOneHandle(SCatalog **pHandle);
|
||||
|
||||
extern SCatalogMgmt gCtgMgmt;
|
||||
extern SCtgDebug gCTGDebug;
|
||||
|
|
|
@ -202,7 +202,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx
|
|||
int32_t code = 0;
|
||||
STableMetaOutput* output = NULL;
|
||||
|
||||
CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, pConn, ctx, pTableMeta));
|
||||
CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, ctx, pTableMeta));
|
||||
if (*pTableMeta || (ctx->flag & CTG_FLAG_ONLY_CACHE)) {
|
||||
goto _return;
|
||||
}
|
||||
|
@ -959,14 +959,14 @@ int32_t catalogGetTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName
|
|||
CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
|
||||
}
|
||||
|
||||
int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta) {
|
||||
int32_t catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta) {
|
||||
CTG_API_ENTER();
|
||||
|
||||
SCtgTbMetaCtx ctx = {0};
|
||||
ctx.pName = (SName*)pTableName;
|
||||
ctx.flag = CTG_FLAG_UNKNOWN_STB | CTG_FLAG_ONLY_CACHE;
|
||||
|
||||
CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
|
||||
CTG_API_LEAVE(ctgGetTbMeta(pCtg, NULL, &ctx, pTableMeta));
|
||||
}
|
||||
|
||||
|
||||
|
@ -981,15 +981,14 @@ int32_t catalogGetSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SNam
|
|||
CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
|
||||
}
|
||||
|
||||
int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
|
||||
STableMeta** pTableMeta) {
|
||||
int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta) {
|
||||
CTG_API_ENTER();
|
||||
|
||||
SCtgTbMetaCtx ctx = {0};
|
||||
ctx.pName = (SName*)pTableName;
|
||||
ctx.flag = CTG_FLAG_STB | CTG_FLAG_ONLY_CACHE;
|
||||
|
||||
CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
|
||||
CTG_API_LEAVE(ctgGetTbMeta(pCtg, NULL, &ctx, pTableMeta));
|
||||
}
|
||||
|
||||
|
||||
|
@ -1114,11 +1113,10 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const
|
|||
CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, pConn, pTableName, pVgroup, NULL));
|
||||
}
|
||||
|
||||
int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
|
||||
SVgroupInfo* pVgroup, bool* exists) {
|
||||
int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists) {
|
||||
CTG_API_ENTER();
|
||||
|
||||
CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, pConn, pTableName, pVgroup, exists));
|
||||
CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, NULL, pTableName, pVgroup, exists));
|
||||
}
|
||||
|
||||
#if 0
|
||||
|
@ -1387,16 +1385,16 @@ _return:
|
|||
CTG_API_LEAVE(code);
|
||||
}
|
||||
|
||||
int32_t catalogChkAuthFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
|
||||
int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type,
|
||||
bool* pass, bool* exists) {
|
||||
CTG_API_ENTER();
|
||||
|
||||
if (NULL == pCtg || NULL == pConn || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) {
|
||||
if (NULL == pCtg || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) {
|
||||
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
|
||||
}
|
||||
|
||||
int32_t code = 0;
|
||||
CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass, exists));
|
||||
CTG_ERR_JRET(ctgChkAuth(pCtg, NULL, user, dbFName, type, pass, exists));
|
||||
|
||||
_return:
|
||||
|
||||
|
|
|
@ -1204,11 +1204,15 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu
|
|||
stbCtx.flag = flag;
|
||||
stbCtx.pName = &stbName;
|
||||
|
||||
taosMemoryFreeClear(pOut->tbMeta);
|
||||
CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta));
|
||||
if (pOut->tbMeta) {
|
||||
STableMeta *stbMeta = NULL;
|
||||
ctgReadTbMetaFromCache(pCtg, &stbCtx, &stbMeta);
|
||||
if (stbMeta && stbMeta->sversion >= pOut->tbMeta->sversion) {
|
||||
ctgDebug("use cached stb meta, tbName:%s", tNameGetTableName(pName));
|
||||
exist = 1;
|
||||
} else {
|
||||
ctgDebug("need to get/update stb meta, tbName:%s", tNameGetTableName(pName));
|
||||
taosMemoryFreeClear(pOut->tbMeta);
|
||||
taosMemoryFreeClear(stbMeta);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1641,7 +1645,7 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask* pTask) {
|
|||
pMsgCtx->pBatchs = pJob->pBatchs;
|
||||
}
|
||||
|
||||
CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, pConn, (SCtgTbMetaCtx*)pTask->taskCtx, (STableMeta**)&pTask->res));
|
||||
CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, (SCtgTbMetaCtx*)pTask->taskCtx, (STableMeta**)&pTask->res));
|
||||
if (pTask->res) {
|
||||
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
|
@ -248,6 +248,8 @@ int32_t ctgAcquireStbMetaFromCache(SCatalog *pCtg, char *dbFName, uint64_t suid,
|
|||
goto _return;
|
||||
}
|
||||
|
||||
taosHashRelease(dbCache->stbCache, stName);
|
||||
|
||||
CTG_LOCK(CTG_READ, &pCache->metaLock);
|
||||
if (NULL == pCache->pMeta) {
|
||||
ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName);
|
||||
|
@ -1550,7 +1552,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
|
|||
char *dbFName = msg->dbFName;
|
||||
SCatalog *pCtg = msg->pCtg;
|
||||
|
||||
if (NULL == dbInfo->vgHash) {
|
||||
if (pCtg->stopUpdate || NULL == dbInfo->vgHash) {
|
||||
goto _return;
|
||||
}
|
||||
|
||||
|
@ -1620,6 +1622,10 @@ int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) {
|
|||
SCtgDropDBMsg *msg = operation->data;
|
||||
SCatalog *pCtg = msg->pCtg;
|
||||
|
||||
if (pCtg->stopUpdate) {
|
||||
goto _return;
|
||||
}
|
||||
|
||||
SCtgDBCache *dbCache = NULL;
|
||||
ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache);
|
||||
if (NULL == dbCache) {
|
||||
|
@ -1646,6 +1652,10 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) {
|
|||
SCtgDropDbVgroupMsg *msg = operation->data;
|
||||
SCatalog *pCtg = msg->pCtg;
|
||||
|
||||
if (pCtg->stopUpdate) {
|
||||
goto _return;
|
||||
}
|
||||
|
||||
SCtgDBCache *dbCache = NULL;
|
||||
ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache);
|
||||
if (NULL == dbCache) {
|
||||
|
@ -1675,6 +1685,10 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) {
|
|||
STableMetaOutput *pMeta = msg->pMeta;
|
||||
SCtgDBCache *dbCache = NULL;
|
||||
|
||||
if (pCtg->stopUpdate) {
|
||||
goto _return;
|
||||
}
|
||||
|
||||
if ((!CTG_IS_META_CTABLE(pMeta->metaType)) && NULL == pMeta->tbMeta) {
|
||||
ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", pMeta->dbFName, pMeta->tbName);
|
||||
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
|
||||
|
@ -1723,6 +1737,10 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) {
|
|||
SCtgDropStbMetaMsg *msg = operation->data;
|
||||
SCatalog *pCtg = msg->pCtg;
|
||||
|
||||
if (pCtg->stopUpdate) {
|
||||
goto _return;
|
||||
}
|
||||
|
||||
SCtgDBCache *dbCache = NULL;
|
||||
ctgGetDBCache(pCtg, msg->dbFName, &dbCache);
|
||||
if (NULL == dbCache) {
|
||||
|
@ -1776,6 +1794,10 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) {
|
|||
SCtgDropTblMetaMsg *msg = operation->data;
|
||||
SCatalog *pCtg = msg->pCtg;
|
||||
|
||||
if (pCtg->stopUpdate) {
|
||||
goto _return;
|
||||
}
|
||||
|
||||
SCtgDBCache *dbCache = NULL;
|
||||
ctgGetDBCache(pCtg, msg->dbFName, &dbCache);
|
||||
if (NULL == dbCache) {
|
||||
|
@ -1819,6 +1841,10 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) {
|
|||
SCtgUpdateUserMsg *msg = operation->data;
|
||||
SCatalog *pCtg = msg->pCtg;
|
||||
|
||||
if (pCtg->stopUpdate) {
|
||||
goto _return;
|
||||
}
|
||||
|
||||
SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user));
|
||||
if (NULL == pUser) {
|
||||
SCtgUserAuth userAuth = {0};
|
||||
|
@ -1872,8 +1898,12 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) {
|
|||
int32_t code = 0;
|
||||
SCtgUpdateEpsetMsg *msg = operation->data;
|
||||
SCatalog *pCtg = msg->pCtg;
|
||||
|
||||
SCtgDBCache *dbCache = NULL;
|
||||
|
||||
if (pCtg->stopUpdate) {
|
||||
goto _return;
|
||||
}
|
||||
|
||||
CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache));
|
||||
if (NULL == dbCache) {
|
||||
ctgDebug("db %s not exist, ignore epset update", msg->dbFName);
|
||||
|
@ -1920,6 +1950,10 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) {
|
|||
STableIndex *pIndex = msg->pIndex;
|
||||
SCtgDBCache *dbCache = NULL;
|
||||
|
||||
if (pCtg->stopUpdate) {
|
||||
goto _return;
|
||||
}
|
||||
|
||||
CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pIndex->dbFName, 0, &dbCache));
|
||||
|
||||
CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex));
|
||||
|
@ -1942,6 +1976,10 @@ int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation) {
|
|||
SCatalog *pCtg = msg->pCtg;
|
||||
SCtgDBCache *dbCache = NULL;
|
||||
|
||||
if (pCtg->stopUpdate) {
|
||||
goto _return;
|
||||
}
|
||||
|
||||
CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache));
|
||||
if (NULL == dbCache) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
@ -2154,7 +2192,7 @@ int32_t ctgStartUpdateThread() {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t ctgGetTbMetaFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) {
|
||||
int32_t ctgGetTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) {
|
||||
if (IS_SYS_DBNAME(ctx->pName->dbname)) {
|
||||
CTG_FLAG_SET_SYS_DB(ctx->flag);
|
||||
}
|
||||
|
|
|
@ -226,28 +226,45 @@ _return:
|
|||
CTG_RET(code);
|
||||
}
|
||||
|
||||
int32_t ctgdEnableDebug(char *option) {
|
||||
int32_t ctgdEnableDebug(char *option, bool enable) {
|
||||
if (0 == strcasecmp(option, "lock")) {
|
||||
gCTGDebug.lockEnable = true;
|
||||
qDebug("lock debug enabled");
|
||||
gCTGDebug.lockEnable = enable;
|
||||
qDebug("catalog lock debug set to %d", enable);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (0 == strcasecmp(option, "cache")) {
|
||||
gCTGDebug.cacheEnable = true;
|
||||
qDebug("cache debug enabled");
|
||||
gCTGDebug.cacheEnable = enable;
|
||||
qDebug("catalog cache debug set to %d", enable);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (0 == strcasecmp(option, "api")) {
|
||||
gCTGDebug.apiEnable = true;
|
||||
qDebug("api debug enabled");
|
||||
gCTGDebug.apiEnable = enable;
|
||||
qDebug("catalog api debug set to %d", enable);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (0 == strcasecmp(option, "meta")) {
|
||||
gCTGDebug.metaEnable = true;
|
||||
qDebug("api debug enabled");
|
||||
gCTGDebug.metaEnable = enable;
|
||||
qDebug("catalog meta debug set to %d", enable);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (0 == strcasecmp(option, "stopUpdate")) {
|
||||
SCatalog *pCtg = NULL;
|
||||
|
||||
void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
|
||||
while (pIter) {
|
||||
pCtg = *(SCatalog **)pIter;
|
||||
|
||||
pCtg->stopUpdate = enable;
|
||||
|
||||
pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
|
||||
}
|
||||
|
||||
qDebug("catalog stopUpdate set to %d", enable);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
@ -256,6 +273,77 @@ int32_t ctgdEnableDebug(char *option) {
|
|||
return TSDB_CODE_CTG_INTERNAL_ERROR;
|
||||
}
|
||||
|
||||
int32_t ctgdHandleDbgCommand(char *command) {
|
||||
if (NULL == command) {
|
||||
CTG_RET(TSDB_CODE_INVALID_PARA);
|
||||
}
|
||||
|
||||
if (strlen(command) > CTG_MAX_COMMAND_LEN) {
|
||||
CTG_RET(TSDB_CODE_INVALID_PARA);
|
||||
}
|
||||
|
||||
char *dup = strdup(command);
|
||||
char *option = NULL;
|
||||
char *param = NULL;
|
||||
|
||||
int32_t i = 0;
|
||||
bool newItem = true;
|
||||
while (*(dup + i)) {
|
||||
if (isspace(*(dup + i))) {
|
||||
*(dup + i) = 0;
|
||||
++i;
|
||||
newItem = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!newItem) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
newItem = false;
|
||||
if (NULL == option) {
|
||||
option = dup + i;
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (NULL == param) {
|
||||
param = dup + i;
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
taosMemoryFree(dup);
|
||||
CTG_RET(TSDB_CODE_INVALID_PARA);
|
||||
}
|
||||
|
||||
bool enable = atoi(param);
|
||||
|
||||
int32_t code = ctgdEnableDebug(option, enable);
|
||||
|
||||
taosMemoryFree(dup);
|
||||
|
||||
CTG_RET(code);
|
||||
}
|
||||
|
||||
int32_t ctgdGetOneHandle(SCatalog **pHandle) {
|
||||
SCatalog *pCtg = NULL;
|
||||
|
||||
void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
|
||||
while (pIter) {
|
||||
pCtg = *(SCatalog **)pIter;
|
||||
|
||||
taosHashCancelIterate(gCtgMgmt.pCluster, pIter);
|
||||
break;
|
||||
}
|
||||
|
||||
*pHandle = pCtg;
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int32_t ctgdGetStatNum(char *option, void *res) {
|
||||
if (0 == strcasecmp(option, "runtime.numOfOpDequeue")) {
|
||||
*(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.numOfOpDequeue);
|
||||
|
|
|
@ -41,7 +41,6 @@
|
|||
namespace {
|
||||
|
||||
extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog *pCatalog, int32_t type);
|
||||
extern "C" int32_t ctgdEnableDebug(char *option);
|
||||
extern "C" int32_t ctgdGetStatNum(char *option, void *res);
|
||||
|
||||
void ctgTestSetRspTableMeta();
|
||||
|
@ -49,6 +48,8 @@ void ctgTestSetRspCTableMeta();
|
|||
void ctgTestSetRspSTableMeta();
|
||||
void ctgTestSetRspMultiSTableMeta();
|
||||
|
||||
extern int32_t clientConnRefPool;
|
||||
|
||||
enum {
|
||||
CTGT_RSP_VGINFO = 1,
|
||||
CTGT_RSP_TBMETA,
|
||||
|
@ -151,10 +152,10 @@ void ctgTestInitLogFile() {
|
|||
qDebugFlag = 159;
|
||||
strcpy(tsLogDir, TD_LOG_DIR_PATH);
|
||||
|
||||
ctgdEnableDebug("api");
|
||||
ctgdEnableDebug("meta");
|
||||
ctgdEnableDebug("cache");
|
||||
ctgdEnableDebug("lock");
|
||||
ctgdEnableDebug("api", true);
|
||||
ctgdEnableDebug("meta", true);
|
||||
ctgdEnableDebug("cache", true);
|
||||
ctgdEnableDebug("lock", true);
|
||||
|
||||
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
|
||||
printf("failed to open log file in directory:%s\n", tsLogDir);
|
||||
|
@ -1204,6 +1205,34 @@ void *ctgTestSetCtableMetaThread(void *param) {
|
|||
}
|
||||
|
||||
|
||||
void ctgTestFetchRows(TAOS_RES *result, int32_t *rows) {
|
||||
TAOS_ROW row;
|
||||
int num_fields = taos_num_fields(result);
|
||||
TAOS_FIELD *fields = taos_fetch_fields(result);
|
||||
char temp[256];
|
||||
|
||||
// fetch the records row by row
|
||||
while ((row = taos_fetch_row(result))) {
|
||||
(*rows)++;
|
||||
memset(temp, 0, sizeof(temp));
|
||||
taos_print_row(temp, row, fields, num_fields);
|
||||
printf("\t[%s]\n", temp);
|
||||
}
|
||||
}
|
||||
|
||||
void ctgTestExecQuery(TAOS * taos, char* sql, bool fetch, int32_t *rows) {
|
||||
TAOS_RES *result = taos_query(taos, sql);
|
||||
int code = taos_errno(result);
|
||||
ASSERT_EQ(code, 0);
|
||||
|
||||
if (fetch) {
|
||||
ctgTestFetchRows(result, rows);
|
||||
}
|
||||
|
||||
taos_free_result(result);
|
||||
}
|
||||
|
||||
|
||||
TEST(tableMeta, normalTable) {
|
||||
struct SCatalog *pCtg = NULL;
|
||||
SVgroupInfo vgInfo = {0};
|
||||
|
@ -1245,7 +1274,7 @@ TEST(tableMeta, normalTable) {
|
|||
|
||||
memset(&vgInfo, 0, sizeof(vgInfo));
|
||||
bool exists = false;
|
||||
code = catalogGetCachedTableHashVgroup(pCtg, mockPointer, &n, &vgInfo, &exists);
|
||||
code = catalogGetCachedTableHashVgroup(pCtg, &n, &vgInfo, &exists);
|
||||
ASSERT_EQ(code, 0);
|
||||
ASSERT_EQ(vgInfo.vgId, 8);
|
||||
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
|
||||
|
@ -1292,7 +1321,7 @@ TEST(tableMeta, normalTable) {
|
|||
taosMemoryFree(tableMeta);
|
||||
|
||||
tableMeta = NULL;
|
||||
catalogGetCachedTableMeta(pCtg, mockPointer, &n, &tableMeta);
|
||||
catalogGetCachedTableMeta(pCtg, &n, &tableMeta);
|
||||
ASSERT_EQ(code, 0);
|
||||
ASSERT_EQ(tableMeta->vgId, 8);
|
||||
ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE);
|
||||
|
@ -1500,7 +1529,7 @@ TEST(tableMeta, superTableCase) {
|
|||
}
|
||||
|
||||
tableMeta = NULL;
|
||||
code = catalogGetCachedSTableMeta(pCtg, mockPointer, &n, &tableMeta);
|
||||
code = catalogGetCachedSTableMeta(pCtg, &n, &tableMeta);
|
||||
ASSERT_EQ(code, 0);
|
||||
ASSERT_EQ(tableMeta->vgId, 0);
|
||||
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
|
||||
|
@ -2772,7 +2801,7 @@ TEST(apiTest, catalogChkAuth_test) {
|
|||
|
||||
bool pass = false;
|
||||
bool exists = false;
|
||||
code = catalogChkAuthFromCache(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
|
||||
code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
|
||||
ASSERT_EQ(code, 0);
|
||||
ASSERT_EQ(exists, false);
|
||||
|
||||
|
@ -2790,7 +2819,7 @@ TEST(apiTest, catalogChkAuth_test) {
|
|||
}
|
||||
}
|
||||
|
||||
code = catalogChkAuthFromCache(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
|
||||
code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
|
||||
ASSERT_EQ(code, 0);
|
||||
ASSERT_EQ(pass, true);
|
||||
ASSERT_EQ(exists, true);
|
||||
|
@ -3063,6 +3092,58 @@ TEST(apiTest, catalogGetDnodeList_test) {
|
|||
catalogDestroy();
|
||||
}
|
||||
|
||||
#ifdef INTEGRATION_TEST
|
||||
TEST(intTest, autoCreateTableTest) {
|
||||
struct SCatalog *pCtg = NULL;
|
||||
|
||||
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
|
||||
ASSERT_TRUE(NULL != taos);
|
||||
|
||||
ctgdEnableDebug("api", true);
|
||||
ctgdEnableDebug("meta", true);
|
||||
ctgdEnableDebug("cache", true);
|
||||
ctgdEnableDebug("lock", true);
|
||||
|
||||
ctgTestExecQuery(taos, "drop database if exists db1", false, NULL);
|
||||
ctgTestExecQuery(taos, "create database db1", false, NULL);
|
||||
ctgTestExecQuery(taos, "create stable db1.st1 (ts timestamp, f1 int) tags(tg1 int)", false, NULL);
|
||||
ctgTestExecQuery(taos, "insert into db1.tb1 using db1.st1 tags(1) values(now, 1)", false, NULL);
|
||||
|
||||
ctgdGetOneHandle(&pCtg);
|
||||
|
||||
while (true) {
|
||||
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
|
||||
if (2 != n) {
|
||||
taosMsleep(50);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t n = 0, m = 0;
|
||||
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
|
||||
|
||||
ctgTestExecQuery(taos, "insert into db1.tb1 using db1.st1 tags(1) values(now, 2)", false, NULL);
|
||||
|
||||
ctgTestExecQuery(taos, "insert into db1.tb1 values(now, 3)", false, NULL);
|
||||
|
||||
taosMsleep(1000);
|
||||
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&m);
|
||||
|
||||
ASSERT_EQ(n, m);
|
||||
|
||||
ctgdEnableDebug("stopUpdate", true);
|
||||
ctgTestExecQuery(taos, "alter table db1.st1 add column f2 double", false, NULL);
|
||||
|
||||
ctgdEnableDebug("stopUpdate", false);
|
||||
|
||||
ctgTestExecQuery(taos, "insert into db1.tb1 (ts, f1) values(now, 4)", false, NULL);
|
||||
|
||||
taos_close(taos);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
|
|
|
@ -96,6 +96,7 @@ extern "C" {
|
|||
#define COMMAND_RESET_LOG "resetLog"
|
||||
#define COMMAND_SCHEDULE_POLICY "schedulePolicy"
|
||||
#define COMMAND_ENABLE_RESCHEDULE "enableReSchedule"
|
||||
#define COMMAND_CATALOG_DEBUG "catalogDebug"
|
||||
|
||||
typedef struct SExplainGroup {
|
||||
int32_t nodeNum;
|
||||
|
|
|
@ -571,6 +571,8 @@ static int32_t execAlterCmd(char* cmd, char* value, bool* processed) {
|
|||
code = schedulerUpdatePolicy(atoi(value));
|
||||
} else if (0 == strcasecmp(cmd, COMMAND_ENABLE_RESCHEDULE)) {
|
||||
code = schedulerEnableReSchedule(atoi(value));
|
||||
} else if (0 == strcasecmp(cmd, COMMAND_CATALOG_DEBUG)) {
|
||||
code = ctgdHandleDbgCommand(value);
|
||||
} else {
|
||||
goto _return;
|
||||
}
|
||||
|
|
|
@ -30,10 +30,10 @@
|
|||
// The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups
|
||||
struct STableListInfo {
|
||||
bool oneTableForEachGroup;
|
||||
int32_t numOfOuputGroups; // the data block will be generated one by one
|
||||
int32_t* groupOffset; // keep the offset value for each group in the tableList
|
||||
int32_t numOfOuputGroups; // the data block will be generated one by one
|
||||
int32_t* groupOffset; // keep the offset value for each group in the tableList
|
||||
SArray* pTableList;
|
||||
SHashObj* map; // speedup acquire the tableQueryInfo by table uid
|
||||
SHashObj* map; // speedup acquire the tableQueryInfo by table uid
|
||||
uint64_t suid;
|
||||
};
|
||||
|
||||
|
@ -421,7 +421,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
|
|||
}
|
||||
|
||||
for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) {
|
||||
SColumnInfoData colInfo = {{0}, 0};
|
||||
SColumnInfoData colInfo = {0, {0}};
|
||||
colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i);
|
||||
blockDataAppendColInfo(pResBlock, &colInfo);
|
||||
}
|
||||
|
@ -582,7 +582,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
|
|||
}
|
||||
|
||||
for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) {
|
||||
SColumnInfoData colInfo = {{0}, 0};
|
||||
SColumnInfoData colInfo = {0, {0}};
|
||||
colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i);
|
||||
blockDataAppendColInfo(pResBlock, &colInfo);
|
||||
}
|
||||
|
@ -925,6 +925,15 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
|
|||
|
||||
SArray* pTbList = getTableNameList(pList);
|
||||
int32_t numOfTables = taosArrayGetSize(pTbList);
|
||||
SHashObj *uHash = NULL;
|
||||
size_t listlen = taosArrayGetSize(list); // len > 0 means there already have uids
|
||||
if (listlen > 0) {
|
||||
uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
|
||||
for (int i = 0; i < listlen; i++) {
|
||||
int64_t *uid = taosArrayGet(list, i);
|
||||
taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i));
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < numOfTables; i++) {
|
||||
char* name = taosArrayGetP(pTbList, i);
|
||||
|
@ -933,9 +942,12 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
|
|||
if (metaGetTableUidByName(metaHandle, name, &uid) == 0) {
|
||||
ETableType tbType = TSDB_TABLE_MAX;
|
||||
if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) {
|
||||
taosArrayPush(list, &uid);
|
||||
if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) {
|
||||
taosArrayPush(list, &uid);
|
||||
}
|
||||
} else {
|
||||
taosArrayDestroy(pTbList);
|
||||
taosHashCleanup(uHash);
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
|
@ -944,6 +956,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
|
|||
}
|
||||
}
|
||||
|
||||
taosHashCleanup(uHash);
|
||||
taosArrayDestroy(pTbList);
|
||||
return 0;
|
||||
}
|
||||
|
@ -1482,10 +1495,6 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray
|
|||
while (i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) {
|
||||
SColumnInfoData* p = taosArrayGet(pCols, i);
|
||||
SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, j);
|
||||
/* if (!outputEveryColumn && pmInfo->reserved) {
|
||||
j++;
|
||||
continue;
|
||||
}*/
|
||||
|
||||
if (p->info.colId == pmInfo->colId) {
|
||||
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->dstSlotId);
|
||||
|
@ -1678,9 +1687,7 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) {
|
|||
return taosArrayGetSize(pTableList->pTableList);
|
||||
}
|
||||
|
||||
uint64_t tableListGetSuid(const STableListInfo* pTableList) {
|
||||
return pTableList->suid;
|
||||
}
|
||||
uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->suid; }
|
||||
|
||||
STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) {
|
||||
if (taosArrayGetSize(pTableList->pTableList) == 0) {
|
||||
|
@ -1718,7 +1725,7 @@ int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t
|
|||
}
|
||||
|
||||
int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalGroupIndex, STableKeyInfo** pKeyInfo,
|
||||
int32_t* size) {
|
||||
int32_t* size) {
|
||||
int32_t total = tableListGetOutputGroups(pTableList);
|
||||
if (ordinalGroupIndex < 0 || ordinalGroupIndex >= total) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
|
@ -1728,7 +1735,7 @@ int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalG
|
|||
// 1. only one group exists, and 2. one table exists for each group.
|
||||
if (total == 1) {
|
||||
*size = tableListGetSize(pTableList);
|
||||
*pKeyInfo = (*size == 0)? NULL:taosArrayGet(pTableList->pTableList, 0);
|
||||
*pKeyInfo = (*size == 0) ? NULL : taosArrayGet(pTableList->pTableList, 0);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else if (total == tableListGetSize(pTableList)) {
|
||||
*size = 1;
|
||||
|
@ -1806,13 +1813,13 @@ void tableListClear(STableListInfo* pTableListInfo) {
|
|||
}
|
||||
|
||||
static int32_t orderbyGroupIdComparFn(const void* p1, const void* p2) {
|
||||
STableKeyInfo* pInfo1 = (STableKeyInfo*) p1;
|
||||
STableKeyInfo* pInfo2 = (STableKeyInfo*) p2;
|
||||
STableKeyInfo* pInfo1 = (STableKeyInfo*)p1;
|
||||
STableKeyInfo* pInfo2 = (STableKeyInfo*)p2;
|
||||
|
||||
if (pInfo1->groupId == pInfo2->groupId) {
|
||||
return 0;
|
||||
} else {
|
||||
return pInfo1->groupId < pInfo2->groupId? -1:1;
|
||||
return pInfo1->groupId < pInfo2->groupId ? -1 : 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1825,12 +1832,12 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
|
|||
SArray* pList = taosArrayInit(4, sizeof(int32_t));
|
||||
|
||||
STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, 0);
|
||||
uint64_t gid = pInfo->groupId;
|
||||
uint64_t gid = pInfo->groupId;
|
||||
|
||||
int32_t start = 0;
|
||||
taosArrayPush(pList, &start);
|
||||
|
||||
for(int32_t i = 1; i < size; ++i) {
|
||||
for (int32_t i = 1; i < size; ++i) {
|
||||
pInfo = taosArrayGet(pTableListInfo->pTableList, i);
|
||||
if (pInfo->groupId != gid) {
|
||||
taosArrayPush(pList, &i);
|
||||
|
@ -1845,16 +1852,17 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
|
|||
return TDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort) {
|
||||
int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group,
|
||||
bool groupSort) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
ASSERT(pTableListInfo->map != NULL);
|
||||
|
||||
bool groupByTbname = groupbyTbname(group);
|
||||
bool groupByTbname = groupbyTbname(group);
|
||||
size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList);
|
||||
if (group == NULL || groupByTbname) {
|
||||
for (int32_t i = 0; i < numOfTables; i++) {
|
||||
STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
|
||||
info->groupId = groupByTbname? info->uid:0;
|
||||
info->groupId = groupByTbname ? info->uid : 0;
|
||||
}
|
||||
|
||||
pTableListInfo->oneTableForEachGroup = groupByTbname;
|
||||
|
@ -1878,7 +1886,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
|
|||
|
||||
// add all table entry in the hash map
|
||||
size_t size = taosArrayGetSize(pTableListInfo->pTableList);
|
||||
for(int32_t i = 0; i < size; ++i) {
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
STableKeyInfo* p = taosArrayGet(pTableListInfo->pTableList, i);
|
||||
taosHashPut(pTableListInfo->map, &p->uid, sizeof(uint64_t), &i, sizeof(int32_t));
|
||||
}
|
||||
|
@ -1889,7 +1897,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
|
|||
int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle,
|
||||
STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond,
|
||||
struct SExecTaskInfo* pTaskInfo) {
|
||||
int64_t st = taosGetTimestampUs();
|
||||
int64_t st = taosGetTimestampUs();
|
||||
const char* idStr = GET_TASKID(pTaskInfo);
|
||||
|
||||
if (pHandle == NULL) {
|
||||
|
@ -1919,7 +1927,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
|
|||
return code;
|
||||
}
|
||||
|
||||
pTaskInfo->cost.groupIdMapTime = (taosGetTimestampUs() - st1)/1000.0;
|
||||
pTaskInfo->cost.groupIdMapTime = (taosGetTimestampUs() - st1) / 1000.0;
|
||||
qDebug("generate group id map completed, elapsed time:%.2f ms %s", pTaskInfo->cost.groupIdMapTime, idStr);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
|
@ -321,7 +321,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
|
|||
}
|
||||
|
||||
int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
|
||||
|
||||
if (isAdd) {
|
||||
qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), pTaskInfo->id.str);
|
||||
|
@ -473,7 +473,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
|
|||
|
||||
qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId);
|
||||
|
||||
_error:
|
||||
_error:
|
||||
// if failed to add ref for all tables in this query, abort current query
|
||||
return code;
|
||||
}
|
||||
|
@ -1027,10 +1027,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
|
|||
|
||||
if (pTableScanInfo->dataReader == NULL) {
|
||||
STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
|
||||
int32_t num = tableListGetSize(pTaskInfo->pTableInfoList);
|
||||
int32_t num = tableListGetSize(pTaskInfo->pTableInfoList);
|
||||
|
||||
if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond, pList, num,
|
||||
&pTableScanInfo->dataReader, NULL) < 0 || pTableScanInfo->dataReader == NULL) {
|
||||
&pTableScanInfo->dataReader, NULL) < 0 ||
|
||||
pTableScanInfo->dataReader == NULL) {
|
||||
ASSERT(0);
|
||||
}
|
||||
}
|
||||
|
@ -1071,14 +1072,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
|
|||
initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo);
|
||||
pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
|
||||
|
||||
if (pTaskInfo->pTableInfoList == NULL) {
|
||||
if (pTaskInfo->pTableInfoList == NULL) {
|
||||
pTaskInfo->pTableInfoList = tableListCreate();
|
||||
}
|
||||
|
||||
tableListAddTableInfo(pTaskInfo->pTableInfoList, mtInfo.uid, 0);
|
||||
|
||||
STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
|
||||
int32_t size = tableListGetSize(pTaskInfo->pTableInfoList);
|
||||
int32_t size = tableListGetSize(pTaskInfo->pTableInfoList);
|
||||
ASSERT(size == 1);
|
||||
|
||||
tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->dataReader, NULL);
|
||||
|
|
|
@ -2783,8 +2783,10 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul
|
|||
*defaultPgsz <<= 1u;
|
||||
}
|
||||
|
||||
// The default buffer for each operator in query is 10MB.
|
||||
// at least four pages need to be in buffer
|
||||
*defaultBufsz = 4096 * 256;
|
||||
// TODO: make this variable to be configurable.
|
||||
*defaultBufsz = 4096 * 2560;
|
||||
if ((*defaultBufsz) <= (*defaultPgsz)) {
|
||||
(*defaultBufsz) = (*defaultPgsz) * 4;
|
||||
}
|
||||
|
@ -2971,7 +2973,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
|
|||
|
||||
return pOperator;
|
||||
|
||||
_error:
|
||||
_error:
|
||||
if (pInfo != NULL) {
|
||||
destroyAggOperatorInfo(pInfo);
|
||||
}
|
||||
|
@ -3187,11 +3189,12 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
|
|||
code = appendDownstream(pOperator, &downstream, 1);
|
||||
return pOperator;
|
||||
|
||||
_error:
|
||||
_error:
|
||||
if (pInfo != NULL) {
|
||||
destroyFillOperatorInfo(pInfo);
|
||||
}
|
||||
|
||||
pTaskInfo->code = code;
|
||||
taosMemoryFreeClear(pOperator);
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
@ -750,6 +750,14 @@ static void destroyPartitionOperatorInfo(void* param) {
|
|||
taosArrayDestroy(pInfo->pGroupColVals);
|
||||
taosMemoryFree(pInfo->keyBuf);
|
||||
taosArrayDestroy(pInfo->sortedGroupArray);
|
||||
|
||||
void* pGroupIter = taosHashIterate(pInfo->pGroupSet, NULL);
|
||||
while (pGroupIter != NULL) {
|
||||
SDataGroupInfo* pGroupInfo = pGroupIter;
|
||||
taosArrayDestroy(pGroupInfo->pPageList);
|
||||
pGroupIter = taosHashIterate(pInfo->pGroupSet, pGroupIter);
|
||||
}
|
||||
|
||||
taosHashCleanup(pInfo->pGroupSet);
|
||||
taosMemoryFree(pInfo->columnOffset);
|
||||
|
||||
|
|
|
@ -1062,15 +1062,18 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi
|
|||
|
||||
int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return NULL;
|
||||
goto _error;
|
||||
}
|
||||
|
||||
STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList;
|
||||
size_t num = tableListGetSize(pTableListInfo);
|
||||
void* pList = tableListGetInfo(pTableListInfo, 0);
|
||||
|
||||
tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str);
|
||||
code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str);
|
||||
cleanupQueryTableDataCond(&cond);
|
||||
if (code != 0) {
|
||||
goto _error;
|
||||
}
|
||||
}
|
||||
|
||||
pInfo->readHandle = *readHandle;
|
||||
|
@ -1164,6 +1167,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
|
|||
GET_TASKID(pTaskInfo));
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
terrno = code;
|
||||
T_LONG_JMP(pTaskInfo->env, code);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
@ -1884,11 +1888,15 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
|
|||
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
|
||||
memcpy(&pTSInfo->cond, &pTaskInfo->streamInfo.tableCond, sizeof(SQueryTableDataCond));
|
||||
if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1) {
|
||||
pTSInfo->cond.startVersion = -1;
|
||||
pTSInfo->cond.startVersion = 0;
|
||||
pTSInfo->cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer1;
|
||||
qDebug("stream recover step 1, from %" PRId64 " to %" PRId64, pTSInfo->cond.startVersion,
|
||||
pTSInfo->cond.endVersion);
|
||||
} else {
|
||||
pTSInfo->cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1;
|
||||
pTSInfo->cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2;
|
||||
qDebug("stream recover step 2, from %" PRId64 " to %" PRId64, pTSInfo->cond.startVersion,
|
||||
pTSInfo->cond.endVersion);
|
||||
}
|
||||
|
||||
/*resetTableScanInfo(pTSInfo, pWin);*/
|
||||
|
@ -1905,11 +1913,15 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
|
|||
if (pBlock != NULL) {
|
||||
calBlockTbName(&pInfo->tbnameCalSup, pBlock);
|
||||
updateInfoFillBlockData(pInfo->pUpdateInfo, pBlock, pInfo->primaryTsIndex);
|
||||
qDebug("stream recover scan get block, rows %d", pBlock->info.rows);
|
||||
return pBlock;
|
||||
}
|
||||
pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE;
|
||||
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
|
||||
pTSInfo->cond.startVersion = 0;
|
||||
tsdbReaderClose(pTSInfo->dataReader);
|
||||
pTSInfo->dataReader = NULL;
|
||||
|
||||
pTSInfo->cond.startVersion = -1;
|
||||
pTSInfo->cond.endVersion = -1;
|
||||
|
||||
return NULL;
|
||||
|
@ -2418,8 +2430,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
|
|||
if (pHandle->initTableReader) {
|
||||
pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
|
||||
pTSInfo->dataReader = NULL;
|
||||
if (tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, pList, num, &pTSInfo->dataReader, NULL) < 0) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
int32_t code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, pList, num, &pTSInfo->dataReader, NULL);
|
||||
if (code != 0) {
|
||||
terrno = code;
|
||||
destroyTableScanOperatorInfo(pTableScanOp);
|
||||
goto _error;
|
||||
}
|
||||
|
@ -4284,131 +4297,6 @@ int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle*
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t createMultipleDataReaders2(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle,
|
||||
STableListInfo* pTableListInfo, int32_t tableStartIdx, int32_t tableEndIdx,
|
||||
STsdbReader** ppReader, const char* idstr) {
|
||||
STsdbReader* pReader = NULL;
|
||||
void* pStart = tableListGetInfo(pTableListInfo, tableStartIdx);
|
||||
int32_t num = tableEndIdx - tableStartIdx + 1;
|
||||
|
||||
int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, pStart, num, &pReader, idstr);
|
||||
if (code != 0) {
|
||||
return code;
|
||||
}
|
||||
|
||||
*ppReader = pReader;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t loadDataBlockFromOneTable2(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
|
||||
SSDataBlock* pBlock, uint32_t* status) {
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
STableMergeScanInfo* pInfo = pOperator->info;
|
||||
|
||||
uint64_t uid = pBlock->info.uid;
|
||||
|
||||
SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
|
||||
|
||||
pCost->totalBlocks += 1;
|
||||
pCost->totalRows += pBlock->info.rows;
|
||||
|
||||
*status = pInfo->dataBlockLoadFlag;
|
||||
if (pTableScanInfo->pFilterNode != NULL ||
|
||||
overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info, pTableScanInfo->cond.order)) {
|
||||
(*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
|
||||
}
|
||||
|
||||
SDataBlockInfo* pBlockInfo = &pBlock->info;
|
||||
taosMemoryFreeClear(pBlock->pBlockAgg);
|
||||
|
||||
if (*status == FUNC_DATA_REQUIRED_FILTEROUT) {
|
||||
qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
|
||||
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
|
||||
pCost->filterOutBlocks += 1;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
|
||||
qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
|
||||
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
|
||||
pCost->skipBlocks += 1;
|
||||
|
||||
// clear all data in pBlock that are set when handing the previous block
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
|
||||
SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
|
||||
pcol->pData = NULL;
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) {
|
||||
pCost->loadBlockStatis += 1;
|
||||
|
||||
bool allColumnsHaveAgg = true;
|
||||
SColumnDataAgg** pColAgg = NULL;
|
||||
STsdbReader* reader = pTableScanInfo->pReader;
|
||||
tsdbRetrieveDatablockSMA(reader, &pColAgg, &allColumnsHaveAgg);
|
||||
|
||||
if (allColumnsHaveAgg == true) {
|
||||
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
|
||||
|
||||
// todo create this buffer during creating operator
|
||||
if (pBlock->pBlockAgg == NULL) {
|
||||
pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES);
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i);
|
||||
if (!pColMatchInfo->needOutput) {
|
||||
continue;
|
||||
}
|
||||
pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else { // failed to load the block sma data, data block statistics does not exist, load data block instead
|
||||
*status = FUNC_DATA_REQUIRED_DATA_LOAD;
|
||||
}
|
||||
}
|
||||
|
||||
ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD);
|
||||
|
||||
pCost->totalCheckedRows += pBlock->info.rows;
|
||||
pCost->loadBlocks += 1;
|
||||
|
||||
STsdbReader* reader = pTableScanInfo->pReader;
|
||||
SArray* pCols = tsdbRetrieveDataBlock(reader, NULL);
|
||||
if (pCols == NULL) {
|
||||
return terrno;
|
||||
}
|
||||
|
||||
relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
|
||||
|
||||
// currently only the tbname pseudo column
|
||||
if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
|
||||
int32_t code =
|
||||
addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo,
|
||||
pTableScanInfo->pseudoSup.numOfExprs, pBlock, pBlock->info.rows, GET_TASKID(pTaskInfo));
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
T_LONG_JMP(pTaskInfo->env, code);
|
||||
}
|
||||
}
|
||||
|
||||
if (pTableScanInfo->pFilterNode != NULL) {
|
||||
int64_t st = taosGetTimestampMs();
|
||||
doFilter(pTableScanInfo->pFilterNode, pBlock, &pTableScanInfo->matchInfo, NULL);
|
||||
|
||||
double el = (taosGetTimestampUs() - st) / 1000.0;
|
||||
pTableScanInfo->readRecorder.filterTime += el;
|
||||
|
||||
if (pBlock->info.rows == 0) {
|
||||
pCost->filterOutBlocks += 1;
|
||||
qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms",
|
||||
GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el);
|
||||
} else {
|
||||
qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el);
|
||||
}
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
// todo refactor
|
||||
static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
|
||||
int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
|
||||
|
@ -4535,7 +4423,7 @@ typedef struct STableMergeScanSortSourceParam {
|
|||
SSDataBlock* inputBlock;
|
||||
} STableMergeScanSortSourceParam;
|
||||
|
||||
static SSDataBlock* getTableDataBlockTemp(void* param) {
|
||||
static SSDataBlock* getTableDataBlockImpl(void* param) {
|
||||
STableMergeScanSortSourceParam* source = param;
|
||||
SOperatorInfo* pOperator = source->pOperator;
|
||||
STableMergeScanInfo* pInfo = pOperator->info;
|
||||
|
@ -4552,7 +4440,11 @@ static SSDataBlock* getTableDataBlockTemp(void* param) {
|
|||
|
||||
void* p = tableListGetInfo(pInfo->tableListInfo, readIdx + pInfo->tableStartIndex);
|
||||
SReadHandle* pHandle = &pInfo->readHandle;
|
||||
tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->pReader, GET_TASKID(pTaskInfo));
|
||||
|
||||
int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->pReader, GET_TASKID(pTaskInfo));
|
||||
if (code != 0) {
|
||||
T_LONG_JMP(pOperator->pTaskInfo->env, code);
|
||||
}
|
||||
|
||||
STsdbReader* reader = pInfo->pReader;
|
||||
while (tsdbNextDataBlock(reader)) {
|
||||
|
@ -4603,55 +4495,6 @@ static SSDataBlock* getTableDataBlockTemp(void* param) {
|
|||
pInfo->pReader = NULL;
|
||||
return NULL;
|
||||
}
|
||||
static SSDataBlock* getTableDataBlock2(void* param) {
|
||||
STableMergeScanSortSourceParam* source = param;
|
||||
SOperatorInfo* pOperator = source->pOperator;
|
||||
int64_t uid = source->uid;
|
||||
SSDataBlock* pBlock = source->inputBlock;
|
||||
STableMergeScanInfo* pTableScanInfo = pOperator->info;
|
||||
|
||||
int64_t st = taosGetTimestampUs();
|
||||
|
||||
blockDataCleanup(pBlock);
|
||||
|
||||
STsdbReader* reader = pTableScanInfo->pReader;
|
||||
while (tsdbTableNextDataBlock(reader, uid)) {
|
||||
if (isTaskKilled(pOperator->pTaskInfo)) {
|
||||
T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
|
||||
}
|
||||
|
||||
// process this data block based on the probabilities
|
||||
bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
|
||||
if (!processThisBlock) {
|
||||
continue;
|
||||
}
|
||||
|
||||
blockDataCleanup(pBlock);
|
||||
|
||||
int32_t rows = 0;
|
||||
tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.uid, &pBlock->info.window);
|
||||
blockDataEnsureCapacity(pBlock, rows);
|
||||
pBlock->info.rows = rows;
|
||||
|
||||
uint32_t status = 0;
|
||||
int32_t code = loadDataBlockFromOneTable2(pOperator, pTableScanInfo, pBlock, &status);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
T_LONG_JMP(pOperator->pTaskInfo->env, code);
|
||||
}
|
||||
|
||||
// current block is filter out according to filter condition, continue load the next block
|
||||
if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
pBlock->info.groupId = getTableGroupId(pOperator->pTaskInfo->pTableInfoList, pBlock->info.uid);
|
||||
pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows;
|
||||
pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
|
||||
|
||||
return pBlock;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static SSDataBlock* getTableDataBlock(void* param) {
|
||||
STableMergeScanSortSourceParam* source = param;
|
||||
|
@ -4761,7 +4604,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
|
|||
pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage,
|
||||
pInfo->pSortInputBlock, pTaskInfo->id.str);
|
||||
|
||||
tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlockTemp, NULL, NULL);
|
||||
tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlockImpl, NULL, NULL);
|
||||
|
||||
// one table has one data block
|
||||
int32_t numOfTable = tableEndIdx - tableStartIdx + 1;
|
||||
|
|
|
@ -579,21 +579,8 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
|
||||
SArray* pColMatchInfo, SOperatorInfo* pOperator) {
|
||||
SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
static void doGetSortedBlockData(SMultiwayMergeOperatorInfo* pInfo, SSortHandle* pHandle, int32_t capacity, SSDataBlock* p) {
|
||||
|
||||
blockDataCleanup(pDataBlock);
|
||||
|
||||
SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
|
||||
if (p == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
blockDataEnsureCapacity(p, capacity);
|
||||
|
||||
_retry:
|
||||
while (1) {
|
||||
STupleHandle* pTupleHandle = NULL;
|
||||
if (pInfo->groupSort) {
|
||||
|
@ -638,22 +625,48 @@ _retry:
|
|||
pInfo->hasGroupId = false;
|
||||
}
|
||||
|
||||
if (p->info.rows > 0) { // todo extract method
|
||||
applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo, pOperator);
|
||||
}
|
||||
|
||||
SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
|
||||
SArray* pColMatchInfo, SOperatorInfo* pOperator) {
|
||||
SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
|
||||
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
blockDataCleanup(pDataBlock);
|
||||
|
||||
SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
|
||||
if (p == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
blockDataEnsureCapacity(p, capacity);
|
||||
|
||||
while (1) {
|
||||
doGetSortedBlockData(pInfo, pHandle, capacity, p);
|
||||
if (p->info.rows == 0) {
|
||||
goto _retry;
|
||||
break;
|
||||
}
|
||||
|
||||
if (p->info.rows > 0) {
|
||||
applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo, pOperator);
|
||||
if (p->info.rows > 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (p->info.rows > 0) {
|
||||
blockDataEnsureCapacity(pDataBlock, p->info.rows);
|
||||
|
||||
int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i);
|
||||
// ASSERT(pColMatchInfo-> == COL_MATCH_FROM_SLOT_ID);
|
||||
|
||||
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
|
||||
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId);
|
||||
colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
|
||||
}
|
||||
|
||||
pInfo->limitInfo.numOfOutputRows += p->info.rows;
|
||||
pDataBlock->info.rows = p->info.rows;
|
||||
pDataBlock->info.groupId = pInfo->groupId;
|
||||
|
|
|
@ -122,14 +122,14 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) {
|
|||
taosGetCpuCores(&numCpuCores);
|
||||
snprintf(thrdPoolSizeEnvItem, 32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores * 2);
|
||||
|
||||
char pathTaosdLdLib[512] = {0};
|
||||
char pathTaosdLdLib[512] = {0};
|
||||
size_t taosdLdLibPathLen = sizeof(pathTaosdLdLib);
|
||||
int ret = uv_os_getenv("LD_LIBRARY_PATH", pathTaosdLdLib, &taosdLdLibPathLen);
|
||||
if (ret != UV_ENOBUFS) {
|
||||
taosdLdLibPathLen = strlen(pathTaosdLdLib);
|
||||
}
|
||||
|
||||
char udfdPathLdLib[1024] = {0};
|
||||
char udfdPathLdLib[1024] = {0};
|
||||
size_t udfdLdLibPathLen = strlen(tsUdfdLdLibPath);
|
||||
strncpy(udfdPathLdLib, tsUdfdLdLibPath, udfdLdLibPathLen);
|
||||
udfdPathLdLib[udfdLdLibPathLen] = ':';
|
||||
|
@ -362,7 +362,7 @@ typedef struct SUdfcProxy {
|
|||
SArray *udfStubs; // SUdfcFuncStub
|
||||
|
||||
uv_mutex_t udfcUvMutex;
|
||||
int8_t initialized;
|
||||
int8_t initialized;
|
||||
} SUdfcProxy;
|
||||
|
||||
SUdfcProxy gUdfcProxy = {0};
|
||||
|
|
|
@ -644,11 +644,13 @@ _return:
|
|||
input.msgType = qwMsg->msgType;
|
||||
code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL);
|
||||
|
||||
if (ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
|
||||
if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
|
||||
void *rsp = NULL;
|
||||
int32_t dataLen = 0;
|
||||
SOutputData sOutput = {0};
|
||||
QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput));
|
||||
if (qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (rsp) {
|
||||
bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd);
|
||||
|
|
|
@ -280,7 +280,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
|
|||
}
|
||||
|
||||
atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows);
|
||||
SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows);
|
||||
SCH_TASK_DLOG("submit succeed, affectedRows:%d, blocks:%d", rsp->affectedRows, rsp->nBlocks);
|
||||
|
||||
SCH_LOCK(SCH_WRITE, &pJob->resLock);
|
||||
if (pJob->execRes.res) {
|
||||
|
|
|
@ -43,6 +43,8 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
|
|||
|
||||
int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq);
|
||||
|
||||
int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet);
|
||||
|
||||
int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId,
|
||||
SEpSet* pEpSet);
|
||||
|
||||
|
|
|
@ -135,8 +135,11 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR
|
|||
((SMsgHead*)buf)->vgId = htonl(pReq->upstreamNodeId);
|
||||
SStreamDispatchRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead));
|
||||
pCont->inputStatus = status;
|
||||
pCont->streamId = pReq->streamId;
|
||||
pCont->taskId = pReq->upstreamTaskId;
|
||||
pCont->streamId = htobe64(pReq->streamId);
|
||||
pCont->upstreamNodeId = htonl(pReq->upstreamNodeId);
|
||||
pCont->upstreamTaskId = htonl(pReq->upstreamTaskId);
|
||||
pCont->downstreamNodeId = htonl(pTask->nodeId);
|
||||
pCont->downstreamTaskId = htonl(pTask->taskId);
|
||||
pRsp->pCont = buf;
|
||||
pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp);
|
||||
tmsgSendRsp(pRsp);
|
||||
|
@ -203,10 +206,10 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp) {
|
||||
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) {
|
||||
ASSERT(pRsp->inputStatus == TASK_OUTPUT_STATUS__NORMAL || pRsp->inputStatus == TASK_OUTPUT_STATUS__BLOCKED);
|
||||
|
||||
qDebug("task %d receive dispatch rsp", pTask->taskId);
|
||||
qDebug("task %d receive dispatch rsp, code: %x", pTask->taskId, code);
|
||||
|
||||
if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1);
|
||||
|
|
|
@ -210,6 +210,46 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) {
|
||||
void* buf = NULL;
|
||||
int32_t code = -1;
|
||||
SRpcMsg msg = {0};
|
||||
|
||||
int32_t tlen;
|
||||
tEncodeSize(tEncodeSStreamTaskCheckReq, pReq, tlen, code);
|
||||
if (code < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
buf = rpcMallocCont(sizeof(SMsgHead) + tlen);
|
||||
if (buf == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
((SMsgHead*)buf)->vgId = htonl(nodeId);
|
||||
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
|
||||
|
||||
SEncoder encoder;
|
||||
tEncoderInit(&encoder, abuf, tlen);
|
||||
if ((code = tEncodeSStreamTaskCheckReq(&encoder, pReq)) < 0) {
|
||||
goto FAIL;
|
||||
}
|
||||
tEncoderClear(&encoder);
|
||||
|
||||
msg.contLen = tlen + sizeof(SMsgHead);
|
||||
msg.pCont = buf;
|
||||
msg.msgType = TDMT_STREAM_TASK_CHECK;
|
||||
|
||||
qDebug("dispatch from task %d to task %d node %d: check msg", pTask->taskId, pReq->downstreamTaskId, nodeId);
|
||||
|
||||
tmsgSendReq(pEpSet, &msg);
|
||||
|
||||
return 0;
|
||||
FAIL:
|
||||
if (buf) rpcFreeCont(buf);
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId,
|
||||
SEpSet* pEpSet) {
|
||||
void* buf = NULL;
|
||||
|
@ -243,7 +283,8 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov
|
|||
|
||||
tmsgSendReq(pEpSet, &msg);
|
||||
|
||||
code = 0;
|
||||
qDebug("dispatch from task %d to task %d node %d: recover finish msg", pTask->taskId, pReq->taskId, vgId);
|
||||
|
||||
return 0;
|
||||
FAIL:
|
||||
if (buf) rpcFreeCont(buf);
|
||||
|
@ -279,7 +320,7 @@ int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* p
|
|||
msg.pCont = buf;
|
||||
msg.msgType = pTask->dispatchMsgType;
|
||||
|
||||
qDebug("dispatch from task %d to task %d node %d", pTask->taskId, pReq->taskId, vgId);
|
||||
qDebug("dispatch from task %d to task %d node %d: data msg", pTask->taskId, pReq->taskId, vgId);
|
||||
|
||||
tmsgSendReq(pEpSet, &msg);
|
||||
|
||||
|
|
|
@ -202,83 +202,83 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch)
|
|||
int32_t streamExecForAll(SStreamTask* pTask) {
|
||||
while (1) {
|
||||
int32_t batchCnt = 1;
|
||||
void* data = NULL;
|
||||
void* input = NULL;
|
||||
while (1) {
|
||||
SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue);
|
||||
if (qItem == NULL) {
|
||||
qDebug("stream task exec over, queue empty, task: %d", pTask->taskId);
|
||||
break;
|
||||
}
|
||||
if (data == NULL) {
|
||||
data = qItem;
|
||||
if (input == NULL) {
|
||||
input = qItem;
|
||||
streamQueueProcessSuccess(pTask->inputQueue);
|
||||
if (pTask->taskLevel == TASK_LEVEL__SINK) {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
void* newRet;
|
||||
if ((newRet = streamMergeQueueItem(data, qItem)) == NULL) {
|
||||
if ((newRet = streamMergeQueueItem(input, qItem)) == NULL) {
|
||||
streamQueueProcessFail(pTask->inputQueue);
|
||||
break;
|
||||
} else {
|
||||
batchCnt++;
|
||||
data = newRet;
|
||||
input = newRet;
|
||||
streamQueueProcessSuccess(pTask->inputQueue);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (pTask->taskStatus == TASK_STATUS__DROPPING) {
|
||||
if (data) streamFreeQitem(data);
|
||||
if (input) streamFreeQitem(input);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (data == NULL) {
|
||||
if (input == NULL) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (pTask->taskLevel == TASK_LEVEL__SINK) {
|
||||
ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK);
|
||||
streamTaskOutput(pTask, data);
|
||||
ASSERT(((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_BLOCK);
|
||||
streamTaskOutput(pTask, input);
|
||||
continue;
|
||||
}
|
||||
|
||||
SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
|
||||
|
||||
qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, batchCnt);
|
||||
streamTaskExecImpl(pTask, data, pRes);
|
||||
streamTaskExecImpl(pTask, input, pRes);
|
||||
qDebug("stream task %d exec end", pTask->taskId);
|
||||
|
||||
if (taosArrayGetSize(pRes) != 0) {
|
||||
SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
|
||||
if (qRes == NULL) {
|
||||
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
|
||||
streamFreeQitem(data);
|
||||
streamFreeQitem(input);
|
||||
return -1;
|
||||
}
|
||||
qRes->type = STREAM_INPUT__DATA_BLOCK;
|
||||
qRes->blocks = pRes;
|
||||
|
||||
if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) {
|
||||
SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
|
||||
if (((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_SUBMIT) {
|
||||
SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)input;
|
||||
qRes->childId = pTask->selfChildId;
|
||||
qRes->sourceVer = pSubmit->ver;
|
||||
} else if (((SStreamQueueItem*)data)->type == STREAM_INPUT__MERGED_SUBMIT) {
|
||||
SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)data;
|
||||
} else if (((SStreamQueueItem*)input)->type == STREAM_INPUT__MERGED_SUBMIT) {
|
||||
SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)input;
|
||||
qRes->childId = pTask->selfChildId;
|
||||
qRes->sourceVer = pMerged->ver;
|
||||
}
|
||||
|
||||
if (streamTaskOutput(pTask, qRes) < 0) {
|
||||
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
|
||||
streamFreeQitem(data);
|
||||
streamFreeQitem(input);
|
||||
taosFreeQitem(qRes);
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
taosArrayDestroy(pRes);
|
||||
}
|
||||
streamFreeQitem(data);
|
||||
streamFreeQitem(input);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -15,6 +15,148 @@
|
|||
|
||||
#include "streamInc.h"
|
||||
|
||||
int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) {
|
||||
qDebug("task %d at node %d launch recover", pTask->taskId, pTask->nodeId);
|
||||
if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
|
||||
atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_PREPARE);
|
||||
streamSetParamForRecover(pTask);
|
||||
streamSourceRecoverPrepareStep1(pTask, version);
|
||||
|
||||
SStreamRecoverStep1Req req;
|
||||
streamBuildSourceRecover1Req(pTask, &req);
|
||||
int32_t len = sizeof(SStreamRecoverStep1Req);
|
||||
|
||||
void* serializedReq = rpcMallocCont(len);
|
||||
if (serializedReq == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
memcpy(serializedReq, &req, len);
|
||||
|
||||
SRpcMsg rpcMsg = {
|
||||
.contLen = len,
|
||||
.pCont = serializedReq,
|
||||
.msgType = TDMT_VND_STREAM_RECOVER_STEP1,
|
||||
};
|
||||
|
||||
if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) {
|
||||
/*ASSERT(0);*/
|
||||
}
|
||||
|
||||
} else if (pTask->taskLevel == TASK_LEVEL__AGG) {
|
||||
atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL);
|
||||
streamSetParamForRecover(pTask);
|
||||
streamAggRecoverPrepare(pTask);
|
||||
} else if (pTask->taskLevel == TASK_LEVEL__SINK) {
|
||||
atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// checkstatus
|
||||
int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) {
|
||||
SStreamTaskCheckReq req = {
|
||||
.streamId = pTask->streamId,
|
||||
.upstreamTaskId = pTask->taskId,
|
||||
.upstreamNodeId = pTask->nodeId,
|
||||
.childId = pTask->selfChildId,
|
||||
};
|
||||
// serialize
|
||||
if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
req.reqId = tGenIdPI64();
|
||||
req.downstreamNodeId = pTask->fixedEpDispatcher.nodeId;
|
||||
req.downstreamTaskId = pTask->fixedEpDispatcher.taskId;
|
||||
pTask->checkReqId = req.reqId;
|
||||
|
||||
qDebug("task %d at node %d check downstream task %d at node %d", pTask->taskId, pTask->nodeId, req.downstreamTaskId,
|
||||
req.downstreamNodeId);
|
||||
streamDispatchOneCheckReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet);
|
||||
} else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
|
||||
int32_t vgSz = taosArrayGetSize(vgInfo);
|
||||
pTask->recoverTryingDownstream = vgSz;
|
||||
pTask->checkReqIds = taosArrayInit(vgSz, sizeof(int64_t));
|
||||
|
||||
for (int32_t i = 0; i < vgSz; i++) {
|
||||
SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
|
||||
req.reqId = tGenIdPI64();
|
||||
taosArrayPush(pTask->checkReqIds, &req.reqId);
|
||||
req.downstreamNodeId = pVgInfo->vgId;
|
||||
req.downstreamTaskId = pVgInfo->taskId;
|
||||
qDebug("task %d at node %d check downstream task %d at node %d (shuffle)", pTask->taskId, pTask->nodeId,
|
||||
req.downstreamTaskId, req.downstreamNodeId);
|
||||
streamDispatchOneCheckReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
|
||||
}
|
||||
} else {
|
||||
qDebug("task %d at node %d direct launch recover since no downstream", pTask->taskId, pTask->nodeId);
|
||||
streamTaskLaunchRecover(pTask, version);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) {
|
||||
SStreamTaskCheckReq req = {
|
||||
.reqId = pRsp->reqId,
|
||||
.streamId = pRsp->streamId,
|
||||
.upstreamTaskId = pRsp->upstreamTaskId,
|
||||
.upstreamNodeId = pRsp->upstreamNodeId,
|
||||
.downstreamTaskId = pRsp->downstreamTaskId,
|
||||
.downstreamNodeId = pRsp->downstreamNodeId,
|
||||
.childId = pRsp->childId,
|
||||
};
|
||||
qDebug("task %d at node %d check downstream task %d at node %d (recheck)", pTask->taskId, pTask->nodeId,
|
||||
req.downstreamTaskId, req.downstreamNodeId);
|
||||
if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet);
|
||||
} else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
|
||||
int32_t vgSz = taosArrayGetSize(vgInfo);
|
||||
for (int32_t i = 0; i < vgSz; i++) {
|
||||
SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
|
||||
if (pVgInfo->taskId == req.downstreamTaskId) {
|
||||
streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pVgInfo->epSet);
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq) {
|
||||
return atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL;
|
||||
}
|
||||
|
||||
int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version) {
|
||||
qDebug("task %d at node %d recv check rsp from task %d at node %d: status %d", pRsp->upstreamTaskId,
|
||||
pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status);
|
||||
if (pRsp->status == 1) {
|
||||
if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
bool found = false;
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pTask->checkReqIds); i++) {
|
||||
int64_t reqId = *(int64_t*)taosArrayGet(pTask->checkReqIds, i);
|
||||
if (reqId == pRsp->reqId) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found) return -1;
|
||||
int32_t left = atomic_sub_fetch_32(&pTask->recoverTryingDownstream, 1);
|
||||
ASSERT(left >= 0);
|
||||
if (left == 0) {
|
||||
taosArrayDestroy(pTask->checkReqIds);
|
||||
streamTaskLaunchRecover(pTask, version);
|
||||
}
|
||||
} else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
if (pRsp->reqId != pTask->checkReqId) return -1;
|
||||
streamTaskLaunchRecover(pTask, version);
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
} else {
|
||||
streamRecheckOneDownstream(pTask, pRsp);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// common
|
||||
int32_t streamSetParamForRecover(SStreamTask* pTask) {
|
||||
void* exec = pTask->exec.executor;
|
||||
|
@ -86,10 +228,7 @@ int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) {
|
|||
// agg
|
||||
int32_t streamAggRecoverPrepare(SStreamTask* pTask) {
|
||||
void* exec = pTask->exec.executor;
|
||||
/*if (qStreamSetParamForRecover(exec) < 0) {*/
|
||||
/*return -1;*/
|
||||
/*}*/
|
||||
pTask->recoverWaitingChild = taosArrayGetSize(pTask->childEpInfo);
|
||||
pTask->recoverWaitingUpstream = taosArrayGetSize(pTask->childEpInfo);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -107,7 +246,7 @@ int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) {
|
|||
|
||||
int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) {
|
||||
if (pTask->taskLevel == TASK_LEVEL__AGG) {
|
||||
int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingChild, 1);
|
||||
int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingUpstream, 1);
|
||||
ASSERT(left >= 0);
|
||||
if (left == 0) {
|
||||
streamAggChildrenRecoverFinish(pTask);
|
||||
|
@ -116,6 +255,60 @@ int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeSStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->reqId) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->downstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->childId) < 0) return -1;
|
||||
tEndEncode(pEncoder);
|
||||
return pEncoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeSStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->reqId) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->downstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->childId) < 0) return -1;
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeSStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pRsp->reqId) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->upstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->upstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->downstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->childId) < 0) return -1;
|
||||
if (tEncodeI8(pEncoder, pRsp->status) < 0) return -1;
|
||||
tEndEncode(pEncoder);
|
||||
return pEncoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeSStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pRsp->reqId) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->upstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->upstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->downstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->childId) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, &pRsp->status) < 0) return -1;
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
|
||||
|
@ -132,79 +325,6 @@ int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishR
|
|||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
#if 0
|
||||
int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1;
|
||||
tEndEncode(pEncoder);
|
||||
return pEncoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeStreamTaskRecoverReq(SDecoder* pDecoder, SStreamTaskRecoverReq* pReq) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1;
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamTaskRecoverRsp* pRsp) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->reqTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->rspTaskId) < 0) return -1;
|
||||
if (tEncodeI8(pEncoder, pRsp->inputStatus) < 0) return -1;
|
||||
tEndEncode(pEncoder);
|
||||
return pEncoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamTaskRecoverRsp* pReq) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->reqTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->rspTaskId) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, &pReq->inputStatus) < 0) return -1;
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeSMStreamTaskRecoverReq(SEncoder* pEncoder, const SMStreamTaskRecoverReq* pReq) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
|
||||
tEndEncode(pEncoder);
|
||||
return pEncoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq* pReq) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1;
|
||||
tEndEncode(pEncoder);
|
||||
return pEncoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pReq) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t tEncodeSStreamCheckpointInfo(SEncoder* pEncoder, const SStreamCheckpointInfo* pCheckpoint) {
|
||||
if (tEncodeI32(pEncoder, pCheckpoint->srcNodeId) < 0) return -1;
|
||||
|
@ -248,308 +368,3 @@ int32_t tDecodeSStreamMultiVgCheckpointInfo(SDecoder* pDecoder, SStreamMultiVgCh
|
|||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#if 0
|
||||
int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq) {
|
||||
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tDecodeSStreamTaskRecoverReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq) {
|
||||
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeSStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamRecoverDownstreamRsp* pRsp) {
|
||||
if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1;
|
||||
int32_t sz = taosArrayGetSize(pRsp->checkpointVer);
|
||||
if (tEncodeI32(pEncoder, sz) < 0) return -1;
|
||||
for (int32_t i = 0; i < sz; i++) {
|
||||
SStreamCheckpointInfo* pInfo = taosArrayGet(pRsp->checkpointVer, i);
|
||||
if (tEncodeSStreamCheckpointInfo(pEncoder, pInfo) < 0) return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamRecoverDownstreamRsp* pRsp) {
|
||||
if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->taskId) < 0) return -1;
|
||||
int32_t sz;
|
||||
if (tDecodeI32(pDecoder, &sz) < 0) return -1;
|
||||
pRsp->checkpointVer = taosArrayInit(sz, sizeof(SStreamCheckpointInfo));
|
||||
if (pRsp->checkpointVer == NULL) return -1;
|
||||
for (int32_t i = 0; i < sz; i++) {
|
||||
SStreamCheckpointInfo info;
|
||||
if (tDecodeSStreamCheckpointInfo(pDecoder, &info) < 0) return -1;
|
||||
taosArrayPush(pRsp->checkpointVer, &info);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t streamSaveStateInfo(SStreamMeta* pMeta, SStreamTask* pTask) {
|
||||
#if 0
|
||||
void* buf = NULL;
|
||||
|
||||
ASSERT(pTask->taskLevel == TASK_LEVEL__SINK);
|
||||
|
||||
SStreamMultiVgCheckpointInfo checkpoint;
|
||||
checkpoint.checkpointId = atomic_fetch_add_32(&pTask->nextCheckId, 1);
|
||||
checkpoint.checkTs = taosGetTimestampMs();
|
||||
checkpoint.streamId = pTask->streamId;
|
||||
checkpoint.taskId = pTask->taskId;
|
||||
checkpoint.checkpointVer = pTask->checkpointInfo;
|
||||
|
||||
int32_t len;
|
||||
int32_t code;
|
||||
tEncodeSize(tEncodeSStreamMultiVgCheckpointInfo, &checkpoint, len, code);
|
||||
if (code < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
buf = taosMemoryCalloc(1, len);
|
||||
if (buf == NULL) {
|
||||
return -1;
|
||||
}
|
||||
SEncoder encoder;
|
||||
tEncoderInit(&encoder, buf, len);
|
||||
tEncodeSStreamMultiVgCheckpointInfo(&encoder, &checkpoint);
|
||||
tEncoderClear(&encoder);
|
||||
|
||||
SStreamCheckpointKey key = {
|
||||
.taskId = pTask->taskId,
|
||||
.checkpointId = checkpoint.checkpointId,
|
||||
};
|
||||
|
||||
if (tdbTbUpsert(pMeta->pStateDb, &key, sizeof(SStreamCheckpointKey), buf, len, &pMeta->txn) < 0) {
|
||||
ASSERT(0);
|
||||
goto FAIL;
|
||||
}
|
||||
|
||||
int32_t sz = taosArrayGetSize(pTask->checkpointInfo);
|
||||
for (int32_t i = 0; i < sz; i++) {
|
||||
SStreamCheckpointInfo* pCheck = taosArrayGet(pTask->checkpointInfo, i);
|
||||
pCheck->stateSaveVer = pCheck->stateProcessedVer;
|
||||
}
|
||||
|
||||
taosMemoryFree(buf);
|
||||
return 0;
|
||||
FAIL:
|
||||
if (buf) taosMemoryFree(buf);
|
||||
return -1;
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamLoadStateInfo(SStreamMeta* pMeta, SStreamTask* pTask) {
|
||||
#if 0
|
||||
void* pVal = NULL;
|
||||
int32_t vLen = 0;
|
||||
if (tdbTbGet(pMeta->pStateDb, &pTask->taskId, sizeof(void*), &pVal, &vLen) < 0) {
|
||||
return -1;
|
||||
}
|
||||
SDecoder decoder;
|
||||
tDecoderInit(&decoder, pVal, vLen);
|
||||
SStreamMultiVgCheckpointInfo aggCheckpoint;
|
||||
tDecodeSStreamMultiVgCheckpointInfo(&decoder, &aggCheckpoint);
|
||||
tDecoderClear(&decoder);
|
||||
|
||||
pTask->nextCheckId = aggCheckpoint.checkpointId + 1;
|
||||
pTask->checkpointInfo = aggCheckpoint.checkpointVer;
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamSaveSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) {
|
||||
ASSERT(pTask->taskLevel == TASK_LEVEL__SINK);
|
||||
return streamSaveStateInfo(pMeta, pTask);
|
||||
}
|
||||
|
||||
int32_t streamRecoverSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) {
|
||||
ASSERT(pTask->taskLevel == TASK_LEVEL__SINK);
|
||||
return streamLoadStateInfo(pMeta, pTask);
|
||||
}
|
||||
|
||||
int32_t streamSaveAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) {
|
||||
ASSERT(pTask->taskLevel == TASK_LEVEL__AGG);
|
||||
// TODO save and copy state
|
||||
|
||||
// save state info
|
||||
if (streamSaveStateInfo(pMeta, pTask) < 0) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#if 0
|
||||
int32_t streamFetchRecoverStatus(SStreamTask* pTask, const SVgroupInfo* pVgInfo) {
|
||||
int32_t taskId = pVgInfo->taskId;
|
||||
int32_t nodeId = pVgInfo->vgId;
|
||||
SStreamRecoverDownstreamReq req = {
|
||||
.streamId = pTask->taskId,
|
||||
.downstreamTaskId = taskId,
|
||||
.taskId = pTask->taskId,
|
||||
};
|
||||
int32_t tlen;
|
||||
int32_t code;
|
||||
tEncodeSize(tEncodeSStreamTaskRecoverReq, &req, tlen, code);
|
||||
if (code < 0) {
|
||||
return -1;
|
||||
}
|
||||
void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + tlen);
|
||||
if (buf == NULL) {
|
||||
return -1;
|
||||
}
|
||||
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
|
||||
SEncoder encoder;
|
||||
tEncoderInit(&encoder, abuf, tlen);
|
||||
if (tEncodeSStreamTaskRecoverReq(&encoder, &req) < 0) {
|
||||
tEncoderClear(&encoder);
|
||||
taosMemoryFree(buf);
|
||||
return -1;
|
||||
}
|
||||
tEncoderClear(&encoder);
|
||||
|
||||
((SMsgHead*)buf)->vgId = htonl(nodeId);
|
||||
SRpcMsg msg = {
|
||||
.pCont = buf, .contLen = sizeof(SMsgHead) + tlen,
|
||||
/*.msgType = */
|
||||
};
|
||||
tmsgSendReq(&pVgInfo->epSet, &msg);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamFetchDownstreamStatus(SStreamMeta* pMeta, SStreamTask* pTask) {
|
||||
// set self status to recover_phase1
|
||||
SStreamRecoverStatus* pRecover;
|
||||
atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_DOWNSTREAM);
|
||||
pRecover = taosHashGet(pMeta->pRecoverStatus, &pTask->taskId, sizeof(int32_t));
|
||||
if (pRecover == NULL) {
|
||||
pRecover = taosMemoryCalloc(1, sizeof(SStreamRecoverStatus));
|
||||
if (pRecover == NULL) {
|
||||
return -1;
|
||||
}
|
||||
pRecover->info = taosArrayInit(0, sizeof(void*));
|
||||
if (pRecover->info == NULL) {
|
||||
taosMemoryFree(pRecover);
|
||||
return -1;
|
||||
}
|
||||
taosHashPut(pMeta->pRecoverStatus, &pTask->taskId, sizeof(int32_t), &pRecover, sizeof(void*));
|
||||
}
|
||||
|
||||
if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
pRecover->totReq = 1;
|
||||
} else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
int32_t numOfDownstream = taosArrayGetSize(pTask->shuffleDispatcher.dbInfo.pVgroupInfos);
|
||||
pRecover->totReq = numOfDownstream;
|
||||
for (int32_t i = 0; i < numOfDownstream; i++) {
|
||||
SVgroupInfo* pVgInfo = taosArrayGet(pTask->shuffleDispatcher.dbInfo.pVgroupInfos, i);
|
||||
streamFetchRecoverStatus(pTask, pVgInfo);
|
||||
}
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if 0
|
||||
int32_t streamProcessFetchStatusRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamRecoverDownstreamRsp* pRsp) {
|
||||
// if failed, set timer and retry
|
||||
// if successful
|
||||
int32_t taskId = pTask->taskId;
|
||||
SStreamRecoverStatus* pRecover = taosHashGet(pMeta->pRecoverStatus, &taskId, sizeof(int32_t));
|
||||
if (pRecover == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
taosArrayPush(pRecover->info, &pRsp->checkpointVer);
|
||||
|
||||
int32_t leftRsp = atomic_sub_fetch_32(&pRecover->waitingRspCnt, 1);
|
||||
ASSERT(leftRsp >= 0);
|
||||
|
||||
if (leftRsp == 0) {
|
||||
ASSERT(taosArrayGetSize(pRecover->info) == pRecover->totReq);
|
||||
|
||||
// srcNodeId -> SStreamCheckpointInfo*
|
||||
SHashObj* pFinalChecks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
|
||||
if (pFinalChecks == NULL) return -1;
|
||||
|
||||
for (int32_t i = 0; i < pRecover->totReq; i++) {
|
||||
SArray* pChecks = taosArrayGetP(pRecover->info, i);
|
||||
int32_t sz = taosArrayGetSize(pChecks);
|
||||
for (int32_t j = 0; j < sz; j++) {
|
||||
SStreamCheckpointInfo* pOneCheck = taosArrayGet(pChecks, j);
|
||||
SStreamCheckpointInfo* pCheck = taosHashGet(pFinalChecks, &pOneCheck->srcNodeId, sizeof(int32_t));
|
||||
if (pCheck == NULL) {
|
||||
pCheck = taosMemoryCalloc(1, sizeof(SStreamCheckpointInfo));
|
||||
pCheck->srcNodeId = pOneCheck->srcNodeId;
|
||||
pCheck->srcChildId = pOneCheck->srcChildId;
|
||||
pCheck->stateProcessedVer = pOneCheck->stateProcessedVer;
|
||||
taosHashPut(pFinalChecks, &pCheck->srcNodeId, sizeof(int32_t), &pCheck, sizeof(void*));
|
||||
} else {
|
||||
pCheck->stateProcessedVer = TMIN(pCheck->stateProcessedVer, pOneCheck->stateProcessedVer);
|
||||
}
|
||||
}
|
||||
}
|
||||
// load local state
|
||||
//
|
||||
// recover
|
||||
//
|
||||
if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
|
||||
qStreamPrepareRecover(pTask->exec.executor, pTask->startVer, pTask->recoverSnapVer);
|
||||
if (streamPipelineExec(pTask, 10000, true) < 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
taosHashCleanup(pFinalChecks);
|
||||
taosHashRemove(pMeta->pRecoverStatus, &taskId, sizeof(int32_t));
|
||||
atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamRecoverAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) {
|
||||
ASSERT(pTask->taskLevel == TASK_LEVEL__AGG);
|
||||
// recover sink level
|
||||
// after all sink level recovered
|
||||
// choose suitable state to recover
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamSaveSourceLevel(SStreamMeta* pMeta, SStreamTask* pTask) {
|
||||
ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE);
|
||||
// TODO: save and copy state
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamRecoverSourceLevel(SStreamMeta* pMeta, SStreamTask* pTask) {
|
||||
ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE);
|
||||
// if totLevel == 3
|
||||
// fetch agg state
|
||||
// recover from local state to agg state, not send msg
|
||||
// recover from agg state to most recent log v1
|
||||
// enable input queue, set status recover_phase2
|
||||
// recover from v1 to queue msg v2, set status normal
|
||||
|
||||
// if totLevel == 2
|
||||
// fetch sink state
|
||||
// recover from local state to sink state v1, send msg
|
||||
// enable input queue, set status recover_phase2
|
||||
// recover from v1 to queue msg v2, set status normal
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamRecoverTask(SStreamTask* pTask) {
|
||||
//
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
|
|
@ -427,7 +427,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
|
|||
|
||||
if (pCtx == NULL || pCtx->pSem == NULL) {
|
||||
if (transMsg.info.ahandle == NULL) {
|
||||
if (REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg);
|
||||
if (pMsg == NULL || REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg);
|
||||
once = true;
|
||||
continue;
|
||||
}
|
||||
|
|
|
@ -127,7 +127,7 @@ static void uvFreeCb(uv_handle_t* handle);
|
|||
|
||||
static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg);
|
||||
|
||||
static void uvPrepareSendData(SSvrMsg* msg, uv_buf_t* wb);
|
||||
static int uvPrepareSendData(SSvrMsg* msg, uv_buf_t* wb);
|
||||
static void uvStartSendResp(SSvrMsg* msg);
|
||||
|
||||
static void uvNotifyLinkBrokenToApp(SSvrConn* conn);
|
||||
|
@ -384,7 +384,7 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) {
|
|||
taosMemoryFree(req);
|
||||
}
|
||||
|
||||
static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
|
||||
static int uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
|
||||
SSvrConn* pConn = smsg->pConn;
|
||||
STransMsg* pMsg = &smsg->msg;
|
||||
if (pMsg->pCont == 0) {
|
||||
|
@ -397,6 +397,13 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
|
|||
pHead->hasEpSet = pMsg->info.hasEpSet;
|
||||
pHead->magicNum = htonl(TRANS_MAGIC_NUM);
|
||||
|
||||
// handle invalid drop_task resp, TD-20098
|
||||
if (pMsg->msgType == TDMT_SCH_DROP_TASK && pMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) {
|
||||
transQueuePop(&pConn->srvMsgs);
|
||||
destroySmsg(smsg);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (pConn->status == ConnNormal) {
|
||||
pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType);
|
||||
if (smsg->type == Release) pHead->msgType = 0;
|
||||
|
@ -431,6 +438,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
|
|||
|
||||
wb->base = (char*)pHead;
|
||||
wb->len = len;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) {
|
||||
|
@ -440,7 +448,9 @@ static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) {
|
|||
}
|
||||
|
||||
uv_buf_t wb;
|
||||
uvPrepareSendData(smsg, &wb);
|
||||
if (uvPrepareSendData(smsg, &wb) < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
transRefSrvHandle(pConn);
|
||||
uv_write_t* req = transReqQueuePush(&pConn->wreqQueue);
|
||||
|
@ -451,8 +461,9 @@ static void uvStartSendResp(SSvrMsg* smsg) {
|
|||
SSvrConn* pConn = smsg->pConn;
|
||||
if (pConn->broken == true) {
|
||||
// persist by
|
||||
transFreeMsg(smsg->msg.pCont);
|
||||
taosMemoryFree(smsg);
|
||||
destroySmsg(smsg);
|
||||
// transFreeMsg(smsg->msg.pCont);
|
||||
// taosMemoryFree(smsg);
|
||||
transUnrefSrvHandle(pConn);
|
||||
return;
|
||||
}
|
||||
|
@ -748,10 +759,11 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
|
|||
return;
|
||||
}
|
||||
transSockInfo2Str(&sockname, pConn->src);
|
||||
struct sockaddr_in addr = *(struct sockaddr_in*)&sockname;
|
||||
|
||||
struct sockaddr_in addr = *(struct sockaddr_in*)&peername;
|
||||
pConn->clientIp = addr.sin_addr.s_addr;
|
||||
pConn->port = ntohs(addr.sin_port);
|
||||
|
||||
uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocRecvBufferCb, uvOnRecvCb);
|
||||
|
||||
} else {
|
||||
|
|
|
@ -340,7 +340,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last ro
|
|||
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_NOT_EXIST, "Table not exists")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_ALREADY_EXIST, "Stable already exists")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST, "Stable not exists")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, "Table schema is old")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error")
|
||||
|
||||
// query
|
||||
|
@ -609,6 +609,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message")
|
|||
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed")
|
||||
|
||||
// stream
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist")
|
||||
|
||||
// TDLite
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS, "Invalid TDLite open flags")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_DIR, "Invalid TDLite open directory")
|
||||
|
|
|
@ -21,46 +21,6 @@
|
|||
int64_t tsRpcQueueMemoryAllowed = 0;
|
||||
int64_t tsRpcQueueMemoryUsed = 0;
|
||||
|
||||
typedef struct STaosQnode STaosQnode;
|
||||
|
||||
typedef struct STaosQnode {
|
||||
STaosQnode *next;
|
||||
STaosQueue *queue;
|
||||
int64_t timestamp;
|
||||
int32_t size;
|
||||
int8_t itype;
|
||||
int8_t reserved[3];
|
||||
char item[];
|
||||
} STaosQnode;
|
||||
|
||||
typedef struct STaosQueue {
|
||||
STaosQnode *head;
|
||||
STaosQnode *tail;
|
||||
STaosQueue *next; // for queue set
|
||||
STaosQset *qset; // for queue set
|
||||
void *ahandle; // for queue set
|
||||
FItem itemFp;
|
||||
FItems itemsFp;
|
||||
TdThreadMutex mutex;
|
||||
int64_t memOfItems;
|
||||
int32_t numOfItems;
|
||||
} STaosQueue;
|
||||
|
||||
typedef struct STaosQset {
|
||||
STaosQueue *head;
|
||||
STaosQueue *current;
|
||||
TdThreadMutex mutex;
|
||||
tsem_t sem;
|
||||
int32_t numOfQueues;
|
||||
int32_t numOfItems;
|
||||
} STaosQset;
|
||||
|
||||
typedef struct STaosQall {
|
||||
STaosQnode *current;
|
||||
STaosQnode *start;
|
||||
int32_t numOfItems;
|
||||
} STaosQall;
|
||||
|
||||
STaosQueue *taosOpenQueue() {
|
||||
STaosQueue *queue = taosMemoryCalloc(1, sizeof(STaosQueue));
|
||||
if (queue == NULL) {
|
||||
|
|
|
@ -36,14 +36,13 @@ int32_t tQWorkerInit(SQWorkerPool *pool) {
|
|||
worker->pool = pool;
|
||||
}
|
||||
|
||||
uInfo("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max);
|
||||
uDebug("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void tQWorkerCleanup(SQWorkerPool *pool) {
|
||||
for (int32_t i = 0; i < pool->max; ++i) {
|
||||
SQWorker *worker = pool->workers + i;
|
||||
// if (worker == NULL) continue;
|
||||
if (taosCheckPthreadValid(worker->thread)) {
|
||||
taosQsetThreadResume(pool->qset);
|
||||
}
|
||||
|
@ -51,7 +50,6 @@ void tQWorkerCleanup(SQWorkerPool *pool) {
|
|||
|
||||
for (int32_t i = 0; i < pool->max; ++i) {
|
||||
SQWorker *worker = pool->workers + i;
|
||||
// if (worker == NULL) continue;
|
||||
if (taosCheckPthreadValid(worker->thread)) {
|
||||
taosThreadJoin(worker->thread, NULL);
|
||||
taosThreadClear(&worker->thread);
|
||||
|
@ -73,11 +71,13 @@ static void *tQWorkerThreadFp(SQWorker *worker) {
|
|||
|
||||
taosBlockSIGPIPE();
|
||||
setThreadName(pool->name);
|
||||
uDebug("worker:%s:%d is running", pool->name, worker->id);
|
||||
worker->pid = taosGetSelfPthreadId();
|
||||
uInfo("worker:%s:%d is running, thread:%08" PRId64, pool->name, worker->id, worker->pid);
|
||||
|
||||
while (1) {
|
||||
if (taosReadQitemFromQset(pool->qset, (void **)&msg, &qinfo) == 0) {
|
||||
uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset);
|
||||
uInfo("worker:%s:%d qset:%p, got no message and exiting, thread:%08" PRId64, pool->name, worker->id, pool->qset,
|
||||
worker->pid);
|
||||
break;
|
||||
}
|
||||
|
||||
|
@ -124,7 +124,7 @@ STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) {
|
|||
}
|
||||
|
||||
taosThreadMutexUnlock(&pool->mutex);
|
||||
uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle);
|
||||
uInfo("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle);
|
||||
|
||||
return queue;
|
||||
}
|
||||
|
@ -191,12 +191,14 @@ static void *tWWorkerThreadFp(SWWorker *worker) {
|
|||
|
||||
taosBlockSIGPIPE();
|
||||
setThreadName(pool->name);
|
||||
uDebug("worker:%s:%d is running", pool->name, worker->id);
|
||||
worker->pid = taosGetSelfPthreadId();
|
||||
uInfo("worker:%s:%d is running, thread:%08" PRId64, pool->name, worker->id, worker->pid);
|
||||
|
||||
while (1) {
|
||||
numOfMsgs = taosReadAllQitemsFromQset(worker->qset, worker->qall, &qinfo);
|
||||
if (numOfMsgs == 0) {
|
||||
uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, worker->qset);
|
||||
uInfo("worker:%s:%d qset:%p, got no message and exiting, thread:%08" PRId64, pool->name, worker->id, worker->qset,
|
||||
worker->pid);
|
||||
break;
|
||||
}
|
||||
|
||||
|
@ -244,7 +246,9 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) {
|
|||
pool->nextId = (pool->nextId + 1) % pool->max;
|
||||
}
|
||||
|
||||
uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle);
|
||||
while (worker->pid <= 0) taosMsleep(10);
|
||||
queue->threadId = worker->pid;
|
||||
uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, queue->threadId);
|
||||
code = 0;
|
||||
|
||||
_OVER:
|
||||
|
|
|
@ -25,7 +25,8 @@ class TDTestCase:
|
|||
'''
|
||||
return
|
||||
|
||||
def init(self, conn, logSql, replicaVarl=1):
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug("start to execute %s" % __file__)
|
||||
tdSql.init(conn.cursor())
|
||||
|
||||
|
|
|
@ -27,7 +27,8 @@ class TDTestCase:
|
|||
'''
|
||||
return
|
||||
|
||||
def init(self, conn, logSql, replicaVarl=1):
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug("start to execute %s" % __file__)
|
||||
tdSql.init(conn.cursor(), logSql)
|
||||
|
||||
|
|
|
@ -24,7 +24,8 @@ class TDTestCase:
|
|||
'''
|
||||
return
|
||||
|
||||
def init(self, conn, logSql, replicaVarl=1):
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug("start to execute %s" % __file__)
|
||||
tdSql.init(conn.cursor(), logSql)
|
||||
|
||||
|
|
|
@ -23,7 +23,8 @@ class TDTestCase:
|
|||
[TD-11510] taosBenchmark test cases
|
||||
"""
|
||||
|
||||
def init(self, conn, logSql, replicaVarl=1):
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug("start to execute %s" % __file__)
|
||||
tdSql.init(conn.cursor(), logSql)
|
||||
|
||||
|
|
|
@ -27,7 +27,8 @@ class TDTestCase:
|
|||
"""
|
||||
return
|
||||
|
||||
def init(self, conn, logSql, replicaVarl=1):
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug("start to execute %s" % __file__)
|
||||
tdSql.init(conn.cursor(), logSql)
|
||||
|
||||
|
|
|
@ -24,7 +24,8 @@ class TDTestCase:
|
|||
'''
|
||||
return
|
||||
|
||||
def init(self, conn, logSql, replicaVarl=1):
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug("start to execute %s" % __file__)
|
||||
tdSql.init(conn.cursor(), logSql)
|
||||
|
||||
|
|
|
@ -24,7 +24,8 @@ class TDTestCase:
|
|||
'''
|
||||
return
|
||||
|
||||
def init(self, conn, logSql, replicaVarl=1):
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug("start to execute %s" % __file__)
|
||||
tdSql.init(conn.cursor(), logSql)
|
||||
|
||||
|
|
|
@ -24,7 +24,8 @@ class TDTestCase:
|
|||
'''
|
||||
return
|
||||
|
||||
def init(self, conn, logSql, replicaVarl=1):
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug("start to execute %s" % __file__)
|
||||
tdSql.init(conn.cursor(), logSql)
|
||||
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue