Merge branch '3.0' into fix/TD-28627
This commit is contained in:
WANG MINGMING
GitHub
commit
64656a1db9
|
|
@ -2,7 +2,7 @@
|
|||
IF (DEFINED VERNUMBER)
|
||||
SET(TD_VER_NUMBER ${VERNUMBER})
|
||||
ELSE ()
|
||||
SET(TD_VER_NUMBER "3.2.3.0.alpha")
|
||||
SET(TD_VER_NUMBER "3.2.4.0.alpha")
|
||||
ENDIF ()
|
||||
|
||||
IF (DEFINED VERCOMPATIBLE)
|
||||
|
|
|
|||
|
|
@ -842,12 +842,12 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"})
|
|||
|
||||
In addition to native connections, the client library also supports subscriptions via websockets.
|
||||
|
||||
The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer).
|
||||
The syntax for creating a consumer is "consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer).
|
||||
|
||||
```python
|
||||
import taosws
|
||||
|
||||
consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
|
||||
consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
|
@ -887,13 +887,13 @@ The `poll` function is used to consume data in tmq. The parameter of the `poll`
|
|||
|
||||
```python
|
||||
while True:
|
||||
res = consumer.poll(1)
|
||||
if not res:
|
||||
message = consumer.poll(1)
|
||||
if not message:
|
||||
continue
|
||||
err = res.error()
|
||||
err = message.error()
|
||||
if err is not None:
|
||||
raise err
|
||||
val = res.value()
|
||||
val = message.value()
|
||||
|
||||
for block in val:
|
||||
print(block.fetchall())
|
||||
|
|
@ -902,16 +902,14 @@ while True:
|
|||
</TabItem>
|
||||
<TabItem value="websocket" label="WebSocket connection">
|
||||
|
||||
The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data.
|
||||
The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out.
|
||||
|
||||
```python
|
||||
while True:
|
||||
res = consumer.poll(timeout=1.0)
|
||||
if not res:
|
||||
message = consumer.poll(1)
|
||||
if not message:
|
||||
continue
|
||||
err = res.error()
|
||||
if err is not None:
|
||||
raise err
|
||||
|
||||
for block in message:
|
||||
for row in block:
|
||||
print(row)
|
||||
|
|
|
|||
|
|
@ -41,16 +41,28 @@ window_clause: {
|
|||
SESSION(ts_col, tol_val)
|
||||
| STATE_WINDOW(col)
|
||||
| INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)]
|
||||
| EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
|
||||
| COUNT_WINDOW(count_val[, sliding_val])
|
||||
}
|
||||
```
|
||||
|
||||
`SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.
|
||||
|
||||
`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns.
|
||||
|
||||
`COUNT_WINDOW` is a counting window that is divided by a fixed number of data rows.`count_val`: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. `count_val` represents the maximum number of data rows contained in each `COUNT_WINDOW`. When the total number of data rows cannot be divided by `count_val`, the number of rows in the last window will be less than `count_val`. `sliding_val`: is a constant that represents the number of window slides, similar to `SLIDING` in `INTERVAL`.
|
||||
|
||||
For example, the following SQL statement creates a stream and automatically creates a supertable named `avg_vol`. The stream has a 1 minute time window that slides forward in 30 second intervals to calculate the average voltage of the meters supertable.
|
||||
|
||||
```sql
|
||||
CREATE STREAM avg_vol_s INTO avg_vol AS
|
||||
SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
|
||||
|
||||
CREATE STREAM streams0 INTO streamt0 AS
|
||||
SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9;
|
||||
|
||||
CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS
|
||||
SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10);
|
||||
```
|
||||
|
||||
## Partitions of Stream
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t
|
|||
|
||||
import Release from "/components/ReleaseV3";
|
||||
|
||||
## 3.2.3.0
|
||||
|
||||
<Release type="tdengine" version="3.2.3.0" />
|
||||
|
||||
## 3.2.2.0
|
||||
|
||||
<Release type="tdengine" version="3.2.2.0" />
|
||||
|
|
|
|||
|
|
@ -856,7 +856,7 @@ taosws `Consumer` API 提供了基于 Websocket 订阅 TMQ 数据的 API。创
|
|||
```python
|
||||
import taosws
|
||||
|
||||
consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
|
||||
consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
|
@ -896,13 +896,13 @@ Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 flo
|
|||
|
||||
```python
|
||||
while True:
|
||||
res = consumer.poll(1)
|
||||
if not res:
|
||||
message = consumer.poll(1)
|
||||
if not message:
|
||||
continue
|
||||
err = res.error()
|
||||
err = message.error()
|
||||
if err is not None:
|
||||
raise err
|
||||
val = res.value()
|
||||
val = message.value()
|
||||
|
||||
for block in val:
|
||||
print(block.fetchall())
|
||||
|
|
@ -911,16 +911,14 @@ while True:
|
|||
</TabItem>
|
||||
<TabItem value="websocket" label="WebSocket 连接">
|
||||
|
||||
Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。
|
||||
Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。
|
||||
|
||||
```python
|
||||
while True:
|
||||
res = consumer.poll(timeout=1.0)
|
||||
if not res:
|
||||
message = consumer.poll(1)
|
||||
if not message:
|
||||
continue
|
||||
err = res.error()
|
||||
if err is not None:
|
||||
raise err
|
||||
|
||||
for block in message:
|
||||
for row in block:
|
||||
print(row)
|
||||
|
|
|
|||
|
|
@ -49,10 +49,14 @@ window_clause: {
|
|||
SESSION(ts_col, tol_val)
|
||||
| STATE_WINDOW(col)
|
||||
| INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)]
|
||||
| EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
|
||||
| COUNT_WINDOW(count_val[, sliding_val])
|
||||
}
|
||||
```
|
||||
|
||||
其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。
|
||||
EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
|
||||
COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。
|
||||
|
||||
窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished)
|
||||
|
||||
|
|
@ -61,6 +65,12 @@ window_clause: {
|
|||
```sql
|
||||
CREATE STREAM avg_vol_s INTO avg_vol AS
|
||||
SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
|
||||
|
||||
CREATE STREAM streams0 INTO streamt0 AS
|
||||
SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9;
|
||||
|
||||
CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS
|
||||
SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10);
|
||||
```
|
||||
|
||||
## 流式计算的 partition
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
|
|||
|
||||
import Release from "/components/ReleaseV3";
|
||||
|
||||
## 3.2.3.0
|
||||
|
||||
<Release type="tdengine" version="3.2.3.0" />
|
||||
|
||||
## 3.2.2.0
|
||||
|
||||
<Release type="tdengine" version="3.2.2.0" />
|
||||
|
|
|
|||
|
|
@ -219,7 +219,6 @@ extern bool tsFilterScalarMode;
|
|||
extern int32_t tsMaxStreamBackendCache;
|
||||
extern int32_t tsPQSortMemThreshold;
|
||||
extern int32_t tsResolveFQDNRetryTime;
|
||||
extern bool tsDisableCount;
|
||||
|
||||
extern bool tsExperimental;
|
||||
// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
|
||||
|
|
|
|||
|
|
@ -581,8 +581,8 @@ typedef struct {
|
|||
};
|
||||
} SSubmitRsp;
|
||||
|
||||
int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
|
||||
int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
|
||||
// int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
|
||||
// int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
|
||||
// void tFreeSSubmitBlkRsp(void* param);
|
||||
void tFreeSSubmitRsp(SSubmitRsp* pRsp);
|
||||
|
||||
|
|
@ -885,8 +885,8 @@ typedef struct {
|
|||
int64_t maxStorage;
|
||||
} SCreateAcctReq, SAlterAcctReq;
|
||||
|
||||
int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
|
||||
int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
|
||||
// int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
|
||||
// int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
char user[TSDB_USER_LEN];
|
||||
|
|
@ -3446,7 +3446,7 @@ int32_t tDeserializeSCreateTagIdxReq(void* buf, int32_t bufLen, SCreateTagIndexR
|
|||
|
||||
typedef SMDropSmaReq SDropTagIndexReq;
|
||||
|
||||
int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq);
|
||||
// int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq);
|
||||
int32_t tDeserializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -3567,8 +3567,8 @@ typedef struct {
|
|||
int8_t igNotExists;
|
||||
} SMDropFullTextReq;
|
||||
|
||||
int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
|
||||
int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
|
||||
// int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
|
||||
// int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
char indexFName[TSDB_INDEX_FNAME_LEN];
|
||||
|
|
|
|||
|
|
@ -433,7 +433,7 @@ int32_t* taosGetErrno();
|
|||
|
||||
//mnode-compact
|
||||
#define TSDB_CODE_MND_INVALID_COMPACT_ID TAOS_DEF_ERROR_CODE(0, 0x04B1)
|
||||
|
||||
#define TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x04B2)
|
||||
|
||||
// vnode
|
||||
// #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x
|
||||
|
|
|
|||
|
|
@ -187,6 +187,8 @@ typedef enum ELogicConditionType {
|
|||
LOGIC_COND_TYPE_NOT,
|
||||
} ELogicConditionType;
|
||||
|
||||
#define TSDB_INT32_ID_LEN 11
|
||||
|
||||
#define TSDB_NAME_DELIMITER_LEN 1
|
||||
|
||||
#define TSDB_UNI_LEN 24
|
||||
|
|
|
|||
|
|
@ -72,40 +72,6 @@ struct STaosQnode {
|
|||
char item[];
|
||||
};
|
||||
|
||||
struct STaosQueue {
|
||||
STaosQnode *head;
|
||||
STaosQnode *tail;
|
||||
STaosQueue *next; // for queue set
|
||||
STaosQset *qset; // for queue set
|
||||
void *ahandle; // for queue set
|
||||
FItem itemFp;
|
||||
FItems itemsFp;
|
||||
TdThreadMutex mutex;
|
||||
int64_t memOfItems;
|
||||
int32_t numOfItems;
|
||||
int64_t threadId;
|
||||
int64_t memLimit;
|
||||
int64_t itemLimit;
|
||||
};
|
||||
|
||||
struct STaosQset {
|
||||
STaosQueue *head;
|
||||
STaosQueue *current;
|
||||
TdThreadMutex mutex;
|
||||
tsem_t sem;
|
||||
int32_t numOfQueues;
|
||||
int32_t numOfItems;
|
||||
};
|
||||
|
||||
struct STaosQall {
|
||||
STaosQnode *current;
|
||||
STaosQnode *start;
|
||||
int32_t numOfItems;
|
||||
int64_t memOfItems;
|
||||
int32_t unAccessedNumOfItems;
|
||||
int64_t unAccessMemOfItems;
|
||||
};
|
||||
|
||||
STaosQueue *taosOpenQueue();
|
||||
void taosCloseQueue(STaosQueue *queue);
|
||||
void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp);
|
||||
|
|
@ -140,6 +106,8 @@ int32_t taosGetQueueNumber(STaosQset *qset);
|
|||
int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo);
|
||||
int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo);
|
||||
void taosResetQsetThread(STaosQset *qset, void *pItem);
|
||||
void taosQueueSetThreadId(STaosQueue *pQueue, int64_t threadId);
|
||||
int64_t taosQueueGetThreadId(STaosQueue *pQueue);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ typedef struct SScalableBf {
|
|||
SArray *bfArray; // array of bloom filters
|
||||
uint32_t growth;
|
||||
uint64_t numBits;
|
||||
uint32_t maxBloomFilters;
|
||||
int8_t status;
|
||||
_hash_fn_t hashFn1;
|
||||
_hash_fn_t hashFn2;
|
||||
} SScalableBf;
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ extern "C" {
|
|||
#define IS_SAME_KEY (maxKV->type == kv->type && maxKV->keyLen == kv->keyLen && memcmp(maxKV->key, kv->key, kv->keyLen) == 0)
|
||||
|
||||
#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \
|
||||
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
|
||||
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH))
|
||||
|
||||
#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
|
||||
|
||||
|
|
|
|||
|
|
@ -20,14 +20,14 @@
|
|||
|
||||
#include "clientSml.h"
|
||||
|
||||
#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH)
|
||||
#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH)
|
||||
#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH)
|
||||
#define IS_COMMA(sql,escapeChar) (*(sql) == COMMA && (*((sql)-1) != SLASH || ((sql)-1 == escapeChar)))
|
||||
#define IS_SPACE(sql,escapeChar) (*(sql) == SPACE && (*((sql)-1) != SLASH || ((sql)-1 == escapeChar)))
|
||||
#define IS_EQUAL(sql,escapeChar) (*(sql) == EQUAL && (*((sql)-1) != SLASH || ((sql)-1 == escapeChar)))
|
||||
|
||||
#define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH))
|
||||
|
||||
#define IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql) \
|
||||
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL))
|
||||
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == SLASH))
|
||||
|
||||
#define PROCESS_SLASH_IN_FIELD_VALUE(key, keyLen) \
|
||||
for (int i = 1; i < keyLen; ++i) { \
|
||||
|
|
@ -198,7 +198,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
|
|||
int cnt = 0;
|
||||
|
||||
while (*sql < sqlEnd) {
|
||||
if (unlikely(IS_SPACE(*sql))) {
|
||||
if (unlikely(IS_SPACE(*sql,NULL))) {
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -207,18 +207,21 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
|
|||
size_t keyLen = 0;
|
||||
bool keyEscaped = false;
|
||||
size_t keyLenEscaped = 0;
|
||||
const char *escapeChar = NULL;
|
||||
|
||||
while (*sql < sqlEnd) {
|
||||
if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
|
||||
if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) {
|
||||
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
|
||||
terrno = TSDB_CODE_SML_INVALID_DATA;
|
||||
return -1;
|
||||
}
|
||||
if (unlikely(IS_EQUAL(*sql))) {
|
||||
if (unlikely(IS_EQUAL(*sql,escapeChar))) {
|
||||
keyLen = *sql - key;
|
||||
(*sql)++;
|
||||
break;
|
||||
}
|
||||
if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
|
||||
escapeChar = *sql;
|
||||
keyLenEscaped++;
|
||||
keyEscaped = true;
|
||||
}
|
||||
|
|
@ -238,15 +241,16 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
|
|||
size_t valueLenEscaped = 0;
|
||||
while (*sql < sqlEnd) {
|
||||
// parse value
|
||||
if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
|
||||
if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) {
|
||||
break;
|
||||
} else if (unlikely(IS_EQUAL(*sql))) {
|
||||
} else if (unlikely(IS_EQUAL(*sql,escapeChar))) {
|
||||
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
|
||||
terrno = TSDB_CODE_SML_INVALID_DATA;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
|
||||
escapeChar = *sql;
|
||||
valueLenEscaped++;
|
||||
valueEscaped = true;
|
||||
}
|
||||
|
|
@ -293,7 +297,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
|
|||
}
|
||||
|
||||
cnt++;
|
||||
if (IS_SPACE(*sql)) {
|
||||
if (IS_SPACE(*sql,escapeChar)) {
|
||||
break;
|
||||
}
|
||||
(*sql)++;
|
||||
|
|
@ -326,7 +330,7 @@ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
|
|||
static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) {
|
||||
int cnt = 0;
|
||||
while (*sql < sqlEnd) {
|
||||
if (unlikely(IS_SPACE(*sql))) {
|
||||
if (unlikely(IS_SPACE(*sql,NULL))) {
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -335,17 +339,19 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
|
|||
size_t keyLen = 0;
|
||||
bool keyEscaped = false;
|
||||
size_t keyLenEscaped = 0;
|
||||
const char *escapeChar = NULL;
|
||||
while (*sql < sqlEnd) {
|
||||
if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
|
||||
if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) {
|
||||
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
|
||||
return TSDB_CODE_SML_INVALID_DATA;
|
||||
}
|
||||
if (unlikely(IS_EQUAL(*sql))) {
|
||||
if (unlikely(IS_EQUAL(*sql,escapeChar))) {
|
||||
keyLen = *sql - key;
|
||||
(*sql)++;
|
||||
break;
|
||||
}
|
||||
if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
|
||||
escapeChar = *sql;
|
||||
keyLenEscaped++;
|
||||
keyEscaped = true;
|
||||
}
|
||||
|
|
@ -363,7 +369,6 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
|
|||
bool valueEscaped = false;
|
||||
size_t valueLenEscaped = 0;
|
||||
int quoteNum = 0;
|
||||
const char *escapeChar = NULL;
|
||||
while (*sql < sqlEnd) {
|
||||
// parse value
|
||||
if (unlikely(*(*sql) == QUOTE && (*(*sql - 1) != SLASH || (*sql - 1) == escapeChar))) {
|
||||
|
|
@ -374,7 +379,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
|
|||
}
|
||||
continue;
|
||||
}
|
||||
if (quoteNum % 2 == 0 && (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql)))) {
|
||||
if (quoteNum % 2 == 0 && (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar)))) {
|
||||
break;
|
||||
}
|
||||
if (IS_SLASH_LETTER_IN_FIELD_VALUE(*sql) && (*sql - 1) != escapeChar) {
|
||||
|
|
@ -437,7 +442,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
|
|||
}
|
||||
|
||||
cnt++;
|
||||
if (IS_SPACE(*sql)) {
|
||||
if (IS_SPACE(*sql,escapeChar)) {
|
||||
break;
|
||||
}
|
||||
(*sql)++;
|
||||
|
|
@ -453,19 +458,18 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
|
|||
elements->measure = sql;
|
||||
// parse measure
|
||||
size_t measureLenEscaped = 0;
|
||||
const char *escapeChar = NULL;
|
||||
while (sql < sqlEnd) {
|
||||
if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) {
|
||||
elements->measureEscaped = true;
|
||||
measureLenEscaped++;
|
||||
sql++;
|
||||
continue;
|
||||
}
|
||||
if (unlikely(IS_COMMA(sql))) {
|
||||
if (unlikely(IS_COMMA(sql,escapeChar) || IS_SPACE(sql,escapeChar))) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (unlikely(IS_SPACE(sql))) {
|
||||
break;
|
||||
if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) {
|
||||
elements->measureEscaped = true;
|
||||
escapeChar = sql;
|
||||
measureLenEscaped++;
|
||||
sql++;
|
||||
continue;
|
||||
}
|
||||
sql++;
|
||||
}
|
||||
|
|
@ -478,9 +482,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
|
|||
// to get measureTagsLen before
|
||||
const char *tmp = sql;
|
||||
while (tmp < sqlEnd) {
|
||||
if (unlikely(IS_SPACE(tmp))) {
|
||||
if (unlikely(IS_SPACE(tmp,escapeChar))) {
|
||||
break;
|
||||
}
|
||||
if(unlikely(IS_SLASH_LETTER_IN_TAG_FIELD_KEY(tmp))){
|
||||
escapeChar = tmp;
|
||||
}
|
||||
tmp++;
|
||||
}
|
||||
elements->measureTagsLen = tmp - elements->measure;
|
||||
|
|
|
|||
|
|
@ -876,12 +876,13 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
|
|||
STaosQall* qall = taosAllocateQall();
|
||||
taosReadAllQitems(pTmq->delayedTask, qall);
|
||||
|
||||
if (qall->numOfItems == 0) {
|
||||
int32_t numOfItems = taosQallItemSize(qall);
|
||||
if (numOfItems == 0) {
|
||||
taosFreeQall(qall);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, qall->numOfItems);
|
||||
tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, numOfItems);
|
||||
int8_t* pTaskType = NULL;
|
||||
taosGetQitem(qall, (void**)&pTaskType);
|
||||
|
||||
|
|
@ -1839,7 +1840,7 @@ static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal
|
|||
}
|
||||
|
||||
static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
|
||||
tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems);
|
||||
tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, taosQallItemSize(tmq->qall));
|
||||
|
||||
while (1) {
|
||||
SMqRspWrapper* pRspWrapper = NULL;
|
||||
|
|
|
|||
|
|
@ -452,20 +452,21 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
|
|||
}
|
||||
|
||||
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
|
||||
int32_t newLen = pSource->varmeta.length;
|
||||
memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows);
|
||||
if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) {
|
||||
char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length);
|
||||
if (pColumnInfoData->varmeta.allocLen < newLen) {
|
||||
char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newLen);
|
||||
if (tmp == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
pColumnInfoData->pData = tmp;
|
||||
pColumnInfoData->varmeta.allocLen = pSource->varmeta.length;
|
||||
pColumnInfoData->varmeta.allocLen = newLen;
|
||||
}
|
||||
|
||||
pColumnInfoData->varmeta.length = pSource->varmeta.length;
|
||||
pColumnInfoData->varmeta.length = newLen;
|
||||
if (pColumnInfoData->pData != NULL && pSource->pData != NULL) {
|
||||
memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length);
|
||||
memcpy(pColumnInfoData->pData, pSource->pData, newLen);
|
||||
}
|
||||
} else {
|
||||
memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows));
|
||||
|
|
@ -1687,7 +1688,29 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) {
|
|||
}
|
||||
|
||||
static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
|
||||
if (n >= total || n == 0) return;
|
||||
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
|
||||
if (pColInfoData->varmeta.length != 0) {
|
||||
int32_t newLen = pColInfoData->varmeta.offset[n];
|
||||
if (-1 == newLen) {
|
||||
for (int i = n - 1; i >= 0; --i) {
|
||||
newLen = pColInfoData->varmeta.offset[i];
|
||||
if (newLen != -1) {
|
||||
if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
|
||||
newLen += getJsonValueLen(pColInfoData->pData + newLen);
|
||||
} else {
|
||||
newLen += varDataTLen(pColInfoData->pData + newLen);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (newLen <= -1) {
|
||||
uFatal("colDataKeepFirstNRows: newLen:%d old:%d", newLen, pColInfoData->varmeta.length);
|
||||
} else {
|
||||
pColInfoData->varmeta.length = newLen;
|
||||
}
|
||||
}
|
||||
// pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
|
||||
memset(&pColInfoData->varmeta.offset[n], 0, total - n);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -269,7 +269,6 @@ int64_t tsStreamBufferSize = 128 * 1024 * 1024;
|
|||
bool tsFilterScalarMode = false;
|
||||
int tsResolveFQDNRetryTime = 100; // seconds
|
||||
int tsStreamAggCnt = 10;
|
||||
bool tsDisableCount = true;
|
||||
|
||||
char tsS3Endpoint[TSDB_FQDN_LEN] = "<endpoint>";
|
||||
char tsS3AccessKey[TSDB_FQDN_LEN] = "<accesskey>";
|
||||
|
|
@ -541,8 +540,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
|
|||
|
||||
if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
|
||||
|
||||
if (cfgAddBool(pCfg, "disableCount", tsDisableCount, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1109,8 +1106,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
|
|||
tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
|
||||
|
||||
tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
|
||||
|
||||
tsDisableCount = cfgGetItem(pCfg, "disableCount")->bval;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1739,8 +1734,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
|
|||
{"shellActivityTimer", &tsShellActivityTimer},
|
||||
{"slowLogThreshold", &tsSlowLogThreshold},
|
||||
{"useAdapter", &tsUseAdapter},
|
||||
{"experimental", &tsExperimental},
|
||||
{"disableCount", &tsDisableCount}};
|
||||
{"experimental", &tsExperimental}};
|
||||
|
||||
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
|
||||
taosCfgSetOption(options, tListLen(options), pItem, false);
|
||||
|
|
|
|||
|
|
@ -1009,19 +1009,19 @@ int32_t tDeserializeSCreateTagIdxReq(void *buf, int32_t bufLen, SCreateTagIndexR
|
|||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
// int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) {
|
||||
// SEncoder encoder = {0};
|
||||
// tEncoderInit(&encoder, buf, bufLen);
|
||||
// if (tStartEncode(&encoder) < 0) return -1;
|
||||
// tEndEncode(&encoder);
|
||||
|
||||
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
|
||||
// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
|
||||
// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tEncoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
// int32_t tlen = encoder.pos;
|
||||
// tEncoderClear(&encoder);
|
||||
// return tlen;
|
||||
// }
|
||||
int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
|
|
@ -1035,6 +1035,7 @@ int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *
|
|||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeSMCreateFullTextReq(void *buf, int32_t bufLen, SMCreateFullTextReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
|
@ -1059,32 +1060,32 @@ void tFreeSMCreateFullTextReq(SMCreateFullTextReq *pReq) {
|
|||
// impl later
|
||||
return;
|
||||
}
|
||||
int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
// int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
|
||||
// SEncoder encoder = {0};
|
||||
// tEncoderInit(&encoder, buf, bufLen);
|
||||
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
// if (tStartEncode(&encoder) < 0) return -1;
|
||||
|
||||
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
|
||||
// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
|
||||
|
||||
if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
|
||||
// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
|
||||
|
||||
tEndEncode(&encoder);
|
||||
int32_t tlen = encoder.pos;
|
||||
tEncoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
|
||||
// tEndEncode(&encoder);
|
||||
// int32_t tlen = encoder.pos;
|
||||
// tEncoderClear(&encoder);
|
||||
// return tlen;
|
||||
// }
|
||||
// int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
|
||||
// SDecoder decoder = {0};
|
||||
// tDecoderInit(&decoder, buf, bufLen);
|
||||
// if (tStartDecode(&decoder) < 0) return -1;
|
||||
// if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
|
||||
// if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
|
||||
|
||||
tEndDecode(&decoder);
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
// tEndDecode(&decoder);
|
||||
// tDecoderClear(&decoder);
|
||||
// return 0;
|
||||
// }
|
||||
|
||||
int32_t tSerializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
|
|
@ -1474,44 +1475,44 @@ void tFreeSStatisReq(SStatisReq *pReq) {
|
|||
taosMemoryFreeClear(pReq->pCont);
|
||||
}
|
||||
|
||||
int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
// int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
|
||||
// SEncoder encoder = {0};
|
||||
// tEncoderInit(&encoder, buf, bufLen);
|
||||
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
// if (tStartEncode(&encoder) < 0) return -1;
|
||||
// if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
|
||||
// if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
|
||||
// if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1;
|
||||
// if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1;
|
||||
// if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1;
|
||||
// if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1;
|
||||
// if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1;
|
||||
// if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1;
|
||||
// tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tEncoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
// int32_t tlen = encoder.pos;
|
||||
// tEncoderClear(&encoder);
|
||||
// return tlen;
|
||||
// }
|
||||
|
||||
int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
// int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
|
||||
// SDecoder decoder = {0};
|
||||
// tDecoderInit(&decoder, buf, bufLen);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1;
|
||||
tEndDecode(&decoder);
|
||||
// if (tStartDecode(&decoder) < 0) return -1;
|
||||
// if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
|
||||
// if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
|
||||
// if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1;
|
||||
// if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1;
|
||||
// if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1;
|
||||
// if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1;
|
||||
// if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1;
|
||||
// if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1;
|
||||
// tEndDecode(&decoder);
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
// tDecoderClear(&decoder);
|
||||
// return 0;
|
||||
// }
|
||||
|
||||
int32_t tSerializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
|
|
@ -5238,11 +5239,11 @@ int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCo
|
|||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
|
||||
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->finished) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -2;
|
||||
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -3;
|
||||
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -4;
|
||||
if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -5;
|
||||
if (tDecodeI32(&decoder, &pReq->finished) < 0) return -6;
|
||||
|
||||
tEndDecode(&decoder);
|
||||
tDecoderClear(&decoder);
|
||||
|
|
@ -7934,64 +7935,64 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
// static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
|
||||
// if (tStartDecode(pDecoder) < 0) return -1;
|
||||
|
||||
if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1;
|
||||
pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
|
||||
if (NULL == pBlock->tblFName) return -1;
|
||||
if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1;
|
||||
if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1;
|
||||
if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1;
|
||||
if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1;
|
||||
// if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1;
|
||||
// if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1;
|
||||
// pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
|
||||
// if (NULL == pBlock->tblFName) return -1;
|
||||
// if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1;
|
||||
// if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1;
|
||||
// if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1;
|
||||
// if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1;
|
||||
|
||||
int32_t meta = 0;
|
||||
if (tDecodeI32(pDecoder, &meta) < 0) return -1;
|
||||
if (meta) {
|
||||
pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
|
||||
if (NULL == pBlock->pMeta) return -1;
|
||||
if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1;
|
||||
} else {
|
||||
pBlock->pMeta = NULL;
|
||||
}
|
||||
// int32_t meta = 0;
|
||||
// if (tDecodeI32(pDecoder, &meta) < 0) return -1;
|
||||
// if (meta) {
|
||||
// pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
|
||||
// if (NULL == pBlock->pMeta) return -1;
|
||||
// if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1;
|
||||
// } else {
|
||||
// pBlock->pMeta = NULL;
|
||||
// }
|
||||
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
// tEndDecode(pDecoder);
|
||||
// return 0;
|
||||
// }
|
||||
|
||||
int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) {
|
||||
int32_t nBlocks = taosArrayGetSize(pRsp->pArray);
|
||||
// int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) {
|
||||
// int32_t nBlocks = taosArrayGetSize(pRsp->pArray);
|
||||
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
// if (tStartEncode(pEncoder) < 0) return -1;
|
||||
|
||||
if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1;
|
||||
if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1;
|
||||
if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1;
|
||||
for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) {
|
||||
if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1;
|
||||
}
|
||||
// if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1;
|
||||
// if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1;
|
||||
// if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1;
|
||||
// for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) {
|
||||
// if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1;
|
||||
// }
|
||||
|
||||
tEndEncode(pEncoder);
|
||||
return 0;
|
||||
}
|
||||
// tEndEncode(pEncoder);
|
||||
// return 0;
|
||||
// }
|
||||
|
||||
int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
// int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) {
|
||||
// if (tStartDecode(pDecoder) < 0) return -1;
|
||||
|
||||
if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1;
|
||||
if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1;
|
||||
if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1;
|
||||
pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks));
|
||||
if (pRsp->pBlocks == NULL) return -1;
|
||||
for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) {
|
||||
if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1;
|
||||
}
|
||||
// if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1;
|
||||
// if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1;
|
||||
// if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1;
|
||||
// pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks));
|
||||
// if (pRsp->pBlocks == NULL) return -1;
|
||||
// for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) {
|
||||
// if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1;
|
||||
// }
|
||||
|
||||
tEndDecode(pDecoder);
|
||||
tDecoderClear(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
// tEndDecode(pDecoder);
|
||||
// tDecoderClear(pDecoder);
|
||||
// return 0;
|
||||
// }
|
||||
|
||||
// void tFreeSSubmitBlkRsp(void *param) {
|
||||
// if (NULL == param) {
|
||||
|
|
|
|||
|
|
@ -194,26 +194,26 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
|
|||
while (pVnode->refCount > 0) taosMsleep(10);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode write queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
|
||||
pVnode->pWriteW.queue->threadId);
|
||||
taosQueueGetThreadId(pVnode->pWriteW.queue));
|
||||
tMultiWorkerCleanup(&pVnode->pWriteW);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode sync queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue,
|
||||
pVnode->pSyncW.queue->threadId);
|
||||
taosQueueGetThreadId(pVnode->pSyncW.queue));
|
||||
tMultiWorkerCleanup(&pVnode->pSyncW);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode sync rd queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue,
|
||||
pVnode->pSyncRdW.queue->threadId);
|
||||
taosQueueGetThreadId(pVnode->pSyncRdW.queue));
|
||||
tMultiWorkerCleanup(&pVnode->pSyncRdW);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
|
||||
pVnode->pApplyW.queue->threadId);
|
||||
taosQueueGetThreadId(pVnode->pApplyW.queue));
|
||||
tMultiWorkerCleanup(&pVnode->pApplyW);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode query queue:%p is empty", pVnode->vgId, pVnode->pQueryQ);
|
||||
while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode fetch queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ,
|
||||
pVnode->pFetchQ->threadId);
|
||||
taosQueueGetThreadId(pVnode->pFetchQ));
|
||||
while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
|
||||
|
||||
tqNotifyClose(pVnode->pImpl->pTq);
|
||||
|
|
|
|||
|
|
@ -365,16 +365,16 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
|
|||
}
|
||||
|
||||
dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
|
||||
pVnode->pWriteW.queue->threadId);
|
||||
taosQueueGetThreadId(pVnode->pWriteW.queue));
|
||||
dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue,
|
||||
pVnode->pSyncW.queue->threadId);
|
||||
taosQueueGetThreadId(pVnode->pSyncW.queue));
|
||||
dInfo("vgId:%d, sync-rd-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue,
|
||||
pVnode->pSyncRdW.queue->threadId);
|
||||
taosQueueGetThreadId(pVnode->pSyncRdW.queue));
|
||||
dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
|
||||
pVnode->pApplyW.queue->threadId);
|
||||
taosQueueGetThreadId(pVnode->pApplyW.queue));
|
||||
dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ);
|
||||
dInfo("vgId:%d, fetch-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ,
|
||||
pVnode->pFetchQ->threadId);
|
||||
taosQueueGetThreadId(pVnode->pFetchQ));
|
||||
dInfo("vgId:%d, stream-queue:%p is alloced", pVnode->vgId, pVnode->pStreamQ);
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -345,6 +345,7 @@ int32_t dmInitClient(SDnode *pDnode) {
|
|||
rpcInit.parent = pDnode;
|
||||
rpcInit.rfp = rpcRfp;
|
||||
rpcInit.compressSize = tsCompressMsgSize;
|
||||
rpcInit.dfp = destroyAhandle;
|
||||
|
||||
rpcInit.retryMinInterval = tsRedirectPeriod;
|
||||
rpcInit.retryStepFactor = tsRedirectFactor;
|
||||
|
|
|
|||
|
|
@ -363,13 +363,15 @@ static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *p
|
|||
}
|
||||
|
||||
static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) {
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "kill-compact");
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact");
|
||||
if (pTrans == NULL) {
|
||||
mError("compact:%" PRId32 ", failed to drop since %s" , pCompact->compactId, terrstr());
|
||||
return -1;
|
||||
}
|
||||
mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId);
|
||||
|
||||
mndTransSetDbName(pTrans, pCompact->dbname, NULL);
|
||||
|
||||
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
|
||||
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
|
||||
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
|
||||
|
|
@ -378,7 +380,7 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
|
|||
}
|
||||
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
|
||||
|
||||
void *pIter = NULL;
|
||||
void *pIter = NULL;
|
||||
while (1) {
|
||||
SCompactDetailObj *pDetail = NULL;
|
||||
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
|
||||
|
|
@ -452,7 +454,7 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){
|
|||
|
||||
code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
char obj[MND_COMPACT_ID_LEN] = {0};
|
||||
char obj[TSDB_INT32_ID_LEN] = {0};
|
||||
sprintf(obj, "%d", pCompact->compactId);
|
||||
|
||||
auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen);
|
||||
|
|
@ -488,13 +490,17 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c
|
|||
sdbRelease(pMnode->pSdb, pDetail);
|
||||
}
|
||||
|
||||
return -1;
|
||||
return TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST;
|
||||
}
|
||||
|
||||
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){
|
||||
SQueryCompactProgressRsp req = {0};
|
||||
if (tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req) != 0) {
|
||||
int32_t code = 0;
|
||||
code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req);
|
||||
if (code != 0) {
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d",
|
||||
code, pReq->pCont, pReq->contLen);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -502,10 +508,10 @@ int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){
|
|||
req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
|
||||
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
int32_t code = -1;
|
||||
|
||||
|
||||
if(mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req) != 0){
|
||||
code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req);
|
||||
if(code != 0){
|
||||
terrno = code;
|
||||
mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
|
||||
req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
|
||||
return -1;
|
||||
|
|
@ -612,15 +618,17 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-compact-progress");
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress");
|
||||
if (pTrans == NULL) {
|
||||
mError("trans:%" PRId32 ", failed to create since %s" , pTrans->id, terrstr());
|
||||
return -1;
|
||||
}
|
||||
mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id);
|
||||
|
||||
|
||||
SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId);
|
||||
|
||||
mndTransSetDbName(pTrans, pCompact->dbname, NULL);
|
||||
|
||||
pIter = NULL;
|
||||
while (1) {
|
||||
SCompactDetailObj *pDetail = NULL;
|
||||
|
|
|
|||
|
|
@ -709,7 +709,8 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
|
|||
|
||||
int32_t code = syncProcessMsg(pMgmt->sync, pMsg);
|
||||
if (code != 0) {
|
||||
mGError("vgId:1, failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr());
|
||||
mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType),
|
||||
terrstr(), code);
|
||||
}
|
||||
|
||||
return code;
|
||||
|
|
|
|||
|
|
@ -789,25 +789,6 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, int8_
|
|||
return code;
|
||||
}
|
||||
|
||||
static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) {
|
||||
SLastCol *pLastCol = NULL;
|
||||
|
||||
char *err = NULL;
|
||||
size_t vlen = 0;
|
||||
SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid};
|
||||
size_t klen = ROCKS_KEY_LEN;
|
||||
char *value = NULL;
|
||||
value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err);
|
||||
if (NULL != err) {
|
||||
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err);
|
||||
rocksdb_free(err);
|
||||
}
|
||||
|
||||
pLastCol = tsdbCacheDeserialize(value);
|
||||
|
||||
return pLastCol;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
int idx;
|
||||
SLastKey key;
|
||||
|
|
@ -1052,6 +1033,25 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC
|
|||
static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols,
|
||||
int nCols, int16_t *slotIds);
|
||||
#ifdef BUILD_NO_CALL
|
||||
static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) {
|
||||
SLastCol *pLastCol = NULL;
|
||||
|
||||
char *err = NULL;
|
||||
size_t vlen = 0;
|
||||
SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid};
|
||||
size_t klen = ROCKS_KEY_LEN;
|
||||
char *value = NULL;
|
||||
value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err);
|
||||
if (NULL != err) {
|
||||
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err);
|
||||
rocksdb_free(err);
|
||||
}
|
||||
|
||||
pLastCol = tsdbCacheDeserialize(value);
|
||||
|
||||
return pLastCol;
|
||||
}
|
||||
|
||||
int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) {
|
||||
rocksdb_writebatch_t *wb = NULL;
|
||||
int32_t code = 0;
|
||||
|
|
@ -1233,10 +1233,10 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
|
|||
int16_t *lastSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t));
|
||||
int16_t *lastrowColIds = taosMemoryMalloc(num_keys * sizeof(int16_t));
|
||||
int16_t *lastrowSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t));
|
||||
SArray* lastTmpColArray = NULL;
|
||||
SArray* lastTmpIndexArray = NULL;
|
||||
SArray* lastrowTmpColArray = NULL;
|
||||
SArray* lastrowTmpIndexArray = NULL;
|
||||
SArray *lastTmpColArray = NULL;
|
||||
SArray *lastTmpIndexArray = NULL;
|
||||
SArray *lastrowTmpColArray = NULL;
|
||||
SArray *lastrowTmpIndexArray = NULL;
|
||||
|
||||
int lastIndex = 0;
|
||||
int lastrowIndex = 0;
|
||||
|
|
@ -1245,7 +1245,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
|
|||
SIdxKey *idxKey = taosArrayGet(remainCols, i);
|
||||
slotIds[i] = pr->pSlotIds[idxKey->idx];
|
||||
if (idxKey->key.ltype == CACHESCAN_RETRIEVE_LAST >> 3) {
|
||||
if(NULL == lastTmpIndexArray) {
|
||||
if (NULL == lastTmpIndexArray) {
|
||||
lastTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t));
|
||||
}
|
||||
taosArrayPush(lastTmpIndexArray, &(i));
|
||||
|
|
@ -1253,7 +1253,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
|
|||
lastSlotIds[lastIndex] = pr->pSlotIds[idxKey->idx];
|
||||
lastIndex++;
|
||||
} else {
|
||||
if(NULL == lastrowTmpIndexArray) {
|
||||
if (NULL == lastrowTmpIndexArray) {
|
||||
lastrowTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t));
|
||||
}
|
||||
taosArrayPush(lastrowTmpIndexArray, &(i));
|
||||
|
|
@ -1265,17 +1265,18 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
|
|||
|
||||
pTmpColArray = taosArrayInit(lastIndex + lastrowIndex, sizeof(SLastCol));
|
||||
|
||||
if(lastTmpIndexArray != NULL) {
|
||||
if (lastTmpIndexArray != NULL) {
|
||||
mergeLastCid(uid, pTsdb, &lastTmpColArray, pr, lastColIds, lastIndex, lastSlotIds);
|
||||
for(int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) {
|
||||
taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i));
|
||||
for (int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) {
|
||||
taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i));
|
||||
}
|
||||
}
|
||||
|
||||
if(lastrowTmpIndexArray != NULL) {
|
||||
if (lastrowTmpIndexArray != NULL) {
|
||||
mergeLastRowCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds);
|
||||
for(int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) {
|
||||
taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastrowTmpIndexArray, i), taosArrayGet(lastrowTmpColArray, i));
|
||||
for (int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) {
|
||||
taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastrowTmpIndexArray, i),
|
||||
taosArrayGet(lastrowTmpColArray, i));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -372,8 +372,8 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
|
|||
|
||||
int32_t code = syncProcessMsg(pVnode->sync, pMsg);
|
||||
if (code != 0) {
|
||||
vGError("vgId:%d, failed to process sync msg:%p type:%s since %s", pVnode->config.vgId, pMsg,
|
||||
TMSG_INFO(pMsg->msgType), terrstr());
|
||||
vGError("vgId:%d, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pVnode->config.vgId, pMsg,
|
||||
TMSG_INFO(pMsg->msgType), terrstr(), code);
|
||||
}
|
||||
|
||||
return code;
|
||||
|
|
|
|||
|
|
@ -94,10 +94,10 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
|
|||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
||||
for (int32_t i = 0; i < pBlock->info.rows;) {
|
||||
int32_t step = pInfo->windowSliding;
|
||||
SCountWindowResult* pBuffInfo = setCountWindowOutputBuff(pExprSup, &pInfo->countSup, &pInfo->pRow);
|
||||
int32_t prevRows = pBuffInfo->winRows;
|
||||
int32_t num = updateCountWindowInfo(i, pBlock->info.rows, pInfo->windowCount, &pBuffInfo->winRows);
|
||||
int32_t step = num;
|
||||
if (prevRows == 0) {
|
||||
pInfo->pRow->win.skey = tsCols[i];
|
||||
}
|
||||
|
|
@ -118,6 +118,8 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
|
|||
if (prevRows <= pInfo->windowSliding) {
|
||||
if (pBuffInfo->winRows > pInfo->windowSliding) {
|
||||
step = pInfo->windowSliding - prevRows;
|
||||
} else {
|
||||
step = pInfo->windowSliding;
|
||||
}
|
||||
} else {
|
||||
step = 0;
|
||||
|
|
|
|||
|
|
@ -1009,6 +1009,22 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
|
|||
pSup->deleteMark = INT64_MAX;
|
||||
pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData;
|
||||
pInfo->ignoreExpiredData = false;
|
||||
} else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT) {
|
||||
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
|
||||
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
|
||||
|
||||
ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);
|
||||
ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0);
|
||||
|
||||
qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark);
|
||||
|
||||
pSup->calTriggerSaved = pSup->calTrigger;
|
||||
pSup->deleteMarkSaved = pSup->deleteMark;
|
||||
pSup->calTrigger = STREAM_TRIGGER_AT_ONCE;
|
||||
pSup->deleteMark = INT64_MAX;
|
||||
pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData;
|
||||
pInfo->ignoreExpiredData = false;
|
||||
qInfo("save stream task:%s, param for state: %d", GET_TASKID(pTaskInfo), pInfo->ignoreExpiredData);
|
||||
}
|
||||
|
||||
// iterate operator tree
|
||||
|
|
|
|||
|
|
@ -3763,8 +3763,9 @@ static int32_t stopSubTablesTableMergeScan(STableMergeScanInfo* pInfo) {
|
|||
|
||||
taosMemoryFree(pSubTblsInfo);
|
||||
pInfo->pSubTablesMergeInfo = NULL;
|
||||
|
||||
taosMemoryTrim(0);
|
||||
}
|
||||
taosMemoryTrim(0);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -76,8 +76,8 @@ char* idxInt2str(int64_t val, char* dst, int radix) {
|
|||
return dst - 1;
|
||||
}
|
||||
__compar_fn_t idxGetCompar(int8_t type) {
|
||||
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY ||
|
||||
type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) {
|
||||
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_NCHAR ||
|
||||
type == TSDB_DATA_TYPE_GEOMETRY) {
|
||||
return (__compar_fn_t)strcmp;
|
||||
}
|
||||
return getComparFunc(type, 0);
|
||||
|
|
@ -108,8 +108,8 @@ static FORCE_INLINE TExeCond tCompareEqual(void* a, void* b, int8_t type) {
|
|||
return tCompare(func, QUERY_TERM, a, b, type);
|
||||
}
|
||||
TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) {
|
||||
if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR ||
|
||||
dtype == TSDB_DATA_TYPE_VARBINARY || dtype == TSDB_DATA_TYPE_GEOMETRY) {
|
||||
if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY ||
|
||||
dtype == TSDB_DATA_TYPE_GEOMETRY) {
|
||||
return tDoCompare(func, cmptype, a, b);
|
||||
}
|
||||
#if 1
|
||||
|
|
@ -290,6 +290,7 @@ int idxUidCompare(const void* a, const void* b) {
|
|||
uint64_t r = *(uint64_t*)b;
|
||||
return l - r;
|
||||
}
|
||||
#ifdef BUILD_NO_CALL
|
||||
int32_t idxConvertData(void* src, int8_t type, void** dst) {
|
||||
int tlen = -1;
|
||||
switch (type) {
|
||||
|
|
@ -372,6 +373,8 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) {
|
|||
// indexMayFillNumbericData(*dst, tlen);
|
||||
return tlen;
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) {
|
||||
if (src == NULL) {
|
||||
*dst = strndup(INDEX_DATA_NULL_STR, (int)strlen(INDEX_DATA_NULL_STR));
|
||||
|
|
|
|||
|
|
@ -1016,10 +1016,6 @@ static int32_t createWindowLogicNodeByCount(SLogicPlanContext* pCxt, SCountWindo
|
|||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
if (!pCxt->pPlanCxt->streamQuery && tsDisableCount) {
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
pWindow->winType = WINDOW_TYPE_COUNT;
|
||||
pWindow->node.groupAction = getGroupAction(pCxt, pSelect);
|
||||
pWindow->node.requireDataOrder =
|
||||
|
|
|
|||
|
|
@ -17,10 +17,14 @@
|
|||
#define _STREAM_BACKEDN_ROCKSDB_H_
|
||||
|
||||
#include "rocksdb/c.h"
|
||||
//#include "streamInt.h"
|
||||
// #include "streamInt.h"
|
||||
#include "streamState.h"
|
||||
#include "tcommon.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct SCfComparator {
|
||||
rocksdb_comparator_t** comp;
|
||||
int32_t numOfComp;
|
||||
|
|
@ -244,11 +248,6 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId);
|
|||
|
||||
int32_t taskDbBuildSnap(void* arg, SArray* pSnap);
|
||||
|
||||
// int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result);
|
||||
|
||||
// STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId);
|
||||
// void taskDbDestroy(void* pDb, bool flush);
|
||||
|
||||
int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId);
|
||||
|
||||
SBkdMgt* bkdMgtCreate(char* path);
|
||||
|
|
@ -258,4 +257,10 @@ int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname);
|
|||
void bkdMgtDestroy(SBkdMgt* bm);
|
||||
|
||||
int32_t taskDbGenChkpUploadData(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** path, SArray* list);
|
||||
#endif
|
||||
|
||||
uint32_t nextPow2(uint32_t x);
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -2788,7 +2788,6 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) {
|
|||
STREAM_STATE_DEL_ROCKSDB(pState, "state", &maxStateKey);
|
||||
return pCur;
|
||||
}
|
||||
#ifdef BUILD_NO_CALL
|
||||
SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) {
|
||||
stDebug("streamStateGetCur_rocksdb");
|
||||
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
|
||||
|
|
@ -2838,7 +2837,6 @@ int32_t streamStateFuncDel_rocksdb(SStreamState* pState, const STupleKey* key) {
|
|||
STREAM_STATE_DEL_ROCKSDB(pState, "func", key);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
// session cf
|
||||
int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) {
|
||||
|
|
@ -3432,7 +3430,6 @@ int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey*
|
|||
SSessionKey tmpKey = *key;
|
||||
int32_t valSize = *pVLen;
|
||||
void* tmp = taosMemoryMalloc(valSize);
|
||||
// tdbRealloc(NULL, valSize);
|
||||
if (!tmp) {
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -3506,13 +3503,11 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi
|
|||
return code;
|
||||
}
|
||||
|
||||
#ifdef BUILD_NO_CALL
|
||||
int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) {
|
||||
int code = 0;
|
||||
STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen);
|
||||
return code;
|
||||
}
|
||||
#endif
|
||||
int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen) {
|
||||
int code = 0;
|
||||
STREAM_STATE_GET_ROCKSDB(pState, "default", key, pVal, pVLen);
|
||||
|
|
@ -3535,10 +3530,10 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co
|
|||
if (pIter == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
size_t klen = 0;
|
||||
rocksdb_iter_seek(pIter, start, strlen(start));
|
||||
while (rocksdb_iter_valid(pIter)) {
|
||||
const char* key = rocksdb_iter_key(pIter, NULL);
|
||||
const char* key = rocksdb_iter_key(pIter, &klen);
|
||||
int32_t vlen = 0;
|
||||
const char* vval = rocksdb_iter_value(pIter, (size_t*)&vlen);
|
||||
char* val = NULL;
|
||||
|
|
@ -3700,6 +3695,8 @@ uint32_t nextPow2(uint32_t x) {
|
|||
x = x | (x >> 16);
|
||||
return x + 1;
|
||||
}
|
||||
|
||||
#ifdef BUILD_NO_CALL
|
||||
int32_t copyFiles(const char* src, const char* dst) {
|
||||
int32_t code = 0;
|
||||
// opt later, just hard link
|
||||
|
|
@ -3739,6 +3736,7 @@ _err:
|
|||
taosCloseDir(&pDir);
|
||||
return code >= 0 ? 0 : -1;
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t isBkdDataMeta(char* name, int32_t len) {
|
||||
const char* pCurrent = "CURRENT";
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ static void streamMetaEnvInit() {
|
|||
streamTimerInit();
|
||||
}
|
||||
|
||||
void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit);}
|
||||
void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); }
|
||||
|
||||
void streamMetaCleanup() {
|
||||
taosCloseRef(streamBackendId);
|
||||
|
|
@ -1104,14 +1104,14 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) {
|
|||
.inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputq.queue)),
|
||||
};
|
||||
|
||||
entry.inputRate = entry.inputQUsed * 100.0 / (2*STREAM_TASK_QUEUE_CAPACITY_IN_SIZE);
|
||||
entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE);
|
||||
if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) {
|
||||
entry.sinkQuota = (*pTask)->outputInfo.pTokenBucket->quotaRate;
|
||||
entry.sinkDataSize = SIZE_IN_MiB((*pTask)->execInfo.sink.dataSize);
|
||||
}
|
||||
|
||||
if ((*pTask)->chkInfo.checkpointingId != 0) {
|
||||
entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId)? 1:0;
|
||||
entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId) ? 1 : 0;
|
||||
entry.checkpointId = (*pTask)->chkInfo.checkpointingId;
|
||||
entry.chkpointTransId = (*pTask)->chkInfo.transId;
|
||||
|
||||
|
|
@ -1172,7 +1172,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) {
|
|||
stDebug("vgId:%d no tasks and no mnd epset, not send stream hb to mnode", pMeta->vgId);
|
||||
}
|
||||
|
||||
_end:
|
||||
_end:
|
||||
streamMetaClearHbMsg(&hbMsg);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
|
@ -1304,28 +1304,28 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) {
|
|||
}
|
||||
|
||||
void streamMetaRLock(SStreamMeta* pMeta) {
|
||||
// stTrace("vgId:%d meta-rlock", pMeta->vgId);
|
||||
// stTrace("vgId:%d meta-rlock", pMeta->vgId);
|
||||
taosThreadRwlockRdlock(&pMeta->lock);
|
||||
}
|
||||
|
||||
void streamMetaRUnLock(SStreamMeta* pMeta) {
|
||||
// stTrace("vgId:%d meta-runlock", pMeta->vgId);
|
||||
// stTrace("vgId:%d meta-runlock", pMeta->vgId);
|
||||
int32_t code = taosThreadRwlockUnlock(&pMeta->lock);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
stError("vgId:%d meta-runlock failed, code:%d", pMeta->vgId, code);
|
||||
} else {
|
||||
// stTrace("vgId:%d meta-runlock completed", pMeta->vgId);
|
||||
// stTrace("vgId:%d meta-runlock completed", pMeta->vgId);
|
||||
}
|
||||
}
|
||||
|
||||
void streamMetaWLock(SStreamMeta* pMeta) {
|
||||
// stTrace("vgId:%d meta-wlock", pMeta->vgId);
|
||||
// stTrace("vgId:%d meta-wlock", pMeta->vgId);
|
||||
taosThreadRwlockWrlock(&pMeta->lock);
|
||||
// stTrace("vgId:%d meta-wlock completed", pMeta->vgId);
|
||||
// stTrace("vgId:%d meta-wlock completed", pMeta->vgId);
|
||||
}
|
||||
|
||||
void streamMetaWUnLock(SStreamMeta* pMeta) {
|
||||
// stTrace("vgId:%d meta-wunlock", pMeta->vgId);
|
||||
// stTrace("vgId:%d meta-wunlock", pMeta->vgId);
|
||||
taosThreadRwlockUnlock(&pMeta->lock);
|
||||
}
|
||||
|
||||
|
|
@ -1395,7 +1395,7 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader)
|
|||
pMeta->sendMsgBeforeClosing = true;
|
||||
}
|
||||
|
||||
pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER;
|
||||
pMeta->role = (isLeader) ? NODE_ROLE_LEADER : NODE_ROLE_FOLLOWER;
|
||||
streamMetaWUnLock(pMeta);
|
||||
|
||||
if (isLeader) {
|
||||
|
|
@ -1531,8 +1531,8 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) {
|
|||
|
||||
bool streamMetaAllTasksReady(const SStreamMeta* pMeta) {
|
||||
int32_t num = taosArrayGetSize(pMeta->pTaskList);
|
||||
for(int32_t i = 0; i < num; ++i) {
|
||||
STaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i);
|
||||
for (int32_t i = 0; i < num; ++i) {
|
||||
STaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i);
|
||||
SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pTaskId, sizeof(*pTaskId));
|
||||
if (ppTask == NULL) {
|
||||
continue;
|
||||
|
|
@ -1633,7 +1633,7 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
|
|||
pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0;
|
||||
|
||||
stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:0x%x (succ:%d) startTs:%" PRId64
|
||||
", readyTs:%" PRId64 " total elapsed time:%.2fs",
|
||||
", readyTs:%" PRId64 " total elapsed time:%.2fs",
|
||||
pMeta->vgId, numOfTotal, taskId, ready, pStartInfo->startTs, pStartInfo->readyTs,
|
||||
pStartInfo->elapsedTime / 1000.0);
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
#define DEFAULT_MAP_CAPACITY 131072
|
||||
#define DEFAULT_MAP_SIZE (DEFAULT_MAP_CAPACITY * 100)
|
||||
#define ROWS_PER_MILLISECOND 1
|
||||
#define MAX_NUM_SCALABLE_BF 100000
|
||||
#define MAX_NUM_SCALABLE_BF 64
|
||||
#define MIN_NUM_SCALABLE_BF 10
|
||||
#define DEFAULT_PREADD_BUCKET 1
|
||||
#define MAX_INTERVAL MILLISECOND_PER_MINUTE
|
||||
|
|
@ -81,7 +81,9 @@ static int64_t adjustInterval(int64_t interval, int32_t precision) {
|
|||
static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t watermark) {
|
||||
if (watermark <= adjInterval) {
|
||||
watermark = TMAX(originInt / adjInterval, 1) * adjInterval;
|
||||
} else if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) {
|
||||
}
|
||||
|
||||
if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) {
|
||||
watermark = MAX_NUM_SCALABLE_BF * adjInterval;
|
||||
}
|
||||
return watermark;
|
||||
|
|
|
|||
|
|
@ -1,40 +1,104 @@
|
|||
|
||||
MESSAGE(STATUS "build stream unit test")
|
||||
|
||||
# GoogleTest requires at least C++11
|
||||
SET(CMAKE_CXX_STANDARD 11)
|
||||
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
|
||||
|
||||
# bloomFilterTest
|
||||
ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp")
|
||||
|
||||
TARGET_LINK_LIBRARIES(streamUpdateTest
|
||||
PUBLIC os util common gtest gtest_main stream executor index
|
||||
#TARGET_LINK_LIBRARIES(streamUpdateTest
|
||||
#PUBLIC os util common gtest gtest_main stream executor index
|
||||
#)
|
||||
|
||||
#TARGET_INCLUDE_DIRECTORIES(
|
||||
#streamUpdateTest
|
||||
#PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
|
||||
#PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
|
||||
#)
|
||||
|
||||
#ADD_EXECUTABLE(checkpointTest checkpointTest.cpp)
|
||||
#TARGET_LINK_LIBRARIES(
|
||||
#checkpointTest
|
||||
#PUBLIC os common gtest stream executor qcom index transport util
|
||||
#)
|
||||
|
||||
#TARGET_INCLUDE_DIRECTORIES(
|
||||
#checkpointTest
|
||||
#PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
|
||||
#)
|
||||
|
||||
#add_executable(backendTest "")
|
||||
|
||||
#target_sources(backendTest
|
||||
#PRIVATE
|
||||
#"backendTest.cpp"
|
||||
#)
|
||||
|
||||
#TARGET_LINK_LIBRARIES(
|
||||
#backendTest
|
||||
#PUBLIC rocksdb
|
||||
#PUBLIC os common gtest stream executor qcom index transport util
|
||||
#)
|
||||
|
||||
#TARGET_INCLUDE_DIRECTORIES(
|
||||
#backendTest
|
||||
#PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
|
||||
#PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
|
||||
#)
|
||||
|
||||
#add_test(
|
||||
#NAME streamUpdateTest
|
||||
#COMMAND streamUpdateTest
|
||||
#)
|
||||
|
||||
#add_test(
|
||||
#NAME checkpointTest
|
||||
#COMMAND checkpointTest
|
||||
#)
|
||||
#add_test(
|
||||
#NAME backendTest
|
||||
#COMMAND backendTest
|
||||
#)
|
||||
|
||||
|
||||
#add_executable(backendTest "")
|
||||
|
||||
#target_sources(backendTest
|
||||
#PUBLIC
|
||||
#"backendTest.cpp"
|
||||
#)
|
||||
|
||||
#target_include_directories(
|
||||
#backendTest
|
||||
#PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
|
||||
#PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
|
||||
#)
|
||||
|
||||
#target_link_libraries(
|
||||
#backendTest
|
||||
#PUBLIC rocksdb
|
||||
#PUBLIC os common gtest stream executor qcom index transport util
|
||||
#)
|
||||
|
||||
|
||||
MESSAGE(STATUS "build parser unit test")
|
||||
|
||||
IF(NOT TD_DARWIN)
|
||||
# GoogleTest requires at least C++11
|
||||
SET(CMAKE_CXX_STANDARD 11)
|
||||
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
|
||||
|
||||
ADD_EXECUTABLE(backendTest ${SOURCE_LIST})
|
||||
TARGET_LINK_LIBRARIES(
|
||||
backendTest
|
||||
PUBLIC rocksdb
|
||||
PUBLIC os common gtest stream executor qcom index transport util vnode
|
||||
)
|
||||
|
||||
TARGET_INCLUDE_DIRECTORIES(
|
||||
streamUpdateTest
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
|
||||
PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
|
||||
)
|
||||
TARGET_INCLUDE_DIRECTORIES(
|
||||
backendTest
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
|
||||
PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
|
||||
)
|
||||
|
||||
ADD_EXECUTABLE(checkpointTest checkpointTest.cpp)
|
||||
TARGET_LINK_LIBRARIES(
|
||||
checkpointTest
|
||||
PUBLIC os common gtest stream executor qcom index transport util
|
||||
)
|
||||
|
||||
TARGET_INCLUDE_DIRECTORIES(
|
||||
checkpointTest
|
||||
PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
|
||||
)
|
||||
|
||||
add_test(
|
||||
NAME streamUpdateTest
|
||||
COMMAND streamUpdateTest
|
||||
)
|
||||
|
||||
add_test(
|
||||
NAME checkpointTest
|
||||
COMMAND checkpointTest
|
||||
)
|
||||
ADD_TEST(
|
||||
NAME backendTest
|
||||
COMMAND backendTest
|
||||
)
|
||||
ENDIF ()
|
||||
|
|
@ -0,0 +1,437 @@
|
|||
#include <gtest/gtest.h>
|
||||
|
||||
#include <taoserror.h>
|
||||
#include <tglobal.h>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include "streamBackendRocksdb.h"
|
||||
#include "streamSnapshot.h"
|
||||
#include "streamState.h"
|
||||
#include "tstream.h"
|
||||
#include "tstreamFileState.h"
|
||||
#include "tstreamUpdate.h"
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wwrite-strings"
|
||||
#pragma GCC diagnostic ignored "-Wunused-function"
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||
#pragma GCC diagnostic ignored "-Wformat"
|
||||
#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
|
||||
#pragma GCC diagnostic ignored "-Wpointer-arith"
|
||||
|
||||
class BackendEnv : public ::testing::Test {
|
||||
protected:
|
||||
virtual void SetUp() {}
|
||||
virtual void TearDown() {}
|
||||
};
|
||||
|
||||
void *backendCreate() {
|
||||
const char *streamPath = "/tmp";
|
||||
void *p = NULL;
|
||||
|
||||
// char *absPath = NULL;
|
||||
// // SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(streamPath, -1, 2);
|
||||
// STaskDbWrapper *p = taskDbOpen((char *)streamPath, (char *)"stream-backend", -1);
|
||||
// ASSERT(p != NULL);
|
||||
return p;
|
||||
}
|
||||
|
||||
SStreamState *stateCreate(const char *path) {
|
||||
SStreamTask *pTask = (SStreamTask *)taosMemoryCalloc(1, sizeof(SStreamTask));
|
||||
pTask->ver = 1024;
|
||||
pTask->id.streamId = 1023;
|
||||
pTask->id.taskId = 1111111;
|
||||
SStreamMeta *pMeta = streamMetaOpen((path), NULL, NULL, 0, 0, NULL);
|
||||
pTask->pMeta = pMeta;
|
||||
|
||||
SStreamState *p = streamStateOpen((char *)path, pTask, true, 32, 32 * 1024);
|
||||
ASSERT(p != NULL);
|
||||
return p;
|
||||
}
|
||||
void *backendOpen() {
|
||||
streamMetaInit();
|
||||
const char *path = "/tmp/backend";
|
||||
SStreamState *p = stateCreate(path);
|
||||
ASSERT(p != NULL);
|
||||
|
||||
// write bacth
|
||||
// default/state/fill/sess/func/parname/partag
|
||||
int32_t size = 100;
|
||||
std::vector<int64_t> tsArray;
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
int64_t ts = taosGetTimestampMs();
|
||||
SWinKey key; // = {.groupId = (uint64_t)(i), .ts = ts};
|
||||
key.groupId = (uint64_t)(i);
|
||||
key.ts = ts;
|
||||
const char *val = "value data";
|
||||
int32_t vlen = strlen(val);
|
||||
streamStatePut_rocksdb(p, &key, (char *)val, vlen);
|
||||
|
||||
tsArray.push_back(ts);
|
||||
}
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
int64_t ts = tsArray[i];
|
||||
SWinKey key = {0}; //{.groupId = (uint64_t)(i), .ts = ts};
|
||||
key.groupId = (uint64_t)(i);
|
||||
key.ts = ts;
|
||||
|
||||
const char *val = "value data";
|
||||
int32_t len = 0;
|
||||
char *newVal = NULL;
|
||||
streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
|
||||
ASSERT(len == strlen(val));
|
||||
}
|
||||
int64_t ts = tsArray[0];
|
||||
SWinKey key = {0}; // {.groupId = (uint64_t)(0), .ts = ts};
|
||||
key.groupId = (uint64_t)(0);
|
||||
key.ts = ts;
|
||||
|
||||
streamStateDel_rocksdb(p, &key);
|
||||
|
||||
streamStateClear_rocksdb(p);
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
int64_t ts = tsArray[i];
|
||||
SWinKey key = {0}; //{.groupId = (uint64_t)(i), .ts = ts};
|
||||
key.groupId = (uint64_t)(i);
|
||||
key.ts = ts;
|
||||
|
||||
const char *val = "value data";
|
||||
int32_t len = 0;
|
||||
char *newVal = NULL;
|
||||
int32_t code = streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
|
||||
ASSERT(code != 0);
|
||||
}
|
||||
tsArray.clear();
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
int64_t ts = taosGetTimestampMs();
|
||||
tsArray.push_back(ts);
|
||||
|
||||
SWinKey key = {0}; //{.groupId = (uint64_t)(i), .ts = ts};
|
||||
key.groupId = (uint64_t)(i);
|
||||
key.ts = ts;
|
||||
|
||||
const char *val = "value data";
|
||||
int32_t vlen = strlen(val);
|
||||
streamStatePut_rocksdb(p, &key, (char *)val, vlen);
|
||||
}
|
||||
|
||||
SWinKey winkey;
|
||||
int32_t code = streamStateGetFirst_rocksdb(p, &key);
|
||||
ASSERT(code == 0);
|
||||
ASSERT(key.ts == tsArray[0]);
|
||||
|
||||
SStreamStateCur *pCurr = streamStateSeekToLast_rocksdb(p);
|
||||
ASSERT(pCurr != NULL);
|
||||
streamStateFreeCur(pCurr);
|
||||
|
||||
winkey.groupId = 0;
|
||||
winkey.ts = tsArray[0];
|
||||
char *val = NULL;
|
||||
int32_t len = 0;
|
||||
|
||||
pCurr = streamStateSeekKeyNext_rocksdb(p, &winkey);
|
||||
ASSERT(pCurr != NULL);
|
||||
|
||||
streamStateFreeCur(pCurr);
|
||||
|
||||
tsArray.clear();
|
||||
for (int i = 0; i < size; i++) {
|
||||
int64_t ts = taosGetTimestampMs();
|
||||
tsArray.push_back(ts);
|
||||
STupleKey key = {0};
|
||||
key.groupId = (uint64_t)(0); //= {.groupId = (uint64_t)(0), .ts = ts, .exprIdx = i};
|
||||
key.ts = ts;
|
||||
key.exprIdx = i;
|
||||
|
||||
const char *val = "Value";
|
||||
int32_t len = strlen(val);
|
||||
streamStateFuncPut_rocksdb(p, &key, val, len);
|
||||
}
|
||||
for (int i = 0; i < size; i++) {
|
||||
STupleKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
|
||||
key.groupId = (uint64_t)(0);
|
||||
key.ts = tsArray[i];
|
||||
key.exprIdx = i;
|
||||
|
||||
char *val = NULL;
|
||||
int32_t len = 0;
|
||||
streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len);
|
||||
ASSERT(len == strlen("Value"));
|
||||
}
|
||||
for (int i = 0; i < size; i++) {
|
||||
STupleKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
|
||||
key.groupId = (uint64_t)(0);
|
||||
key.ts = tsArray[i];
|
||||
key.exprIdx = i;
|
||||
|
||||
char *val = NULL;
|
||||
int32_t len = 0;
|
||||
streamStateFuncDel_rocksdb(p, &key);
|
||||
}
|
||||
|
||||
// session put
|
||||
tsArray.clear();
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
SSessionKey key = {0}; //{.win = {.skey = i, .ekey = i}, .groupId = (uint64_t)(0)};
|
||||
key.win.skey = i;
|
||||
key.win.ekey = i;
|
||||
key.groupId = (uint64_t)(0);
|
||||
tsArray.push_back(i);
|
||||
|
||||
const char *val = "Value";
|
||||
int32_t len = strlen(val);
|
||||
streamStateSessionPut_rocksdb(p, &key, val, len);
|
||||
|
||||
char *pval = NULL;
|
||||
ASSERT(0 == streamStateSessionGet_rocksdb(p, &key, (void **)&pval, &len));
|
||||
ASSERT(strncmp(pval, val, len) == 0);
|
||||
}
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
SSessionKey key = {0}; //{.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)};
|
||||
key.win.skey = tsArray[i];
|
||||
key.win.ekey = tsArray[i];
|
||||
key.groupId = (uint64_t)(0);
|
||||
|
||||
const char *val = "Value";
|
||||
int32_t len = strlen(val);
|
||||
|
||||
char *pval = NULL;
|
||||
ASSERT(0 == streamStateSessionGet_rocksdb(p, &key, (void **)&pval, &len));
|
||||
ASSERT(strncmp(pval, val, len) == 0);
|
||||
taosMemoryFreeClear(pval);
|
||||
}
|
||||
|
||||
pCurr = streamStateSessionSeekToLast_rocksdb(p, 0);
|
||||
ASSERT(pCurr != NULL);
|
||||
|
||||
{
|
||||
SSessionKey key;
|
||||
memset(&key, 0, sizeof(key));
|
||||
char *val = NULL;
|
||||
int32_t vlen = 0;
|
||||
code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen);
|
||||
ASSERT(code == 0);
|
||||
pCurr = streamStateSessionSeekKeyPrev_rocksdb(p, &key);
|
||||
|
||||
code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen);
|
||||
ASSERT(code == 0);
|
||||
|
||||
ASSERT(key.groupId == 0 && key.win.ekey == tsArray[tsArray.size() - 2]);
|
||||
|
||||
pCurr = streamStateSessionSeekKeyNext_rocksdb(p, &key);
|
||||
code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen);
|
||||
ASSERT(code == 0);
|
||||
ASSERT(vlen == strlen("Value"));
|
||||
ASSERT(key.groupId == 0 && key.win.skey == tsArray[tsArray.size() - 1]);
|
||||
|
||||
ASSERT(0 == streamStateSessionAddIfNotExist_rocksdb(p, &key, 10, (void **)&val, &len));
|
||||
|
||||
ASSERT(0 ==
|
||||
streamStateStateAddIfNotExist_rocksdb(p, &key, (char *)"key", strlen("key"), NULL, (void **)&val, &len));
|
||||
}
|
||||
for (int i = 0; i < size; i++) {
|
||||
SSessionKey key = {0}; //{.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)};
|
||||
key.win.skey = tsArray[i];
|
||||
key.win.ekey = tsArray[i];
|
||||
key.groupId = (uint64_t)(0);
|
||||
|
||||
const char *val = "Value";
|
||||
int32_t len = strlen(val);
|
||||
|
||||
char *pval = NULL;
|
||||
ASSERT(0 == streamStateSessionDel_rocksdb(p, &key));
|
||||
}
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]};
|
||||
key.groupId = (uint64_t)(i);
|
||||
key.ts = tsArray[i];
|
||||
const char *val = "Value";
|
||||
int32_t vlen = strlen(val);
|
||||
ASSERT(streamStateFillPut_rocksdb(p, &key, val, vlen) == 0);
|
||||
}
|
||||
for (int i = 0; i < size; i++) {
|
||||
SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]};
|
||||
key.groupId = (uint64_t)(i);
|
||||
key.ts = tsArray[i];
|
||||
char *val = NULL;
|
||||
int32_t vlen = 0;
|
||||
ASSERT(streamStateFillGet_rocksdb(p, &key, (void **)&val, &vlen) == 0);
|
||||
taosMemoryFreeClear(val);
|
||||
}
|
||||
{
|
||||
SWinKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[0]};
|
||||
key.groupId = (uint64_t)(0);
|
||||
key.ts = tsArray[0];
|
||||
SStreamStateCur *pCurr = streamStateFillGetCur_rocksdb(p, &key);
|
||||
ASSERT(pCurr != NULL);
|
||||
|
||||
char *val = NULL;
|
||||
int32_t vlen = 0;
|
||||
ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen));
|
||||
ASSERT(vlen == strlen("Value"));
|
||||
streamStateFreeCur(pCurr);
|
||||
|
||||
pCurr = streamStateFillSeekKeyNext_rocksdb(p, &key);
|
||||
ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen));
|
||||
ASSERT(vlen == strlen("Value") && key.groupId == 1 && key.ts == tsArray[1]);
|
||||
|
||||
key.groupId = 1;
|
||||
key.ts = tsArray[1];
|
||||
|
||||
pCurr = streamStateFillSeekKeyPrev_rocksdb(p, &key);
|
||||
ASSERT(pCurr != NULL);
|
||||
ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen));
|
||||
|
||||
ASSERT(vlen == strlen("Value") && key.groupId == 0 && key.ts == tsArray[0]);
|
||||
}
|
||||
|
||||
for (int i = 0; i < size - 1; i++) {
|
||||
SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]};
|
||||
key.groupId = (uint64_t)(i);
|
||||
key.ts = tsArray[i];
|
||||
char *val = NULL;
|
||||
int32_t vlen = 0;
|
||||
ASSERT(streamStateFillDel_rocksdb(p, &key) == 0);
|
||||
taosMemoryFreeClear(val);
|
||||
}
|
||||
streamStateSessionClear_rocksdb(p);
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
char tbname[TSDB_TABLE_NAME_LEN] = {0};
|
||||
sprintf(tbname, "%s_%d", "tbname", i);
|
||||
ASSERT(0 == streamStatePutParName_rocksdb(p, i, tbname));
|
||||
}
|
||||
for (int i = 0; i < size; i++) {
|
||||
char *val = NULL;
|
||||
ASSERT(0 == streamStateGetParName_rocksdb(p, i, (void **)&val));
|
||||
ASSERT(strncmp(val, "tbname", strlen("tbname")) == 0);
|
||||
taosMemoryFree(val);
|
||||
}
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
char tbname[TSDB_TABLE_NAME_LEN] = {0};
|
||||
sprintf(tbname, "%s_%d", "tbname", i);
|
||||
ASSERT(0 == streamStatePutParName_rocksdb(p, i, tbname));
|
||||
}
|
||||
for (int i = 0; i < size; i++) {
|
||||
char *val = NULL;
|
||||
ASSERT(0 == streamStateGetParName_rocksdb(p, i, (void **)&val));
|
||||
ASSERT(strncmp(val, "tbname", strlen("tbname")) == 0);
|
||||
taosMemoryFree(val);
|
||||
}
|
||||
for (int i = 0; i < size; i++) {
|
||||
char key[128] = {0};
|
||||
sprintf(key, "tbname_%d", i);
|
||||
char val[128] = {0};
|
||||
sprintf(val, "val_%d", i);
|
||||
code = streamDefaultPut_rocksdb(p, key, val, strlen(val));
|
||||
ASSERT(code == 0);
|
||||
}
|
||||
for (int i = 0; i < size; i++) {
|
||||
char key[128] = {0};
|
||||
sprintf(key, "tbname_%d", i);
|
||||
|
||||
char *val = NULL;
|
||||
int32_t len = 0;
|
||||
code = streamDefaultGet_rocksdb(p, key, (void **)&val, &len);
|
||||
ASSERT(code == 0);
|
||||
}
|
||||
SArray *result = taosArrayInit(8, sizeof(void *));
|
||||
streamDefaultIterGet_rocksdb(p, "tbname", "tbname_99", result);
|
||||
ASSERT(taosArrayGetSize(result) >= 0);
|
||||
|
||||
return p;
|
||||
// streamStateClose((SStreamState *)p, true);
|
||||
}
|
||||
TEST_F(BackendEnv, checkOpen) {
|
||||
SStreamState *p = (SStreamState *)backendOpen();
|
||||
int64_t tsStart = taosGetTimestampMs();
|
||||
{
|
||||
void *pBatch = streamStateCreateBatch();
|
||||
int32_t size = 0;
|
||||
for (int i = 0; i < size; i++) {
|
||||
char key[128] = {0};
|
||||
sprintf(key, "key_%d", i);
|
||||
char val[128] = {0};
|
||||
sprintf(val, "val_%d", i);
|
||||
streamStatePutBatch(p, "default", (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val,
|
||||
(int32_t)(strlen(val)), tsStart + 100000);
|
||||
}
|
||||
streamStatePutBatch_rocksdb(p, pBatch);
|
||||
streamStateDestroyBatch(pBatch);
|
||||
}
|
||||
{
|
||||
void *pBatch = streamStateCreateBatch();
|
||||
int32_t size = 0;
|
||||
char valBuf[256] = {0};
|
||||
for (int i = 0; i < size; i++) {
|
||||
char key[128] = {0};
|
||||
sprintf(key, "key_%d", i);
|
||||
char val[128] = {0};
|
||||
sprintf(val, "val_%d", i);
|
||||
streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val,
|
||||
(int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf);
|
||||
}
|
||||
streamStatePutBatch_rocksdb(p, pBatch);
|
||||
streamStateDestroyBatch(pBatch);
|
||||
}
|
||||
// do checkpoint 2
|
||||
taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2);
|
||||
{
|
||||
void *pBatch = streamStateCreateBatch();
|
||||
int32_t size = 0;
|
||||
char valBuf[256] = {0};
|
||||
for (int i = 0; i < size; i++) {
|
||||
char key[128] = {0};
|
||||
sprintf(key, "key_%d", i);
|
||||
char val[128] = {0};
|
||||
sprintf(val, "val_%d", i);
|
||||
streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val,
|
||||
(int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf);
|
||||
}
|
||||
streamStatePutBatch_rocksdb(p, pBatch);
|
||||
streamStateDestroyBatch(pBatch);
|
||||
}
|
||||
|
||||
taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 3);
|
||||
|
||||
const char *path = "/tmp/backend/stream";
|
||||
const char *dump = "/tmp/backend/stream/dump";
|
||||
// taosMkDir(dump);
|
||||
taosMulMkDir(dump);
|
||||
SBkdMgt *mgt = bkdMgtCreate((char *)path);
|
||||
SArray *result = taosArrayInit(4, sizeof(void *));
|
||||
bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump);
|
||||
|
||||
bkdMgtDestroy(mgt);
|
||||
streamStateClose((SStreamState *)p, true);
|
||||
taosRemoveDir(path);
|
||||
}
|
||||
|
||||
TEST_F(BackendEnv, backendChkp) { const char *path = "/tmp"; }
|
||||
|
||||
typedef struct BdKV {
|
||||
uint32_t k;
|
||||
uint32_t v;
|
||||
} BdKV;
|
||||
|
||||
BdKV kvDict[] = {{0, 2}, {1, 2}, {15, 16}, {31, 32}, {56, 64}, {100, 128},
|
||||
{200, 256}, {500, 512}, {1000, 1024}, {2000, 2048}, {3000, 4096}};
|
||||
|
||||
TEST_F(BackendEnv, backendUtil) {
|
||||
for (int i = 0; i < sizeof(kvDict) / sizeof(kvDict[0]); i++) {
|
||||
ASSERT_EQ(nextPow2((uint32_t)(kvDict[i].k)), kvDict[i].v);
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
|
|
@ -25,46 +25,49 @@
|
|||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||
|
||||
// tsSnodeAddress = "";
|
||||
// tsS3StreamEnabled = 0;
|
||||
|
||||
#include "cos.h"
|
||||
#include "rsync.h"
|
||||
#include "streamInt.h"
|
||||
#include "cos.h"
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
// int main(int argc, char **argv) {
|
||||
// testing::InitGoogleTest(&argc, argv);
|
||||
|
||||
if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) {
|
||||
printf("error");
|
||||
}
|
||||
if (s3Init() < 0) {
|
||||
return -1;
|
||||
}
|
||||
strcpy(tsSnodeAddress, "127.0.0.1");
|
||||
int ret = RUN_ALL_TESTS();
|
||||
s3CleanUp();
|
||||
return ret;
|
||||
}
|
||||
// if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) {
|
||||
// printf("error");
|
||||
// }
|
||||
// if (s3Init() < 0) {
|
||||
// return -1;
|
||||
// }
|
||||
// strcpy(tsSnodeAddress, "127.0.0.1");
|
||||
// int ret = RUN_ALL_TESTS();
|
||||
// s3CleanUp();
|
||||
// return ret;
|
||||
// }
|
||||
|
||||
TEST(testCase, checkpointUpload_Test) {
|
||||
stopRsync();
|
||||
startRsync();
|
||||
// stopRsync();
|
||||
// startRsync();
|
||||
|
||||
taosSsleep(5);
|
||||
char* id = "2013892036";
|
||||
|
||||
uploadCheckpoint(id, "/root/offset/");
|
||||
// uploadCheckpoint(id, "/root/offset/");
|
||||
}
|
||||
|
||||
TEST(testCase, checkpointDownload_Test) {
|
||||
char* id = "2013892036";
|
||||
downloadCheckpoint(id, "/root/offset/download/");
|
||||
// downloadCheckpoint(id, "/root/offset/download/");
|
||||
}
|
||||
|
||||
TEST(testCase, checkpointDelete_Test) {
|
||||
char* id = "2013892036";
|
||||
deleteCheckpoint(id);
|
||||
// deleteCheckpoint(id);
|
||||
}
|
||||
|
||||
TEST(testCase, checkpointDeleteFile_Test) {
|
||||
char* id = "2013892036";
|
||||
deleteCheckpointFile(id, "offset-ver0");
|
||||
// deleteCheckpointFile(id, "offset-ver0");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,10 +14,7 @@ class StreamStateEnv : public ::testing::Test {
|
|||
streamMetaInit();
|
||||
backend = streamBackendInit(path, 0, 0);
|
||||
}
|
||||
virtual void TearDown() {
|
||||
streamMetaCleanup();
|
||||
// indexClose(index);
|
||||
}
|
||||
virtual void TearDown() { streamMetaCleanup(); }
|
||||
|
||||
const char *path = TD_TMP_DIR_PATH "stream";
|
||||
void *backend;
|
||||
|
|
@ -50,6 +47,14 @@ bool equalSBF(SScalableBf *left, SScalableBf *right) {
|
|||
}
|
||||
|
||||
TEST(TD_STREAM_UPDATE_TEST, update) {
|
||||
const char *streamPath = "/tmp";
|
||||
|
||||
char *absPath = NULL;
|
||||
void *p = NULL;
|
||||
// SBackendWrapper *p = streamBackendInit(streamPath, -1, 2);
|
||||
// p = taskDbOpen((char *)streamPath, (char *)"test", -1);
|
||||
p = bkdMgtCreate((char *)streamPath);
|
||||
|
||||
// const int64_t interval = 20 * 1000;
|
||||
// const int64_t watermark = 10 * 60 * 1000;
|
||||
// SUpdateInfo *pSU = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark);
|
||||
|
|
|
|||
|
|
@ -1343,7 +1343,7 @@ ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->raftCfg
|
|||
int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) {
|
||||
int32_t ret = 0;
|
||||
if (syncIsInit()) {
|
||||
taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, pSyncNode, syncEnv()->pTimerManager,
|
||||
taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, (void*)pSyncNode->rid, syncEnv()->pTimerManager,
|
||||
&pSyncNode->pPingTimer);
|
||||
atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser);
|
||||
} else {
|
||||
|
|
@ -1415,8 +1415,8 @@ void syncNodeResetElectTimer(SSyncNode* pSyncNode) {
|
|||
static int32_t syncNodeDoStartHeartbeatTimer(SSyncNode* pSyncNode) {
|
||||
int32_t ret = 0;
|
||||
if (syncIsInit()) {
|
||||
taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, syncEnv()->pTimerManager,
|
||||
&pSyncNode->pHeartbeatTimer);
|
||||
taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, (void*)pSyncNode->rid,
|
||||
syncEnv()->pTimerManager, &pSyncNode->pHeartbeatTimer);
|
||||
atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser);
|
||||
} else {
|
||||
sError("vgId:%d, start heartbeat timer error, sync env is stop", pSyncNode->vgId);
|
||||
|
|
@ -2153,7 +2153,11 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex
|
|||
static void syncNodeEqPingTimer(void* param, void* tmrId) {
|
||||
if (!syncIsInit()) return;
|
||||
|
||||
SSyncNode* pNode = param;
|
||||
int64_t rid = (int64_t)param;
|
||||
SSyncNode* pNode = syncNodeAcquire(rid);
|
||||
|
||||
if (pNode == NULL) return;
|
||||
|
||||
if (atomic_load_64(&pNode->pingTimerLogicClockUser) <= atomic_load_64(&pNode->pingTimerLogicClock)) {
|
||||
SRpcMsg rpcMsg = {0};
|
||||
int32_t code = syncBuildTimeout(&rpcMsg, SYNC_TIMEOUT_PING, atomic_load_64(&pNode->pingTimerLogicClock),
|
||||
|
|
@ -2173,7 +2177,8 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) {
|
|||
}
|
||||
|
||||
_out:
|
||||
taosTmrReset(syncNodeEqPingTimer, pNode->pingTimerMS, pNode, syncEnv()->pTimerManager, &pNode->pPingTimer);
|
||||
taosTmrReset(syncNodeEqPingTimer, pNode->pingTimerMS, (void*)pNode->rid, syncEnv()->pTimerManager,
|
||||
&pNode->pPingTimer);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2224,7 +2229,11 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) {
|
|||
static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {
|
||||
if (!syncIsInit()) return;
|
||||
|
||||
SSyncNode* pNode = param;
|
||||
int64_t rid = (int64_t)param;
|
||||
SSyncNode* pNode = syncNodeAcquire(rid);
|
||||
|
||||
if (pNode == NULL) return;
|
||||
|
||||
if (pNode->totalReplicaNum > 1) {
|
||||
if (atomic_load_64(&pNode->heartbeatTimerLogicClockUser) <= atomic_load_64(&pNode->heartbeatTimerLogicClock)) {
|
||||
SRpcMsg rpcMsg = {0};
|
||||
|
|
@ -2245,7 +2254,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {
|
|||
}
|
||||
|
||||
_out:
|
||||
taosTmrReset(syncNodeEqHeartbeatTimer, pNode->heartbeatTimerMS, pNode, syncEnv()->pTimerManager,
|
||||
taosTmrReset(syncNodeEqHeartbeatTimer, pNode->heartbeatTimerMS, (void*)pNode->rid, syncEnv()->pTimerManager,
|
||||
&pNode->pHeartbeatTimer);
|
||||
|
||||
} else {
|
||||
|
|
@ -3385,4 +3394,4 @@ bool syncNodeCanChange(SSyncNode* pSyncNode) {
|
|||
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -316,7 +316,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
|
|||
}
|
||||
|
||||
// 3. Try to Recycle a page
|
||||
if (!pPage && !pCache->lru.pLruPrev->isAnchor) {
|
||||
if (!pPageH && !pPage && !pCache->lru.pLruPrev->isAnchor) {
|
||||
pPage = pCache->lru.pLruPrev;
|
||||
tdbPCacheRemovePageFromHash(pCache, pPage);
|
||||
tdbPCachePinPage(pCache, pPage);
|
||||
|
|
|
|||
|
|
@ -256,21 +256,21 @@ void transAsyncPoolDestroy(SAsyncPool* pool);
|
|||
int transAsyncSend(SAsyncPool* pool, queue* mq);
|
||||
bool transAsyncPoolIsEmpty(SAsyncPool* pool);
|
||||
|
||||
#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc) \
|
||||
do { \
|
||||
for (int i = 0; i < pool->nAsync; i++) { \
|
||||
uv_async_t* async = &(pool->asyncs[i]); \
|
||||
SAsyncItem* item = async->data; \
|
||||
while (!QUEUE_IS_EMPTY(&item->qmsg)) { \
|
||||
tTrace("destroy msg in async pool "); \
|
||||
queue* h = QUEUE_HEAD(&item->qmsg); \
|
||||
QUEUE_REMOVE(h); \
|
||||
msgType* msg = QUEUE_DATA(h, msgType, q); \
|
||||
if (msg != NULL) { \
|
||||
freeFunc(msg); \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc, param) \
|
||||
do { \
|
||||
for (int i = 0; i < pool->nAsync; i++) { \
|
||||
uv_async_t* async = &(pool->asyncs[i]); \
|
||||
SAsyncItem* item = async->data; \
|
||||
while (!QUEUE_IS_EMPTY(&item->qmsg)) { \
|
||||
tTrace("destroy msg in async pool "); \
|
||||
queue* h = QUEUE_HEAD(&item->qmsg); \
|
||||
QUEUE_REMOVE(h); \
|
||||
msgType* msg = QUEUE_DATA(h, msgType, q); \
|
||||
if (msg != NULL) { \
|
||||
freeFunc(msg, param); \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define ASYNC_CHECK_HANDLE(exh1, id) \
|
||||
|
|
|
|||
|
|
@ -191,6 +191,15 @@ static void httpDestroyMsg(SHttpMsg* msg) {
|
|||
taosMemoryFree(msg->cont);
|
||||
taosMemoryFree(msg);
|
||||
}
|
||||
static void httpDestroyMsgWrapper(void* cont, void* param) {
|
||||
httpDestroyMsg((SHttpMsg*)cont);
|
||||
// if (msg == NULL) return;
|
||||
|
||||
// taosMemoryFree(msg->server);
|
||||
// taosMemoryFree(msg->uri);
|
||||
// taosMemoryFree(msg->cont);
|
||||
// taosMemoryFree(msg);
|
||||
}
|
||||
|
||||
static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) {
|
||||
SHttpMsg *msg = NULL, *quitMsg = NULL;
|
||||
|
|
@ -554,7 +563,7 @@ void transHttpEnvDestroy() {
|
|||
httpSendQuit();
|
||||
taosThreadJoin(load->thread, NULL);
|
||||
|
||||
TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg);
|
||||
TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsgWrapper, NULL);
|
||||
transAsyncPoolDestroy(load->asyncPool);
|
||||
uv_loop_close(load->loop);
|
||||
taosMemoryFree(load->loop);
|
||||
|
|
|
|||
|
|
@ -219,6 +219,8 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq,
|
|||
/// NULL,cliHandleUpdate};
|
||||
|
||||
static FORCE_INLINE void destroyCmsg(void* cmsg);
|
||||
|
||||
static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param);
|
||||
static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg);
|
||||
static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst);
|
||||
static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx);
|
||||
|
|
@ -1963,7 +1965,17 @@ static FORCE_INLINE void destroyCmsg(void* arg) {
|
|||
transFreeMsg(pMsg->msg.pCont);
|
||||
taosMemoryFree(pMsg);
|
||||
}
|
||||
|
||||
static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param) {
|
||||
SCliMsg* pMsg = arg;
|
||||
if (pMsg == NULL) {
|
||||
return;
|
||||
}
|
||||
if (param != NULL) {
|
||||
SCliThrd* pThrd = param;
|
||||
if (pThrd->destroyAhandleFp) (*pThrd->destroyAhandleFp)(pMsg->msg.info.ahandle);
|
||||
}
|
||||
destroyCmsg(pMsg);
|
||||
}
|
||||
static FORCE_INLINE void destroyCmsgAndAhandle(void* param) {
|
||||
if (param == NULL) return;
|
||||
|
||||
|
|
@ -2057,7 +2069,7 @@ static void destroyThrdObj(SCliThrd* pThrd) {
|
|||
taosThreadJoin(pThrd->thread, NULL);
|
||||
CLI_RELEASE_UV(pThrd->loop);
|
||||
taosThreadMutexDestroy(&pThrd->msgMtx);
|
||||
TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsg);
|
||||
TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsgWrapper, (void*)pThrd);
|
||||
transAsyncPoolDestroy(pThrd->asyncPool);
|
||||
|
||||
transDQDestroy(pThrd->delayQueue, destroyCmsgAndAhandle);
|
||||
|
|
|
|||
|
|
@ -159,7 +159,7 @@ static void uvStartSendResp(SSvrMsg* msg);
|
|||
|
||||
static void uvNotifyLinkBrokenToApp(SSvrConn* conn);
|
||||
|
||||
static FORCE_INLINE void destroySmsg(SSvrMsg* smsg);
|
||||
static FORCE_INLINE void destroySmsg(SSvrMsg* smsg);
|
||||
static FORCE_INLINE SSvrConn* createConn(void* hThrd);
|
||||
static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/);
|
||||
static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn);
|
||||
|
|
@ -671,7 +671,8 @@ static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) {
|
|||
transFreeMsg(smsg->msg.pCont);
|
||||
taosMemoryFree(smsg);
|
||||
}
|
||||
static void destroyAllConn(SWorkThrd* pThrd) {
|
||||
static FORCE_INLINE void destroySmsgWrapper(void* smsg, void* param) { destroySmsg((SSvrMsg*)smsg); }
|
||||
static void destroyAllConn(SWorkThrd* pThrd) {
|
||||
tTrace("thread %p destroy all conn ", pThrd);
|
||||
while (!QUEUE_IS_EMPTY(&pThrd->conn)) {
|
||||
queue* h = QUEUE_HEAD(&pThrd->conn);
|
||||
|
|
@ -1394,7 +1395,7 @@ void destroyWorkThrd(SWorkThrd* pThrd) {
|
|||
}
|
||||
taosThreadJoin(pThrd->thread, NULL);
|
||||
SRV_RELEASE_UV(pThrd->loop);
|
||||
TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsg);
|
||||
TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL);
|
||||
transAsyncPoolDestroy(pThrd->asyncPool);
|
||||
|
||||
uvWhiteListDestroy(pThrd->pWhiteList);
|
||||
|
|
|
|||
|
|
@ -89,12 +89,14 @@ static int32_t taosArrayResize(SArray* pArray) {
|
|||
int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) {
|
||||
if (newCap > pArray->capacity) {
|
||||
float factor = BOUNDARY_BIG_FACTOR;
|
||||
if(newCap * pArray->elemSize > BOUNDARY_SIZE){
|
||||
if (newCap * pArray->elemSize > BOUNDARY_SIZE) {
|
||||
factor = BOUNDARY_SMALL_FACTOR;
|
||||
}
|
||||
|
||||
size_t tsize = (pArray->capacity * factor);
|
||||
while (newCap > tsize) {
|
||||
tsize = (tsize * factor);
|
||||
size_t newSize = (tsize * factor);
|
||||
tsize = (newSize == tsize) ? (tsize + 2) : newSize;
|
||||
}
|
||||
|
||||
pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize);
|
||||
|
|
|
|||
|
|
@ -329,6 +329,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VIEW_NOT_EXIST, "view not exists in db
|
|||
|
||||
//mnode-compact
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COMPACT_ID, "Invalid compact id")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST, "compact detail doesn't exist")
|
||||
|
||||
// dnode
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_OFFLINE, "Dnode is offline")
|
||||
|
|
|
|||
|
|
@ -21,6 +21,40 @@
|
|||
int64_t tsRpcQueueMemoryAllowed = 0;
|
||||
int64_t tsRpcQueueMemoryUsed = 0;
|
||||
|
||||
struct STaosQueue {
|
||||
STaosQnode *head;
|
||||
STaosQnode *tail;
|
||||
STaosQueue *next; // for queue set
|
||||
STaosQset *qset; // for queue set
|
||||
void *ahandle; // for queue set
|
||||
FItem itemFp;
|
||||
FItems itemsFp;
|
||||
TdThreadMutex mutex;
|
||||
int64_t memOfItems;
|
||||
int32_t numOfItems;
|
||||
int64_t threadId;
|
||||
int64_t memLimit;
|
||||
int64_t itemLimit;
|
||||
};
|
||||
|
||||
struct STaosQset {
|
||||
STaosQueue *head;
|
||||
STaosQueue *current;
|
||||
TdThreadMutex mutex;
|
||||
tsem_t sem;
|
||||
int32_t numOfQueues;
|
||||
int32_t numOfItems;
|
||||
};
|
||||
|
||||
struct STaosQall {
|
||||
STaosQnode *current;
|
||||
STaosQnode *start;
|
||||
int32_t numOfItems;
|
||||
int64_t memOfItems;
|
||||
int32_t unAccessedNumOfItems;
|
||||
int64_t unAccessMemOfItems;
|
||||
};
|
||||
|
||||
void taosSetQueueMemoryCapacity(STaosQueue *queue, int64_t cap) { queue->memLimit = cap; }
|
||||
void taosSetQueueCapacity(STaosQueue *queue, int64_t size) { queue->itemLimit = size; }
|
||||
|
||||
|
|
@ -497,6 +531,12 @@ int64_t taosQallUnAccessedMemSize(STaosQall *qall) { return qall->unAccessMemOfI
|
|||
void taosResetQitems(STaosQall *qall) { qall->current = qall->start; }
|
||||
int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; }
|
||||
|
||||
void taosQueueSetThreadId(STaosQueue* pQueue, int64_t threadId) {
|
||||
pQueue->threadId = threadId;
|
||||
}
|
||||
|
||||
int64_t taosQueueGetThreadId(STaosQueue *pQueue) { return pQueue->threadId; }
|
||||
|
||||
#if 0
|
||||
|
||||
void taosResetQsetThread(STaosQset *qset, void *pItem) {
|
||||
|
|
|
|||
|
|
@ -20,6 +20,9 @@
|
|||
|
||||
#define DEFAULT_GROWTH 2
|
||||
#define DEFAULT_TIGHTENING_RATIO 0.5
|
||||
#define DEFAULT_MAX_BLOOMFILTERS 4
|
||||
#define SBF_INVALID -1
|
||||
#define SBF_VALID 0
|
||||
|
||||
static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate);
|
||||
|
||||
|
|
@ -32,6 +35,8 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) {
|
|||
if (pSBf == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS;
|
||||
pSBf->status = SBF_VALID;
|
||||
pSBf->numBits = 0;
|
||||
pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *));
|
||||
if (tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO) == NULL) {
|
||||
|
|
@ -45,6 +50,9 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) {
|
|||
}
|
||||
|
||||
int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
|
||||
if (pSBf->status == SBF_INVALID) {
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
int32_t size = taosArrayGetSize(pSBf->bfArray);
|
||||
SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
|
||||
ASSERT(pNormalBf);
|
||||
|
|
@ -52,6 +60,7 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le
|
|||
pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth,
|
||||
pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO);
|
||||
if (pNormalBf == NULL) {
|
||||
pSBf->status = SBF_INVALID;
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
}
|
||||
|
|
@ -59,6 +68,9 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le
|
|||
}
|
||||
|
||||
int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
|
||||
if (pSBf->status == SBF_INVALID) {
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len);
|
||||
uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len);
|
||||
int32_t size = taosArrayGetSize(pSBf->bfArray);
|
||||
|
|
@ -74,6 +86,7 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
|
|||
pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth,
|
||||
pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO);
|
||||
if (pNormalBf == NULL) {
|
||||
pSBf->status = SBF_INVALID;
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
}
|
||||
|
|
@ -81,6 +94,9 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
|
|||
}
|
||||
|
||||
int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
|
||||
if (pSBf->status == SBF_INVALID) {
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len);
|
||||
uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len);
|
||||
int32_t size = taosArrayGetSize(pSBf->bfArray);
|
||||
|
|
@ -93,6 +109,10 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32
|
|||
}
|
||||
|
||||
static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate) {
|
||||
if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
SBloomFilter *pNormalBf = tBloomFilterInit(expectedEntries, errorRate);
|
||||
if (pNormalBf == NULL) {
|
||||
return NULL;
|
||||
|
|
@ -128,6 +148,8 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) {
|
|||
}
|
||||
if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1;
|
||||
if (tEncodeU64(pEncoder, pSBf->numBits) < 0) return -1;
|
||||
if (tEncodeU32(pEncoder, pSBf->maxBloomFilters) < 0) return -1;
|
||||
if (tEncodeI8(pEncoder, pSBf->status) < 0) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -150,6 +172,8 @@ SScalableBf *tScalableBfDecode(SDecoder *pDecoder) {
|
|||
}
|
||||
if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error;
|
||||
if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) goto _error;
|
||||
if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) goto _error;
|
||||
if (tDecodeI8(pDecoder, &pSBf->status) < 0) goto _error;
|
||||
return pSBf;
|
||||
|
||||
_error:
|
||||
|
|
|
|||
|
|
@ -417,9 +417,9 @@ _OVER:
|
|||
return NULL;
|
||||
} else {
|
||||
while (worker->pid <= 0) taosMsleep(10);
|
||||
queue->threadId = worker->pid;
|
||||
uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle,
|
||||
queue->threadId);
|
||||
|
||||
taosQueueSetThreadId(queue, worker->pid);
|
||||
uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, worker->pid);
|
||||
return queue;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -238,7 +238,8 @@
|
|||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True
|
||||
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1
|
||||
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db-removewal.py -N 2 -n 1
|
||||
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb-removewal.py -N 6 -n 3
|
||||
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1
|
||||
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3
|
||||
#,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3
|
||||
|
|
@ -1085,6 +1086,7 @@
|
|||
,,y,script,./test.sh -f tsim/parser/join_multivnode.sim
|
||||
,,y,script,./test.sh -f tsim/parser/join.sim
|
||||
,,y,script,./test.sh -f tsim/parser/last_cache.sim
|
||||
,,y,script,./test.sh -f tsim/parser/last_both.sim
|
||||
,,y,script,./test.sh -f tsim/parser/last_groupby.sim
|
||||
,,y,script,./test.sh -f tsim/parser/lastrow.sim
|
||||
,,y,script,./test.sh -f tsim/parser/lastrow2.sim
|
||||
|
|
@ -1211,6 +1213,7 @@
|
|||
,,y,script,./test.sh -f tsim/stream/deleteState.sim
|
||||
,,y,script,./test.sh -f tsim/stream/distributeInterval0.sim
|
||||
,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim
|
||||
,,y,script,./test.sh -f tsim/stream/distributeMultiLevelInterval0.sim
|
||||
,,y,script,./test.sh -f tsim/stream/distributeSession0.sim
|
||||
,,y,script,./test.sh -f tsim/stream/drop_stream.sim
|
||||
,,y,script,./test.sh -f tsim/stream/event0.sim
|
||||
|
|
|
|||
|
|
@ -0,0 +1,150 @@
|
|||
system sh/stop_dnodes.sh
|
||||
system sh/deploy.sh -n dnode1 -i 1
|
||||
system sh/exec.sh -n dnode1 -s start
|
||||
sql connect
|
||||
|
||||
print ======================== dnode1 start
|
||||
$db = testdb
|
||||
sql drop database if exists $db
|
||||
sql create database $db cachemodel 'both' minrows 10 stt_trigger 1
|
||||
sql use $db
|
||||
|
||||
sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int)
|
||||
sql create table tb1 using st2 tags (1);
|
||||
sql create table tb2 using st2 tags (2);
|
||||
sql create table tb3 using st2 tags (3);
|
||||
sql create table tb4 using st2 tags (4);
|
||||
sql create table tb5 using st2 tags (1);
|
||||
sql create table tb6 using st2 tags (2);
|
||||
sql create table tb7 using st2 tags (3);
|
||||
sql create table tb8 using st2 tags (4);
|
||||
sql create table tb9 using st2 tags (5);
|
||||
sql create table tba using st2 tags (5);
|
||||
sql create table tbb using st2 tags (5);
|
||||
sql create table tbc using st2 tags (5);
|
||||
sql create table tbd using st2 tags (5);
|
||||
sql create table tbe using st2 tags (5);
|
||||
sql create table tbf using st2 tags (5);
|
||||
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.000",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.001",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.002",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.003",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.004",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.005",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.006",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.007",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.008",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.009",28, 29, '30', -1005)
|
||||
sql delete from tb9 where ts = "2021-05-09 10:12:26.000"
|
||||
sql flush database $db
|
||||
|
||||
sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000)
|
||||
sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000)
|
||||
sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000)
|
||||
|
||||
sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3', -1001)
|
||||
sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001)
|
||||
sql insert into tb2 values ("2021-05-11 10:11:15",-6, -7, '-8', -3001)
|
||||
|
||||
sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002)
|
||||
sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002)
|
||||
sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002)
|
||||
|
||||
sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003)
|
||||
sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003)
|
||||
sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003)
|
||||
|
||||
sql insert into tb5 values ("2021-05-09 10:12:22",19, 20, '21', -1004)
|
||||
sql insert into tb6 values ("2021-05-11 10:12:23",22, 23, NULL, -2004)
|
||||
sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004)
|
||||
sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004)
|
||||
|
||||
sql insert into tba values ("2021-05-10 10:12:27",31, 32, NULL, -2005)
|
||||
sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005)
|
||||
sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005)
|
||||
sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL )
|
||||
|
||||
sql drop table tbf;
|
||||
sql alter table st2 add column c1 int;
|
||||
sql alter table st2 drop column c1;
|
||||
|
||||
run tsim/parser/last_cache_query.sim
|
||||
|
||||
sql flush database $db
|
||||
system sh/exec.sh -n dnode1 -s stop -x SIGINT
|
||||
system sh/exec.sh -n dnode1 -s start
|
||||
|
||||
run tsim/parser/last_cache_query.sim
|
||||
|
||||
system sh/exec.sh -n dnode1 -s stop -x SIGINT
|
||||
system sh/exec.sh -n dnode1 -s start
|
||||
|
||||
sql drop database if exists $db
|
||||
sql create database $db minrows 10 stt_trigger 1
|
||||
sql use $db
|
||||
|
||||
sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int)
|
||||
sql create table tb1 using st2 tags (1);
|
||||
sql create table tb2 using st2 tags (2);
|
||||
sql create table tb3 using st2 tags (3);
|
||||
sql create table tb4 using st2 tags (4);
|
||||
sql create table tb5 using st2 tags (1);
|
||||
sql create table tb6 using st2 tags (2);
|
||||
sql create table tb7 using st2 tags (3);
|
||||
sql create table tb8 using st2 tags (4);
|
||||
sql create table tb9 using st2 tags (5);
|
||||
sql create table tba using st2 tags (5);
|
||||
sql create table tbb using st2 tags (5);
|
||||
sql create table tbc using st2 tags (5);
|
||||
sql create table tbd using st2 tags (5);
|
||||
sql create table tbe using st2 tags (5);
|
||||
sql create table tbf using st2 tags (5);
|
||||
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.000",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.001",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.002",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.003",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.004",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.005",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.006",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.007",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.008",28, 29, '30', -1005)
|
||||
sql insert into tb9 values ("2021-05-09 10:12:26.009",28, 29, '30', -1005)
|
||||
sql delete from tb9 where ts = "2021-05-09 10:12:26.000"
|
||||
sql flush database $db
|
||||
|
||||
sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000)
|
||||
sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000)
|
||||
sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000)
|
||||
|
||||
sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3', -1001)
|
||||
sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001)
|
||||
sql insert into tb2 values ("2021-05-11 10:11:15",-6, -7, '-8', -3001)
|
||||
|
||||
sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002)
|
||||
sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002)
|
||||
sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002)
|
||||
|
||||
sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003)
|
||||
sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003)
|
||||
sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003)
|
||||
|
||||
sql insert into tb5 values ("2021-05-09 10:12:22",19, 20, '21', -1004)
|
||||
sql insert into tb6 values ("2021-05-11 10:12:23",22, 23, NULL, -2004)
|
||||
sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004)
|
||||
sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004)
|
||||
|
||||
sql insert into tba values ("2021-05-10 10:12:27",31, 32, NULL, -2005)
|
||||
sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005)
|
||||
sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005)
|
||||
sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL )
|
||||
|
||||
sql drop table tbf
|
||||
sql alter database $db cachemodel 'both'
|
||||
sql alter database $db cachesize 2
|
||||
sleep 11000
|
||||
|
||||
run tsim/parser/last_cache_query.sim
|
||||
|
||||
system sh/exec.sh -n dnode1 -s stop -x SIGINT
|
||||
|
|
@ -357,6 +357,112 @@ if $data45 != 5 then
|
|||
return -1
|
||||
endi
|
||||
|
||||
sql select last_row(*), id from st2 group by id order by id
|
||||
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
|
||||
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
|
||||
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
|
||||
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
|
||||
print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49
|
||||
|
||||
if $rows != 5 then
|
||||
return -1
|
||||
endi
|
||||
if $data00 != @21-05-12 10:10:12.000@ then
|
||||
return -1
|
||||
endi
|
||||
if $data01 != 6 then
|
||||
return -1
|
||||
endi
|
||||
if $data02 != NULL then
|
||||
print $data02
|
||||
return -1
|
||||
endi
|
||||
if $data03 != NULL then
|
||||
return -1
|
||||
endi
|
||||
if $data04 != @70-01-01 07:59:57.000@ then
|
||||
return -1
|
||||
endi
|
||||
if $data05 != 1 then
|
||||
return -1
|
||||
endi
|
||||
if $data10 != @21-05-11 10:12:23.000@ then
|
||||
return -1
|
||||
endi
|
||||
if $data11 != 22 then
|
||||
return -1
|
||||
endi
|
||||
if $data12 != 23.000000000 then
|
||||
print $data02
|
||||
return -1
|
||||
endi
|
||||
if $data13 != NULL then
|
||||
return -1
|
||||
endi
|
||||
if $data14 != @70-01-01 07:59:58.-04@ then
|
||||
return -1
|
||||
endi
|
||||
if $data15 != 2 then
|
||||
return -1
|
||||
endi
|
||||
if $data20 != @21-05-10 10:12:24.000@ then
|
||||
return -1
|
||||
endi
|
||||
if $data21 != 24 then
|
||||
return -1
|
||||
endi
|
||||
if $data22 != NULL then
|
||||
print expect NULL actual: $data22
|
||||
return -1
|
||||
endi
|
||||
if $data23 != 25 then
|
||||
return -1
|
||||
endi
|
||||
if $data24 != @70-01-01 07:59:57.-04@ then =
|
||||
return -1
|
||||
endi
|
||||
if $data25 != 3 then
|
||||
return -1
|
||||
endi
|
||||
if $data30 != @21-05-11 10:12:25.000@ then
|
||||
return -1
|
||||
endi
|
||||
if $data31 != 26 then
|
||||
return -1
|
||||
endi
|
||||
if $data32 != NULL then
|
||||
print $data02
|
||||
return -1
|
||||
endi
|
||||
if $data33 != 27 then
|
||||
return -1
|
||||
endi
|
||||
if $data34 != @70-01-01 07:59:56.-04@ then
|
||||
return -1
|
||||
endi
|
||||
if $data35 != 4 then
|
||||
return -1
|
||||
endi
|
||||
if $data40 != @21-05-11 10:12:29.000@ then
|
||||
return -1
|
||||
endi
|
||||
if $data41 != 36 then
|
||||
return -1
|
||||
endi
|
||||
if $data42 != 37.000000000 then
|
||||
print $data02
|
||||
return -1
|
||||
endi
|
||||
if $data43 != NULL then
|
||||
return -1
|
||||
endi
|
||||
if $data44 != @70-01-01 07:59:56.-05@ then
|
||||
return -1
|
||||
endi
|
||||
if $data45 != 5 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
print "test tbn"
|
||||
sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp)
|
||||
sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000)
|
||||
|
|
@ -386,3 +492,5 @@ if $data04 != @70-01-01 07:59:57.000@ then
|
|||
return -1
|
||||
endi
|
||||
|
||||
sql alter table tbn add column c1 int;
|
||||
sql alter table tbn drop column c1;
|
||||
|
|
|
|||
|
|
@ -9,8 +9,6 @@ print =============== create database
|
|||
sql create database test vgroups 1;
|
||||
sql use test;
|
||||
|
||||
sql alter local 'disableCount' '0' ;
|
||||
|
||||
sql create table t1(ts timestamp, a int, b int , c int, d double);
|
||||
|
||||
sql insert into t1 values(1648791213000,0,1,1,1.0);
|
||||
|
|
|
|||
|
|
@ -9,8 +9,6 @@ print =============== create database
|
|||
sql create database test vgroups 4;
|
||||
sql use test;
|
||||
|
||||
sql alter local 'disableCount' '0' ;
|
||||
|
||||
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
|
||||
sql create table t1 using st tags(1,1,1);
|
||||
sql create table t2 using st tags(2,2,2);
|
||||
|
|
|
|||
|
|
@ -9,8 +9,6 @@ print =============== create database
|
|||
sql create database test vgroups 1;
|
||||
sql use test;
|
||||
|
||||
sql alter local 'disableCount' '0' ;
|
||||
|
||||
sql create table t1(ts timestamp, a int, b int , c int, d double);
|
||||
|
||||
sql insert into t1 values(1648791213000,0,1,1,1.0);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,267 @@
|
|||
system sh/stop_dnodes.sh
|
||||
system sh/deploy.sh -n dnode1 -i 1
|
||||
|
||||
system sh/cfg.sh -n dnode1 -c streamAggCnt -v 2
|
||||
|
||||
system sh/exec.sh -n dnode1 -s start
|
||||
sleep 50
|
||||
sql connect
|
||||
|
||||
|
||||
|
||||
print ===== step1
|
||||
sql drop stream if exists streams1;
|
||||
sql drop database if exists test;
|
||||
sql create database test vgroups 4;
|
||||
sql use test;
|
||||
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
|
||||
sql create table ts1 using st tags(1,1,1);
|
||||
sql create table ts2 using st tags(2,2,2);
|
||||
sql create table ts3 using st tags(3,2,2);
|
||||
sql create table ts4 using st tags(4,2,2);
|
||||
sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4 from st interval(10s);
|
||||
|
||||
sleep 1000
|
||||
|
||||
sql insert into ts1 values(1648791213000,1,1,3,4.1);
|
||||
sql insert into ts1 values(1648791223000,2,2,3,1.1);
|
||||
sql insert into ts1 values(1648791233000,3,3,3,2.1);
|
||||
sql insert into ts1 values(1648791243000,4,4,3,3.1);
|
||||
|
||||
sql insert into ts2 values(1648791213000,1,5,3,4.1);
|
||||
sql insert into ts2 values(1648791223000,2,6,3,1.1);
|
||||
sql insert into ts2 values(1648791233000,3,7,3,2.1);
|
||||
sql insert into ts2 values(1648791243000,4,8,3,3.1);
|
||||
|
||||
|
||||
$loop_count = 0
|
||||
loop0:
|
||||
|
||||
$loop_count = $loop_count + 1
|
||||
if $loop_count == 10 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sleep 1000
|
||||
print 2 select * from streamt1;
|
||||
sql select * from streamt1;
|
||||
|
||||
print $data00 $data01 $data02 $data03
|
||||
print $data10 $data11 $data12 $data13
|
||||
print $data20 $data21 $data22 $data23
|
||||
print $data30 $data31 $data32 $data33
|
||||
print $data40 $data41 $data42 $data43
|
||||
|
||||
if $rows != 4 then
|
||||
print =====rows=$rows
|
||||
goto loop0
|
||||
endi
|
||||
|
||||
if $data01 != 2 then
|
||||
print =====data01=$data01
|
||||
goto loop0
|
||||
endi
|
||||
|
||||
if $data11 != 2 then
|
||||
print =====data11=$data11
|
||||
goto loop0
|
||||
endi
|
||||
|
||||
if $data21 != 2 then
|
||||
print =====data21=$data21
|
||||
goto loop0
|
||||
endi
|
||||
|
||||
if $data31 != 2 then
|
||||
print =====data31=$data31
|
||||
goto loop0
|
||||
endi
|
||||
|
||||
|
||||
sql insert into ts1 values(1648791213000,1,9,3,4.1);
|
||||
|
||||
$loop_count = 0
|
||||
loop1:
|
||||
|
||||
$loop_count = $loop_count + 1
|
||||
if $loop_count == 10 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sleep 1000
|
||||
print 2 select * from streamt1;
|
||||
sql select * from streamt1;
|
||||
|
||||
print $data00 $data01 $data02 $data03
|
||||
print $data10 $data11 $data12 $data13
|
||||
print $data20 $data21 $data22 $data23
|
||||
print $data30 $data31 $data32 $data33
|
||||
print $data40 $data41 $data42 $data43
|
||||
|
||||
if $rows != 4 then
|
||||
print =====rows=$rows
|
||||
goto loop1
|
||||
endi
|
||||
|
||||
if $data01 != 2 then
|
||||
print =====data01=$data01
|
||||
goto loop1
|
||||
endi
|
||||
|
||||
if $data11 != 2 then
|
||||
print =====data11=$data11
|
||||
goto loop1
|
||||
endi
|
||||
|
||||
if $data21 != 2 then
|
||||
print =====data21=$data21
|
||||
goto loop1
|
||||
endi
|
||||
|
||||
if $data31 != 2 then
|
||||
print =====data31=$data31
|
||||
goto loop1
|
||||
endi
|
||||
|
||||
sql delete from ts2 where ts = 1648791243000 ;
|
||||
|
||||
$loop_count = 0
|
||||
loop2:
|
||||
|
||||
$loop_count = $loop_count + 1
|
||||
if $loop_count == 10 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sleep 1000
|
||||
print 2 select * from streamt1;
|
||||
sql select * from streamt1;
|
||||
|
||||
print $data00 $data01 $data02 $data03
|
||||
print $data10 $data11 $data12 $data13
|
||||
print $data20 $data21 $data22 $data23
|
||||
print $data30 $data31 $data32 $data33
|
||||
print $data40 $data41 $data42 $data43
|
||||
|
||||
if $rows != 4 then
|
||||
print =====rows=$rows
|
||||
goto loop2
|
||||
endi
|
||||
|
||||
if $data01 != 2 then
|
||||
print =====data01=$data01
|
||||
goto loop2
|
||||
endi
|
||||
|
||||
if $data11 != 2 then
|
||||
print =====data11=$data11
|
||||
goto loop2
|
||||
endi
|
||||
|
||||
if $data21 != 2 then
|
||||
print =====data21=$data21
|
||||
goto loop2
|
||||
endi
|
||||
|
||||
if $data31 != 1 then
|
||||
print =====data31=$data31
|
||||
goto loop2
|
||||
endi
|
||||
|
||||
sql delete from ts2 where ts = 1648791223000 ;
|
||||
|
||||
$loop_count = 0
|
||||
loop3:
|
||||
|
||||
$loop_count = $loop_count + 1
|
||||
if $loop_count == 10 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sleep 1000
|
||||
print 2 select * from streamt1;
|
||||
sql select * from streamt1;
|
||||
|
||||
print $data00 $data01 $data02 $data03
|
||||
print $data10 $data11 $data12 $data13
|
||||
print $data20 $data21 $data22 $data23
|
||||
print $data30 $data31 $data32 $data33
|
||||
print $data40 $data41 $data42 $data43
|
||||
|
||||
if $rows != 4 then
|
||||
print =====rows=$rows
|
||||
goto loop3
|
||||
endi
|
||||
|
||||
if $data01 != 2 then
|
||||
print =====data01=$data01
|
||||
goto loop3
|
||||
endi
|
||||
|
||||
if $data11 != 1 then
|
||||
print =====data11=$data11
|
||||
goto loop3
|
||||
endi
|
||||
|
||||
if $data21 != 2 then
|
||||
print =====data21=$data21
|
||||
goto loop3
|
||||
endi
|
||||
|
||||
if $data31 != 1 then
|
||||
print =====data31=$data31
|
||||
goto loop3
|
||||
endi
|
||||
|
||||
|
||||
sql insert into ts1 values(1648791233001,3,9,3,2.1);
|
||||
|
||||
$loop_count = 0
|
||||
loop4:
|
||||
|
||||
$loop_count = $loop_count + 1
|
||||
if $loop_count == 10 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sleep 1000
|
||||
print 2 select * from streamt1;
|
||||
sql select * from streamt1;
|
||||
|
||||
print $data00 $data01 $data02 $data03
|
||||
print $data10 $data11 $data12 $data13
|
||||
print $data20 $data21 $data22 $data23
|
||||
print $data30 $data31 $data32 $data33
|
||||
print $data40 $data41 $data42 $data43
|
||||
|
||||
if $rows != 4 then
|
||||
print =====rows=$rows
|
||||
goto loop4
|
||||
endi
|
||||
|
||||
if $data01 != 2 then
|
||||
print =====data01=$data01
|
||||
goto loop4
|
||||
endi
|
||||
|
||||
if $data11 != 1 then
|
||||
print =====data11=$data11
|
||||
goto loop4
|
||||
endi
|
||||
|
||||
if $data21 != 3 then
|
||||
print =====data21=$data21
|
||||
goto loop4
|
||||
endi
|
||||
|
||||
if $data31 != 1 then
|
||||
print =====data31=$data31
|
||||
goto loop4
|
||||
endi
|
||||
|
||||
sql select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5, avg(d) from st interval(10s);
|
||||
|
||||
|
||||
print ===== over
|
||||
|
||||
system sh/stop_dnodes.sh
|
||||
|
|
@ -122,135 +122,7 @@ class TDTestCase:
|
|||
tdLog.debug(f"redistributeSql:{redistributeSql}")
|
||||
tdSql.query(redistributeSql)
|
||||
tdLog.debug("redistributeSql ok")
|
||||
|
||||
def tmqCase1(self):
|
||||
tdLog.printNoPrefix("======== test case 1: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 1000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 60,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 0}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
# expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
# tdSql.query(queryString)
|
||||
# expectRowsList.append(tdSql.getRows())
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 0
|
||||
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
tdLog.info("create ctb1")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
|
||||
|
||||
tmqCom.getStartConsumeNotifyFromTmqsim()
|
||||
tmqCom.getStartCommitNotifyFromTmqsim()
|
||||
|
||||
#restart dnode & remove wal
|
||||
self.restartAndRemoveWal()
|
||||
|
||||
# redistribute vgroup
|
||||
self.redistributeVgroups();
|
||||
|
||||
tdLog.info("create ctb2")
|
||||
paraDict['ctbPrefix'] = "ctbn"
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict)
|
||||
pInsertThread.join()
|
||||
pInsertThread1.join()
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
if expectrowcnt / 2 > resultList[0]:
|
||||
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0]))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
# tmqCom.checkFileContent(consumerId, queryString)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 1 end ...... ")
|
||||
|
||||
def tmqCase2(self):
|
||||
tdLog.printNoPrefix("======== test case 2: ")
|
||||
paraDict = {'dbName':'dbt'}
|
||||
|
||||
ntbName = "ntb"
|
||||
|
||||
topicNameList = ['topic2']
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName)
|
||||
tdLog.info("create nomal table sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
|
||||
tdLog.info("create topics from nomal table")
|
||||
queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName)
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
#restart dnode & remove wal
|
||||
self.restartAndRemoveWal()
|
||||
|
||||
# redistribute vgroup
|
||||
self.redistributeVgroups();
|
||||
|
||||
sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName)
|
||||
tdLog.info("alter table sql: %s"%sqlString)
|
||||
tdSql.error(sqlString)
|
||||
expectRows = 0
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
time.sleep(1)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 2 end ...... ")
|
||||
|
||||
|
||||
def tmqCase3(self):
|
||||
tdLog.printNoPrefix("======== test case 3: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
|
|
@ -330,12 +202,90 @@ class TDTestCase:
|
|||
|
||||
tdLog.printNoPrefix("======== test case 3 end ...... ")
|
||||
|
||||
def tmqCaseDbname(self):
|
||||
tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stbn',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 1000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 10,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 0}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic4']
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create stb")
|
||||
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
|
||||
|
||||
tdLog.info("create ctb")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
|
||||
ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
|
||||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("create topics from database ")
|
||||
queryString = "database %s "%(paraDict['dbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 0
|
||||
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
time.sleep(1)
|
||||
# restart dnode & remove wal
|
||||
self.restartAndRemoveWal()
|
||||
|
||||
# redistribute vgroup
|
||||
self.redistributeVgroups()
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
expectRows = 2
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
time.sleep(6)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ")
|
||||
|
||||
def run(self):
|
||||
self.prepareTestEnv()
|
||||
self.tmqCase1()
|
||||
self.tmqCase2()
|
||||
self.prepareTestEnv()
|
||||
self.tmqCase3()
|
||||
self.prepareTestEnv()
|
||||
self.tmqCaseDbname()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
|
|
@ -0,0 +1,266 @@
|
|||
|
||||
import taos
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import os
|
||||
import threading
|
||||
import math
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
from util.common import *
|
||||
from util.cluster import *
|
||||
sys.path.append("./7-tmq")
|
||||
from tmqCommon import *
|
||||
|
||||
class TDTestCase:
|
||||
def __init__(self):
|
||||
self.vgroups = 1
|
||||
self.ctbNum = 10
|
||||
self.rowsPerTbl = 1000
|
||||
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor(), True)
|
||||
|
||||
def getDataPath(self):
|
||||
selfPath = tdCom.getBuildPath()
|
||||
|
||||
return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*';
|
||||
|
||||
def prepareTestEnv(self):
|
||||
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 1000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 60,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 0}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
tdCom.drop_all_db()
|
||||
tmqCom.initConsumerTable()
|
||||
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar)
|
||||
tdLog.info("create stb")
|
||||
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
|
||||
# tdLog.info("create ctb")
|
||||
# tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
# ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
# tdLog.info("insert data")
|
||||
# tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
|
||||
# ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
|
||||
# startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
# tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
# tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def restartAndRemoveWal(self):
|
||||
tdDnodes = cluster.dnodes
|
||||
tdSql.query("select * from information_schema.ins_vnodes")
|
||||
for result in tdSql.queryResult:
|
||||
if result[2] == 'dbt':
|
||||
tdLog.debug("dnode is %d"%(result[0]))
|
||||
dnodeId = result[0]
|
||||
vnodeId = result[1]
|
||||
|
||||
tdDnodes[dnodeId - 1].stoptaosd()
|
||||
time.sleep(1)
|
||||
dataPath = self.getDataPath()
|
||||
dataPath = dataPath%(dnodeId,vnodeId)
|
||||
os.system('rm -rf ' + dataPath)
|
||||
tdLog.debug("dataPath:%s"%dataPath)
|
||||
tdDnodes[dnodeId - 1].starttaosd()
|
||||
time.sleep(1)
|
||||
break
|
||||
tdLog.debug("restart dnode ok")
|
||||
|
||||
def redistributeVgroups(self):
|
||||
dnodesList = []
|
||||
tdSql.query("show dnodes")
|
||||
for result in tdSql.queryResult:
|
||||
dnodesList.append(result[0])
|
||||
print("dnodeList:",dnodesList)
|
||||
tdSql.query("select * from information_schema.ins_vnodes")
|
||||
vnodeId = 0
|
||||
for result in tdSql.queryResult:
|
||||
if result[2] == 'dbt':
|
||||
tdLog.debug("dnode is %d"%(result[0]))
|
||||
dnodesList.remove(result[0])
|
||||
vnodeId = result[1]
|
||||
print("its all data",dnodesList)
|
||||
# if self.replicaVar == 1:
|
||||
# redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0])
|
||||
# else:
|
||||
redistributeSql = f"redistribute vgroup {vnodeId} "
|
||||
for vgdnode in dnodesList:
|
||||
redistributeSql += f"dnode {vgdnode} "
|
||||
print(redistributeSql)
|
||||
|
||||
tdLog.debug(f"redistributeSql:{redistributeSql}")
|
||||
tdSql.query(redistributeSql)
|
||||
tdLog.debug("redistributeSql ok")
|
||||
|
||||
def tmqCase1(self):
|
||||
tdLog.printNoPrefix("======== test case 1: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 1000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 60,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 0}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
# expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
# tdSql.query(queryString)
|
||||
# expectRowsList.append(tdSql.getRows())
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 0
|
||||
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
tdLog.info("create ctb1")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
|
||||
|
||||
tmqCom.getStartConsumeNotifyFromTmqsim()
|
||||
tmqCom.getStartCommitNotifyFromTmqsim()
|
||||
|
||||
#restart dnode & remove wal
|
||||
self.restartAndRemoveWal()
|
||||
|
||||
# redistribute vgroup
|
||||
self.redistributeVgroups();
|
||||
|
||||
tdLog.info("create ctb2")
|
||||
paraDict['ctbPrefix'] = "ctbn"
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict)
|
||||
pInsertThread.join()
|
||||
pInsertThread1.join()
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
if expectrowcnt / 2 > resultList[0]:
|
||||
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0]))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
# tmqCom.checkFileContent(consumerId, queryString)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 1 end ...... ")
|
||||
|
||||
def tmqCase2(self):
|
||||
tdLog.printNoPrefix("======== test case 2: ")
|
||||
paraDict = {'dbName':'dbt'}
|
||||
|
||||
ntbName = "ntb"
|
||||
|
||||
topicNameList = ['topic2']
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName)
|
||||
tdLog.info("create nomal table sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
|
||||
tdLog.info("create topics from nomal table")
|
||||
queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName)
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
#restart dnode & remove wal
|
||||
self.restartAndRemoveWal()
|
||||
|
||||
# redistribute vgroup
|
||||
self.redistributeVgroups();
|
||||
|
||||
sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName)
|
||||
tdLog.info("alter table sql: %s"%sqlString)
|
||||
tdSql.error(sqlString)
|
||||
expectRows = 0
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
time.sleep(1)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 2 end ...... ")
|
||||
|
||||
def run(self):
|
||||
self.prepareTestEnv()
|
||||
self.tmqCase1()
|
||||
self.tmqCase2()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
event = threading.Event()
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -75,7 +75,7 @@ bool shellIsEmptyCommand(const char *cmd) {
|
|||
|
||||
int32_t shellRunSingleCommand(char *command) {
|
||||
shellCmdkilled = false;
|
||||
|
||||
|
||||
if (shellIsEmptyCommand(command)) {
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -1019,7 +1019,7 @@ void shellReadHistory() {
|
|||
|
||||
char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1);
|
||||
int32_t read_size = 0;
|
||||
while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) {
|
||||
while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) > 0) {
|
||||
line[read_size - 1] = '\0';
|
||||
taosMemoryFree(pHistory->hist[pHistory->hend]);
|
||||
pHistory->hist[pHistory->hend] = taosStrdup(line);
|
||||
|
|
@ -1315,7 +1315,7 @@ int32_t shellExecute() {
|
|||
shellSetConn(shell.conn, runOnce);
|
||||
shellReadHistory();
|
||||
|
||||
if(shell.args.is_bi_mode) {
|
||||
if(shell.args.is_bi_mode) {
|
||||
// need set bi mode
|
||||
printf("Set BI mode is true.\n");
|
||||
#ifndef WEBSOCKET
|
||||
|
|
|
|||
|
|
@ -1018,7 +1018,7 @@ int sml_escape_Test() {
|
|||
ASSERT(numFields == 5);
|
||||
ASSERT(strncmp(fields[1].name, "inode\"i,= s_used", sizeof("inode\"i,= s_used") - 1) == 0);
|
||||
ASSERT(strncmp(fields[2].name, "total", sizeof("total") - 1) == 0);
|
||||
ASSERT(strncmp(fields[3].name, "inode\"i,= s_f\\\\ree", sizeof("inode\"i,= s_f\\\\ree") - 1) == 0);
|
||||
ASSERT(strncmp(fields[3].name, "inode\"i,= s_f\\ree", sizeof("inode\"i,= s_f\\ree") - 1) == 0);
|
||||
ASSERT(strncmp(fields[4].name, "dev\"i,= ce", sizeof("dev\"i,= ce") - 1) == 0);
|
||||
|
||||
TAOS_ROW row = NULL;
|
||||
|
|
@ -1044,6 +1044,88 @@ int sml_escape_Test() {
|
|||
return code;
|
||||
}
|
||||
|
||||
// test field with end of escape
|
||||
int sml_escape1_Test() {
|
||||
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
|
||||
|
||||
TAOS_RES *pRes = taos_query(taos, "create database if not exists db_escape");
|
||||
taos_free_result(pRes);
|
||||
|
||||
pRes = taos_query(taos, "use db_escape");
|
||||
taos_free_result(pRes);
|
||||
|
||||
const char *sql[] = {
|
||||
"stab,t1\\=1 c1=3,c2=\"32fw\" 1661943970000000000",
|
||||
"stab,t1=1\\ c1=3,c2=\"32fw\" 1661943980000000000",
|
||||
"stab,t1=1 c1\\=3,c2=\"32fw\" 1661943990000000000",
|
||||
};
|
||||
for(int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++){
|
||||
pRes = taos_schemaless_insert(taos, (char**)&sql[i], 1, TSDB_SML_LINE_PROTOCOL, 0);
|
||||
int code = taos_errno(pRes);
|
||||
ASSERT(code);
|
||||
}
|
||||
|
||||
const char *sql1[] = {
|
||||
"stab\\,t1=1 c1=3,c2=\"32fw\" 1661943960000000000",
|
||||
"stab\\\\,t1=1 c1=3,c2=\"32fw\" 1661943960000000000",
|
||||
"stab,t1\\\\=1 c1=3,c2=\"32fw\" 1661943970000000000",
|
||||
"stab,t1=1\\\\ c1=3,c2=\"32fw\" 1661943980000000000",
|
||||
"stab,t1=1 c1\\\\=3,c2=\"32fw\" 1661943990000000000",
|
||||
};
|
||||
pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, 0);
|
||||
printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes));
|
||||
int code = taos_errno(pRes);
|
||||
ASSERT(!code);
|
||||
ASSERT(taos_affected_rows(pRes) == 5);
|
||||
taos_free_result(pRes);
|
||||
|
||||
pRes = taos_query(taos, "select * from stab"); //check stable name
|
||||
ASSERT(pRes);
|
||||
int fieldNum = taos_field_count(pRes);
|
||||
ASSERT(fieldNum == 6);
|
||||
printf("fieldNum:%d\n", fieldNum);
|
||||
|
||||
int numFields = taos_num_fields(pRes);
|
||||
TAOS_FIELD *fields = taos_fetch_fields(pRes);
|
||||
ASSERT(numFields == 6);
|
||||
ASSERT(strncmp(fields[1].name, "c1", sizeof("c1") - 1) == 0);
|
||||
ASSERT(strncmp(fields[2].name, "c2", sizeof("c2") - 1) == 0);
|
||||
ASSERT(strncmp(fields[3].name, "c1\\", sizeof("c1\\") - 1) == 0);
|
||||
ASSERT(strncmp(fields[4].name, "t1\\", sizeof("t1\\") - 1) == 0);
|
||||
ASSERT(strncmp(fields[5].name, "t1", sizeof("t1") - 1) == 0);
|
||||
|
||||
TAOS_ROW row = NULL;
|
||||
int32_t rowIndex = 0;
|
||||
while ((row = taos_fetch_row(pRes)) != NULL) {
|
||||
int64_t ts = *(int64_t *)row[0];
|
||||
|
||||
if (ts == 1661943970000) {
|
||||
ASSERT(*(double *)row[1] == 3);
|
||||
ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0);
|
||||
ASSERT(row[3] == NULL);
|
||||
ASSERT(strncmp(row[4], "1", sizeof("1") - 1) == 0);
|
||||
ASSERT(row[5] == NULL);
|
||||
}else if (ts == 1661943980000) {
|
||||
ASSERT(*(double *)row[1] == 3);
|
||||
ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0);
|
||||
ASSERT(row[3] == NULL);
|
||||
ASSERT(row[4] == NULL);
|
||||
ASSERT(strncmp(row[5], "1\\", sizeof("1\\") - 1) == 0);
|
||||
}else if (ts == 1661943990000) {
|
||||
ASSERT(row[1] == NULL);
|
||||
ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0);
|
||||
ASSERT(*(double *)row[3] == 3);
|
||||
ASSERT(row[4] == NULL);
|
||||
ASSERT(strncmp(row[5], "1", sizeof("1") - 1) == 0);
|
||||
}
|
||||
rowIndex++;
|
||||
}
|
||||
taos_free_result(pRes);
|
||||
taos_close(taos);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
int sml_19221_Test() {
|
||||
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
|
||||
|
||||
|
|
@ -1775,17 +1857,14 @@ int main(int argc, char *argv[]) {
|
|||
ASSERT(ret);
|
||||
ret = sml_escape_Test();
|
||||
ASSERT(!ret);
|
||||
ret = sml_escape1_Test();
|
||||
ASSERT(!ret);
|
||||
ret = sml_ts3116_Test();
|
||||
ASSERT(!ret);
|
||||
ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file
|
||||
ASSERT(!ret);
|
||||
ret = sml_ts3303_Test();
|
||||
ASSERT(!ret);
|
||||
|
||||
// for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){
|
||||
// printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i])));
|
||||
// }
|
||||
// int ret = 0;
|
||||
ret = sml_ttl_Test();
|
||||
ASSERT(!ret);
|
||||
ret = sml_ts2164_Test();
|
||||
|
|
|
|||
Loading…
Reference in New Issue