diff --git a/cmake/cmake.version b/cmake/cmake.version index 0e35fa316f..4abc854e71 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.2.3.0.alpha") + SET(TD_VER_NUMBER "3.2.4.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/docs/en/08-client-libraries/07-python.mdx b/docs/en/08-client-libraries/07-python.mdx index aacfd0fe53..3110afcf10 100644 --- a/docs/en/08-client-libraries/07-python.mdx +++ b/docs/en/08-client-libraries/07-python.mdx @@ -842,12 +842,12 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) In addition to native connections, the client library also supports subscriptions via websockets. -The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer). +The syntax for creating a consumer is "consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer). ```python import taosws -consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) +consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) ``` @@ -887,13 +887,13 @@ The `poll` function is used to consume data in tmq. The parameter of the `poll` ```python while True: - res = consumer.poll(1) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() + err = message.error() if err is not None: raise err - val = res.value() + val = message.value() for block in val: print(block.fetchall()) @@ -902,16 +902,14 @@ while True: -The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data. +The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. ```python while True: - res = consumer.poll(timeout=1.0) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() - if err is not None: - raise err + for block in message: for row in block: print(row) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index 6f2343d347..e1bf18c854 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -41,16 +41,28 @@ window_clause: { SESSION(ts_col, tol_val) | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] + | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition + | COUNT_WINDOW(count_val[, sliding_val]) } ``` `SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically. +`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns. + +`COUNT_WINDOW` is a counting window that is divided by a fixed number of data rows.`count_val`: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. `count_val` represents the maximum number of data rows contained in each `COUNT_WINDOW`. When the total number of data rows cannot be divided by `count_val`, the number of rows in the last window will be less than `count_val`. `sliding_val`: is a constant that represents the number of window slides, similar to `SLIDING` in `INTERVAL`. + For example, the following SQL statement creates a stream and automatically creates a supertable named `avg_vol`. The stream has a 1 minute time window that slides forward in 30 second intervals to calculate the average voltage of the meters supertable. ```sql CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + +CREATE STREAM streams0 INTO streamt0 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9; + +CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10); ``` ## Partitions of Stream diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index f5a4789976..902e62de73 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t import Release from "/components/ReleaseV3"; +## 3.2.3.0 + + + ## 3.2.2.0 diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index e7160ab094..71dc82316e 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -856,7 +856,7 @@ taosws `Consumer` API 提供了基于 Websocket 订阅 TMQ 数据的 API。创 ```python import taosws -consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) +consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) ``` @@ -896,13 +896,13 @@ Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 flo ```python while True: - res = consumer.poll(1) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() + err = message.error() if err is not None: raise err - val = res.value() + val = message.value() for block in val: print(block.fetchall()) @@ -911,16 +911,14 @@ while True: -Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。 +Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。 ```python while True: - res = consumer.poll(timeout=1.0) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() - if err is not None: - raise err + for block in message: for row in block: print(row) diff --git a/docs/zh/12-taos-sql/12-distinguished.md b/docs/zh/12-taos-sql/12-distinguished.md index 8ae3f900f4..f73919a970 100755 --- a/docs/zh/12-taos-sql/12-distinguished.md +++ b/docs/zh/12-taos-sql/12-distinguished.md @@ -49,6 +49,7 @@ window_clause: { | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [FILL(fill_mod_and_val)] | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition + | COUNT_WINDOW(count_val[, sliding_val]) } ``` @@ -180,6 +181,19 @@ select _wstart, _wend, count(*) from t event_window start with c1 > 0 end with c ![TDengine Database 事件窗口示意图](./event_window.webp) +### 计数窗口 + +计数窗口按固定的数据行数来划分窗口。默认将数据按时间戳排序，再按照count_val的值，将数据划分为多个窗口，然后做聚合计算。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。sliding_val是常量，表示窗口滑动的数量，类似于 interval的SLIDING。 + +以下面的 SQL 语句为例，计数窗口切分如图所示： +```sql +select _wstart, _wend, count(*) from t count_window(4); +``` + +![count_window](https://github.com/taosdata/TDengine/assets/38781207/ff86278e-cbff-477f-a866-6e5cc150b712) + + + ### 时间戳伪列 窗口聚合查询结果中，如果 SQL 语句中没有指定输出查询结果中的时间戳列，那么最终结果中不会自动包含窗口的时间列信息。如果需要在结果中输出聚合结果所对应的时间窗口信息，需要在 SELECT 子句中使用时间戳相关的伪列: 时间窗口起始时间 (\_WSTART), 时间窗口结束时间 (\_WEND), 时间窗口持续时间 (\_WDURATION), 以及查询整体窗口相关的伪列: 查询窗口起始时间(\_QSTART) 和查询窗口结束时间(\_QEND)。需要注意的是时间窗口起始时间和结束时间均是闭区间，时间窗口持续时间是数据当前时间分辨率下的数值。例如，如果当前数据库的时间分辨率是毫秒，那么结果中 500 就表示当前时间窗口的持续时间是 500毫秒 (500 ms)。 diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 979fc436b9..8868b728f8 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -49,10 +49,14 @@ window_clause: { SESSION(ts_col, tol_val) | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] + | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition + | COUNT_WINDOW(count_val[, sliding_val]) } ``` 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 +EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。 +COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished) @@ -61,6 +65,12 @@ window_clause: { ```sql CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + +CREATE STREAM streams0 INTO streamt0 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9; + +CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10); ``` ## 流式计算的 partition diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index b0a81e01a1..1c51f934fe 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.2.3.0 + + + ## 3.2.2.0 diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 469bc6227e..93f17fa887 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -219,7 +219,6 @@ extern bool tsFilterScalarMode; extern int32_t tsMaxStreamBackendCache; extern int32_t tsPQSortMemThreshold; extern int32_t tsResolveFQDNRetryTime; -extern bool tsDisableCount; extern bool tsExperimental; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) @@ -234,10 +233,10 @@ int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer); struct SConfig *taosGetCfg(); -void taosSetAllDebugFlag(int32_t flag); -void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal); -void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); -int8_t taosGranted(int8_t type); +void taosSetGlobalDebugFlag(int32_t flag); +void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal); +void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); +int8_t taosGranted(int8_t type); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 36242ddea8..db2c51fb01 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -581,8 +581,8 @@ typedef struct { }; } SSubmitRsp; -int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); -int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); +// int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); +// int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); // void tFreeSSubmitBlkRsp(void* param); void tFreeSSubmitRsp(SSubmitRsp* pRsp); @@ -885,8 +885,8 @@ typedef struct { int64_t maxStorage; } SCreateAcctReq, SAlterAcctReq; -int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); -int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); +// int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); +// int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); typedef struct { char user[TSDB_USER_LEN]; @@ -3446,7 +3446,7 @@ int32_t tDeserializeSCreateTagIdxReq(void* buf, int32_t bufLen, SCreateTagIndexR typedef SMDropSmaReq SDropTagIndexReq; -int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); +// int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); int32_t tDeserializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); typedef struct { @@ -3567,8 +3567,8 @@ typedef struct { int8_t igNotExists; } SMDropFullTextReq; -int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); -int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); +// int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); +// int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); typedef struct { char indexFName[TSDB_INDEX_FNAME_LEN]; @@ -3820,6 +3820,7 @@ typedef struct { uint32_t phyLen; char* sql; char* msg; + int8_t source; } SVDeleteReq; int32_t tSerializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq); @@ -3841,6 +3842,7 @@ typedef struct SDeleteRes { char tableFName[TSDB_TABLE_NAME_LEN]; char tsColName[TSDB_COL_NAME_LEN]; int64_t ctimeMs; // fill by vnode + int8_t source; } SDeleteRes; int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index e66ffe5521..3433f98d38 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -223,10 +223,10 @@ typedef struct SStoreTqReader { bool (*tqReaderCurrentBlockConsumed)(); struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it - int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it +// int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it int32_t (*tqReaderSetSubmitMsg)(); // todo remove it - bool (*tqReaderNextBlockFilterOut)(); +// bool (*tqReaderNextBlockFilterOut)(); } SStoreTqReader; typedef struct SStoreSnapshotFn { diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 958d63349d..952af3c443 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -78,6 +78,7 @@ typedef struct SSchedulerReq { void* chkKillParam; SExecResult* pExecRes; void** pFetchRes; + int8_t source; } SSchedulerReq; int32_t schedulerInit(void); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 64ce735843..5f3761d7b7 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -56,7 +56,6 @@ extern "C" { #define STREAM_EXEC_T_RESTART_ALL_TASKS (-4) #define STREAM_EXEC_T_STOP_ALL_TASKS (-5) #define STREAM_EXEC_T_RESUME_TASK (-6) -#define STREAM_EXEC_T_UPDATE_TASK_EPSET (-7) typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; @@ -783,11 +782,14 @@ bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask); bool streamTaskSetSchedStatusWait(SStreamTask* pTask); int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask); int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask); -int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt, bool metaLock); +int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt); int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event); -int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event); -void streamTaskRestoreStatus(SStreamTask* pTask); + +typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param); +int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param); +int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param); +int32_t streamTaskRestoreStatus(SStreamTask* pTask); int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp, SRpcHandleInfo* pRpcInfo, int32_t taskId); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 556479f547..3a689fc63d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -433,7 +433,7 @@ int32_t* taosGetErrno(); //mnode-compact #define TSDB_CODE_MND_INVALID_COMPACT_ID TAOS_DEF_ERROR_CODE(0, 0x04B1) - +#define TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x04B2) // vnode // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x diff --git a/include/util/tdef.h b/include/util/tdef.h index 8371ee24bb..e2d1beb5a5 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -187,6 +187,8 @@ typedef enum ELogicConditionType { LOGIC_COND_TYPE_NOT, } ELogicConditionType; +#define TSDB_INT32_ID_LEN 11 + #define TSDB_NAME_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 diff --git a/include/util/tqueue.h b/include/util/tqueue.h index 9f09bd2930..eb887596d0 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -72,40 +72,6 @@ struct STaosQnode { char item[]; }; -struct STaosQueue { - STaosQnode *head; - STaosQnode *tail; - STaosQueue *next; // for queue set - STaosQset *qset; // for queue set - void *ahandle; // for queue set - FItem itemFp; - FItems itemsFp; - TdThreadMutex mutex; - int64_t memOfItems; - int32_t numOfItems; - int64_t threadId; - int64_t memLimit; - int64_t itemLimit; -}; - -struct STaosQset { - STaosQueue *head; - STaosQueue *current; - TdThreadMutex mutex; - tsem_t sem; - int32_t numOfQueues; - int32_t numOfItems; -}; - -struct STaosQall { - STaosQnode *current; - STaosQnode *start; - int32_t numOfItems; - int64_t memOfItems; - int32_t unAccessedNumOfItems; - int64_t unAccessMemOfItems; -}; - STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); @@ -140,6 +106,8 @@ int32_t taosGetQueueNumber(STaosQset *qset); int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo); int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo); void taosResetQsetThread(STaosQset *qset, void *pItem); +void taosQueueSetThreadId(STaosQueue *pQueue, int64_t threadId); +int64_t taosQueueGetThreadId(STaosQueue *pQueue); #ifdef __cplusplus } diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index 2cf170cf04..d3ce2eb23b 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -26,6 +26,8 @@ typedef struct SScalableBf { SArray *bfArray; // array of bloom filters uint32_t growth; uint64_t numBits; + uint32_t maxBloomFilters; + int8_t status; _hash_fn_t hashFn1; _hash_fn_t hashFn2; } SScalableBf; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 989c6614a6..6650d4c8b3 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -284,6 +284,7 @@ typedef struct SRequestObj { void* pWrapper; SMetaData parseMeta; char* effectiveUser; + int8_t source; } SRequestObj; typedef struct SSyncQueryParam { @@ -306,10 +307,10 @@ void doFreeReqResultInfo(SReqResultInfo* pResInfo); int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); -TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly); +TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source); TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, int64_t reqid); -void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly); +void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source); void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int64_t reqid); void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param); @@ -354,6 +355,7 @@ SRequestObj* acquireRequest(int64_t rid); int32_t releaseRequest(int64_t rid); int32_t removeRequest(int64_t rid); void doDestroyRequest(void* p); +int64_t removeFromMostPrevReq(SRequestObj* pRequest); char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index b732abffb1..122914fd34 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -80,7 +80,7 @@ extern "C" { #define IS_SAME_KEY (maxKV->type == kv->type && maxKV->keyLen == kv->keyLen && memcmp(maxKV->key, kv->key, kv->keyLen) == 0) #define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ - (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE)) + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH)) #define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 1df50a51da..6c20813118 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -385,6 +385,33 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); } +/// return the most previous req ref id +int64_t removeFromMostPrevReq(SRequestObj* pRequest) { + int64_t mostPrevReqRefId = pRequest->self; + SRequestObj* pTmp = pRequest; + while (pTmp->relation.prevRefId) { + pTmp = acquireRequest(pTmp->relation.prevRefId); + if (pTmp) { + mostPrevReqRefId = pTmp->self; + releaseRequest(mostPrevReqRefId); + } else { + break; + } + } + removeRequest(mostPrevReqRefId); + return mostPrevReqRefId; +} + +void destroyNextReq(int64_t nextRefId) { + if (nextRefId) { + SRequestObj* pObj = acquireRequest(nextRefId); + if (pObj) { + releaseRequest(nextRefId); + releaseRequest(nextRefId); + } + } +} + void destroySubRequests(SRequestObj *pRequest) { int32_t reqIdx = -1; SRequestObj *pReqList[16] = {NULL}; @@ -435,7 +462,7 @@ void doDestroyRequest(void *p) { uint64_t reqId = pRequest->requestId; tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest); - destroySubRequests(pRequest); + int64_t nextReqRefId = pRequest->relation.nextRefId; taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); @@ -471,6 +498,7 @@ void doDestroyRequest(void *p) { taosMemoryFreeClear(pRequest->sqlstr); taosMemoryFree(pRequest); tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest); + destroyNextReq(nextReqRefId); } void destroyRequest(SRequestObj *pRequest) { @@ -479,7 +507,7 @@ void destroyRequest(SRequestObj *pRequest) { } taos_stop_query(pRequest); - removeRequest(pRequest->self); + removeFromMostPrevReq(pRequest); } void taosStopQueryImpl(SRequestObj *pRequest) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index de4f9f16e4..94da1e1998 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -743,6 +743,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList .chkKillFp = chkRequestKilled, .chkKillParam = (void*)pRequest->self, .pExecRes = &res, + .source = pRequest->source, }; int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob); @@ -1212,6 +1213,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat .chkKillFp = chkRequestKilled, .chkKillParam = (void*)pRequest->self, .pExecRes = NULL, + .source = pRequest->source, }; code = schedulerExecJob(&req, &pRequest->body.queryJob); taosArrayDestroy(pNodeList); @@ -2475,7 +2477,7 @@ void syncQueryFn(void* param, void* res, int32_t code) { tsem_post(&pParam->sem); } -void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) { +void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source) { if (sql == NULL || NULL == fp) { terrno = TSDB_CODE_INVALID_PARA; if (fp) { @@ -2501,6 +2503,7 @@ void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, return; } + pRequest->source = source; pRequest->body.queryFp = fp; doAsyncQuery(pRequest, false); } @@ -2535,7 +2538,7 @@ void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_ doAsyncQuery(pRequest, false); } -TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { +TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; @@ -2550,7 +2553,7 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { } tsem_init(¶m->sem, 0, 0); - taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly); + taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, source); tsem_wait(¶m->sem); SRequestObj* pRequest = NULL; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 275ca0d2aa..47adb40eaa 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -402,7 +402,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { return pResInfo->userFields; } -TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false); } +TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false, TD_REQ_FROM_APP); } TAOS_RES *taos_query_with_reqid(TAOS *taos, const char *sql, int64_t reqid) { return taosQueryImplWithReqid(taos, sql, false, reqid); } @@ -828,7 +828,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { } int taos_validate_sql(TAOS *taos, const char *sql) { - TAOS_RES *pObj = taosQueryImpl(taos, sql, true); + TAOS_RES *pObj = taosQueryImpl(taos, sql, true, TD_REQ_FROM_APP); int code = taos_errno(pObj); @@ -1126,7 +1126,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { int64_t connId = *(int64_t *)taos; tscDebug("taos_query_a start with sql:%s", sql); - taosAsyncQueryImpl(connId, sql, fp, param, false); + taosAsyncQueryImpl(connId, sql, fp, param, false, TD_REQ_FROM_APP); tscDebug("taos_query_a end with sql:%s", sql); } @@ -1254,54 +1254,34 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } void restartAsyncQuery(SRequestObj *pRequest, int32_t code) { - int32_t reqIdx = 0; - SRequestObj *pReqList[16] = {NULL}; - SRequestObj *pUserReq = NULL; - pReqList[0] = pRequest; - uint64_t tmpRefId = 0; - SRequestObj *pTmp = pRequest; - while (pTmp->relation.prevRefId) { - tmpRefId = pTmp->relation.prevRefId; - pTmp = acquireRequest(tmpRefId); - if (pTmp) { - pReqList[++reqIdx] = pTmp; - releaseRequest(tmpRefId); - } else { - tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId); + tscInfo("restart request: %s p: %p", pRequest->sqlstr, pRequest); + SRequestObj* pUserReq = pRequest; + acquireRequest(pRequest->self); + while (pUserReq) { + if (pUserReq->self == pUserReq->relation.userRefId || pUserReq->relation.userRefId == 0) { break; - } - } - - tmpRefId = pRequest->relation.nextRefId; - while (tmpRefId) { - pTmp = acquireRequest(tmpRefId); - if (pTmp) { - tmpRefId = pTmp->relation.nextRefId; - removeRequest(pTmp->self); - releaseRequest(pTmp->self); } else { - tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId); - break; + int64_t nextRefId = pUserReq->relation.nextRefId; + releaseRequest(pUserReq->self); + if (nextRefId) { + pUserReq = acquireRequest(nextRefId); + } } } - - for (int32_t i = reqIdx; i >= 0; i--) { - destroyCtxInRequest(pReqList[i]); - if (pReqList[i]->relation.userRefId == pReqList[i]->self || 0 == pReqList[i]->relation.userRefId) { - pUserReq = pReqList[i]; - } else { - removeRequest(pReqList[i]->self); - } - } - + bool hasSubRequest = pUserReq != pRequest || pRequest->relation.prevRefId != 0; if (pUserReq) { + destroyCtxInRequest(pUserReq); pUserReq->prevCode = code; memset(&pUserReq->relation, 0, sizeof(pUserReq->relation)); } else { - tscError("user req is missing"); + tscError("User req is missing"); + removeFromMostPrevReq(pRequest); return; } - + if (hasSubRequest) + removeFromMostPrevReq(pRequest); + else + releaseRequest(pUserReq->self); doAsyncQuery(pUserReq, true); } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index f143624bab..adc8c361cd 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1256,7 +1256,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName, req.tsColName, req.skey, req.tsColName, req.ekey); - TAOS_RES* res = taos_query(taos, sql); + TAOS_RES* res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX); SRequestObj* pRequest = (SRequestObj*)res; code = pRequest->code; if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_PAR_GET_META_ERROR) { diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 0c610a4611..7535cbfd0c 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -20,14 +20,14 @@ #include "clientSml.h" -#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) -#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) -#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) +#define IS_COMMA(sql,escapeChar) (*(sql) == COMMA && (*((sql)-1) != SLASH || ((sql)-1 == escapeChar))) +#define IS_SPACE(sql,escapeChar) (*(sql) == SPACE && (*((sql)-1) != SLASH || ((sql)-1 == escapeChar))) +#define IS_EQUAL(sql,escapeChar) (*(sql) == EQUAL && (*((sql)-1) != SLASH || ((sql)-1 == escapeChar))) #define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH)) #define IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql) \ - (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL)) + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == SLASH)) #define PROCESS_SLASH_IN_FIELD_VALUE(key, keyLen) \ for (int i = 1; i < keyLen; ++i) { \ @@ -198,7 +198,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ int cnt = 0; while (*sql < sqlEnd) { - if (unlikely(IS_SPACE(*sql))) { + if (unlikely(IS_SPACE(*sql,NULL))) { break; } @@ -207,18 +207,21 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ size_t keyLen = 0; bool keyEscaped = false; size_t keyLenEscaped = 0; + const char *escapeChar = NULL; + while (*sql < sqlEnd) { - if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); terrno = TSDB_CODE_SML_INVALID_DATA; return -1; } - if (unlikely(IS_EQUAL(*sql))) { + if (unlikely(IS_EQUAL(*sql,escapeChar))) { keyLen = *sql - key; (*sql)++; break; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + escapeChar = *sql; keyLenEscaped++; keyEscaped = true; } @@ -238,15 +241,16 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ size_t valueLenEscaped = 0; while (*sql < sqlEnd) { // parse value - if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { break; - } else if (unlikely(IS_EQUAL(*sql))) { + } else if (unlikely(IS_EQUAL(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); terrno = TSDB_CODE_SML_INVALID_DATA; return -1; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + escapeChar = *sql; valueLenEscaped++; valueEscaped = true; } @@ -293,7 +297,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ } cnt++; - if (IS_SPACE(*sql)) { + if (IS_SPACE(*sql,escapeChar)) { break; } (*sql)++; @@ -326,7 +330,7 @@ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) { int cnt = 0; while (*sql < sqlEnd) { - if (unlikely(IS_SPACE(*sql))) { + if (unlikely(IS_SPACE(*sql,NULL))) { break; } @@ -335,17 +339,19 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL size_t keyLen = 0; bool keyEscaped = false; size_t keyLenEscaped = 0; + const char *escapeChar = NULL; while (*sql < sqlEnd) { - if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); return TSDB_CODE_SML_INVALID_DATA; } - if (unlikely(IS_EQUAL(*sql))) { + if (unlikely(IS_EQUAL(*sql,escapeChar))) { keyLen = *sql - key; (*sql)++; break; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + escapeChar = *sql; keyLenEscaped++; keyEscaped = true; } @@ -363,7 +369,6 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL bool valueEscaped = false; size_t valueLenEscaped = 0; int quoteNum = 0; - const char *escapeChar = NULL; while (*sql < sqlEnd) { // parse value if (unlikely(*(*sql) == QUOTE && (*(*sql - 1) != SLASH || (*sql - 1) == escapeChar))) { @@ -374,7 +379,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } continue; } - if (quoteNum % 2 == 0 && (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql)))) { + if (quoteNum % 2 == 0 && (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar)))) { break; } if (IS_SLASH_LETTER_IN_FIELD_VALUE(*sql) && (*sql - 1) != escapeChar) { @@ -437,7 +442,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } cnt++; - if (IS_SPACE(*sql)) { + if (IS_SPACE(*sql,escapeChar)) { break; } (*sql)++; @@ -453,19 +458,18 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine elements->measure = sql; // parse measure size_t measureLenEscaped = 0; + const char *escapeChar = NULL; while (sql < sqlEnd) { - if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) { - elements->measureEscaped = true; - measureLenEscaped++; - sql++; - continue; - } - if (unlikely(IS_COMMA(sql))) { + if (unlikely(IS_COMMA(sql,escapeChar) || IS_SPACE(sql,escapeChar))) { break; } - if (unlikely(IS_SPACE(sql))) { - break; + if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) { + elements->measureEscaped = true; + escapeChar = sql; + measureLenEscaped++; + sql++; + continue; } sql++; } @@ -478,9 +482,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine // to get measureTagsLen before const char *tmp = sql; while (tmp < sqlEnd) { - if (unlikely(IS_SPACE(tmp))) { + if (unlikely(IS_SPACE(tmp,escapeChar))) { break; } + if(unlikely(IS_SLASH_LETTER_IN_TAG_FIELD_KEY(tmp))){ + escapeChar = tmp; + } tmp++; } elements->measureTagsLen = tmp - elements->measure; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 9b74456da2..0270ae9657 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -876,12 +876,13 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { STaosQall* qall = taosAllocateQall(); taosReadAllQitems(pTmq->delayedTask, qall); - if (qall->numOfItems == 0) { + int32_t numOfItems = taosQallItemSize(qall); + if (numOfItems == 0) { taosFreeQall(qall); return TSDB_CODE_SUCCESS; } - tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, qall->numOfItems); + tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, numOfItems); int8_t* pTaskType = NULL; taosGetQitem(qall, (void**)&pTaskType); @@ -1009,19 +1010,8 @@ int32_t tmq_unsubscribe(tmq_t* tmq) { } taosSsleep(2); // sleep 2s for hb to send offset and rows to server - int32_t rsp; - int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); - while (1) { - rsp = tmq_subscribe(tmq, lst); - if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) { - break; - } else { - retryCnt++; - taosMsleep(500); - } - } - + int32_t rsp = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); return rsp; } @@ -1271,10 +1261,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } int32_t retryCnt = 0; - while (syncAskEp(tmq) != 0) { - if (retryCnt++ > MAX_RETRY_COUNT) { + while ((code = syncAskEp(tmq)) != 0) { + if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes", tmq->consumerId); - code = TSDB_CODE_MND_CONSUMER_NOT_READY; goto FAIL; } @@ -1839,7 +1828,7 @@ static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal } static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { - tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems); + tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, taosQallItemSize(tmq->qall)); while (1) { SMqRspWrapper* pRspWrapper = NULL; @@ -2147,26 +2136,19 @@ int32_t tmq_consumer_close(tmq_t* tmq) { if (tmq->status == TMQ_CONSUMER_STATUS__READY) { // if auto commit is set, commit before close consumer. Otherwise, do nothing. if (tmq->autoCommit) { - int32_t rsp = tmq_commit_sync(tmq, NULL); - if (rsp != 0) { - return rsp; + int32_t code = tmq_commit_sync(tmq, NULL); + if (code != 0) { + return code; } } taosSsleep(2); // sleep 2s for hb to send offset and rows to server - int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); - while (1) { - int32_t rsp = tmq_subscribe(tmq, lst); - if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) { - break; - } else { - retryCnt++; - taosMsleep(500); - } - } - + int32_t code = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); + if (code != 0) { + return code; + } } else { tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId); } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9f55b67ea3..f4455be206 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -452,20 +452,21 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + int32_t newLen = pSource->varmeta.length; memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); - if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { - char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); + if (pColumnInfoData->varmeta.allocLen < newLen) { + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newLen); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } pColumnInfoData->pData = tmp; - pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; + pColumnInfoData->varmeta.allocLen = newLen; } - pColumnInfoData->varmeta.length = pSource->varmeta.length; + pColumnInfoData->varmeta.length = newLen; if (pColumnInfoData->pData != NULL && pSource->pData != NULL) { - memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); + memcpy(pColumnInfoData->pData, pSource->pData, newLen); } } else { memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); @@ -1687,7 +1688,29 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) { } static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { + if (n >= total || n == 0) return; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + if (pColInfoData->varmeta.length != 0) { + int32_t newLen = pColInfoData->varmeta.offset[n]; + if (-1 == newLen) { + for (int i = n - 1; i >= 0; --i) { + newLen = pColInfoData->varmeta.offset[i]; + if (newLen != -1) { + if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { + newLen += getJsonValueLen(pColInfoData->pData + newLen); + } else { + newLen += varDataTLen(pColInfoData->pData + newLen); + } + break; + } + } + } + if (newLen <= -1) { + uFatal("colDataKeepFirstNRows: newLen:%d old:%d", newLen, pColInfoData->varmeta.length); + } else { + pColInfoData->varmeta.length = newLen; + } + } // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3d8c7c7f8b..89d3b88955 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -58,7 +58,7 @@ int32_t tsNumOfMnodeQueryThreads = 4; int32_t tsNumOfMnodeFetchThreads = 1; int32_t tsNumOfMnodeReadThreads = 1; int32_t tsNumOfVnodeQueryThreads = 4; -float tsRatioOfVnodeStreamThreads = 1.5F; +float tsRatioOfVnodeStreamThreads = 0.5F; int32_t tsNumOfVnodeFetchThreads = 4; int32_t tsNumOfVnodeRsmaThreads = 2; int32_t tsNumOfQnodeQueryThreads = 4; @@ -269,7 +269,6 @@ int64_t tsStreamBufferSize = 128 * 1024 * 1024; bool tsFilterScalarMode = false; int tsResolveFQDNRetryTime = 100; // seconds int tsStreamAggCnt = 1000; -bool tsDisableCount = true; char tsS3Endpoint[TSDB_FQDN_LEN] = ""; char tsS3AccessKey[TSDB_FQDN_LEN] = ""; @@ -541,8 +540,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - - if (cfgAddBool(pCfg, "disableCount", tsDisableCount, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; return 0; } @@ -589,7 +586,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfSupportVnodes = tsNumOfCores * 2; tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); - if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) @@ -705,7 +702,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - + if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; @@ -1109,8 +1106,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32; tsExperimental = cfgGetItem(pCfg, "experimental")->bval; - - tsDisableCount = cfgGetItem(pCfg, "disableCount")->bval; return 0; } @@ -1174,7 +1169,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMonitorLogProtocol = cfgGetItem(pCfg, "monitorLogProtocol")->bval; tsMonitorIntervalForBasic = cfgGetItem(pCfg, "monitorIntervalForBasic")->i32; tsMonitorForceV2 = cfgGetItem(pCfg, "monitorForceV2")->i32; - + tsEnableAudit = cfgGetItem(pCfg, "audit")->bval; tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval; tsAuditInterval = cfgGetItem(pCfg, "auditInterval")->i32; @@ -1263,6 +1258,8 @@ static int32_t taosSetReleaseCfg(SConfig *pCfg) { return 0; } int32_t taosSetReleaseCfg(SConfig *pCfg); #endif +static void taosSetAllDebugFlag(SConfig *pCfg, int32_t flag); + int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { if (tsCfg == NULL) osDefaultInit(); @@ -1307,7 +1304,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetServerLogCfg(pCfg); } - taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); + taosSetAllDebugFlag(pCfg, cfgGetItem(pCfg, "debugFlag")->i32); if (taosMulModeMkDir(tsLogDir, 0777, true) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -1356,6 +1353,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile if (taosAddClientLogCfg(tsCfg) != 0) return -1; if (taosAddServerLogCfg(tsCfg) != 0) return -1; } + taosAddSystemCfg(tsCfg); if (taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) { @@ -1382,10 +1380,12 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile if (taosSetTfsCfg(tsCfg) != 0) return -1; if (taosSetS3Cfg(tsCfg) != 0) return -1; } + taosSetSystemCfg(tsCfg); + if (taosSetFileHandlesLimit() != 0) return -1; - taosSetAllDebugFlag(cfgGetItem(tsCfg, "debugFlag")->i32); + taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32); cfgDumpCfg(tsCfg, tsc, false); @@ -1478,7 +1478,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { } if (strncasecmp(name, "debugFlag", 9) == 0) { - taosSetAllDebugFlag(pItem->i32); + taosSetAllDebugFlag(pCfg, pItem->i32); return 0; } @@ -1552,7 +1552,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { switch (lowcaseName[0]) { case 'd': { if (strcasecmp("debugFlag", name) == 0) { - taosSetAllDebugFlag(pItem->i32); + taosSetAllDebugFlag(pCfg, pItem->i32); matched = true; } break; @@ -1737,8 +1737,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { {"shellActivityTimer", &tsShellActivityTimer}, {"slowLogThreshold", &tsSlowLogThreshold}, {"useAdapter", &tsUseAdapter}, - {"experimental", &tsExperimental}, - {"disableCount", &tsDisableCount}}; + {"experimental", &tsExperimental}}; if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { taosCfgSetOption(options, tListLen(options), pItem, false); @@ -1777,11 +1776,13 @@ static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag taosSetDebugFlag(pFlagPtr, name, flag); } -void taosSetAllDebugFlag(int32_t flag) { +void taosSetGlobalDebugFlag(int32_t flag) { taosSetAllDebugFlag(tsCfg, flag); } + +static void taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) { if (flag <= 0) return; SArray *noNeedToSetVars = NULL; - SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag"); + SConfigItem *pItem = cfgGetItem(pCfg, "debugFlag"); if (pItem != NULL) { pItem->i32 = flag; noNeedToSetVars = pItem->array; @@ -1831,4 +1832,4 @@ int8_t taosGranted(int8_t type) { break; } return 0; -} \ No newline at end of file +} diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 4640c20e07..9138d7c983 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1009,19 +1009,19 @@ int32_t tDeserializeSCreateTagIdxReq(void *buf, int32_t bufLen, SCreateTagIndexR tDecoderClear(&decoder); return 0; } -int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - tEndEncode(&encoder); +// int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); +// if (tStartEncode(&encoder) < 0) return -1; +// tEndEncode(&encoder); - if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; - if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -1035,6 +1035,7 @@ int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq * return 0; } + int32_t tSerializeSMCreateFullTextReq(void *buf, int32_t bufLen, SMCreateFullTextReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -1059,32 +1060,32 @@ void tFreeSMCreateFullTextReq(SMCreateFullTextReq *pReq) { // impl later return; } -int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; +// if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; - if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; - tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} -int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; +// tEndEncode(&encoder); +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } +// int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) { SEncoder encoder = {0}; @@ -1474,44 +1475,44 @@ void tFreeSStatisReq(SStatisReq *pReq) { taosMemoryFreeClear(pReq->pCont); } -int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1; - if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1; - if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1; - tEndEncode(&encoder); +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1; +// if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1; +// tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } -int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1; +// if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) { SEncoder encoder = {0}; @@ -5238,11 +5239,11 @@ int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCo if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->finished) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -2; + if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -3; + if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -4; + if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -5; + if (tDecodeI32(&decoder, &pReq->finished) < 0) return -6; tEndDecode(&decoder); tDecoderClear(&decoder); @@ -7191,6 +7192,7 @@ int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1; if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (tEncodeBinary(&encoder, pReq->msg, pReq->phyLen) < 0) return -1; + if (tEncodeI8(&encoder, pReq->source) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -7227,6 +7229,9 @@ int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->msg, &msgLen) < 0) return -1; pReq->phyLen = msgLen; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->source) < 0) return -1; + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -7934,64 +7939,64 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl return 0; } -static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { - if (tStartDecode(pDecoder) < 0) return -1; +// static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { +// if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; - if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; - pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); - if (NULL == pBlock->tblFName) return -1; - if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; - if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; - if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; - if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; +// if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; +// if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; +// pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); +// if (NULL == pBlock->tblFName) return -1; +// if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; +// if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; - int32_t meta = 0; - if (tDecodeI32(pDecoder, &meta) < 0) return -1; - if (meta) { - pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pBlock->pMeta) return -1; - if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1; - } else { - pBlock->pMeta = NULL; - } +// int32_t meta = 0; +// if (tDecodeI32(pDecoder, &meta) < 0) return -1; +// if (meta) { +// pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pBlock->pMeta) return -1; +// if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1; +// } else { +// pBlock->pMeta = NULL; +// } - tEndDecode(pDecoder); - return 0; -} +// tEndDecode(pDecoder); +// return 0; +// } -int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) { - int32_t nBlocks = taosArrayGetSize(pRsp->pArray); +// int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) { +// int32_t nBlocks = taosArrayGetSize(pRsp->pArray); - if (tStartEncode(pEncoder) < 0) return -1; +// if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1; - if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1; - if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1; - for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { - if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1; - } +// if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1; +// if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1; +// if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1; +// for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { +// if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1; +// } - tEndEncode(pEncoder); - return 0; -} +// tEndEncode(pEncoder); +// return 0; +// } -int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) { - if (tStartDecode(pDecoder) < 0) return -1; +// int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) { +// if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1; - pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks)); - if (pRsp->pBlocks == NULL) return -1; - for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) { - if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1; - } +// if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1; +// pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks)); +// if (pRsp->pBlocks == NULL) return -1; +// for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) { +// if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1; +// } - tEndDecode(pDecoder); - tDecoderClear(pDecoder); - return 0; -} +// tEndDecode(pDecoder); +// tDecoderClear(pDecoder); +// return 0; +// } // void tFreeSSubmitBlkRsp(void *param) { // if (NULL == param) { @@ -8426,6 +8431,7 @@ int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) { if (tEncodeCStr(pCoder, pRes->tableFName) < 0) return -1; if (tEncodeCStr(pCoder, pRes->tsColName) < 0) return -1; if (tEncodeI64(pCoder, pRes->ctimeMs) < 0) return -1; + if (tEncodeI8(pCoder, pRes->source) < 0) return -1; return 0; } @@ -8450,6 +8456,9 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) { if (!tDecodeIsEnd(pCoder)) { if (tDecodeI64(pCoder, &pRes->ctimeMs) < 0) return -1; } + if (!tDecodeIsEnd(pCoder)) { + if (tDecodeI8(pCoder, &pRes->source) < 0) return -1; + } return 0; } diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 1508d88def..f9456f1729 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -68,7 +68,7 @@ static struct { int64_t startTime; } global = {0}; -static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143); } +static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetGlobalDebugFlag(143); } static void dmSetAssert(int32_t signum, void *sigInfo, void *context) { tsAssert = 1; } static void dmStopDnode(int signum, void *sigInfo, void *context) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index be88e8b3fd..3dfc2bd96f 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -194,26 +194,26 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) while (pVnode->refCount > 0) taosMsleep(10); dInfo("vgId:%d, wait for vnode write queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue, - pVnode->pWriteW.queue->threadId); + taosQueueGetThreadId(pVnode->pWriteW.queue)); tMultiWorkerCleanup(&pVnode->pWriteW); dInfo("vgId:%d, wait for vnode sync queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue, - pVnode->pSyncW.queue->threadId); + taosQueueGetThreadId(pVnode->pSyncW.queue)); tMultiWorkerCleanup(&pVnode->pSyncW); dInfo("vgId:%d, wait for vnode sync rd queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue, - pVnode->pSyncRdW.queue->threadId); + taosQueueGetThreadId(pVnode->pSyncRdW.queue)); tMultiWorkerCleanup(&pVnode->pSyncRdW); dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue, - pVnode->pApplyW.queue->threadId); + taosQueueGetThreadId(pVnode->pApplyW.queue)); tMultiWorkerCleanup(&pVnode->pApplyW); dInfo("vgId:%d, wait for vnode query queue:%p is empty", pVnode->vgId, pVnode->pQueryQ); while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); dInfo("vgId:%d, wait for vnode fetch queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ, - pVnode->pFetchQ->threadId); + taosQueueGetThreadId(pVnode->pFetchQ)); while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); tqNotifyClose(pVnode->pImpl->pTq); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 8b80527447..a6abe5ab4d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -365,16 +365,16 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { } dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue, - pVnode->pWriteW.queue->threadId); + taosQueueGetThreadId(pVnode->pWriteW.queue)); dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue, - pVnode->pSyncW.queue->threadId); + taosQueueGetThreadId(pVnode->pSyncW.queue)); dInfo("vgId:%d, sync-rd-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue, - pVnode->pSyncRdW.queue->threadId); + taosQueueGetThreadId(pVnode->pSyncRdW.queue)); dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue, - pVnode->pApplyW.queue->threadId); + taosQueueGetThreadId(pVnode->pApplyW.queue)); dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ); dInfo("vgId:%d, fetch-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ, - pVnode->pFetchQ->threadId); + taosQueueGetThreadId(pVnode->pFetchQ)); dInfo("vgId:%d, stream-queue:%p is alloced", pVnode->vgId, pVnode->pStreamQ); return 0; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 1a31f08801..77760b16f4 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -345,6 +345,7 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.parent = pDnode; rpcInit.rfp = rpcRfp; rpcInit.compressSize = tsCompressMsgSize; + rpcInit.dfp = destroyAhandle; rpcInit.retryMinInterval = tsRedirectPeriod; rpcInit.retryStepFactor = tsRedirectFactor; diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 57fd187da3..5307ff4b05 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -111,7 +111,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo); -int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans); +int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans); SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId); int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId); diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index deaaf7f2af..75b4531dbb 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -363,13 +363,15 @@ static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *p } static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "kill-compact"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact"); if (pTrans == NULL) { mError("compact:%" PRId32 ", failed to drop since %s" , pCompact->compactId, terrstr()); return -1; } mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId); + mndTransSetDbName(pTrans, pCompact->dbname, NULL); + SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); @@ -378,7 +380,7 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - void *pIter = NULL; + void *pIter = NULL; while (1) { SCompactDetailObj *pDetail = NULL; pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail); @@ -452,7 +454,7 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){ code = TSDB_CODE_ACTION_IN_PROGRESS; - char obj[MND_COMPACT_ID_LEN] = {0}; + char obj[TSDB_INT32_ID_LEN] = {0}; sprintf(obj, "%d", pCompact->compactId); auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen); @@ -488,13 +490,17 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c sdbRelease(pMnode->pSdb, pDetail); } - return -1; + return TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST; } int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){ SQueryCompactProgressRsp req = {0}; - if (tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req) != 0) { + int32_t code = 0; + code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req); + if (code != 0) { terrno = TSDB_CODE_INVALID_MSG; + mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", + code, pReq->pCont, pReq->contLen); return -1; } @@ -502,10 +508,10 @@ int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){ req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished); SMnode *pMnode = pReq->info.node; - int32_t code = -1; - - if(mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req) != 0){ + code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req); + if(code != 0){ + terrno = code; mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished); return -1; @@ -612,15 +618,17 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { return 0; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-compact-progress"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress"); if (pTrans == NULL) { mError("trans:%" PRId32 ", failed to create since %s" , pTrans->id, terrstr()); return -1; } mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id); - + SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId); + mndTransSetDbName(pTrans, pCompact->dbname, NULL); + pIter = NULL; while (1) { SCompactDetailObj *pDetail = NULL; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 79a5f5fd83..befb6d3521 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -610,7 +610,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { for(int32_t j = 0; j < tagSize; j++){ SJson* item = tjsonGetArrayItem(arrayTag, j); - *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); + *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); tjsonGetStringValue(item, "name", *(labels + j)); *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN); @@ -626,7 +626,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { for(int32_t j = 0; j < metricLen; j++){ SJson *item = tjsonGetArrayItem(metrics, j); - char name[MONITOR_METRIC_NAME_LEN] = {0}; + char name[MONITOR_METRIC_NAME_LEN] = {0}; tjsonGetStringValue(item, "name", name); double value = 0; @@ -636,7 +636,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { tjsonGetDoubleValue(item, "type", &type); int32_t metricNameLen = strlen(name) + strlen(tableName) + 2; - char* metricName = taosMemoryMalloc(metricNameLen); + char* metricName = taosMemoryMalloc(metricNameLen); memset(metricName, 0, metricNameLen); sprintf(metricName, "%s:%s", tableName, name); @@ -669,7 +669,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { else{ mTrace("get metric from registry:%p", metric); } - + if(type == 0){ taos_counter_add(metric, value, (const char**)sample_labels); } @@ -689,7 +689,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { taosMemoryFreeClear(labels); } } - + } code = 0; @@ -1409,24 +1409,6 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { if (strcasecmp(cfgReq.config, "resetlog") == 0) { strcpy(dcfgReq.config, "resetlog"); #ifdef TD_ENTERPRISE - } else if (strncasecmp(cfgReq.config, "supportvnodes", 13) == 0) { - int32_t optLen = strlen("supportvnodes"); - int32_t flag = -1; - int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; - - if (flag < 0 || flag > 4096) { - mError("dnode:%d, failed to config supportVnodes since value:%d. Valid range: [0, 4096]", cfgReq.dnodeId, flag); - terrno = TSDB_CODE_OUT_OF_RANGE; - goto _err_out; - } - if (flag == 0) { - flag = tsNumOfCores * 2; - } - flag = TMAX(flag, 2); - - strcpy(dcfgReq.config, "supportvnodes"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) { int32_t optLen = strlen("s3blocksize"); int32_t flag = -1; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index b188d314d9..69a0bd477d 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -709,7 +709,8 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { int32_t code = syncProcessMsg(pMgmt->sync, pMsg); if (code != 0) { - mGError("vgId:1, failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); + mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType), + terrstr(), code); } return code; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3ef2f64df7..50e3dd0d03 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -721,6 +721,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } + // add into buffer firstly + // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already. taosThreadMutexLock(&execInfo.lock); mDebug("stream stream:%s tasks register into node list", createReq.name); saveStreamTasksInfo(&streamObj, &execInfo); @@ -1811,7 +1813,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid, pStream->name, pTrans->id); - int32_t code = mndStreamSetUpdateEpsetAction(pStream, pChangeInfo, pTrans); + int32_t code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans); // todo: not continue, drop all and retry again if (code != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 1ae85a2cc6..a124b4052c 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -115,6 +115,7 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) { char buf[256] = {0}; EPSET_TO_STR(&entry.epset, buf); + mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf); taosArrayPush(pVgroupListSnapshot, &entry); sdbRelease(pSdb, pVgroup); @@ -300,7 +301,10 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa return code; } - mDebug("pause node:%d, epset:%d", pTask->info.nodeId, epset.numOfEps); + char buf[256] = {0}; + EPSET_TO_STR(&epset, buf); + mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); + code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0); if (code != 0) { taosMemoryFree(pReq); @@ -462,14 +466,22 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha return TSDB_CODE_SUCCESS; } -static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) { +static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) { void *pBuf = NULL; int32_t len = 0; streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id); - int32_t code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet, 0); + SEpSet epset = {0}; + bool hasEpset = false; + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS || !hasEpset) { + terrno = code; + return code; + } + + code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(pBuf); } @@ -478,14 +490,14 @@ static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroup } // build trans to update the epset -int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) { +int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) { mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid); taosWLockLatch(&pStream->lock); SStreamTaskIter *pIter = createStreamTaskIter(pStream); while (streamTaskIterNextTask(pIter)) { SStreamTask *pTask = streamTaskIterGetCurrent(pIter); - int32_t code = doSetUpdateTaskAction(pTrans, pTask, pInfo); + int32_t code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo); if (code != TSDB_CODE_SUCCESS) { destroyStreamTaskIter(pIter); taosWUnLockLatch(&pStream->lock); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index e610161544..9fdb4993bd 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -106,7 +106,7 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" -#define VNODE_METRIC_SQL_COUNT "taos_sql_req:count" +#define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count" #define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" #define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 011e62cb89..47900d540c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -917,6 +917,22 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask } } +int32_t handleStep2Async(SStreamTask* pStreamTask, void* param) { + STQ* pTq = param; + + SStreamMeta* pMeta = pStreamTask->pMeta; + STaskId hId = pStreamTask->hTaskInfo.id; + SStreamTask* pTask = streamMetaAcquireTask(pStreamTask->pMeta, hId.streamId, hId.taskId); + if (pTask == NULL) { + // todo handle error + } + + doStartFillhistoryStep2(pTask, pStreamTask, pTq); + + streamMetaReleaseTask(pMeta, pTask); + return 0; +} + // this function should be executed by only one thread, so we set an sentinel to protect this function int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont; @@ -1007,37 +1023,27 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { // the following procedure should be executed, no matter status is stop/pause or not tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, pTask->execInfo.step1El); - if (pTask->info.fillHistory) { - SStreamTask* pStreamTask = NULL; + ASSERT(pTask->info.fillHistory == 1); - // 1. get the related stream task - pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); - if (pStreamTask == NULL) { - tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s", - pTask->streamTaskId.taskId, pTask->id.idStr); + // 1. get the related stream task + SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); + if (pStreamTask == NULL) { + tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s", + pTask->streamTaskId.taskId, pTask->id.idStr); tqDebug("s-task:%s fill-history task set status to be dropping and drop it", id); streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); - atomic_store_32(&pTask->status.inScanHistorySentinel, 0); - streamMetaReleaseTask(pMeta, pTask); - return -1; - } - - ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE); - - code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); - if (code == TSDB_CODE_SUCCESS) { - doStartFillhistoryStep2(pTask, pStreamTask, pTq); - } else { - tqError("s-task:%s failed to halt s-task:%s, not launch step2", id, pStreamTask->id.idStr); - } - - streamMetaReleaseTask(pMeta, pStreamTask); - } else { - ASSERT(0); + atomic_store_32(&pTask->status.inScanHistorySentinel, 0); + streamMetaReleaseTask(pMeta, pTask); + return -1; } + ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE); + code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, handleStep2Async, pTq); + + streamMetaReleaseTask(pMeta, pStreamTask); + atomic_store_32(&pTask->status.inScanHistorySentinel, 0); streamMetaReleaseTask(pMeta, pTask); return code; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 7a737fbb5d..110cac152d 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -368,24 +368,11 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con } } -// todo ignore the error in wal? bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { SWalReader* pWalReader = pReader->pWalReader; - SSDataBlock* pDataBlock = NULL; uint64_t st = taosGetTimestampMs(); while (1) { - // try next message in wal file - if (walNextValidMsg(pWalReader) < 0) { - return false; - } - - void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); - int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); - int64_t ver = pWalReader->pHead->head.version; - - tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver); - pReader->nextBlk = 0; int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < numOfBlocks) { tqTrace("tq reader next data block %d/%d, len:%d %" PRId64, pReader->nextBlk, numOfBlocks, pReader->msg.msgLen, @@ -400,33 +387,32 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { tqTrace("tq reader return submit block, uid:%" PRId64, pSubmitTbData->uid); SSDataBlock* pRes = NULL; int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); - if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { - if (pDataBlock == NULL) { - pDataBlock = createOneDataBlock(pRes, true); - } else { - blockDataMerge(pDataBlock, pRes); - } + if (code == TSDB_CODE_SUCCESS) { + return true; } } else { pReader->nextBlk += 1; tqTrace("tq reader discard submit block, uid:%" PRId64 ", continue", pSubmitTbData->uid); } } + tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); pReader->msg.msgStr = NULL; - if (pDataBlock != NULL) { - blockDataCleanup(pReader->pResBlock); - copyDataBlock(pReader->pResBlock, pDataBlock); - blockDataDestroy(pDataBlock); - return true; - } else { - qTrace("stream scan return empty, all %d submit blocks consumed, %s", numOfBlocks, id); - } - if (taosGetTimestampMs() - st > 1000) { return false; } + + // try next message in wal file + if (walNextValidMsg(pWalReader) < 0) { + return false; + } + + void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); + int64_t ver = pWalReader->pHead->head.version; + tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver); + pReader->nextBlk = 0; } } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 280c110711..73508202d9 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -28,8 +28,8 @@ static int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDurat // extract data blocks(submit/delete) from WAL, and add them into the input queue for all the sources tasks. int32_t tqScanWal(STQ* pTq) { - int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; + int32_t vgId = pMeta->vgId; int64_t st = taosGetTimestampMs(); tqDebug("vgId:%d continue to check if data in wal are available, scanCounter:%d", vgId, pMeta->scanInfo.scanCounter); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index dad1211326..6029575e2c 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -263,8 +263,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) { PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq) } else if (pHead->msgType == TDMT_VND_DELETE) { - fetchVer++; - continue; + PROCESS_EXCLUDED_MSG(SDeleteRes, tDecodeDeleteRes) } } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index a2d45062b9..9bfdd70477 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -142,8 +142,10 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); if (ppHTask == NULL || *ppHTask == NULL) { - tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", - vgId, req.taskId); + tqError( + "vgId:%d failed to acquire fill-history task:0x%x when handling update, may have been dropped already, rel " + "stream task:0x%x", + vgId, (uint32_t)pTask->hTaskInfo.id.taskId, req.taskId); CLEAR_RELATED_FILLHISTORY_TASK(pTask); } else { tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); @@ -612,23 +614,35 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; + int32_t vgId = pMeta->vgId; + STaskId hTaskId = {0}; - int32_t vgId = pMeta->vgId; tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); - if (pTask != NULL) { - // drop the related fill-history task firstly + streamMetaWLock(pMeta); + + STaskId id = {.streamId = pReq->streamId, .taskId = pReq->taskId}; + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if ((ppTask != NULL) && ((*ppTask) != NULL)) { + streamMetaAcquireOneTask(*ppTask); + SStreamTask* pTask = *ppTask; + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - STaskId* pHTaskId = &pTask->hTaskInfo.id; - streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); - tqDebug("s-task:0x%x vgId:%d drop fill-history task:0x%x firstly", pReq->taskId, vgId, - (int32_t)pHTaskId->taskId); + hTaskId.streamId = pTask->hTaskInfo.id.streamId; + hTaskId.taskId = pTask->hTaskInfo.id.taskId; } + + streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); streamMetaReleaseTask(pMeta, pTask); } - streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt, true); + streamMetaWUnLock(pMeta); + + // drop the related fill-history task firstly + if (hTaskId.taskId != 0 && hTaskId.streamId != 0) { + streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); + tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); + } // drop the stream task now streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); @@ -865,7 +879,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg){ pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); if (pHistoryTask == NULL) { tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 - ", it may have been dropped already", + ", it may have been dropped already", pMeta->vgId, pTask->hTaskInfo.id.taskId); streamMetaReleaseTask(pMeta, pTask); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 1ef2a451a7..0848fd0076 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -789,25 +789,6 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, int8_ return code; } -static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) { - SLastCol *pLastCol = NULL; - - char *err = NULL; - size_t vlen = 0; - SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid}; - size_t klen = ROCKS_KEY_LEN; - char *value = NULL; - value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err); - if (NULL != err) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); - rocksdb_free(err); - } - - pLastCol = tsdbCacheDeserialize(value); - - return pLastCol; -} - typedef struct { int idx; SLastKey key; @@ -1052,6 +1033,25 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, int nCols, int16_t *slotIds); #ifdef BUILD_NO_CALL +static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) { + SLastCol *pLastCol = NULL; + + char *err = NULL; + size_t vlen = 0; + SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid}; + size_t klen = ROCKS_KEY_LEN; + char *value = NULL; + value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err); + if (NULL != err) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); + rocksdb_free(err); + } + + pLastCol = tsdbCacheDeserialize(value); + + return pLastCol; +} + int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { rocksdb_writebatch_t *wb = NULL; int32_t code = 0; @@ -1233,10 +1233,10 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr int16_t *lastSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); int16_t *lastrowColIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); int16_t *lastrowSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); - SArray* lastTmpColArray = NULL; - SArray* lastTmpIndexArray = NULL; - SArray* lastrowTmpColArray = NULL; - SArray* lastrowTmpIndexArray = NULL; + SArray *lastTmpColArray = NULL; + SArray *lastTmpIndexArray = NULL; + SArray *lastrowTmpColArray = NULL; + SArray *lastrowTmpIndexArray = NULL; int lastIndex = 0; int lastrowIndex = 0; @@ -1245,7 +1245,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr SIdxKey *idxKey = taosArrayGet(remainCols, i); slotIds[i] = pr->pSlotIds[idxKey->idx]; if (idxKey->key.ltype == CACHESCAN_RETRIEVE_LAST >> 3) { - if(NULL == lastTmpIndexArray) { + if (NULL == lastTmpIndexArray) { lastTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t)); } taosArrayPush(lastTmpIndexArray, &(i)); @@ -1253,7 +1253,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr lastSlotIds[lastIndex] = pr->pSlotIds[idxKey->idx]; lastIndex++; } else { - if(NULL == lastrowTmpIndexArray) { + if (NULL == lastrowTmpIndexArray) { lastrowTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t)); } taosArrayPush(lastrowTmpIndexArray, &(i)); @@ -1265,17 +1265,18 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr pTmpColArray = taosArrayInit(lastIndex + lastrowIndex, sizeof(SLastCol)); - if(lastTmpIndexArray != NULL) { + if (lastTmpIndexArray != NULL) { mergeLastCid(uid, pTsdb, &lastTmpColArray, pr, lastColIds, lastIndex, lastSlotIds); - for(int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) { - taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i)); + for (int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) { + taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i)); } } - if(lastrowTmpIndexArray != NULL) { + if (lastrowTmpIndexArray != NULL) { mergeLastRowCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds); - for(int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) { - taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastrowTmpIndexArray, i), taosArrayGet(lastrowTmpColArray, i)); + for (int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) { + taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastrowTmpIndexArray, i), + taosArrayGet(lastrowTmpColArray, i)); } } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index aff7e4642b..16abe69def 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -128,12 +128,12 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it - pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it +// pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it pTq->tqGetResultBlock = tqGetResultBlock; - pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; +// pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; pTq->tqGetResultBlockTime = tqGetResultBlockTime; pTq->tqGetStreamExecProgress = tqGetStreamExecInfo; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 5f4b7b8442..f9f2ae6b21 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -372,8 +372,8 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int32_t code = syncProcessMsg(pVnode->sync, pMsg); if (code != 0) { - vGError("vgId:%d, failed to process sync msg:%p type:%s since %s", pVnode->config.vgId, pMsg, - TMSG_INFO(pMsg->msgType), terrstr()); + vGError("vgId:%d, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pVnode->config.vgId, pMsg, + TMSG_INFO(pMsg->msgType), terrstr(), code); } return code; diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c index 1d2a55fac8..1f38264644 100644 --- a/source/libs/executor/src/countwindowoperator.c +++ b/source/libs/executor/src/countwindowoperator.c @@ -94,10 +94,10 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pBlock->info.rows;) { - int32_t step = pInfo->windowSliding; SCountWindowResult* pBuffInfo = setCountWindowOutputBuff(pExprSup, &pInfo->countSup, &pInfo->pRow); int32_t prevRows = pBuffInfo->winRows; int32_t num = updateCountWindowInfo(i, pBlock->info.rows, pInfo->windowCount, &pBuffInfo->winRows); + int32_t step = num; if (prevRows == 0) { pInfo->pRow->win.skey = tsCols[i]; } @@ -118,6 +118,8 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) { if (prevRows <= pInfo->windowSliding) { if (pBuffInfo->winRows > pInfo->windowSliding) { step = pInfo->windowSliding - prevRows; + } else { + step = pInfo->windowSliding; } } else { step = 0; diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c index 9e7f1144f8..e7c9c17a0b 100755 --- a/source/libs/executor/src/dynqueryctrloperator.c +++ b/source/libs/executor/src/dynqueryctrloperator.c @@ -456,7 +456,8 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS code = pInfo->stbJoin.basic.srcScan[1] ? buildSingleTableScanOperatorParam(&pSrcParam1, 1, rightVg, rightUid) : buildExchangeOperatorParam(&pSrcParam1, 1, rightVg, rightUid); } } - + + bool initParam = pSrcParam0 ? true : false; if (TSDB_CODE_SUCCESS == code) { code = buildGroupCacheOperatorParam(&pGcParam0, 0, *leftVg, *leftUid, pPost->leftNeedCache, pSrcParam0); pSrcParam0 = NULL; @@ -466,7 +467,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS pSrcParam1 = NULL; } if (TSDB_CODE_SUCCESS == code) { - code = buildMergeJoinOperatorParam(ppParam, pSrcParam0 ? true : false, pGcParam0, pGcParam1); + code = buildMergeJoinOperatorParam(ppParam, initParam, pGcParam0, pGcParam1); } if (TSDB_CODE_SUCCESS != code) { if (pSrcParam0) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 2534c5e9f0..831fd4e883 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1009,6 +1009,22 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { pSup->deleteMark = INT64_MAX; pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; pInfo->ignoreExpiredData = false; + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT) { + SStreamCountAggOperatorInfo* pInfo = pOperator->info; + STimeWindowAggSupp* pSup = &pInfo->twAggSup; + + ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); + + qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); + + pSup->calTriggerSaved = pSup->calTrigger; + pSup->deleteMarkSaved = pSup->deleteMark; + pSup->calTrigger = STREAM_TRIGGER_AT_ONCE; + pSup->deleteMark = INT64_MAX; + pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; + pInfo->ignoreExpiredData = false; + qInfo("save stream task:%s, param for state: %d", GET_TASKID(pTaskInfo), pInfo->ignoreExpiredData); } // iterate operator tree diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index ecc2526dd9..e1aa75d828 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -32,12 +32,6 @@ #include "operator.h" #include "querytask.h" - -#define FILL_POS_INVALID 0 -#define FILL_POS_START 1 -#define FILL_POS_MID 2 -#define FILL_POS_END 3 - typedef struct STimeRange { TSKEY skey; TSKEY ekey; @@ -474,1139 +468,3 @@ _error: taosMemoryFreeClear(pOperator); return NULL; } - -TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { - STimeWindow win = {.skey = ts, .ekey = ts}; - getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); - return win.skey; -} - -TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { - STimeWindow win = {.skey = ts, .ekey = ts}; - getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC); - return win.skey; -} - -void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { - colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull); -} - -SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) { - if (!pRaw || !pRaw->pRowVal) { - return NULL; - } - char* pData = (char*)pRaw->pRowVal; - SResultCellData* pCell = pRaw->pRowVal; - for (int32_t i = 0; i < index; i++) { - pData += (pCell->bytes + sizeof(SResultCellData)); - pCell = (SResultCellData*)pData; - } - return pCell; -} - -void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end) { - for (int32_t i = start; i < end; i++) { - destroyExprInfo(pFillCol[i].pExpr, 1); - taosVariantDestroy(&pFillCol[i].fillVal); - } - taosMemoryFreeClear(pFillCol[start].pExpr); - taosMemoryFree(pFillCol); - return NULL; -} - -void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { - pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); - tSimpleHashCleanup(pFillSup->pResMap); - pFillSup->pResMap = NULL; - cleanupExprSupp(&pFillSup->notFillExprSup); - if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) { - taosMemoryFree(pFillSup->cur.pRowVal); - } - taosMemoryFree(pFillSup->prev.pRowVal); - taosMemoryFree(pFillSup->next.pRowVal); - taosMemoryFree(pFillSup->nextNext.pRowVal); - - taosMemoryFree(pFillSup); - return NULL; -} - -void destroySPoint(void* ptr) { - SPoint* point = (SPoint*) ptr; - taosMemoryFreeClear(point->val); -} - -void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { - taosArrayDestroyEx(pFillLinear->pEndPoints, destroySPoint); - taosArrayDestroyEx(pFillLinear->pNextEndPoints, destroySPoint); - taosMemoryFree(pFillLinear); - return NULL; -} -void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { - if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_SET_VALUE_F || - pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { - taosMemoryFreeClear(pFillInfo->pResRow->pRowVal); - taosMemoryFreeClear(pFillInfo->pResRow); - } - pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); - taosArrayDestroy(pFillInfo->delRanges); - taosMemoryFree(pFillInfo); - return NULL; -} - -static void destroyStreamFillOperatorInfo(void* param) { - SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; - pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); - pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); - pInfo->pRes = blockDataDestroy(pInfo->pRes); - pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); - pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList); - taosMemoryFree(pInfo); -} - -static void resetFillWindow(SResultRowData* pRowData) { - pRowData->key = INT64_MIN; - taosMemoryFreeClear(pRowData->pRowVal); -} - -void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) { - if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) { - resetFillWindow(&pFillSup->cur); - } else { - pFillSup->cur.key = INT64_MIN; - pFillSup->cur.pRowVal = NULL; - } - resetFillWindow(&pFillSup->prev); - resetFillWindow(&pFillSup->next); - resetFillWindow(&pFillSup->nextNext); -} - -void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - - void* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState, pAPI); - - SWinKey key = {.ts = ts, .groupId = groupId}; - int32_t curVLen = 0; - - int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); - ASSERT(code == TSDB_CODE_SUCCESS); - pFillSup->cur.key = key.ts; -} - -void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - void* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState, pAPI); - - SWinKey key = {.ts = ts, .groupId = groupId}; - void* curVal = NULL; - int32_t curVLen = 0; - int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); - ASSERT(code == TSDB_CODE_SUCCESS); - pFillSup->cur.key = key.ts; - pFillSup->cur.pRowVal = curVal; - - SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key); - SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId}; - void* preVal = NULL; - int32_t preVLen = 0; - code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); - - if (code == TSDB_CODE_SUCCESS) { - pFillSup->prev.key = preKey.ts; - pFillSup->prev.pRowVal = preVal; - - code = pAPI->stateStore.streamStateCurNext(pState, pCur); - ASSERT(code == TSDB_CODE_SUCCESS); - - code = pAPI->stateStore.streamStateCurNext(pState, pCur); - if (code != TSDB_CODE_SUCCESS) { - pAPI->stateStore.streamStateFreeCur(pCur); - pCur = NULL; - } - } else { - pAPI->stateStore.streamStateFreeCur(pCur); - pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key); - } - - SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId}; - void* nextVal = NULL; - int32_t nextVLen = 0; - code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); - if (code == TSDB_CODE_SUCCESS) { - pFillSup->next.key = nextKey.ts; - pFillSup->next.pRowVal = nextVal; - if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { - code = pAPI->stateStore.streamStateCurNext(pState, pCur); - if (code == TSDB_CODE_SUCCESS) { - SWinKey nextNextKey = {.groupId = groupId}; - void* nextNextVal = NULL; - int32_t nextNextVLen = 0; - code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); - if (code == TSDB_CODE_SUCCESS) { - pFillSup->nextNext.key = nextNextKey.ts; - pFillSup->nextNext.pRowVal = nextNextVal; - } - } - } - } - pAPI->stateStore.streamStateFreeCur(pCur); -} - -static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; } -static bool hasNextWindow(SStreamFillSupporter* pFillSup) { return pFillSup->next.key != INT64_MIN; } -static bool hasNextNextWindow(SStreamFillSupporter* pFillSup) { - return pFillSup->nextNext.key != INT64_MIN; - return false; -} - -static void transBlockToResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SResultRowData* pRowVal) { - int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - SResultCellData* pCell = getResultCell(pRowVal, i); - if (!colDataIsNull_s(pColData, rowId)) { - pCell->isNull = false; - pCell->type = pColData->info.type; - pCell->bytes = pColData->info.bytes; - char* val = colDataGetData(pColData, rowId); - if (IS_VAR_DATA_TYPE(pCell->type)) { - memcpy(pCell->pData, val, varDataTLen(val)); - } else { - memcpy(pCell->pData, val, pCell->bytes); - } - } else { - pCell->isNull = true; - } - } - pRowVal->key = ts; -} - -static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pRowVal, SArray* pDelta, - SFillColInfo* pFillCol, int32_t numOfCol, int32_t winCount, int32_t order) { - for (int32_t i = 0; i < numOfCol; i++) { - if (!pFillCol[i].notFillCol) { - int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - char* var = colDataGetData(pCol, rowId); - double start = 0; - GET_TYPED_DATA(start, double, pCol->info.type, var); - SResultCellData* pCell = getResultCell(pRowVal, slotId); - double end = 0; - GET_TYPED_DATA(end, double, pCell->type, pCell->pData); - double delta = 0; - if (order == TSDB_ORDER_ASC) { - delta = (end - start) / winCount; - } else { - delta = (start - end) / winCount; - } - taosArraySet(pDelta, slotId, &delta); - } - } -} - -static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, - int32_t numOfCol) { - for (int32_t i = 0; i < numOfCol; i++) { - if (!pFillCol[i].notFillCol) { - int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); - SResultCellData* pECell = getResultCell(pEndRow, slotId); - SPoint* pPoint = taosArrayGet(pEndPoins, slotId); - pPoint->key = pEndRow->key; - memcpy(pPoint->val, pECell->pData, pECell->bytes); - } - } -} - -static void setFillInfoStart(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { - ts = taosTimeAdd(ts, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - pFillInfo->start = ts; -} - -static void setFillInfoEnd(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { - ts = taosTimeAdd(ts, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision); - pFillInfo->end = ts; -} - -static void setFillKeyInfo(TSKEY start, TSKEY end, SInterval* pInterval, SStreamFillInfo* pFillInfo) { - setFillInfoStart(start, pInterval, pFillInfo); - pFillInfo->current = pFillInfo->start; - setFillInfoEnd(end, pInterval, pFillInfo); -} - -void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { - if (!hasPrevWindow(pFillSup) || !hasNextWindow(pFillSup)) { - pFillInfo->needFill = false; - return; - } - - TSKEY realStart = taosTimeAdd(pFillSup->prev.key, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - - pFillInfo->needFill = true; - pFillInfo->start = realStart; - pFillInfo->current = pFillInfo->start; - pFillInfo->end = end; - pFillInfo->pos = FILL_POS_INVALID; - switch (pFillInfo->type) { - case TSDB_FILL_NULL: - case TSDB_FILL_NULL_F: - case TSDB_FILL_SET_VALUE: - case TSDB_FILL_SET_VALUE_F: - break; - case TSDB_FILL_PREV: - pFillInfo->pResRow = &pFillSup->prev; - break; - case TSDB_FILL_NEXT: - pFillInfo->pResRow = &pFillSup->next; - break; - case TSDB_FILL_LINEAR: { - setFillKeyInfo(pFillSup->prev.key, pFillSup->next.key, &pFillSup->interval, pFillInfo); - pFillInfo->pLinearInfo->hasNext = false; - pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, - pFillSup->numOfAllCols); - pFillInfo->pResRow = &pFillSup->prev; - pFillInfo->pLinearInfo->winIndex = 0; - } break; - default: - ASSERT(0); - break; - } -} - -void copyNotFillExpData(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { - for (int32_t i = pFillSup->numOfFillCols; i < pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); - SResultCellData* pCurCell = getResultCell(&pFillSup->cur, slotId); - pCell->isNull = pCurCell->isNull; - if (!pCurCell->isNull) { - memcpy(pCell->pData, pCurCell->pData, pCell->bytes); - } - } -} - -void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillSupporter* pFillSup, - SStreamFillInfo* pFillInfo) { - pFillInfo->preRowKey = pFillSup->cur.key; - if (!hasPrevWindow(pFillSup) && !hasNextWindow(pFillSup)) { - pFillInfo->needFill = false; - pFillInfo->pos = FILL_POS_START; - return; - } - TSKEY prevWKey = INT64_MIN; - TSKEY nextWKey = INT64_MIN; - if (hasPrevWindow(pFillSup)) { - prevWKey = pFillSup->prev.key; - } - if (hasNextWindow(pFillSup)) { - nextWKey = pFillSup->next.key; - } - - pFillInfo->needFill = true; - pFillInfo->pos = FILL_POS_INVALID; - switch (pFillInfo->type) { - case TSDB_FILL_NULL: - case TSDB_FILL_NULL_F: - case TSDB_FILL_SET_VALUE: - case TSDB_FILL_SET_VALUE_F: { - if (pFillSup->prev.key == pFillInfo->preRowKey) { - resetFillWindow(&pFillSup->prev); - } - if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { - if (pFillSup->next.key == pFillInfo->nextRowKey) { - pFillInfo->preRowKey = INT64_MIN; - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - } else { - pFillInfo->needFill = false; - pFillInfo->pos = FILL_POS_START; - } - } else if (hasPrevWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - } else { - setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_START; - } - copyNotFillExpData(pFillSup, pFillInfo); - } break; - case TSDB_FILL_PREV: { - if (hasNextWindow(pFillSup) && ((pFillSup->next.key != pFillInfo->nextRowKey) || - (pFillSup->next.key == pFillInfo->nextRowKey && hasNextNextWindow(pFillSup)) || - (pFillSup->next.key == pFillInfo->nextRowKey && !hasPrevWindow(pFillSup)))) { - setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_START; - resetFillWindow(&pFillSup->prev); - pFillSup->prev.key = pFillSup->cur.key; - pFillSup->prev.pRowVal = pFillSup->cur.pRowVal; - } else if (hasPrevWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - pFillInfo->preRowKey = INT64_MIN; - } - pFillInfo->pResRow = &pFillSup->prev; - } break; - case TSDB_FILL_NEXT: { - if (hasPrevWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - resetFillWindow(&pFillSup->next); - pFillSup->next.key = pFillSup->cur.key; - pFillSup->next.pRowVal = pFillSup->cur.pRowVal; - pFillInfo->preRowKey = INT64_MIN; - } else { - ASSERT(hasNextWindow(pFillSup)); - setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_START; - } - pFillInfo->pResRow = &pFillSup->next; - } break; - case TSDB_FILL_LINEAR: { - pFillInfo->pLinearInfo->winIndex = 0; - if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_MID; - pFillInfo->pLinearInfo->nextEnd = nextWKey; - calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, - pFillSup->numOfAllCols); - pFillInfo->pResRow = &pFillSup->prev; - - calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pNextEndPoints, pFillSup->pAllColInfo, - pFillSup->numOfAllCols); - pFillInfo->pLinearInfo->hasNext = true; - } else if (hasPrevWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, - pFillSup->numOfAllCols); - pFillInfo->pResRow = &pFillSup->prev; - pFillInfo->pLinearInfo->hasNext = false; - } else { - ASSERT(hasNextWindow(pFillSup)); - setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_START; - pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, - pFillSup->numOfAllCols); - pFillInfo->pResRow = &pFillSup->cur; - pFillInfo->pLinearInfo->hasNext = false; - } - } break; - default: - ASSERT(0); - break; - } - ASSERT(pFillInfo->pos != FILL_POS_INVALID); -} - -static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId) { - SWinKey key = {.groupId = groupId, .ts = ts}; - if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) { - return false; - } - tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); - return true; -} - -static bool buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock) { - if (pBlock->info.rows >= pBlock->info.capacity) { - return false; - } - uint64_t groupId = pBlock->info.id.groupId; - if (pFillSup->hasDelete && !checkResult(pFillSup, ts, groupId)) { - return true; - } - for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); - SFillInfo tmpInfo = { - .currentKey = ts, - .order = TSDB_ORDER_ASC, - .interval = pFillSup->interval, - }; - bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows); - if (!filled) { - SResultCellData* pCell = getResultCell(pResRow, slotId); - setRowCell(pColData, pBlock->info.rows, pCell); - } - } - pBlock->info.rows++; - return true; -} - -static bool hasRemainCalc(SStreamFillInfo* pFillInfo) { - if (pFillInfo->current != INT64_MIN && pFillInfo->current <= pFillInfo->end) { - return true; - } - return false; -} - -static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { - while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { - STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; - if (inWinRange(&pFillSup->winRange, &st)) { - buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); - } - pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - } -} - -static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { - while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { - uint64_t groupId = pBlock->info.id.groupId; - SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; - STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; - if ( ( pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId) ) || !inWinRange(&pFillSup->winRange, &st) ) { - pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - pFillInfo->pLinearInfo->winIndex++; - continue; - } - pFillInfo->pLinearInfo->winIndex++; - for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; - SFillInfo tmp = { - .currentKey = pFillInfo->current, - .order = TSDB_ORDER_ASC, - .interval = pFillSup->interval, - }; - - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); - int16_t type = pColData->info.type; - SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); - int32_t index = pBlock->info.rows; - if (pFillCol->notFillCol) { - bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index); - if (!filled) { - setRowCell(pColData, index, pCell); - } - } else { - if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { - colDataSetNULL(pColData, index); - continue; - } - SPoint* pEnd = taosArrayGet(pFillInfo->pLinearInfo->pEndPoints, slotId); - double vCell = 0; - SPoint start = {0}; - start.key = pFillInfo->pResRow->key; - start.val = pCell->pData; - - SPoint cur = {0}; - cur.key = pFillInfo->current; - cur.val = taosMemoryCalloc(1, pCell->bytes); - taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type); - colDataSetVal(pColData, index, (const char*)cur.val, false); - destroySPoint(&cur); - } - } - pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - pBlock->info.rows++; - } -} - -static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - - SWinKey key = {.groupId = groupId, .ts = pRow->key}; - int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); - qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code); - ASSERT(code == TSDB_CODE_SUCCESS); -} - -static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { - if (pFillInfo->needFill == false) { - buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); - return; - } - - if (pFillInfo->pos == FILL_POS_START) { - if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { - pFillInfo->pos = FILL_POS_INVALID; - } - } - if (pFillInfo->type != TSDB_FILL_LINEAR) { - doStreamFillNormal(pFillSup, pFillInfo, pRes); - } else { - doStreamFillLinear(pFillSup, pFillInfo, pRes); - - if (pFillInfo->pos == FILL_POS_MID) { - if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { - pFillInfo->pos = FILL_POS_INVALID; - } - } - - if (pFillInfo->current > pFillInfo->end && pFillInfo->pLinearInfo->hasNext) { - pFillInfo->pLinearInfo->hasNext = false; - pFillInfo->pLinearInfo->winIndex = 0; - taosArraySwap(pFillInfo->pLinearInfo->pEndPoints, pFillInfo->pLinearInfo->pNextEndPoints); - pFillInfo->pResRow = &pFillSup->cur; - setFillKeyInfo(pFillSup->cur.key, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo); - doStreamFillLinear(pFillSup, pFillInfo, pRes); - } - } - if (pFillInfo->pos == FILL_POS_END) { - if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { - pFillInfo->pos = FILL_POS_INVALID; - } - } -} - -void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol, - int32_t rowId, uint64_t groupId, int32_t rowSize) { - TSKEY ts = tsCol[rowId]; - pFillInfo->nextRowKey = ts; - SResultRowData tmpNextRow = {.key = ts}; - tmpNextRow.pRowVal = taosMemoryCalloc(1, rowSize); - transBlockToResultRow(pBlock, rowId, ts, &tmpNextRow); - keepResultInDiscBuf(pOperator, groupId, &tmpNextRow, rowSize); - taosMemoryFreeClear(tmpNextRow.pRowVal); -} - -static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, - SSDataBlock* pBlock, TSKEY* tsCol, int32_t rowId, SSDataBlock* pRes) { - uint64_t groupId = pBlock->info.id.groupId; - getWindowFromDiscBuf(pOperator, tsCol[rowId], groupId, pFillSup); - if (pFillSup->prev.key == pFillInfo->preRowKey) { - resetFillWindow(&pFillSup->prev); - } - setFillValueInfo(pBlock, tsCol[rowId], rowId, pFillSup, pFillInfo); - doStreamFillRange(pFillInfo, pFillSup, pRes); -} - -static void doStreamFillImpl(SOperatorInfo* pOperator) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStreamFillSupporter* pFillSup = pInfo->pFillSup; - SStreamFillInfo* pFillInfo = pInfo->pFillInfo; - SSDataBlock* pBlock = pInfo->pSrcBlock; - uint64_t groupId = pBlock->info.id.groupId; - SSDataBlock* pRes = pInfo->pRes; - SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); - TSKEY* tsCol = (TSKEY*)pTsCol->pData; - pRes->info.id.groupId = groupId; - pInfo->srcRowIndex++; - - if (pInfo->srcRowIndex == 0) { - keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); - pInfo->srcRowIndex++; - } - - while (pInfo->srcRowIndex < pBlock->info.rows) { - TSKEY ts = tsCol[pInfo->srcRowIndex]; - keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); - doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); - if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) { - blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); - return; - } - pInfo->srcRowIndex++; - } - doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); - blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); - blockDataCleanup(pInfo->pSrcBlock); -} - -static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { - SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; - void* pState = pOp->pTaskInfo->streamInfo.pState; - - SSDataBlock* pBlock = delRes; - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); - SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false); - colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false); - colDataSetNULL(pUidCol, pBlock->info.rows); - colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); - colDataSetNULL(pCalStartCol, pBlock->info.rows); - colDataSetNULL(pCalEndCol, pBlock->info.rows); - - SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - - void* tbname = NULL; - pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); - if (tbname == NULL) { - colDataSetNULL(pTableCol, pBlock->info.rows); - } else { - char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; - STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); - colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); - pAPI->stateStore.streamStateFreeVal(tbname); - } - - pBlock->info.rows++; -} - -static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, - SSDataBlock* delRes) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SStreamFillSupporter* pFillSup = pInfo->pFillSup; - if (hasPrevWindow(pFillSup)) { - TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval); - buildDeleteRange(pOperator, start, endTs, groupId, delRes); - } else if (hasNextWindow(pFillSup)) { - TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval); - buildDeleteRange(pOperator, startTs, end, groupId, delRes); - } else { - buildDeleteRange(pOperator, startTs, endTs, groupId, delRes); - } -} - -static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - SStreamFillOperatorInfo* pInfo = pOperator->info; - getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); - setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); - SWinKey key = {.ts = startTs, .groupId = groupId}; - if (!pInfo->pFillInfo->needFill) { - pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); - buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); - } else { - STimeRange tw = { - .skey = startTs, - .ekey = endTs, - .groupId = groupId, - }; - taosArrayPush(pInfo->pFillInfo->delRanges, &tw); - while (key.ts <= endTs) { - key.ts = taosTimeAdd(key.ts, pInfo->pFillSup->interval.sliding, pInfo->pFillSup->interval.slidingUnit, - pInfo->pFillSup->interval.precision); - tSimpleHashPut(pInfo->pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); - } - } -} - -static void doDeleteFillFinalize(SOperatorInfo* pOperator) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - - SStreamFillOperatorInfo* pInfo = pOperator->info; - SStreamFillInfo* pFillInfo = pInfo->pFillInfo; - int32_t size = taosArrayGetSize(pFillInfo->delRanges); - tSimpleHashClear(pInfo->pFillSup->pResMap); - for (; pFillInfo->delIndex < size; pFillInfo->delIndex++) { - STimeRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex); - if (pInfo->pRes->info.id.groupId != 0 && pInfo->pRes->info.id.groupId != range->groupId) { - return; - } - getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup); - setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo); - if (pInfo->pFillInfo->needFill) { - doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); - pInfo->pRes->info.id.groupId = range->groupId; - } - SWinKey key = {.ts = range->skey, .groupId = range->groupId}; - pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); - } -} - -static void doDeleteFillResult(SOperatorInfo* pOperator) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - - SStreamFillOperatorInfo* pInfo = pOperator->info; - SStreamFillInfo* pFillInfo = pInfo->pFillInfo; - SSDataBlock* pBlock = pInfo->pSrcDelBlock; - - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* tsStarts = (TSKEY*)pStartCol->pData; - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* groupIds = (uint64_t*)pGroupCol->pData; - while (pInfo->srcDelRowIndex < pBlock->info.rows) { - TSKEY ts = tsStarts[pInfo->srcDelRowIndex]; - TSKEY endTs = ts; - uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; - SWinKey key = {.ts = ts, .groupId = groupId}; - SStreamStateCur* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); - - if (!pCur) { - pInfo->srcDelRowIndex++; - continue; - } - - SWinKey nextKey = {.groupId = groupId, .ts = ts}; - while (pInfo->srcDelRowIndex < pBlock->info.rows) { - TSKEY delTs = tsStarts[pInfo->srcDelRowIndex]; - uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex]; - int32_t code = TSDB_CODE_SUCCESS; - if (groupId != delGroupId) { - break; - } - if (delTs > nextKey.ts) { - break; - } - - SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; - if (delTs == nextKey.ts) { - code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); - if (code == TSDB_CODE_SUCCESS) { - code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, NULL, NULL); - } - // ts will be deleted later - if (delTs != ts) { - pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); - pAPI->stateStore.streamStateFreeCur(pCur); - pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); - } - endTs = TMAX(delTs, nextKey.ts - 1); - if (code != TSDB_CODE_SUCCESS) { - break; - } - } - pInfo->srcDelRowIndex++; - } - - pAPI->stateStore.streamStateFreeCur(pCur); - doDeleteFillResultImpl(pOperator, ts, endTs, groupId); - } - - pFillInfo->current = pFillInfo->end + 1; -} - -static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { - tSimpleHashClear(pInfo->pFillSup->pResMap); - pInfo->pFillSup->hasDelete = false; - taosArrayClear(pInfo->pFillInfo->delRanges); - pInfo->pFillInfo->delIndex = 0; -} - -static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SExprSupp* pSup = &pOperator->exprSupp; - - blockDataCleanup(pDstBlock); - blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); - setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); - projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); - - pDstBlock->info.rows = 0; - pSup = &pInfo->pFillSup->notFillExprSup; - setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); - projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); - pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId; - - blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); -} - -static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - blockDataCleanup(pInfo->pRes); - if (hasRemainCalc(pInfo->pFillInfo) || - (pInfo->pFillInfo->pos != FILL_POS_INVALID && pInfo->pFillInfo->needFill == true)) { - doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); - if (pInfo->pRes->info.rows > 0) { - printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); - return pInfo->pRes; - } - } - if (pOperator->status == OP_RES_TO_RETURN) { - doDeleteFillFinalize(pOperator); - if (pInfo->pRes->info.rows > 0) { - printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); - return pInfo->pRes; - } - setOperatorCompleted(pOperator); - resetStreamFillInfo(pInfo); - return NULL; - } - - SSDataBlock* fillResult = NULL; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - while (1) { - if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows || pInfo->pSrcBlock->info.rows == 0) { - // If there are delete datablocks, we receive them first. - SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0); - if (pBlock == NULL) { - pOperator->status = OP_RES_TO_RETURN; - pInfo->pFillInfo->preRowKey = INT64_MIN; - if (pInfo->pRes->info.rows > 0) { - printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); - return pInfo->pRes; - } - break; - } - printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo)); - - if (pInfo->pFillInfo->curGroupId != pBlock->info.id.groupId) { - pInfo->pFillInfo->curGroupId = pBlock->info.id.groupId; - pInfo->pFillInfo->preRowKey = INT64_MIN; - } - - pInfo->pFillSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; - if (pInfo->pFillSup->winRange.ekey <= 0) { - pInfo->pFillSup->winRange.ekey = INT64_MAX; - } - - switch (pBlock->info.type) { - case STREAM_RETRIEVE: - return pBlock; - case STREAM_DELETE_RESULT: { - pInfo->pSrcDelBlock = pBlock; - pInfo->srcDelRowIndex = 0; - blockDataCleanup(pInfo->pDelRes); - pInfo->pFillSup->hasDelete = true; - doDeleteFillResult(pOperator); - if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); - return pInfo->pDelRes; - } - continue; - } break; - case STREAM_NORMAL: - case STREAM_INVALID: - case STREAM_PULL_DATA: { - doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); - memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); - pInfo->srcRowIndex = -1; - } break; - case STREAM_CHECKPOINT: - case STREAM_CREATE_CHILD_TABLE: { - return pBlock; - } break; - default: - ASSERTS(false, "invalid SSDataBlock type"); - } - } - - doStreamFillImpl(pOperator); - doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); - memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN); - pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; - if (pInfo->pRes->info.rows > 0) { - break; - } - } - if (pOperator->status == OP_RES_TO_RETURN) { - doDeleteFillFinalize(pOperator); - } - - if (pInfo->pRes->info.rows == 0) { - setOperatorCompleted(pOperator); - resetStreamFillInfo(pInfo); - return NULL; - } - - pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; - printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); - return pInfo->pRes; -} - -static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { - pFillSup->rowSize = sizeof(SResultCellData) * pFillSup->numOfAllCols; - for (int i = 0; i < pFillSup->numOfAllCols; i++) { - SFillColInfo* pCol = &pFillSup->pAllColInfo[i]; - SResSchema* pSchema = &pCol->pExpr->base.resSchema; - pFillSup->rowSize += pSchema->bytes; - } - pFillSup->next.key = INT64_MIN; - pFillSup->nextNext.key = INT64_MIN; - pFillSup->prev.key = INT64_MIN; - pFillSup->cur.key = INT64_MIN; - pFillSup->next.pRowVal = NULL; - pFillSup->nextNext.pRowVal = NULL; - pFillSup->prev.pRowVal = NULL; - pFillSup->cur.pRowVal = NULL; - - return TSDB_CODE_SUCCESS; -} - -static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, - SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { - SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); - if (!pFillSup) { - return NULL; - } - pFillSup->numOfFillCols = numOfFillCols; - int32_t numOfNotFillCols = 0; - SExprInfo* noFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); - pFillSup->pAllColInfo = createFillColInfo(pFillExprInfo, pFillSup->numOfFillCols, noFillExprInfo, numOfNotFillCols, - (const SNodeListNode*)(pPhyFillNode->pValues)); - pFillSup->type = convertFillType(pPhyFillNode->mode); - pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols; - pFillSup->interval = *pInterval; - pFillSup->pAPI = pAPI; - - int32_t code = initResultBuf(pFillSup); - if (code != TSDB_CODE_SUCCESS) { - destroyStreamFillSupporter(pFillSup); - return NULL; - } - - SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); - code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore); - if (code != TSDB_CODE_SUCCESS) { - destroyStreamFillSupporter(pFillSup); - return NULL; - } - - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pFillSup->pResMap = tSimpleHashInit(16, hashFn); - pFillSup->hasDelete = false; - return pFillSup; -} - -SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { - SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo)); - pFillInfo->start = INT64_MIN; - pFillInfo->current = INT64_MIN; - pFillInfo->end = INT64_MIN; - pFillInfo->preRowKey = INT64_MIN; - pFillInfo->needFill = false; - pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo)); - pFillInfo->pLinearInfo->hasNext = false; - pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - pFillInfo->pLinearInfo->pEndPoints = NULL; - pFillInfo->pLinearInfo->pNextEndPoints = NULL; - if (pFillSup->type == TSDB_FILL_LINEAR) { - pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); - pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); - for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) { - SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); - SPoint value = {0}; - value.val = taosMemoryCalloc(1, pColData->info.bytes); - taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value); - - value.val = taosMemoryCalloc(1, pColData->info.bytes); - taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value); - } - } - pFillInfo->pLinearInfo->winIndex = 0; - - pFillInfo->pResRow = NULL; - if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_SET_VALUE_F || - pFillSup->type == TSDB_FILL_NULL || pFillSup->type == TSDB_FILL_NULL_F) { - pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData)); - pFillInfo->pResRow->key = INT64_MIN; - pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize); - for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { - SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); - SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i); - pCell->bytes = pColData->info.bytes; - pCell->type = pColData->info.type; - } - } - - pFillInfo->type = pFillSup->type; - pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange)); - pFillInfo->delIndex = 0; - pFillInfo->curGroupId = 0; - return pFillInfo; -} - -SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, - SExecTaskInfo* pTaskInfo) { - SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; - int32_t numOfFillCols = 0; - SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); - pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); - if (!pInfo->pFillSup) { - goto _error; - } - - initResultSizeInfo(&pOperator->resultInfo, 4096); - pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity); - - pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes); - if (!pInfo->pFillInfo) { - goto _error; - } - - if (pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE || pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE_F) { - for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); - SVariant* pVar = &(pFillCol->fillVal); - if (pCell->type == TSDB_DATA_TYPE_FLOAT) { - float v = 0; - GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); - SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else if (IS_FLOAT_TYPE(pCell->type)) { - double v = 0; - GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); - SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else if (IS_INTEGER_TYPE(pCell->type)) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); - SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else { - pCell->isNull = true; - } - } - } else if (pInfo->pFillInfo->type == TSDB_FILL_NULL || pInfo->pFillInfo->type == TSDB_FILL_NULL_F) { - for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); - pCell->isNull = true; - } - } - - pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); - blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity); - - pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; - pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; - - int32_t numOfOutputCols = 0; - int32_t code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, - &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->srcRowIndex = -1; - setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, - pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); - setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); - - code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - return pOperator; - -_error: - destroyStreamFillOperatorInfo(pInfo); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8cc2f72adb..9024f7a341 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3763,8 +3763,9 @@ static int32_t stopSubTablesTableMergeScan(STableMergeScanInfo* pInfo) { taosMemoryFree(pSubTblsInfo); pInfo->pSubTablesMergeInfo = NULL; + + taosMemoryTrim(0); } - taosMemoryTrim(0); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index f9c7b51316..720734431f 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -390,10 +390,12 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato buf = taosDecodeFixedI32(buf, &mapSize); for (int32_t i = 0; i < mapSize; i++) { SSessionKey key = {0}; - SResultWindowInfo winfo = {0}; + SCountWindowInfo curWin = {0}; buf = decodeSSessionKey(buf, &key); - buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL}; + setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); + buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize); + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); } // 2.twAggSup @@ -694,6 +696,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->recvGetAll = false; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT; + setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, true, + OP_NOT_OPENED, pInfo, pTaskInfo); // for stream void* buff = NULL; int32_t len = 0; @@ -704,8 +708,6 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys doStreamCountDecodeOpState(buff, len, pOperator, true); taosMemoryFree(buff); } - setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, true, - OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamCountAgg, NULL, destroyStreamCountAggOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState); diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 4a2fe2416f..11d8d1487a 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -406,6 +406,7 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato if (!pInfo) { return buf; } + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; // 4.checksum int32_t dataLen = len - sizeof(uint32_t); @@ -423,6 +424,8 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); + pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap, + (void**)&winfo.pStatePos, &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } @@ -735,6 +738,8 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->reCkBlock = false; pInfo->recvGetAll = false; + setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED, + pInfo, pTaskInfo); // for stream void* buff = NULL; int32_t len = 0; @@ -746,8 +751,6 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys taosMemoryFree(buff); } - setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED, - pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState); diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c new file mode 100644 index 0000000000..b54802e2c6 --- /dev/null +++ b/source/libs/executor/src/streamfilloperator.c @@ -0,0 +1,1181 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "filter.h" +#include "os.h" +#include "query.h" +#include "taosdef.h" +#include "tmsg.h" +#include "ttypes.h" + +#include "executorInt.h" +#include "tcommon.h" +#include "thash.h" +#include "ttime.h" + +#include "function.h" +#include "querynodes.h" +#include "tdatablock.h" +#include "tfill.h" +#include "operator.h" +#include "querytask.h" + + +#define FILL_POS_INVALID 0 +#define FILL_POS_START 1 +#define FILL_POS_MID 2 +#define FILL_POS_END 3 + +typedef struct STimeRange { + TSKEY skey; + TSKEY ekey; + uint64_t groupId; +} STimeRange; + +TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { + STimeWindow win = {.skey = ts, .ekey = ts}; + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); + return win.skey; +} + +TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { + STimeWindow win = {.skey = ts, .ekey = ts}; + getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC); + return win.skey; +} + +void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { + colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull); +} + +SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) { + if (!pRaw || !pRaw->pRowVal) { + return NULL; + } + char* pData = (char*)pRaw->pRowVal; + SResultCellData* pCell = pRaw->pRowVal; + for (int32_t i = 0; i < index; i++) { + pData += (pCell->bytes + sizeof(SResultCellData)); + pCell = (SResultCellData*)pData; + } + return pCell; +} + +void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end) { + for (int32_t i = start; i < end; i++) { + destroyExprInfo(pFillCol[i].pExpr, 1); + taosVariantDestroy(&pFillCol[i].fillVal); + } + taosMemoryFreeClear(pFillCol[start].pExpr); + taosMemoryFree(pFillCol); + return NULL; +} + +void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { + pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); + tSimpleHashCleanup(pFillSup->pResMap); + pFillSup->pResMap = NULL; + cleanupExprSupp(&pFillSup->notFillExprSup); + if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) { + taosMemoryFree(pFillSup->cur.pRowVal); + } + taosMemoryFree(pFillSup->prev.pRowVal); + taosMemoryFree(pFillSup->next.pRowVal); + taosMemoryFree(pFillSup->nextNext.pRowVal); + + taosMemoryFree(pFillSup); + return NULL; +} + +void destroySPoint(void* ptr) { + SPoint* point = (SPoint*) ptr; + taosMemoryFreeClear(point->val); +} + +void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { + taosArrayDestroyEx(pFillLinear->pEndPoints, destroySPoint); + taosArrayDestroyEx(pFillLinear->pNextEndPoints, destroySPoint); + taosMemoryFree(pFillLinear); + return NULL; +} +void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { + if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_SET_VALUE_F || + pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { + taosMemoryFreeClear(pFillInfo->pResRow->pRowVal); + taosMemoryFreeClear(pFillInfo->pResRow); + } + pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); + taosArrayDestroy(pFillInfo->delRanges); + taosMemoryFree(pFillInfo); + return NULL; +} + +static void destroyStreamFillOperatorInfo(void* param) { + SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; + pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); + pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); + pInfo->pRes = blockDataDestroy(pInfo->pRes); + pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList); + taosMemoryFree(pInfo); +} + +static void resetFillWindow(SResultRowData* pRowData) { + pRowData->key = INT64_MIN; + taosMemoryFreeClear(pRowData->pRowVal); +} + +void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) { + if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) { + resetFillWindow(&pFillSup->cur); + } else { + pFillSup->cur.key = INT64_MIN; + pFillSup->cur.pRowVal = NULL; + } + resetFillWindow(&pFillSup->prev); + resetFillWindow(&pFillSup->next); + resetFillWindow(&pFillSup->nextNext); +} + +void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + void* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState, pAPI); + + SWinKey key = {.ts = ts, .groupId = groupId}; + int32_t curVLen = 0; + + int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); + ASSERT(code == TSDB_CODE_SUCCESS); + pFillSup->cur.key = key.ts; +} + +void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + void* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState, pAPI); + + SWinKey key = {.ts = ts, .groupId = groupId}; + void* curVal = NULL; + int32_t curVLen = 0; + int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); + ASSERT(code == TSDB_CODE_SUCCESS); + pFillSup->cur.key = key.ts; + pFillSup->cur.pRowVal = curVal; + + SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key); + SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId}; + void* preVal = NULL; + int32_t preVLen = 0; + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); + + if (code == TSDB_CODE_SUCCESS) { + pFillSup->prev.key = preKey.ts; + pFillSup->prev.pRowVal = preVal; + + code = pAPI->stateStore.streamStateCurNext(pState, pCur); + ASSERT(code == TSDB_CODE_SUCCESS); + + code = pAPI->stateStore.streamStateCurNext(pState, pCur); + if (code != TSDB_CODE_SUCCESS) { + pAPI->stateStore.streamStateFreeCur(pCur); + pCur = NULL; + } + } else { + pAPI->stateStore.streamStateFreeCur(pCur); + pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key); + } + + SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId}; + void* nextVal = NULL; + int32_t nextVLen = 0; + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); + if (code == TSDB_CODE_SUCCESS) { + pFillSup->next.key = nextKey.ts; + pFillSup->next.pRowVal = nextVal; + if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { + code = pAPI->stateStore.streamStateCurNext(pState, pCur); + if (code == TSDB_CODE_SUCCESS) { + SWinKey nextNextKey = {.groupId = groupId}; + void* nextNextVal = NULL; + int32_t nextNextVLen = 0; + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); + if (code == TSDB_CODE_SUCCESS) { + pFillSup->nextNext.key = nextNextKey.ts; + pFillSup->nextNext.pRowVal = nextNextVal; + } + } + } + } + pAPI->stateStore.streamStateFreeCur(pCur); +} + +static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; } +static bool hasNextWindow(SStreamFillSupporter* pFillSup) { return pFillSup->next.key != INT64_MIN; } +static bool hasNextNextWindow(SStreamFillSupporter* pFillSup) { + return pFillSup->nextNext.key != INT64_MIN; + return false; +} + +static void transBlockToResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SResultRowData* pRowVal) { + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); + SResultCellData* pCell = getResultCell(pRowVal, i); + if (!colDataIsNull_s(pColData, rowId)) { + pCell->isNull = false; + pCell->type = pColData->info.type; + pCell->bytes = pColData->info.bytes; + char* val = colDataGetData(pColData, rowId); + if (IS_VAR_DATA_TYPE(pCell->type)) { + memcpy(pCell->pData, val, varDataTLen(val)); + } else { + memcpy(pCell->pData, val, pCell->bytes); + } + } else { + pCell->isNull = true; + } + } + pRowVal->key = ts; +} + +static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pRowVal, SArray* pDelta, + SFillColInfo* pFillCol, int32_t numOfCol, int32_t winCount, int32_t order) { + for (int32_t i = 0; i < numOfCol; i++) { + if (!pFillCol[i].notFillCol) { + int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + char* var = colDataGetData(pCol, rowId); + double start = 0; + GET_TYPED_DATA(start, double, pCol->info.type, var); + SResultCellData* pCell = getResultCell(pRowVal, slotId); + double end = 0; + GET_TYPED_DATA(end, double, pCell->type, pCell->pData); + double delta = 0; + if (order == TSDB_ORDER_ASC) { + delta = (end - start) / winCount; + } else { + delta = (start - end) / winCount; + } + taosArraySet(pDelta, slotId, &delta); + } + } +} + +static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, + int32_t numOfCol) { + for (int32_t i = 0; i < numOfCol; i++) { + if (!pFillCol[i].notFillCol) { + int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); + SResultCellData* pECell = getResultCell(pEndRow, slotId); + SPoint* pPoint = taosArrayGet(pEndPoins, slotId); + pPoint->key = pEndRow->key; + memcpy(pPoint->val, pECell->pData, pECell->bytes); + } + } +} + +static void setFillInfoStart(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { + ts = taosTimeAdd(ts, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + pFillInfo->start = ts; +} + +static void setFillInfoEnd(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { + ts = taosTimeAdd(ts, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision); + pFillInfo->end = ts; +} + +static void setFillKeyInfo(TSKEY start, TSKEY end, SInterval* pInterval, SStreamFillInfo* pFillInfo) { + setFillInfoStart(start, pInterval, pFillInfo); + pFillInfo->current = pFillInfo->start; + setFillInfoEnd(end, pInterval, pFillInfo); +} + +void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { + if (!hasPrevWindow(pFillSup) || !hasNextWindow(pFillSup)) { + pFillInfo->needFill = false; + return; + } + + TSKEY realStart = taosTimeAdd(pFillSup->prev.key, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + + pFillInfo->needFill = true; + pFillInfo->start = realStart; + pFillInfo->current = pFillInfo->start; + pFillInfo->end = end; + pFillInfo->pos = FILL_POS_INVALID; + switch (pFillInfo->type) { + case TSDB_FILL_NULL: + case TSDB_FILL_NULL_F: + case TSDB_FILL_SET_VALUE: + case TSDB_FILL_SET_VALUE_F: + break; + case TSDB_FILL_PREV: + pFillInfo->pResRow = &pFillSup->prev; + break; + case TSDB_FILL_NEXT: + pFillInfo->pResRow = &pFillSup->next; + break; + case TSDB_FILL_LINEAR: { + setFillKeyInfo(pFillSup->prev.key, pFillSup->next.key, &pFillSup->interval, pFillInfo); + pFillInfo->pLinearInfo->hasNext = false; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); + pFillInfo->pResRow = &pFillSup->prev; + pFillInfo->pLinearInfo->winIndex = 0; + } break; + default: + ASSERT(0); + break; + } +} + +void copyNotFillExpData(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { + for (int32_t i = pFillSup->numOfFillCols; i < pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); + SResultCellData* pCurCell = getResultCell(&pFillSup->cur, slotId); + pCell->isNull = pCurCell->isNull; + if (!pCurCell->isNull) { + memcpy(pCell->pData, pCurCell->pData, pCell->bytes); + } + } +} + +void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillSupporter* pFillSup, + SStreamFillInfo* pFillInfo) { + pFillInfo->preRowKey = pFillSup->cur.key; + if (!hasPrevWindow(pFillSup) && !hasNextWindow(pFillSup)) { + pFillInfo->needFill = false; + pFillInfo->pos = FILL_POS_START; + return; + } + TSKEY prevWKey = INT64_MIN; + TSKEY nextWKey = INT64_MIN; + if (hasPrevWindow(pFillSup)) { + prevWKey = pFillSup->prev.key; + } + if (hasNextWindow(pFillSup)) { + nextWKey = pFillSup->next.key; + } + + pFillInfo->needFill = true; + pFillInfo->pos = FILL_POS_INVALID; + switch (pFillInfo->type) { + case TSDB_FILL_NULL: + case TSDB_FILL_NULL_F: + case TSDB_FILL_SET_VALUE: + case TSDB_FILL_SET_VALUE_F: { + if (pFillSup->prev.key == pFillInfo->preRowKey) { + resetFillWindow(&pFillSup->prev); + } + if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { + if (pFillSup->next.key == pFillInfo->nextRowKey) { + pFillInfo->preRowKey = INT64_MIN; + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + } else { + pFillInfo->needFill = false; + pFillInfo->pos = FILL_POS_START; + } + } else if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + } else { + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + } + copyNotFillExpData(pFillSup, pFillInfo); + } break; + case TSDB_FILL_PREV: { + if (hasNextWindow(pFillSup) && ((pFillSup->next.key != pFillInfo->nextRowKey) || + (pFillSup->next.key == pFillInfo->nextRowKey && hasNextNextWindow(pFillSup)) || + (pFillSup->next.key == pFillInfo->nextRowKey && !hasPrevWindow(pFillSup)))) { + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + resetFillWindow(&pFillSup->prev); + pFillSup->prev.key = pFillSup->cur.key; + pFillSup->prev.pRowVal = pFillSup->cur.pRowVal; + } else if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + pFillInfo->preRowKey = INT64_MIN; + } + pFillInfo->pResRow = &pFillSup->prev; + } break; + case TSDB_FILL_NEXT: { + if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + resetFillWindow(&pFillSup->next); + pFillSup->next.key = pFillSup->cur.key; + pFillSup->next.pRowVal = pFillSup->cur.pRowVal; + pFillInfo->preRowKey = INT64_MIN; + } else { + ASSERT(hasNextWindow(pFillSup)); + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + } + pFillInfo->pResRow = &pFillSup->next; + } break; + case TSDB_FILL_LINEAR: { + pFillInfo->pLinearInfo->winIndex = 0; + if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_MID; + pFillInfo->pLinearInfo->nextEnd = nextWKey; + calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); + pFillInfo->pResRow = &pFillSup->prev; + + calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pNextEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); + pFillInfo->pLinearInfo->hasNext = true; + } else if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); + pFillInfo->pResRow = &pFillSup->prev; + pFillInfo->pLinearInfo->hasNext = false; + } else { + ASSERT(hasNextWindow(pFillSup)); + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); + pFillInfo->pResRow = &pFillSup->cur; + pFillInfo->pLinearInfo->hasNext = false; + } + } break; + default: + ASSERT(0); + break; + } + ASSERT(pFillInfo->pos != FILL_POS_INVALID); +} + +static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId) { + SWinKey key = {.groupId = groupId, .ts = ts}; + if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) { + return false; + } + tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + return true; +} + +static bool buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock) { + if (pBlock->info.rows >= pBlock->info.capacity) { + return false; + } + uint64_t groupId = pBlock->info.id.groupId; + if (pFillSup->hasDelete && !checkResult(pFillSup, ts, groupId)) { + return true; + } + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); + SFillInfo tmpInfo = { + .currentKey = ts, + .order = TSDB_ORDER_ASC, + .interval = pFillSup->interval, + }; + bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows); + if (!filled) { + SResultCellData* pCell = getResultCell(pResRow, slotId); + setRowCell(pColData, pBlock->info.rows, pCell); + } + } + pBlock->info.rows++; + return true; +} + +static bool hasRemainCalc(SStreamFillInfo* pFillInfo) { + if (pFillInfo->current != INT64_MIN && pFillInfo->current <= pFillInfo->end) { + return true; + } + return false; +} + +static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { + while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { + STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; + if (inWinRange(&pFillSup->winRange, &st)) { + buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); + } + pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + } +} + +static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { + while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { + uint64_t groupId = pBlock->info.id.groupId; + SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; + STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; + if ( ( pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId) ) || !inWinRange(&pFillSup->winRange, &st) ) { + pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + pFillInfo->pLinearInfo->winIndex++; + continue; + } + pFillInfo->pLinearInfo->winIndex++; + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; + SFillInfo tmp = { + .currentKey = pFillInfo->current, + .order = TSDB_ORDER_ASC, + .interval = pFillSup->interval, + }; + + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); + int16_t type = pColData->info.type; + SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); + int32_t index = pBlock->info.rows; + if (pFillCol->notFillCol) { + bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index); + if (!filled) { + setRowCell(pColData, index, pCell); + } + } else { + if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { + colDataSetNULL(pColData, index); + continue; + } + SPoint* pEnd = taosArrayGet(pFillInfo->pLinearInfo->pEndPoints, slotId); + double vCell = 0; + SPoint start = {0}; + start.key = pFillInfo->pResRow->key; + start.val = pCell->pData; + + SPoint cur = {0}; + cur.key = pFillInfo->current; + cur.val = taosMemoryCalloc(1, pCell->bytes); + taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type); + colDataSetVal(pColData, index, (const char*)cur.val, false); + destroySPoint(&cur); + } + } + pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + pBlock->info.rows++; + } +} + +static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + SWinKey key = {.groupId = groupId, .ts = pRow->key}; + int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); + qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code); + ASSERT(code == TSDB_CODE_SUCCESS); +} + +static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { + if (pFillInfo->needFill == false) { + buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + return; + } + + if (pFillInfo->pos == FILL_POS_START) { + if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { + pFillInfo->pos = FILL_POS_INVALID; + } + } + if (pFillInfo->type != TSDB_FILL_LINEAR) { + doStreamFillNormal(pFillSup, pFillInfo, pRes); + } else { + doStreamFillLinear(pFillSup, pFillInfo, pRes); + + if (pFillInfo->pos == FILL_POS_MID) { + if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { + pFillInfo->pos = FILL_POS_INVALID; + } + } + + if (pFillInfo->current > pFillInfo->end && pFillInfo->pLinearInfo->hasNext) { + pFillInfo->pLinearInfo->hasNext = false; + pFillInfo->pLinearInfo->winIndex = 0; + taosArraySwap(pFillInfo->pLinearInfo->pEndPoints, pFillInfo->pLinearInfo->pNextEndPoints); + pFillInfo->pResRow = &pFillSup->cur; + setFillKeyInfo(pFillSup->cur.key, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo); + doStreamFillLinear(pFillSup, pFillInfo, pRes); + } + } + if (pFillInfo->pos == FILL_POS_END) { + if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { + pFillInfo->pos = FILL_POS_INVALID; + } + } +} + +void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol, + int32_t rowId, uint64_t groupId, int32_t rowSize) { + TSKEY ts = tsCol[rowId]; + pFillInfo->nextRowKey = ts; + SResultRowData tmpNextRow = {.key = ts}; + tmpNextRow.pRowVal = taosMemoryCalloc(1, rowSize); + transBlockToResultRow(pBlock, rowId, ts, &tmpNextRow); + keepResultInDiscBuf(pOperator, groupId, &tmpNextRow, rowSize); + taosMemoryFreeClear(tmpNextRow.pRowVal); +} + +static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, + SSDataBlock* pBlock, TSKEY* tsCol, int32_t rowId, SSDataBlock* pRes) { + uint64_t groupId = pBlock->info.id.groupId; + getWindowFromDiscBuf(pOperator, tsCol[rowId], groupId, pFillSup); + if (pFillSup->prev.key == pFillInfo->preRowKey) { + resetFillWindow(&pFillSup->prev); + } + setFillValueInfo(pBlock, tsCol[rowId], rowId, pFillSup, pFillInfo); + doStreamFillRange(pFillInfo, pFillSup, pRes); +} + +static void doStreamFillImpl(SOperatorInfo* pOperator) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamFillSupporter* pFillSup = pInfo->pFillSup; + SStreamFillInfo* pFillInfo = pInfo->pFillInfo; + SSDataBlock* pBlock = pInfo->pSrcBlock; + uint64_t groupId = pBlock->info.id.groupId; + SSDataBlock* pRes = pInfo->pRes; + SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); + TSKEY* tsCol = (TSKEY*)pTsCol->pData; + pRes->info.id.groupId = groupId; + pInfo->srcRowIndex++; + + if (pInfo->srcRowIndex == 0) { + keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); + pInfo->srcRowIndex++; + } + + while (pInfo->srcRowIndex < pBlock->info.rows) { + TSKEY ts = tsCol[pInfo->srcRowIndex]; + keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); + doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); + if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) { + blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + return; + } + pInfo->srcRowIndex++; + } + doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); + blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + blockDataCleanup(pInfo->pSrcBlock); +} + +static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { + SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; + void* pState = pOp->pTaskInfo->streamInfo.pState; + + SSDataBlock* pBlock = delRes; + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false); + colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false); + colDataSetNULL(pUidCol, pBlock->info.rows); + colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); + colDataSetNULL(pCalStartCol, pBlock->info.rows); + colDataSetNULL(pCalEndCol, pBlock->info.rows); + + SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + + void* tbname = NULL; + pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); + if (tbname == NULL) { + colDataSetNULL(pTableCol, pBlock->info.rows); + } else { + char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; + STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); + colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + pAPI->stateStore.streamStateFreeVal(tbname); + } + + pBlock->info.rows++; +} + +static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, + SSDataBlock* delRes) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SStreamFillSupporter* pFillSup = pInfo->pFillSup; + if (hasPrevWindow(pFillSup)) { + TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval); + buildDeleteRange(pOperator, start, endTs, groupId, delRes); + } else if (hasNextWindow(pFillSup)) { + TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval); + buildDeleteRange(pOperator, startTs, end, groupId, delRes); + } else { + buildDeleteRange(pOperator, startTs, endTs, groupId, delRes); + } +} + +static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamFillOperatorInfo* pInfo = pOperator->info; + getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); + setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); + SWinKey key = {.ts = startTs, .groupId = groupId}; + if (!pInfo->pFillInfo->needFill) { + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); + } else { + STimeRange tw = { + .skey = startTs, + .ekey = endTs, + .groupId = groupId, + }; + taosArrayPush(pInfo->pFillInfo->delRanges, &tw); + while (key.ts <= endTs) { + key.ts = taosTimeAdd(key.ts, pInfo->pFillSup->interval.sliding, pInfo->pFillSup->interval.slidingUnit, + pInfo->pFillSup->interval.precision); + tSimpleHashPut(pInfo->pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + } + } +} + +static void doDeleteFillFinalize(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + SStreamFillOperatorInfo* pInfo = pOperator->info; + SStreamFillInfo* pFillInfo = pInfo->pFillInfo; + int32_t size = taosArrayGetSize(pFillInfo->delRanges); + tSimpleHashClear(pInfo->pFillSup->pResMap); + for (; pFillInfo->delIndex < size; pFillInfo->delIndex++) { + STimeRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex); + if (pInfo->pRes->info.id.groupId != 0 && pInfo->pRes->info.id.groupId != range->groupId) { + return; + } + getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup); + setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo); + if (pInfo->pFillInfo->needFill) { + doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); + pInfo->pRes->info.id.groupId = range->groupId; + } + SWinKey key = {.ts = range->skey, .groupId = range->groupId}; + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + } +} + +static void doDeleteFillResult(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + SStreamFillOperatorInfo* pInfo = pOperator->info; + SStreamFillInfo* pFillInfo = pInfo->pFillInfo; + SSDataBlock* pBlock = pInfo->pSrcDelBlock; + + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* tsStarts = (TSKEY*)pStartCol->pData; + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* groupIds = (uint64_t*)pGroupCol->pData; + while (pInfo->srcDelRowIndex < pBlock->info.rows) { + TSKEY ts = tsStarts[pInfo->srcDelRowIndex]; + TSKEY endTs = ts; + uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; + SWinKey key = {.ts = ts, .groupId = groupId}; + SStreamStateCur* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); + + if (!pCur) { + pInfo->srcDelRowIndex++; + continue; + } + + SWinKey nextKey = {.groupId = groupId, .ts = ts}; + while (pInfo->srcDelRowIndex < pBlock->info.rows) { + TSKEY delTs = tsStarts[pInfo->srcDelRowIndex]; + uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex]; + int32_t code = TSDB_CODE_SUCCESS; + if (groupId != delGroupId) { + break; + } + if (delTs > nextKey.ts) { + break; + } + + SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; + if (delTs == nextKey.ts) { + code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); + if (code == TSDB_CODE_SUCCESS) { + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, NULL, NULL); + } + // ts will be deleted later + if (delTs != ts) { + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); + pAPI->stateStore.streamStateFreeCur(pCur); + pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); + } + endTs = TMAX(delTs, nextKey.ts - 1); + if (code != TSDB_CODE_SUCCESS) { + break; + } + } + pInfo->srcDelRowIndex++; + } + + pAPI->stateStore.streamStateFreeCur(pCur); + doDeleteFillResultImpl(pOperator, ts, endTs, groupId); + } + + pFillInfo->current = pFillInfo->end + 1; +} + +static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { + tSimpleHashClear(pInfo->pFillSup->pResMap); + pInfo->pFillSup->hasDelete = false; + taosArrayClear(pInfo->pFillInfo->delRanges); + pInfo->pFillInfo->delIndex = 0; +} + +static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SExprSupp* pSup = &pOperator->exprSupp; + + blockDataCleanup(pDstBlock); + blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); + setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); + projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + + pDstBlock->info.rows = 0; + pSup = &pInfo->pFillSup->notFillExprSup; + setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); + projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId; + + blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); +} + +static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + blockDataCleanup(pInfo->pRes); + if (hasRemainCalc(pInfo->pFillInfo) || + (pInfo->pFillInfo->pos != FILL_POS_INVALID && pInfo->pFillInfo->needFill == true)) { + doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); + if (pInfo->pRes->info.rows > 0) { + printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pRes; + } + } + if (pOperator->status == OP_RES_TO_RETURN) { + doDeleteFillFinalize(pOperator); + if (pInfo->pRes->info.rows > 0) { + printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pRes; + } + setOperatorCompleted(pOperator); + resetStreamFillInfo(pInfo); + return NULL; + } + + SSDataBlock* fillResult = NULL; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + while (1) { + if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows || pInfo->pSrcBlock->info.rows == 0) { + // If there are delete datablocks, we receive them first. + SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0); + if (pBlock == NULL) { + pOperator->status = OP_RES_TO_RETURN; + pInfo->pFillInfo->preRowKey = INT64_MIN; + if (pInfo->pRes->info.rows > 0) { + printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pRes; + } + break; + } + printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo)); + + if (pInfo->pFillInfo->curGroupId != pBlock->info.id.groupId) { + pInfo->pFillInfo->curGroupId = pBlock->info.id.groupId; + pInfo->pFillInfo->preRowKey = INT64_MIN; + } + + pInfo->pFillSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; + if (pInfo->pFillSup->winRange.ekey <= 0) { + pInfo->pFillSup->winRange.ekey = INT64_MAX; + } + + switch (pBlock->info.type) { + case STREAM_RETRIEVE: + return pBlock; + case STREAM_DELETE_RESULT: { + pInfo->pSrcDelBlock = pBlock; + pInfo->srcDelRowIndex = 0; + blockDataCleanup(pInfo->pDelRes); + pInfo->pFillSup->hasDelete = true; + doDeleteFillResult(pOperator); + if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pDelRes; + } + continue; + } break; + case STREAM_NORMAL: + case STREAM_INVALID: + case STREAM_PULL_DATA: { + doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); + memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); + pInfo->srcRowIndex = -1; + } break; + case STREAM_CHECKPOINT: + case STREAM_CREATE_CHILD_TABLE: { + return pBlock; + } break; + default: + ASSERTS(false, "invalid SSDataBlock type"); + } + } + + doStreamFillImpl(pOperator); + doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); + memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN); + pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; + if (pInfo->pRes->info.rows > 0) { + break; + } + } + if (pOperator->status == OP_RES_TO_RETURN) { + doDeleteFillFinalize(pOperator); + } + + if (pInfo->pRes->info.rows == 0) { + setOperatorCompleted(pOperator); + resetStreamFillInfo(pInfo); + return NULL; + } + + pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; + printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pRes; +} + +static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { + pFillSup->rowSize = sizeof(SResultCellData) * pFillSup->numOfAllCols; + for (int i = 0; i < pFillSup->numOfAllCols; i++) { + SFillColInfo* pCol = &pFillSup->pAllColInfo[i]; + SResSchema* pSchema = &pCol->pExpr->base.resSchema; + pFillSup->rowSize += pSchema->bytes; + } + pFillSup->next.key = INT64_MIN; + pFillSup->nextNext.key = INT64_MIN; + pFillSup->prev.key = INT64_MIN; + pFillSup->cur.key = INT64_MIN; + pFillSup->next.pRowVal = NULL; + pFillSup->nextNext.pRowVal = NULL; + pFillSup->prev.pRowVal = NULL; + pFillSup->cur.pRowVal = NULL; + + return TSDB_CODE_SUCCESS; +} + +static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, + SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { + SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); + if (!pFillSup) { + return NULL; + } + pFillSup->numOfFillCols = numOfFillCols; + int32_t numOfNotFillCols = 0; + SExprInfo* noFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); + pFillSup->pAllColInfo = createFillColInfo(pFillExprInfo, pFillSup->numOfFillCols, noFillExprInfo, numOfNotFillCols, + (const SNodeListNode*)(pPhyFillNode->pValues)); + pFillSup->type = convertFillType(pPhyFillNode->mode); + pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols; + pFillSup->interval = *pInterval; + pFillSup->pAPI = pAPI; + + int32_t code = initResultBuf(pFillSup); + if (code != TSDB_CODE_SUCCESS) { + destroyStreamFillSupporter(pFillSup); + return NULL; + } + + SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); + code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore); + if (code != TSDB_CODE_SUCCESS) { + destroyStreamFillSupporter(pFillSup); + return NULL; + } + + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pFillSup->pResMap = tSimpleHashInit(16, hashFn); + pFillSup->hasDelete = false; + return pFillSup; +} + +SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { + SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo)); + pFillInfo->start = INT64_MIN; + pFillInfo->current = INT64_MIN; + pFillInfo->end = INT64_MIN; + pFillInfo->preRowKey = INT64_MIN; + pFillInfo->needFill = false; + pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo)); + pFillInfo->pLinearInfo->hasNext = false; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + pFillInfo->pLinearInfo->pEndPoints = NULL; + pFillInfo->pLinearInfo->pNextEndPoints = NULL; + if (pFillSup->type == TSDB_FILL_LINEAR) { + pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); + pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); + for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) { + SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); + SPoint value = {0}; + value.val = taosMemoryCalloc(1, pColData->info.bytes); + taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value); + + value.val = taosMemoryCalloc(1, pColData->info.bytes); + taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value); + } + } + pFillInfo->pLinearInfo->winIndex = 0; + + pFillInfo->pResRow = NULL; + if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_SET_VALUE_F || + pFillSup->type == TSDB_FILL_NULL || pFillSup->type == TSDB_FILL_NULL_F) { + pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData)); + pFillInfo->pResRow->key = INT64_MIN; + pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize); + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); + SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i); + pCell->bytes = pColData->info.bytes; + pCell->type = pColData->info.type; + } + } + + pFillInfo->type = pFillSup->type; + pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange)); + pFillInfo->delIndex = 0; + pFillInfo->curGroupId = 0; + return pFillInfo; +} + +SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, + SExecTaskInfo* pTaskInfo) { + SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; + int32_t numOfFillCols = 0; + SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); + pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); + if (!pInfo->pFillSup) { + goto _error; + } + + initResultSizeInfo(&pOperator->resultInfo, 4096); + pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); + pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity); + + pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes); + if (!pInfo->pFillInfo) { + goto _error; + } + + if (pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE || pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE_F) { + for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); + SVariant* pVar = &(pFillCol->fillVal); + if (pCell->type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); + SET_TYPED_DATA(pCell->pData, pCell->type, v); + } else if (IS_FLOAT_TYPE(pCell->type)) { + double v = 0; + GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); + SET_TYPED_DATA(pCell->pData, pCell->type, v); + } else if (IS_INTEGER_TYPE(pCell->type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + SET_TYPED_DATA(pCell->pData, pCell->type, v); + } else { + pCell->isNull = true; + } + } + } else if (pInfo->pFillInfo->type == TSDB_FILL_NULL || pInfo->pFillInfo->type == TSDB_FILL_NULL_F) { + for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); + pCell->isNull = true; + } + } + + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); + blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity); + + pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; + pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; + + int32_t numOfOutputCols = 0; + int32_t code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, + &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->srcRowIndex = -1; + setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, + pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + return pOperator; + +_error: + destroyStreamFillOperatorInfo(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 2838d005ab..7a1bb2729c 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2533,7 +2533,6 @@ int32_t encodeSResultWindowInfo(void** buf, SResultWindowInfo* key, int32_t outL void* decodeSResultWindowInfo(void* buf, SResultWindowInfo* key, int32_t outLen) { buf = taosDecodeFixedBool(buf, &key->isOutput); - key->pStatePos->pRowBuff = NULL; buf = decodeSSessionKey(buf, &key->sessionWin); return buf; } @@ -2591,6 +2590,7 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera if (!pInfo) { return buf; } + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; // 5.checksum if (isParent) { @@ -2609,6 +2609,8 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); + pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap, + (void**)&winfo.pStatePos, &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } @@ -2992,6 +2994,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->recvGetAll = false; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; + setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, + OP_NOT_OPENED, pInfo, pTaskInfo); // for stream void* buff = NULL; int32_t len = 0; @@ -3002,8 +3006,6 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh doStreamSessionDecodeOpState(buff, len, pOperator, true); taosMemoryFree(buff); } - setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, - OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); @@ -3538,6 +3540,7 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato if (!pInfo) { return buf; } + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; // 5.checksum if (isParent) { @@ -3556,6 +3559,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); + pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, + pAggSup->stateKeySize, compareStateKey, + (void**)&winfo.pStatePos, &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } @@ -3873,6 +3879,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); pInfo->recvGetAll = false; + setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, + pInfo, pTaskInfo); // for stream void* buff = NULL; int32_t len = 0; @@ -3884,8 +3892,6 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys taosMemoryFree(buff); } - setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, - pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 9ff903cdb9..10220426a3 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -1199,6 +1199,18 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { return code; } +static void freeSSortSource(SSortSource* source) { + if (NULL == source) return; + if (source->param && !source->onlyRef) { + taosMemoryFree(source->param); + } + if (!source->onlyRef && source->src.pBlock) { + blockDataDestroy(source->src.pBlock); + source->src.pBlock = NULL; + } + taosMemoryFree(source); +} + static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { int32_t code = 0; size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; @@ -1231,14 +1243,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { code = blockDataMerge(pHandle->pDataBlock, pBlock); if (code != TSDB_CODE_SUCCESS) { - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); - } - if (!source->onlyRef && source->src.pBlock) { - blockDataDestroy(source->src.pBlock); - source->src.pBlock = NULL; - } - taosMemoryFree(source); + freeSSortSource(source); return code; } @@ -1248,15 +1253,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { int64_t p = taosGetTimestampUs(); code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); if (code != 0) { - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); - } - if (!source->onlyRef && source->src.pBlock) { - blockDataDestroy(source->src.pBlock); - source->src.pBlock = NULL; - } - - taosMemoryFree(source); + freeSSortSource(source); return code; } @@ -1265,16 +1262,13 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { if (pHandle->pqMaxRows > 0) blockDataKeepFirstNRows(pHandle->pDataBlock, pHandle->pqMaxRows); code = doAddToBuf(pHandle->pDataBlock, pHandle); if (code != TSDB_CODE_SUCCESS) { + freeSSortSource(source); return code; } } } - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); - } - - taosMemoryFree(source); + freeSSortSource(source); if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) { size_t size = blockDataGetSize(pHandle->pDataBlock); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 1313221952..b7b9f1cc9f 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -76,8 +76,8 @@ char* idxInt2str(int64_t val, char* dst, int radix) { return dst - 1; } __compar_fn_t idxGetCompar(int8_t type) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || - type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_NCHAR || + type == TSDB_DATA_TYPE_GEOMETRY) { return (__compar_fn_t)strcmp; } return getComparFunc(type, 0); @@ -108,8 +108,8 @@ static FORCE_INLINE TExeCond tCompareEqual(void* a, void* b, int8_t type) { return tCompare(func, QUERY_TERM, a, b, type); } TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) { - if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || - dtype == TSDB_DATA_TYPE_VARBINARY || dtype == TSDB_DATA_TYPE_GEOMETRY) { + if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY || + dtype == TSDB_DATA_TYPE_GEOMETRY) { return tDoCompare(func, cmptype, a, b); } #if 1 @@ -290,6 +290,7 @@ int idxUidCompare(const void* a, const void* b) { uint64_t r = *(uint64_t*)b; return l - r; } +#ifdef BUILD_NO_CALL int32_t idxConvertData(void* src, int8_t type, void** dst) { int tlen = -1; switch (type) { @@ -372,6 +373,8 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) { // indexMayFillNumbericData(*dst, tlen); return tlen; } +#endif + int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { if (src == NULL) { *dst = strndup(INDEX_DATA_NULL_STR, (int)strlen(INDEX_DATA_NULL_STR)); diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index 55b08775b4..c3ed0112c5 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -216,7 +216,7 @@ int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self regfree(&r); return 0; } - +/* const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts, char *format) { taos_metric_formatter_clear(self->metric_formatter); taos_metric_formatter_load_metrics(self->metric_formatter, self->collectors, ts, format); @@ -229,7 +229,7 @@ const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char return taos_string_builder_str(self->string_builder_batch); } - +*/ int taos_collector_registry_clear_batch(taos_collector_registry_t *self){ return taos_string_builder_clear(self->string_builder_batch); } diff --git a/source/libs/monitorfw/src/taos_gauge.c b/source/libs/monitorfw/src/taos_gauge.c index 74d2665194..7793f4c464 100644 --- a/source/libs/monitorfw/src/taos_gauge.c +++ b/source/libs/monitorfw/src/taos_gauge.c @@ -38,7 +38,7 @@ int taos_gauge_destroy(taos_gauge_t *self) { self = NULL; return r; } - +/* int taos_gauge_inc(taos_gauge_t *self, const char **label_values) { TAOS_ASSERT(self != NULL); if (self == NULL) return 1; @@ -86,7 +86,7 @@ int taos_gauge_sub(taos_gauge_t *self, double r_value, const char **label_values if (sample == NULL) return 1; return taos_metric_sample_sub(sample, r_value); } - +*/ int taos_gauge_set(taos_gauge_t *self, double r_value, const char **label_values) { TAOS_ASSERT(self != NULL); if (self == NULL) return 1; diff --git a/source/libs/monitorfw/src/taos_metric_formatter.c b/source/libs/monitorfw/src/taos_metric_formatter.c index a9f35c3e8d..53012935ba 100644 --- a/source/libs/monitorfw/src/taos_metric_formatter.c +++ b/source/libs/monitorfw/src/taos_metric_formatter.c @@ -63,7 +63,7 @@ int taos_metric_formatter_destroy(taos_metric_formatter_t *self) { self = NULL; return ret; } - +/* int taos_metric_formatter_load_help(taos_metric_formatter_t *self, const char *name, const char *help) { TAOS_ASSERT(self != NULL); if (self == NULL) return 1; @@ -105,7 +105,7 @@ int taos_metric_formatter_load_type(taos_metric_formatter_t *self, const char *n return taos_string_builder_add_char(self->string_builder, '\n'); } - +*/ int taos_metric_formatter_load_l_value(taos_metric_formatter_t *self, const char *name, const char *suffix, size_t label_count, const char **label_keys, const char **label_values) { TAOS_ASSERT(self != NULL); @@ -156,7 +156,7 @@ int taos_metric_formatter_load_l_value(taos_metric_formatter_t *self, const char } return 0; } - +/* int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric_sample_t *sample, char *ts, char *format) { TAOS_ASSERT(self != NULL); @@ -185,7 +185,7 @@ int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric return taos_string_builder_add_char(self->string_builder, '\n'); } - +*/ int taos_metric_formatter_clear(taos_metric_formatter_t *self) { TAOS_ASSERT(self != NULL); return taos_string_builder_clear(self->string_builder); @@ -204,7 +204,7 @@ char *taos_metric_formatter_dump(taos_metric_formatter_t *self) { } return data; } - +/* int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format) { TAOS_ASSERT(self != NULL); if (self == NULL) return 1; @@ -255,3 +255,4 @@ int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t } return r; } +*/ \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index ca6ea30028..c6d817b513 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -91,6 +91,7 @@ int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { return 0; } +/* int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { TAOS_ASSERT(self != NULL); if (self->type != TAOS_GAUGE) { @@ -99,7 +100,8 @@ int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { } #ifdef C11_ATOMIC - /*_Atomic*/ double old = atomic_load(&self->r_value); + ///_Atomic/ + double old = atomic_load(&self->r_value); for (;;) { _Atomic double new = ATOMIC_VAR_INIT(old - r_value); if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { @@ -116,6 +118,7 @@ int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { return 0; } +*/ int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) { if (self->type != TAOS_GAUGE && self->type != TAOS_COUNTER) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0f9f978515..6db68f54c7 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4010,6 +4010,26 @@ static int32_t translateEventWindow(STranslateContext* pCxt, SSelectStmt* pSelec } static int32_t translateCountWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow; + if (pCountWin->windowCount <= 1) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Size of Count window must exceed 1."); + } + + if (pCountWin->windowSliding <= 0) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Size of Count window must exceed 0."); + } + + if (pCountWin->windowSliding > pCountWin->windowCount) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "sliding value no larger than the count value."); + } + + if (pCountWin->windowCount > INT32_MAX) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Size of Count window must less than 2147483647(INT32_MAX)."); + } if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TIMELINE_QUERY, @@ -4528,7 +4548,7 @@ static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = getQueryTimeRange(pCxt, pSelect->pWhere, &pSelect->timeRange); } - if (pSelect->pWhere != NULL) { + if (pSelect->pWhere != NULL && pCxt->pParseCxt->topicQuery == false) { setTableVgroupsFromEqualTbnameCond(pCxt, pSelect); } return code; @@ -7829,29 +7849,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm if (pStmt->pOptions->ignoreExpired != 1) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Ignore expired data of Count window must be 1."); - } - - SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow; - if (pCountWin->windowCount <= 1) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Size of Count window must exceed 1."); } - - if (pCountWin->windowSliding <= 0) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Size of Count window must exceed 0."); - } - - if (pCountWin->windowSliding > pCountWin->windowCount) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "sliding value no larger than the count value."); - } - - if (pCountWin->windowCount > INT32_MAX) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Size of Count window must less than 2147483647(INT32_MAX)."); - } - } return TSDB_CODE_SUCCESS; @@ -8138,7 +8136,9 @@ static int32_t adjustTagsForCreateTable(STranslateContext* pCxt, SCreateStreamSt SColumnDefNode* pDef = (SColumnDefNode*)pTagDef; if (!dataTypeEqual(&pDef->dataType, &((SExprNode*)pTagExpr)->resType)) { SNode* pFunc = NULL; - int32_t code = createCastFunc(pCxt, pTagExpr, pDef->dataType, &pFunc); + SDataType defType = pDef->dataType; + defType.bytes = calcTypeBytes(defType); + int32_t code = createCastFunc(pCxt, pTagExpr, defType, &pFunc); if (TSDB_CODE_SUCCESS != code) { return code; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 0f5ddf926d..37cdc31ae0 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1016,10 +1016,6 @@ static int32_t createWindowLogicNodeByCount(SLogicPlanContext* pCxt, SCountWindo return TSDB_CODE_OUT_OF_MEMORY; } - if (!pCxt->pPlanCxt->streamQuery && tsDisableCount) { - return TSDB_CODE_FAILED; - } - pWindow->winType = WINDOW_TYPE_COUNT; pWindow->node.groupAction = getGroupAction(pCxt, pSelect); pWindow->node.requireDataOrder = diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index d9a7475f59..9664be1da2 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -274,7 +274,7 @@ static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) { } } - if (WINDOW_TYPE_STATE == pWindow->winType) { + if (WINDOW_TYPE_STATE == pWindow->winType || WINDOW_TYPE_COUNT == pWindow->winType) { if (!streamQuery) { return stbSplHasMultiTbScan(streamQuery, pNode); } else { diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 66ec460861..faa90dcbf8 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -715,6 +715,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SD uint64_t tId = req.taskId; int64_t rId = 0; int32_t eId = -1; + pRes->source = req.source; SQWMsg qwMsg = {.node = node, .msg = req.msg, .msgLen = req.phyLen, .connInfo = pMsg->info}; QW_SCH_TASK_DLOG("processDelete start, node:%p, handle:%p, sql:%s", node, pMsg->info.handle, req.sql); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 1aa21e73a3..d129b0024f 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -304,6 +304,7 @@ typedef struct SSchJob { SSchResInfo userRes; char *sql; SQueryProfileSummary summary; + int8_t source; } SSchJob; typedef struct SSchTaskCtx { diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index e50ec64d54..48aab63ba3 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -746,6 +746,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->chkKillParam = pReq->chkKillParam; pJob->userRes.execFp = pReq->execFp; pJob->userRes.cbParam = pReq->cbParam; + pJob->source = pReq->source; if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) { qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1c0b31109e..43ea418851 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -940,6 +940,10 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery SCH_ERR_JRET(schGenerateCallBackInfo(pJob, pTask, msg, msgSize, msgType, trans, isHb, &pMsgSendInfo)); SCH_ERR_JRET(schUpdateSendTargetInfo(pMsgSendInfo, addr, pTask)); + if (isHb && persistHandle && trans->pHandle == 0) { + trans->pHandle = rpcAllocHandle(); + } + if (pJob && pTask) { SCH_TASK_DLOG("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType), addr->nodeId, epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle); @@ -1086,6 +1090,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.sqlLen = strlen(pJob->sql); req.sql = (char *)pJob->sql; req.msg = pTask->msg; + req.source = pJob->source; msgSize = tSerializeSVDeleteReq(NULL, 0, &req); msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 03f70604b7..1dc1db8e9c 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -17,10 +17,14 @@ #define _STREAM_BACKEDN_ROCKSDB_H_ #include "rocksdb/c.h" -//#include "streamInt.h" +// #include "streamInt.h" #include "streamState.h" #include "tcommon.h" +#ifdef __cplusplus +extern "C" { +#endif + typedef struct SCfComparator { rocksdb_comparator_t** comp; int32_t numOfComp; @@ -244,11 +248,6 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId); int32_t taskDbBuildSnap(void* arg, SArray* pSnap); -// int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); - -// STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId); -// void taskDbDestroy(void* pDb, bool flush); - int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); SBkdMgt* bkdMgtCreate(char* path); @@ -258,4 +257,10 @@ int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname); void bkdMgtDestroy(SBkdMgt* bm); int32_t taskDbGenChkpUploadData(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** path, SArray* list); -#endif \ No newline at end of file + +uint32_t nextPow2(uint32_t x); +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 87f63b48ed..d0055d5400 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -99,6 +99,7 @@ void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration); int32_t streamDispatchStreamBlock(SStreamTask* pTask); void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups); int32_t getNumOfDispatchBranch(SStreamTask* pTask); +void clearBufferedDispatchMsg(SStreamTask* pTask); int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg); diff --git a/source/libs/stream/inc/streamsm.h b/source/libs/stream/inc/streamsm.h index 22e1c4497b..47e0ce1b55 100644 --- a/source/libs/stream/inc/streamsm.h +++ b/source/libs/stream/inc/streamsm.h @@ -26,21 +26,21 @@ extern "C" { typedef int32_t (*__state_trans_fn)(SStreamTask*); typedef int32_t (*__state_trans_succ_fn)(SStreamTask*); -typedef struct SAttachedEventInfo { +typedef struct SFutureHandleEventInfo { ETaskStatus status; // required status that this event can be handled EStreamTaskEvent event; // the delayed handled event void* pParam; - void* pFn; -} SAttachedEventInfo; + __state_trans_user_fn callBackFn; +} SFutureHandleEventInfo; typedef struct STaskStateTrans { - bool autoInvokeEndFn; - SStreamTaskState state; - EStreamTaskEvent event; - SStreamTaskState next; - __state_trans_fn pAction; - __state_trans_succ_fn pSuccAction; - SAttachedEventInfo attachEvent; + bool autoInvokeEndFn; + SStreamTaskState state; + EStreamTaskEvent event; + SStreamTaskState next; + __state_trans_fn pAction; + __state_trans_succ_fn pSuccAction; + SFutureHandleEventInfo attachEvent; } STaskStateTrans; struct SStreamTaskSM { diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f173157da6..5dafad8951 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -906,6 +906,7 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { return 0; } +#ifdef BUILD_NO_CALL static int32_t chkpIdComp(const void* a, const void* b) { int64_t x = *(int64_t*)a; int64_t y = *(int64_t*)b; @@ -964,6 +965,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) { taosMemoryFree(chkpPath); return 0; } +#endif #ifdef BUILD_NO_CALL int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*** ppHandle, SArray* refs) { @@ -2788,7 +2790,6 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) { STREAM_STATE_DEL_ROCKSDB(pState, "state", &maxStateKey); return pCur; } -#ifdef BUILD_NO_CALL SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { stDebug("streamStateGetCur_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; @@ -2838,7 +2839,6 @@ int32_t streamStateFuncDel_rocksdb(SStreamState* pState, const STupleKey* key) { STREAM_STATE_DEL_ROCKSDB(pState, "func", key); return 0; } -#endif // session cf int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { @@ -3432,7 +3432,6 @@ int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* SSessionKey tmpKey = *key; int32_t valSize = *pVLen; void* tmp = taosMemoryMalloc(valSize); - // tdbRealloc(NULL, valSize); if (!tmp) { return -1; } @@ -3506,13 +3505,11 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi return code; } -#ifdef BUILD_NO_CALL int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) { int code = 0; STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen); return code; } -#endif int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen) { int code = 0; STREAM_STATE_GET_ROCKSDB(pState, "default", key, pVal, pVLen); @@ -3535,10 +3532,10 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co if (pIter == NULL) { return -1; } - + size_t klen = 0; rocksdb_iter_seek(pIter, start, strlen(start)); while (rocksdb_iter_valid(pIter)) { - const char* key = rocksdb_iter_key(pIter, NULL); + const char* key = rocksdb_iter_key(pIter, &klen); int32_t vlen = 0; const char* vval = rocksdb_iter_value(pIter, (size_t*)&vlen); char* val = NULL; @@ -3700,6 +3697,8 @@ uint32_t nextPow2(uint32_t x) { x = x | (x >> 16); return x + 1; } + +#ifdef BUILD_NO_CALL int32_t copyFiles(const char* src, const char* dst) { int32_t code = 0; // opt later, just hard link @@ -3739,6 +3738,7 @@ _err: taosCloseDir(&pDir); return code >= 0 ? 0 : -1; } +#endif int32_t isBkdDataMeta(char* name, int32_t len) { const char* pCurrent = "CURRENT"; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 78b914c3db..dc790b5b2d 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -315,6 +315,16 @@ int32_t getNumOfDispatchBranch(SStreamTask* pTask) { : taosArrayGetSize(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos); } +void clearBufferedDispatchMsg(SStreamTask* pTask) { + SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo; + if (pMsgInfo->pData != NULL) { + destroyDispatchMsg(pMsgInfo->pData, getNumOfDispatchBranch(pTask)); + } + + pMsgInfo->pData = NULL; + pMsgInfo->dispatchMsgType = 0; +} + static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pData) { int32_t code = 0; int32_t numOfBlocks = taosArrayGetSize(pData->blocks); @@ -678,8 +688,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { // todo deal with only partially success dispatch case atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 0); if (terrno == TSDB_CODE_APP_IS_STOPPING) { // in case of this error, do not retry anymore - destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); - pTask->msgInfo.pData = NULL; + clearBufferedDispatchMsg(pTask); return code; } @@ -740,6 +749,8 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + ASSERT(dataStrLen > 0); + void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) return -1; @@ -936,15 +947,12 @@ void streamClearChkptReadyMsg(SStreamTask* pTask) { // this message has been sent successfully, let's try next one. static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId) { stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); - destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); - bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); if (delayDispatch) { pTask->chkInfo.dispatchCheckpointTrigger = true; } - pTask->msgInfo.pData = NULL; - pTask->msgInfo.dispatchMsgType = 0; + clearBufferedDispatchMsg(pTask); int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs; @@ -1084,7 +1092,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } else { // this message has been sent successfully, let's try next one. pTask->msgInfo.retryCount = 0; - // transtate msg has been sent to downstream successfully. let's transfer the fill-history task state + // trans-state msg has been sent to downstream successfully. let's transfer the fill-history task state if (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__TRANS_STATE) { stDebug("s-task:%s dispatch transtate msgId:%d to downstream successfully, start to transfer state", id, msgId); ASSERT(pTask->info.fillHistory == 1); @@ -1093,6 +1101,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (code != TSDB_CODE_SUCCESS) { // todo: do nothing if error happens } + clearBufferedDispatchMsg(pTask); + // now ready for next data output atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); } else { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index bac6022834..cb51d90fba 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -353,7 +353,8 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { ASSERT(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP); } else { - ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP); + ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__PAUSE || status == TASK_STATUS__DROPPING || + status == TASK_STATUS__STOP); int32_t code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", id, @@ -410,6 +411,12 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { return TSDB_CODE_SUCCESS; } +static int32_t haltCallback(SStreamTask* pTask, void* param) { + streamTaskOpenAllUpstreamInput(pTask); + streamTaskSendCheckpointReq(pTask); + return TSDB_CODE_SUCCESS; +} + int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { int32_t code = TSDB_CODE_SUCCESS; SStreamMeta* pMeta = pTask->pMeta; @@ -419,11 +426,12 @@ int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { int32_t level = pTask->info.taskLevel; if (level == TASK_LEVEL__AGG || level == TASK_LEVEL__SOURCE) { // do transfer task operator states. code = streamDoTransferStateToStreamTask(pTask); - } else { // no state transfer for sink tasks, and drop fill-history task, followed by opening inputQ of sink task. + } else { + // no state transfer for sink tasks, and drop fill-history task, followed by opening inputQ of sink task. SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); if (pStreamTask != NULL) { // halt the related stream sink task - code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); + code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, haltCallback, NULL); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", pTask->id.idStr, pStreamTask->id.idStr, tstrerror(code)); @@ -432,9 +440,6 @@ int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { } else { stDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, pTask->id.idStr); } - - streamTaskOpenAllUpstreamInput(pStreamTask); - streamTaskSendCheckpointReq(pStreamTask); streamMetaReleaseTask(pMeta, pStreamTask); } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index b35f401cb9..a072ee1f6f 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -70,7 +70,7 @@ static void streamMetaEnvInit() { streamTimerInit(); } -void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit);} +void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } void streamMetaCleanup() { taosCloseRef(streamBackendId); @@ -669,6 +669,13 @@ static void doRemoveIdFromList(SStreamMeta* pMeta, int32_t num, SStreamTaskId* i } } +static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + streamTaskSendCheckpointSourceRsp(pTask); + } + return 0; +} + int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { SStreamTask* pTask = NULL; @@ -687,7 +694,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t } // handle the dropping event - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_DROPPING); + streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL); } else { stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId); streamMetaWUnLock(pMeta); @@ -1104,14 +1111,14 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { .inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputq.queue)), }; - entry.inputRate = entry.inputQUsed * 100.0 / (2*STREAM_TASK_QUEUE_CAPACITY_IN_SIZE); + entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE); if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) { entry.sinkQuota = (*pTask)->outputInfo.pTokenBucket->quotaRate; entry.sinkDataSize = SIZE_IN_MiB((*pTask)->execInfo.sink.dataSize); } if ((*pTask)->chkInfo.checkpointingId != 0) { - entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId)? 1:0; + entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId) ? 1 : 0; entry.checkpointId = (*pTask)->chkInfo.checkpointingId; entry.chkpointTransId = (*pTask)->chkInfo.transId; @@ -1172,7 +1179,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { stDebug("vgId:%d no tasks and no mnd epset, not send stream hb to mnode", pMeta->vgId); } - _end: +_end: streamMetaClearHbMsg(&hbMsg); return TSDB_CODE_SUCCESS; } @@ -1304,28 +1311,28 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { } void streamMetaRLock(SStreamMeta* pMeta) { -// stTrace("vgId:%d meta-rlock", pMeta->vgId); + // stTrace("vgId:%d meta-rlock", pMeta->vgId); taosThreadRwlockRdlock(&pMeta->lock); } void streamMetaRUnLock(SStreamMeta* pMeta) { -// stTrace("vgId:%d meta-runlock", pMeta->vgId); + // stTrace("vgId:%d meta-runlock", pMeta->vgId); int32_t code = taosThreadRwlockUnlock(&pMeta->lock); if (code != TSDB_CODE_SUCCESS) { stError("vgId:%d meta-runlock failed, code:%d", pMeta->vgId, code); } else { -// stTrace("vgId:%d meta-runlock completed", pMeta->vgId); + // stTrace("vgId:%d meta-runlock completed", pMeta->vgId); } } void streamMetaWLock(SStreamMeta* pMeta) { -// stTrace("vgId:%d meta-wlock", pMeta->vgId); + // stTrace("vgId:%d meta-wlock", pMeta->vgId); taosThreadRwlockWrlock(&pMeta->lock); -// stTrace("vgId:%d meta-wlock completed", pMeta->vgId); + // stTrace("vgId:%d meta-wlock completed", pMeta->vgId); } void streamMetaWUnLock(SStreamMeta* pMeta) { -// stTrace("vgId:%d meta-wunlock", pMeta->vgId); + // stTrace("vgId:%d meta-wunlock", pMeta->vgId); taosThreadRwlockUnlock(&pMeta->lock); } @@ -1395,7 +1402,7 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) pMeta->sendMsgBeforeClosing = true; } - pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER; + pMeta->role = (isLeader) ? NODE_ROLE_LEADER : NODE_ROLE_FOLLOWER; streamMetaWUnLock(pMeta); if (isLeader) { @@ -1531,8 +1538,8 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { bool streamMetaAllTasksReady(const SStreamMeta* pMeta) { int32_t num = taosArrayGetSize(pMeta->pTaskList); - for(int32_t i = 0; i < num; ++i) { - STaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); + for (int32_t i = 0; i < num; ++i) { + STaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pTaskId, sizeof(*pTaskId)); if (ppTask == NULL) { continue; @@ -1633,7 +1640,7 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3 pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0; stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:0x%x (succ:%d) startTs:%" PRId64 - ", readyTs:%" PRId64 " total elapsed time:%.2fs", + ", readyTs:%" PRId64 " total elapsed time:%.2fs", pMeta->vgId, numOfTotal, taskId, ready, pStartInfo->startTs, pStartInfo->readyTs, pStartInfo->elapsedTime / 1000.0); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 3d0241df75..723f04c499 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -156,6 +156,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pKey = *pDestWinKey; goto _end; } @@ -167,6 +168,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pKey = *pDestWinKey; goto _end; } @@ -380,6 +382,14 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co (*pWins) = pWinStates; } + if (size > 0 && index == -1) { + SRowBuffPos* pPos = taosArrayGetP(pWinStates, 0); + SSessionKey* pWin = (SSessionKey*)pPos->pKey; + if (pWinKey->win.skey == pWin->win.skey) { + index = 0; + } + } + if (index >= 0) { pCur = createSessionStateCursor(pFileState); pCur->buffIndex = index; @@ -387,6 +397,7 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co *pIndex = index; } } + return pCur; } @@ -666,6 +677,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *key = *pDestWinKey; goto _end; } @@ -679,6 +691,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *key = *pDestWinKey; goto _end; } @@ -771,6 +784,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pWinKey = *pDestWinKey; goto _end; } @@ -799,6 +813,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pWinKey = *pDestWinKey; goto _end; } diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 6112a208c6..cb340ade32 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -385,7 +385,7 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) { void doProcessDownstreamReadyRsp(SStreamTask* pTask) { EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; - streamTaskOnHandleEventSuccess(pTask->status.pSM, event); + streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); int64_t initTs = pTask->execInfo.init; int64_t startTs = pTask->execInfo.start; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 9639921c77..45a9a68d3d 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -39,7 +39,7 @@ static int32_t doUpdateTaskEpset(SStreamTask* pTask, int32_t nodeId, SEpSet* pEp stDebug("s-task:0x%x (vgId:%d) self node epset is updated %s", pTask->id.taskId, nodeId, buf); } - // check for the dispath info and the upstream task info + // check for the dispatch info and the upstream task info int32_t level = pTask->info.taskLevel; if (level == TASK_LEVEL__SOURCE) { streamTaskUpdateDownstreamInfo(pTask, nodeId, pEpSet); @@ -412,9 +412,7 @@ void tFreeStreamTask(SStreamTask* pTask) { pTask->pReadyMsgList = taosArrayDestroy(pTask->pReadyMsgList); if (pTask->msgInfo.pData != NULL) { - destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); - pTask->msgInfo.pData = NULL; - pTask->msgInfo.dispatchMsgType = 0; + clearBufferedDispatchMsg(pTask); } if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { @@ -624,6 +622,7 @@ void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDo void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet) { char buf[512] = {0}; EPSET_TO_STR(pEpSet, buf); + int32_t id = pTask->id.taskId; int8_t type = pTask->outputInfo.type; if (type == TASK_OUTPUT__SHUFFLE_DISPATCH) { @@ -635,8 +634,8 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE if (pVgInfo->vgId == nodeId) { epsetAssign(&pVgInfo->epSet, pEpSet); - stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", pTask->id.taskId, - pVgInfo->taskId, nodeId, buf); + stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", id, pVgInfo->taskId, nodeId, + buf); break; } } @@ -644,11 +643,9 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE STaskDispatcherFixed* pDispatcher = &pTask->outputInfo.fixedDispatcher; if (pDispatcher->nodeId == nodeId) { epsetAssign(&pDispatcher->epSet, pEpSet); - stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpSet:%s", pTask->id.taskId, - pDispatcher->taskId, nodeId, buf); + stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", id, pDispatcher->taskId, nodeId, + buf); } - } else { - // do nothing } } @@ -766,21 +763,13 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) { return status; } -int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool metaLock) { - if (pTask == NULL) { - return TSDB_CODE_SUCCESS; - } - +int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) { SStreamMeta* pMeta = pTask->pMeta; STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId}; if (pTask->info.fillHistory == 0) { return TSDB_CODE_SUCCESS; } - if (metaLock) { - streamMetaWLock(pMeta); - } - SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &sTaskId, sizeof(sTaskId)); if (ppStreamTask != NULL) { stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr, @@ -798,10 +787,6 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool taosThreadMutexUnlock(&(*ppStreamTask)->lock); } - if (metaLock) { - streamMetaWUnLock(pMeta); - } - return TSDB_CODE_SUCCESS; } @@ -869,8 +854,8 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc) pDst->chkpointTransId = pSrc->chkpointTransId; } -void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) { - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_PAUSE); +static int32_t taskPauseCallback(SStreamTask* pTask, void* param) { + SStreamMeta* pMeta = pTask->pMeta; int32_t num = atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1); stInfo("vgId:%d s-task:%s pause stream task. pause task num:%d", pMeta->vgId, pTask->id.idStr, num); @@ -882,24 +867,24 @@ void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) { } stDebug("vgId:%d s-task:%s set pause flag and pause task", pMeta->vgId, pTask->id.idStr); + return TSDB_CODE_SUCCESS; +} + +void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) { + streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_PAUSE, taskPauseCallback, NULL); } void streamTaskResume(SStreamTask* pTask) { SStreamTaskState prevState = *streamTaskGetStatus(pTask); - SStreamMeta* pMeta = pTask->pMeta; - if (prevState.state == TASK_STATUS__PAUSE || prevState.state == TASK_STATUS__HALT) { - streamTaskRestoreStatus(pTask); - - char* pNew = streamTaskGetStatus(pTask)->name; - if (prevState.state == TASK_STATUS__PAUSE) { - int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); - stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num); - } else { - stInfo("s-task:%s status:%s resume from %s", pTask->id.idStr, pNew, prevState.name); - } + SStreamMeta* pMeta = pTask->pMeta; + int32_t code = streamTaskRestoreStatus(pTask); + if (code == TSDB_CODE_SUCCESS) { + char* pNew = streamTaskGetStatus(pTask)->name; + int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); + stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num); } else { - stDebug("s-task:%s status:%s not in pause/halt status, no need to resume", pTask->id.idStr, prevState.name); + stInfo("s-task:%s status:%s no need to resume, paused task(s):%d", pTask->id.idStr, prevState.name, pMeta->numOfPausedTasks); } } diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 83e71c42bc..6aa215586a 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -59,20 +59,23 @@ static int32_t streamTaskInitStatus(SStreamTask* pTask); static int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask); static int32_t initStateTransferTable(); static void doInitStateTransferTable(void); -static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask); static STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event, __state_trans_fn fn, __state_trans_succ_fn succFn, - SAttachedEventInfo* pEventInfo, bool autoInvoke); + SFutureHandleEventInfo* pEventInfo); static int32_t dummyFn(SStreamTask* UNUSED_PARAM(p)) { return TSDB_CODE_SUCCESS; } -static int32_t attachEvent(SStreamTask* pTask, SAttachedEventInfo* pEvtInfo) { +static int32_t attachWaitedEvent(SStreamTask* pTask, SFutureHandleEventInfo* pEvtInfo) { char* p = streamTaskGetStatus(pTask)->name; stDebug("s-task:%s status:%s attach event:%s required status:%s, since not allowed to handle it", pTask->id.idStr, p, GET_EVT_NAME(pEvtInfo->event), StreamTaskStatusList[pEvtInfo->status].name); - taosArrayPush(pTask->status.pSM->pWaitingEventList, pEvtInfo); + + SArray* pList = pTask->status.pSM->pWaitingEventList; + taosArrayPush(pList, pEvtInfo); + + stDebug("s-task:%s add into waiting list, total waiting events:%d", pTask->id.idStr, (int32_t)taosArrayGetSize(pList)); return 0; } @@ -85,18 +88,6 @@ int32_t streamTaskInitStatus(SStreamTask* pTask) { return 0; } -static int32_t streamTaskDoCheckpoint(SStreamTask* pTask) { - stDebug("s-task:%s start to do checkpoint", pTask->id.idStr); - return 0; -} - -int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask) { - if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - streamTaskSendCheckpointSourceRsp(pTask); - } - return 0; -} - int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask) { if (!HAS_RELATED_FILLHISTORY_TASK(pTask)) { stError("s-task:%s no related fill-history task, since it may have been dropped already", pTask->id.idStr); @@ -170,9 +161,11 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr, pEventName, el, pSM->prev.state.name, pSM->current.name); - SAttachedEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0); + ASSERT(taosArrayGetSize(pSM->pWaitingEventList) == 1); - // OK, let's handle the attached event, since the task has reached the required status now + SFutureHandleEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0); + + // OK, let's handle the waiting event, since the task has reached the required status now if (pSM->current.state == pEvtInfo->status) { stDebug("s-task:%s handle the event:%s in waiting list, state:%s", pTask->id.idStr, GET_EVT_NAME(pEvtInfo->event), pSM->current.name); @@ -189,7 +182,7 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, code = pNextTrans->pAction(pSM->pTask); if (pNextTrans->autoInvokeEndFn) { - return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event); + return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event, pEvtInfo->callBackFn, pEvtInfo->pParam); } else { return code; } @@ -203,30 +196,61 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, return code; } -void streamTaskRestoreStatus(SStreamTask* pTask) { +static int32_t removeEventInWaitingList(SStreamTask* pTask, EStreamTaskEvent event) { SStreamTaskSM* pSM = pTask->status.pSM; + bool removed = false; taosThreadMutexLock(&pTask->lock); - ASSERT(pSM->pActiveTrans == NULL); - ASSERT(pSM->current.state == TASK_STATUS__PAUSE || pSM->current.state == TASK_STATUS__HALT); + int32_t num = taosArrayGetSize(pSM->pWaitingEventList); + for (int32_t i = 0; i < num; ++i) { + SFutureHandleEventInfo* pInfo = taosArrayGet(pSM->pWaitingEventList, i); + if (pInfo->event == event) { + taosArrayRemove(pSM->pWaitingEventList, i); + stDebug("s-task:%s pause event in waiting list not be handled yet, remove it from waiting list, remaining:%d", + pTask->id.idStr, pInfo->event); + removed = true; + break; + } + } - SStreamTaskState state = pSM->current; - pSM->current = pSM->prev.state; - - pSM->prev.state = state; - pSM->prev.evt = 0; - - pSM->startTs = taosGetTimestampMs(); - - if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { - stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); - doHandleWaitingEvent(pSM, "restore-pause/halt", pTask); - } else { - stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); + if (!removed) { + stDebug("s-task:%s failed to remove event:%s in waiting list", pTask->id.idStr, StreamTaskEventList[event].name); } taosThreadMutexUnlock(&pTask->lock); + return TSDB_CODE_SUCCESS; +} + +int32_t streamTaskRestoreStatus(SStreamTask* pTask) { + SStreamTaskSM* pSM = pTask->status.pSM; + int32_t code = 0; + + taosThreadMutexLock(&pTask->lock); + + if (pSM->current.state == TASK_STATUS__PAUSE && pSM->pActiveTrans == NULL) { + SStreamTaskState state = pSM->current; + pSM->current = pSM->prev.state; + + pSM->prev.state = state; + pSM->prev.evt = 0; + + pSM->startTs = taosGetTimestampMs(); + + if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { + stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr, + pSM->prev.state.name, pSM->current.name); + doHandleWaitingEvent(pSM, "restore-pause/halt", pTask); + } else { + stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); + } + } else { + removeEventInWaitingList(pTask, TASK_EVENT_PAUSE); + code = -1; // failed to restore the status + } + + taosThreadMutexUnlock(&pTask->lock); + return code; } SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask) { @@ -242,7 +266,7 @@ SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask) { } pSM->pTask = pTask; - pSM->pWaitingEventList = taosArrayInit(4, sizeof(SAttachedEventInfo)); + pSM->pWaitingEventList = taosArrayInit(4, sizeof(SFutureHandleEventInfo)); if (pSM->pWaitingEventList == NULL) { taosMemoryFree(pSM); @@ -273,7 +297,7 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt const char* id = pTask->id.idStr; if (pTrans->attachEvent.event != 0) { - attachEvent(pTask, &pTrans->attachEvent); + attachWaitedEvent(pTask, &pTrans->attachEvent); taosThreadMutexUnlock(&pTask->lock); while (1) { @@ -303,7 +327,32 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt // todo handle error code; if (pTrans->autoInvokeEndFn) { - streamTaskOnHandleEventSuccess(pSM, event); + streamTaskOnHandleEventSuccess(pSM, event, NULL, NULL); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t doHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans, __state_trans_user_fn callbackFn, void* param) { + SStreamTask* pTask = pSM->pTask; + if (pTrans->attachEvent.event != 0) { + SFutureHandleEventInfo info = pTrans->attachEvent; + info.pParam = param; + info.callBackFn = callbackFn; + + attachWaitedEvent(pTask, &info); + taosThreadMutexUnlock(&pTask->lock); + } else { // override current active trans + pSM->pActiveTrans = pTrans; + pSM->startTs = taosGetTimestampMs(); + taosThreadMutexUnlock(&pTask->lock); + + int32_t code = pTrans->pAction(pTask); + // todo handle error code; + + if (pTrans->autoInvokeEndFn) { + streamTaskOnHandleEventSuccess(pSM, event, callbackFn, param); } } @@ -349,6 +398,45 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) { return code; } +int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) { + int32_t code = TSDB_CODE_SUCCESS; + SStreamTask* pTask = pSM->pTask; + STaskStateTrans* pTrans = NULL; + + while (1) { + taosThreadMutexLock(&pTask->lock); + + if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { + EStreamTaskEvent evt = pSM->pActiveTrans->event; + taosThreadMutexUnlock(&pTask->lock); + + stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", + pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); + taosMsleep(100); + } else { + // no active event trans exists, handle this event directly + pTrans = streamTaskFindTransform(pSM->current.state, event); + if (pTrans == NULL) { + stDebug("s-task:%s failed to handle event:%s, status:%s", pTask->id.idStr, GET_EVT_NAME(event), pSM->current.name); + taosThreadMutexUnlock(&pTask->lock); + return TSDB_CODE_STREAM_INVALID_STATETRANS; + } + + if (pSM->pActiveTrans != NULL) { + // currently in some state transfer procedure, not auto invoke transfer, quit from this procedure + stDebug("s-task:%s event:%s handle procedure quit, status %s -> %s failed, handle event %s now", + pTask->id.idStr, GET_EVT_NAME(pSM->pActiveTrans->event), pSM->current.name, + pSM->pActiveTrans->next.name, GET_EVT_NAME(event)); + } + + code = doHandleEventAsync(pSM, event, pTrans, callbackFn, param); + break; + } + } + + return code; +} + static void keepPrevInfo(SStreamTaskSM* pSM) { STaskStateTrans* pTrans = pSM->pActiveTrans; @@ -356,8 +444,9 @@ static void keepPrevInfo(SStreamTaskSM* pSM) { pSM->prev.evt = pTrans->event; } -int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event) { +int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) { SStreamTask* pTask = pSM->pTask; + const char* id = pTask->id.idStr; // do update the task status taosThreadMutexLock(&pTask->lock); @@ -369,16 +458,16 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even s == TASK_STATUS__UNINIT || s == TASK_STATUS__READY); // the pSM->prev.evt may be 0, so print string is not appropriate. - stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", pTask->id.idStr, - GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); + stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", id, GET_EVT_NAME(event), + pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } if (pTrans->event != event) { - stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", pTask->id.idStr, - GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pTrans->event)); + stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", id, GET_EVT_NAME(event), + pSM->current.name, GET_EVT_NAME(pTrans->event)); taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } @@ -388,16 +477,31 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even pSM->current = pTrans->next; pSM->pActiveTrans = NULL; + // todo remove it // on success callback, add into lock if necessary, or maybe we should add an option for this? pTrans->pSuccAction(pTask); + taosThreadMutexUnlock(&pTask->lock); + + // todo: add parameter to control lock + // after handling the callback function assigned by invoker, go on handling the waiting tasks + if (callbackFn != NULL) { + stDebug("s-task:%s start to handle user-specified callback fn for event:%s", id, GET_EVT_NAME(pTrans->event)); + callbackFn(pSM->pTask, param); + + stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event)); + } + + taosThreadMutexLock(&pTask->lock); + + // tasks in waiting list if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask); } else { taosThreadMutexUnlock(&pTask->lock); int64_t el = (taosGetTimestampMs() - pSM->startTs); - stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr, + stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", id, GET_EVT_NAME(pTrans->event), el, pSM->prev.state.name, pSM->current.name); } @@ -453,7 +557,7 @@ void streamTaskSetStatusReady(SStreamTask* pTask) { } STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event, __state_trans_fn fn, - __state_trans_succ_fn succFn, SAttachedEventInfo* pEventInfo, bool autoInvoke) { + __state_trans_succ_fn succFn, SFutureHandleEventInfo* pEventInfo) { STaskStateTrans trans = {0}; trans.state = StreamTaskStatusList[current]; trans.next = StreamTaskStatusList[next]; @@ -468,7 +572,7 @@ STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStr trans.pAction = (fn != NULL) ? fn : dummyFn; trans.pSuccAction = (succFn != NULL) ? succFn : dummyFn; - trans.autoInvokeEndFn = autoInvoke; + trans.autoInvokeEndFn = (fn == NULL); return trans; } @@ -482,93 +586,93 @@ void doInitStateTransferTable(void) { streamTaskSMTrans = taosArrayInit(8, sizeof(STaskStateTrans)); // initialization event handle - STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, false, false); + STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, NULL); taosArrayPush(streamTaskSMTrans, &trans); // scan-history related event - trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); // halt stream task, from other task status - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); taosArrayPush(streamTaskSMTrans, &trans); - SAttachedEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT}; + SFutureHandleEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT}; - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true); + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); + trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); taosArrayPush(streamTaskSMTrans, &trans); // checkpoint related event - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, streamTaskDoCheckpoint, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, streamTaskDoCheckpoint, NULL, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); // pause & resume related event handle - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - info = (SAttachedEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE}; - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); + info = (SFutureHandleEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE}; + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); // resume is completed by restore status of state-machine // stop related event - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); // dropping related event - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); } //clang-format on \ No newline at end of file diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 454ed4297c..764bf6e026 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -22,7 +22,7 @@ #define DEFAULT_MAP_CAPACITY 131072 #define DEFAULT_MAP_SIZE (DEFAULT_MAP_CAPACITY * 100) #define ROWS_PER_MILLISECOND 1 -#define MAX_NUM_SCALABLE_BF 100000 +#define MAX_NUM_SCALABLE_BF 64 #define MIN_NUM_SCALABLE_BF 10 #define DEFAULT_PREADD_BUCKET 1 #define MAX_INTERVAL MILLISECOND_PER_MINUTE @@ -81,7 +81,9 @@ static int64_t adjustInterval(int64_t interval, int32_t precision) { static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t watermark) { if (watermark <= adjInterval) { watermark = TMAX(originInt / adjInterval, 1) * adjInterval; - } else if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) { + } + + if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) { watermark = MAX_NUM_SCALABLE_BF * adjInterval; } return watermark; diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index c90e05bcf6..c472207b27 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -1,40 +1,104 @@ -MESSAGE(STATUS "build stream unit test") - -# GoogleTest requires at least C++11 -SET(CMAKE_CXX_STANDARD 11) -AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) # bloomFilterTest -ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp") -TARGET_LINK_LIBRARIES(streamUpdateTest - PUBLIC os util common gtest gtest_main stream executor index +#TARGET_LINK_LIBRARIES(streamUpdateTest + #PUBLIC os util common gtest gtest_main stream executor index + #) + +#TARGET_INCLUDE_DIRECTORIES( + #streamUpdateTest + #PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" + #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +#) + +#ADD_EXECUTABLE(checkpointTest checkpointTest.cpp) +#TARGET_LINK_LIBRARIES( + #checkpointTest + #PUBLIC os common gtest stream executor qcom index transport util +#) + +#TARGET_INCLUDE_DIRECTORIES( + #checkpointTest + #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +#) + +#add_executable(backendTest "") + +#target_sources(backendTest + #PRIVATE + #"backendTest.cpp" +#) + +#TARGET_LINK_LIBRARIES( + #backendTest + #PUBLIC rocksdb + #PUBLIC os common gtest stream executor qcom index transport util +#) + +#TARGET_INCLUDE_DIRECTORIES( + #backendTest + #PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" + #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +#) + +#add_test( + #NAME streamUpdateTest + #COMMAND streamUpdateTest +#) + +#add_test( + #NAME checkpointTest + #COMMAND checkpointTest +#) +#add_test( + #NAME backendTest + #COMMAND backendTest +#) + + +#add_executable(backendTest "") + +#target_sources(backendTest + #PUBLIC + #"backendTest.cpp" +#) + +#target_include_directories( + #backendTest + #PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" + #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +#) + +#target_link_libraries( + #backendTest + #PUBLIC rocksdb + #PUBLIC os common gtest stream executor qcom index transport util +#) + + +MESSAGE(STATUS "build parser unit test") + +IF(NOT TD_DARWIN) + # GoogleTest requires at least C++11 + SET(CMAKE_CXX_STANDARD 11) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + ADD_EXECUTABLE(backendTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES( + backendTest + PUBLIC rocksdb + PUBLIC os common gtest stream executor qcom index transport util vnode ) -TARGET_INCLUDE_DIRECTORIES( - streamUpdateTest - PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" - PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" -) + TARGET_INCLUDE_DIRECTORIES( + backendTest + PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" + PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" + ) -ADD_EXECUTABLE(checkpointTest checkpointTest.cpp) -TARGET_LINK_LIBRARIES( - checkpointTest - PUBLIC os common gtest stream executor qcom index transport util -) - -TARGET_INCLUDE_DIRECTORIES( - checkpointTest - PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" -) - -add_test( - NAME streamUpdateTest - COMMAND streamUpdateTest -) - -add_test( - NAME checkpointTest - COMMAND checkpointTest -) \ No newline at end of file + ADD_TEST( + NAME backendTest + COMMAND backendTest + ) +ENDIF () \ No newline at end of file diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp new file mode 100644 index 0000000000..c9b981e5f9 --- /dev/null +++ b/source/libs/stream/test/backendTest.cpp @@ -0,0 +1,461 @@ +#include + +#include +#include +#include +#include +#include "streamBackendRocksdb.h" +#include "streamSnapshot.h" +#include "streamState.h" +#include "tstream.h" +#include "tstreamFileState.h" +#include "tstreamUpdate.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +class BackendEnv : public ::testing::Test { + protected: + virtual void SetUp() {} + virtual void TearDown() {} +}; + +void *backendCreate() { + const char *streamPath = "/tmp"; + void * p = NULL; + + // char *absPath = NULL; + // // SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(streamPath, -1, 2); + // STaskDbWrapper *p = taskDbOpen((char *)streamPath, (char *)"stream-backend", -1); + // ASSERT(p != NULL); + return p; +} + +SStreamState *stateCreate(const char *path) { + SStreamTask *pTask = (SStreamTask *)taosMemoryCalloc(1, sizeof(SStreamTask)); + pTask->ver = 1024; + pTask->id.streamId = 1023; + pTask->id.taskId = 1111111; + SStreamMeta *pMeta = streamMetaOpen((path), NULL, NULL, 0, 0, NULL); + pTask->pMeta = pMeta; + + SStreamState *p = streamStateOpen((char *)path, pTask, true, 32, 32 * 1024); + ASSERT(p != NULL); + return p; +} +void *backendOpen() { + streamMetaInit(); + const char * path = "/tmp/backend"; + SStreamState *p = stateCreate(path); + ASSERT(p != NULL); + + // write bacth + // default/state/fill/sess/func/parname/partag + int32_t size = 100; + std::vector tsArray; + for (int32_t i = 0; i < size; i++) { + int64_t ts = taosGetTimestampMs(); + SWinKey key; // = {.groupId = (uint64_t)(i), .ts = ts}; + key.groupId = (uint64_t)(i); + key.ts = ts; + const char *val = "value data"; + int32_t vlen = strlen(val); + streamStatePut_rocksdb(p, &key, (char *)val, vlen); + + tsArray.push_back(ts); + } + for (int32_t i = 0; i < size; i++) { + int64_t ts = tsArray[i]; + SWinKey key = {0}; //{.groupId = (uint64_t)(i), .ts = ts}; + key.groupId = (uint64_t)(i); + key.ts = ts; + + const char *val = "value data"; + int32_t len = 0; + char * newVal = NULL; + streamStateGet_rocksdb(p, &key, (void **)&newVal, &len); + ASSERT(len == strlen(val)); + } + int64_t ts = tsArray[0]; + SWinKey key = {0}; // {.groupId = (uint64_t)(0), .ts = ts}; + key.groupId = (uint64_t)(0); + key.ts = ts; + + streamStateDel_rocksdb(p, &key); + + streamStateClear_rocksdb(p); + + for (int i = 0; i < size; i++) { + int64_t ts = tsArray[i]; + SWinKey key = {0}; //{.groupId = (uint64_t)(i), .ts = ts}; + key.groupId = (uint64_t)(i); + key.ts = ts; + + const char *val = "value data"; + int32_t len = 0; + char * newVal = NULL; + int32_t code = streamStateGet_rocksdb(p, &key, (void **)&newVal, &len); + ASSERT(code != 0); + } + tsArray.clear(); + + for (int i = 0; i < size; i++) { + int64_t ts = taosGetTimestampMs(); + tsArray.push_back(ts); + + SWinKey key = {0}; //{.groupId = (uint64_t)(i), .ts = ts}; + key.groupId = (uint64_t)(i); + key.ts = ts; + + const char *val = "value data"; + int32_t vlen = strlen(val); + streamStatePut_rocksdb(p, &key, (char *)val, vlen); + } + + SWinKey winkey; + int32_t code = streamStateGetFirst_rocksdb(p, &key); + ASSERT(code == 0); + ASSERT(key.ts == tsArray[0]); + + SStreamStateCur *pCurr = streamStateSeekToLast_rocksdb(p); + ASSERT(pCurr != NULL); + streamStateFreeCur(pCurr); + + winkey.groupId = 0; + winkey.ts = tsArray[0]; + char * val = NULL; + int32_t len = 0; + + pCurr = streamStateSeekKeyNext_rocksdb(p, &winkey); + ASSERT(pCurr != NULL); + + streamStateFreeCur(pCurr); + + tsArray.clear(); + for (int i = 0; i < size; i++) { + int64_t ts = taosGetTimestampMs(); + tsArray.push_back(ts); + STupleKey key = {0}; + key.groupId = (uint64_t)(0); //= {.groupId = (uint64_t)(0), .ts = ts, .exprIdx = i}; + key.ts = ts; + key.exprIdx = i; + + const char *val = "Value"; + int32_t len = strlen(val); + streamStateFuncPut_rocksdb(p, &key, val, len); + } + for (int i = 0; i < size; i++) { + STupleKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i}; + key.groupId = (uint64_t)(0); + key.ts = tsArray[i]; + key.exprIdx = i; + + char * val = NULL; + int32_t len = 0; + streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len); + ASSERT(len == strlen("Value")); + } + for (int i = 0; i < size; i++) { + STupleKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i}; + key.groupId = (uint64_t)(0); + key.ts = tsArray[i]; + key.exprIdx = i; + + char * val = NULL; + int32_t len = 0; + streamStateFuncDel_rocksdb(p, &key); + } + + // session put + tsArray.clear(); + + for (int i = 0; i < size; i++) { + SSessionKey key = {0}; //{.win = {.skey = i, .ekey = i}, .groupId = (uint64_t)(0)}; + key.win.skey = i; + key.win.ekey = i; + key.groupId = (uint64_t)(0); + tsArray.push_back(i); + + const char *val = "Value"; + int32_t len = strlen(val); + streamStateSessionPut_rocksdb(p, &key, val, len); + + char *pval = NULL; + ASSERT(0 == streamStateSessionGet_rocksdb(p, &key, (void **)&pval, &len)); + ASSERT(strncmp(pval, val, len) == 0); + } + + for (int i = 0; i < size; i++) { + SSessionKey key = {0}; //{.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)}; + key.win.skey = tsArray[i]; + key.win.ekey = tsArray[i]; + key.groupId = (uint64_t)(0); + + const char *val = "Value"; + int32_t len = strlen(val); + + char *pval = NULL; + ASSERT(0 == streamStateSessionGet_rocksdb(p, &key, (void **)&pval, &len)); + ASSERT(strncmp(pval, val, len) == 0); + taosMemoryFreeClear(pval); + } + + pCurr = streamStateSessionSeekToLast_rocksdb(p, 0); + ASSERT(pCurr != NULL); + + { + SSessionKey key; + memset(&key, 0, sizeof(key)); + char * val = NULL; + int32_t vlen = 0; + code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen); + ASSERT(code == 0); + pCurr = streamStateSessionSeekKeyPrev_rocksdb(p, &key); + + code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen); + ASSERT(code == 0); + + ASSERT(key.groupId == 0 && key.win.ekey == tsArray[tsArray.size() - 2]); + + pCurr = streamStateSessionSeekKeyNext_rocksdb(p, &key); + code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen); + ASSERT(code == 0); + ASSERT(vlen == strlen("Value")); + ASSERT(key.groupId == 0 && key.win.skey == tsArray[tsArray.size() - 1]); + + ASSERT(0 == streamStateSessionAddIfNotExist_rocksdb(p, &key, 10, (void **)&val, &len)); + + ASSERT(0 == + streamStateStateAddIfNotExist_rocksdb(p, &key, (char *)"key", strlen("key"), NULL, (void **)&val, &len)); + } + for (int i = 0; i < size; i++) { + SSessionKey key = {0}; //{.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)}; + key.win.skey = tsArray[i]; + key.win.ekey = tsArray[i]; + key.groupId = (uint64_t)(0); + + const char *val = "Value"; + int32_t len = strlen(val); + + char *pval = NULL; + ASSERT(0 == streamStateSessionDel_rocksdb(p, &key)); + } + + for (int i = 0; i < size; i++) { + SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; + key.groupId = (uint64_t)(i); + key.ts = tsArray[i]; + const char *val = "Value"; + int32_t vlen = strlen(val); + ASSERT(streamStateFillPut_rocksdb(p, &key, val, vlen) == 0); + } + for (int i = 0; i < size; i++) { + SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; + key.groupId = (uint64_t)(i); + key.ts = tsArray[i]; + char * val = NULL; + int32_t vlen = 0; + ASSERT(streamStateFillGet_rocksdb(p, &key, (void **)&val, &vlen) == 0); + taosMemoryFreeClear(val); + } + { + SWinKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[0]}; + key.groupId = (uint64_t)(0); + key.ts = tsArray[0]; + SStreamStateCur *pCurr = streamStateFillGetCur_rocksdb(p, &key); + ASSERT(pCurr != NULL); + + char * val = NULL; + int32_t vlen = 0; + ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen)); + ASSERT(vlen == strlen("Value")); + streamStateFreeCur(pCurr); + + pCurr = streamStateFillSeekKeyNext_rocksdb(p, &key); + ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen)); + ASSERT(vlen == strlen("Value") && key.groupId == 1 && key.ts == tsArray[1]); + + key.groupId = 1; + key.ts = tsArray[1]; + + pCurr = streamStateFillSeekKeyPrev_rocksdb(p, &key); + ASSERT(pCurr != NULL); + ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen)); + + ASSERT(vlen == strlen("Value") && key.groupId == 0 && key.ts == tsArray[0]); + } + + for (int i = 0; i < size - 1; i++) { + SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; + key.groupId = (uint64_t)(i); + key.ts = tsArray[i]; + char * val = NULL; + int32_t vlen = 0; + ASSERT(streamStateFillDel_rocksdb(p, &key) == 0); + taosMemoryFreeClear(val); + } + streamStateSessionClear_rocksdb(p); + + for (int i = 0; i < size; i++) { + char tbname[TSDB_TABLE_NAME_LEN] = {0}; + sprintf(tbname, "%s_%d", "tbname", i); + ASSERT(0 == streamStatePutParName_rocksdb(p, i, tbname)); + } + for (int i = 0; i < size; i++) { + char *val = NULL; + ASSERT(0 == streamStateGetParName_rocksdb(p, i, (void **)&val)); + ASSERT(strncmp(val, "tbname", strlen("tbname")) == 0); + taosMemoryFree(val); + } + + for (int i = 0; i < size; i++) { + char tbname[TSDB_TABLE_NAME_LEN] = {0}; + sprintf(tbname, "%s_%d", "tbname", i); + ASSERT(0 == streamStatePutParName_rocksdb(p, i, tbname)); + } + for (int i = 0; i < size; i++) { + char *val = NULL; + ASSERT(0 == streamStateGetParName_rocksdb(p, i, (void **)&val)); + ASSERT(strncmp(val, "tbname", strlen("tbname")) == 0); + taosMemoryFree(val); + } + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "tbname_%d", i); + char val[128] = {0}; + sprintf(val, "val_%d", i); + code = streamDefaultPut_rocksdb(p, key, val, strlen(val)); + ASSERT(code == 0); + } + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "tbname_%d", i); + + char * val = NULL; + int32_t len = 0; + code = streamDefaultGet_rocksdb(p, key, (void **)&val, &len); + ASSERT(code == 0); + } + SArray *result = taosArrayInit(8, sizeof(void *)); + streamDefaultIterGet_rocksdb(p, "tbname", "tbname_99", result); + ASSERT(taosArrayGetSize(result) >= 0); + + return p; + // streamStateClose((SStreamState *)p, true); +} +TEST_F(BackendEnv, checkOpen) { + SStreamState *p = (SStreamState *)backendOpen(); + int64_t tsStart = taosGetTimestampMs(); + { + void * pBatch = streamStateCreateBatch(); + int32_t size = 0; + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "key_%d", i); + char val[128] = {0}; + sprintf(val, "val_%d", i); + streamStatePutBatch(p, "default", (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val, + (int32_t)(strlen(val)), tsStart + 100000); + } + streamStatePutBatch_rocksdb(p, pBatch); + streamStateDestroyBatch(pBatch); + } + { + void * pBatch = streamStateCreateBatch(); + int32_t size = 0; + char valBuf[256] = {0}; + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "key_%d", i); + char val[128] = {0}; + sprintf(val, "val_%d", i); + streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val, + (int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf); + } + streamStatePutBatch_rocksdb(p, pBatch); + streamStateDestroyBatch(pBatch); + } + // do checkpoint 2 + taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2); + { + void * pBatch = streamStateCreateBatch(); + int32_t size = 0; + char valBuf[256] = {0}; + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "key_%d", i); + char val[128] = {0}; + sprintf(val, "val_%d", i); + streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val, + (int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf); + } + streamStatePutBatch_rocksdb(p, pBatch); + streamStateDestroyBatch(pBatch); + } + + taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 3); + + const char *path = "/tmp/backend/stream"; + const char *dump = "/tmp/backend/stream/dump"; + // taosMkDir(dump); + taosMulMkDir(dump); + SBkdMgt *mgt = bkdMgtCreate((char *)path); + SArray * result = taosArrayInit(4, sizeof(void *)); + bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump); + + taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 4); + + taosArrayClear(result); + bkdMgtGetDelta(mgt, p->pTdbState->idstr, 4, result, (char *)dump); + bkdMgtDestroy(mgt); + streamStateClose((SStreamState *)p, true); + // { + // taosRemoveDir("/tmp/backend"); + // const char * path = "/tmp/backend"; + // SStreamState *p = stateCreate(path); + // } + taosRemoveDir(path); + // streamStateClose((SStreamState *)p, true); +} + +TEST_F(BackendEnv, backendChkp) { const char *path = "/tmp"; } + +typedef struct BdKV { + uint32_t k; + uint32_t v; +} BdKV; + +BdKV kvDict[] = {{0, 2}, {1, 2}, {15, 16}, {31, 32}, {56, 64}, {100, 128}, + {200, 256}, {500, 512}, {1000, 1024}, {2000, 2048}, {3000, 4096}}; + +TEST_F(BackendEnv, backendUtil) { + for (int i = 0; i < sizeof(kvDict) / sizeof(kvDict[0]); i++) { + ASSERT_EQ(nextPow2((uint32_t)(kvDict[i].k)), kvDict[i].v); + } +} +TEST_F(BackendEnv, oldBackendInit) { + const char *path = "/tmp/backend1"; + taosMulMkDir(path); + { + SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(path, 10, 10); + streamBackendCleanup((void *)p); + } + { + SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(path, 10, 10); + streamBackendCleanup((void *)p); + } + + taosRemoveDir(path); +} + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp index 0dc2cc13f5..0caad479e5 100644 --- a/source/libs/stream/test/checkpointTest.cpp +++ b/source/libs/stream/test/checkpointTest.cpp @@ -25,46 +25,49 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" +// tsSnodeAddress = ""; +// tsS3StreamEnabled = 0; + +#include "cos.h" #include "rsync.h" #include "streamInt.h" -#include "cos.h" -int main(int argc, char **argv) { - testing::InitGoogleTest(&argc, argv); +// int main(int argc, char **argv) { +// testing::InitGoogleTest(&argc, argv); - if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) { - printf("error"); - } - if (s3Init() < 0) { - return -1; - } - strcpy(tsSnodeAddress, "127.0.0.1"); - int ret = RUN_ALL_TESTS(); - s3CleanUp(); - return ret; -} +// if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) { +// printf("error"); +// } +// if (s3Init() < 0) { +// return -1; +// } +// strcpy(tsSnodeAddress, "127.0.0.1"); +// int ret = RUN_ALL_TESTS(); +// s3CleanUp(); +// return ret; +// } TEST(testCase, checkpointUpload_Test) { - stopRsync(); - startRsync(); + // stopRsync(); + // startRsync(); taosSsleep(5); char* id = "2013892036"; - uploadCheckpoint(id, "/root/offset/"); + // uploadCheckpoint(id, "/root/offset/"); } TEST(testCase, checkpointDownload_Test) { char* id = "2013892036"; - downloadCheckpoint(id, "/root/offset/download/"); + // downloadCheckpoint(id, "/root/offset/download/"); } TEST(testCase, checkpointDelete_Test) { char* id = "2013892036"; - deleteCheckpoint(id); + // deleteCheckpoint(id); } TEST(testCase, checkpointDeleteFile_Test) { char* id = "2013892036"; - deleteCheckpointFile(id, "offset-ver0"); + // deleteCheckpointFile(id, "offset-ver0"); } diff --git a/source/libs/stream/test/tstreamUpdateTest.cpp b/source/libs/stream/test/tstreamUpdateTest.cpp index 1b999e5fb0..59171876ff 100644 --- a/source/libs/stream/test/tstreamUpdateTest.cpp +++ b/source/libs/stream/test/tstreamUpdateTest.cpp @@ -14,10 +14,7 @@ class StreamStateEnv : public ::testing::Test { streamMetaInit(); backend = streamBackendInit(path, 0, 0); } - virtual void TearDown() { - streamMetaCleanup(); - // indexClose(index); - } + virtual void TearDown() { streamMetaCleanup(); } const char *path = TD_TMP_DIR_PATH "stream"; void *backend; @@ -50,6 +47,14 @@ bool equalSBF(SScalableBf *left, SScalableBf *right) { } TEST(TD_STREAM_UPDATE_TEST, update) { + const char *streamPath = "/tmp"; + + char *absPath = NULL; + void *p = NULL; + // SBackendWrapper *p = streamBackendInit(streamPath, -1, 2); + // p = taskDbOpen((char *)streamPath, (char *)"test", -1); + p = bkdMgtCreate((char *)streamPath); + // const int64_t interval = 20 * 1000; // const int64_t watermark = 10 * 60 * 1000; // SUpdateInfo *pSU = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index e06ea70f70..7ff6116137 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1343,7 +1343,7 @@ ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->raftCfg int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { int32_t ret = 0; if (syncIsInit()) { - taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, pSyncNode, syncEnv()->pTimerManager, + taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, (void*)pSyncNode->rid, syncEnv()->pTimerManager, &pSyncNode->pPingTimer); atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); } else { @@ -1415,8 +1415,8 @@ void syncNodeResetElectTimer(SSyncNode* pSyncNode) { static int32_t syncNodeDoStartHeartbeatTimer(SSyncNode* pSyncNode) { int32_t ret = 0; if (syncIsInit()) { - taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, syncEnv()->pTimerManager, - &pSyncNode->pHeartbeatTimer); + taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, (void*)pSyncNode->rid, + syncEnv()->pTimerManager, &pSyncNode->pHeartbeatTimer); atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); } else { sError("vgId:%d, start heartbeat timer error, sync env is stop", pSyncNode->vgId); @@ -2153,7 +2153,11 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex static void syncNodeEqPingTimer(void* param, void* tmrId) { if (!syncIsInit()) return; - SSyncNode* pNode = param; + int64_t rid = (int64_t)param; + SSyncNode* pNode = syncNodeAcquire(rid); + + if (pNode == NULL) return; + if (atomic_load_64(&pNode->pingTimerLogicClockUser) <= atomic_load_64(&pNode->pingTimerLogicClock)) { SRpcMsg rpcMsg = {0}; int32_t code = syncBuildTimeout(&rpcMsg, SYNC_TIMEOUT_PING, atomic_load_64(&pNode->pingTimerLogicClock), @@ -2173,7 +2177,8 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { } _out: - taosTmrReset(syncNodeEqPingTimer, pNode->pingTimerMS, pNode, syncEnv()->pTimerManager, &pNode->pPingTimer); + taosTmrReset(syncNodeEqPingTimer, pNode->pingTimerMS, (void*)pNode->rid, syncEnv()->pTimerManager, + &pNode->pPingTimer); } } @@ -2224,7 +2229,11 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { if (!syncIsInit()) return; - SSyncNode* pNode = param; + int64_t rid = (int64_t)param; + SSyncNode* pNode = syncNodeAcquire(rid); + + if (pNode == NULL) return; + if (pNode->totalReplicaNum > 1) { if (atomic_load_64(&pNode->heartbeatTimerLogicClockUser) <= atomic_load_64(&pNode->heartbeatTimerLogicClock)) { SRpcMsg rpcMsg = {0}; @@ -2245,7 +2254,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { } _out: - taosTmrReset(syncNodeEqHeartbeatTimer, pNode->heartbeatTimerMS, pNode, syncEnv()->pTimerManager, + taosTmrReset(syncNodeEqHeartbeatTimer, pNode->heartbeatTimerMS, (void*)pNode->rid, syncEnv()->pTimerManager, &pNode->pHeartbeatTimer); } else { @@ -3385,4 +3394,4 @@ bool syncNodeCanChange(SSyncNode* pSyncNode) { return true; } -#endif \ No newline at end of file +#endif diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 3ee65f11dd..455128e6ec 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -316,7 +316,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) } // 3. Try to Recycle a page - if (!pPage && !pCache->lru.pLruPrev->isAnchor) { + if (!pPageH && !pPage && !pCache->lru.pLruPrev->isAnchor) { pPage = pCache->lru.pLruPrev; tdbPCacheRemovePageFromHash(pCache, pPage); tdbPCachePinPage(pCache, pPage); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 5b18d56d70..c010e31320 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -256,21 +256,21 @@ void transAsyncPoolDestroy(SAsyncPool* pool); int transAsyncSend(SAsyncPool* pool, queue* mq); bool transAsyncPoolIsEmpty(SAsyncPool* pool); -#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc) \ - do { \ - for (int i = 0; i < pool->nAsync; i++) { \ - uv_async_t* async = &(pool->asyncs[i]); \ - SAsyncItem* item = async->data; \ - while (!QUEUE_IS_EMPTY(&item->qmsg)) { \ - tTrace("destroy msg in async pool "); \ - queue* h = QUEUE_HEAD(&item->qmsg); \ - QUEUE_REMOVE(h); \ - msgType* msg = QUEUE_DATA(h, msgType, q); \ - if (msg != NULL) { \ - freeFunc(msg); \ - } \ - } \ - } \ +#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc, param) \ + do { \ + for (int i = 0; i < pool->nAsync; i++) { \ + uv_async_t* async = &(pool->asyncs[i]); \ + SAsyncItem* item = async->data; \ + while (!QUEUE_IS_EMPTY(&item->qmsg)) { \ + tTrace("destroy msg in async pool "); \ + queue* h = QUEUE_HEAD(&item->qmsg); \ + QUEUE_REMOVE(h); \ + msgType* msg = QUEUE_DATA(h, msgType, q); \ + if (msg != NULL) { \ + freeFunc(msg, param); \ + } \ + } \ + } \ } while (0) #define ASYNC_CHECK_HANDLE(exh1, id) \ diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 6de10cbb9e..c4ca39c323 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -191,6 +191,15 @@ static void httpDestroyMsg(SHttpMsg* msg) { taosMemoryFree(msg->cont); taosMemoryFree(msg); } +static void httpDestroyMsgWrapper(void* cont, void* param) { + httpDestroyMsg((SHttpMsg*)cont); + // if (msg == NULL) return; + + // taosMemoryFree(msg->server); + // taosMemoryFree(msg->uri); + // taosMemoryFree(msg->cont); + // taosMemoryFree(msg); +} static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) { SHttpMsg *msg = NULL, *quitMsg = NULL; @@ -554,7 +563,7 @@ void transHttpEnvDestroy() { httpSendQuit(); taosThreadJoin(load->thread, NULL); - TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg); + TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsgWrapper, NULL); transAsyncPoolDestroy(load->asyncPool); uv_loop_close(load->loop); taosMemoryFree(load->loop); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index e937d2e65e..2e35aecd08 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -219,6 +219,8 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, /// NULL,cliHandleUpdate}; static FORCE_INLINE void destroyCmsg(void* cmsg); + +static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param); static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg); static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst); static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx); @@ -582,8 +584,8 @@ void* destroyConnPool(SCliThrd* pThrd) { static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) { void* pool = pThrd->pool; - SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key) + 1); STrans* pTranInst = pThrd->pTransInst; + SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key) + 1); if (plist == NULL) { SConnList list = {0}; taosHashPut((SHashObj*)pool, key, strlen(key) + 1, (void*)&list, sizeof(list)); @@ -865,17 +867,18 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { QUEUE_INIT(&conn->q); conn->broken = true; + if (conn->list == NULL) { + conn->list = taosHashGet((SHashObj*)pThrd->pool, conn->dstAddr, strlen(conn->dstAddr)); + } - if (conn->list != NULL) { - SConnList* connList = conn->list; - connList->list->numOfConn--; - connList->size--; - } else { - if (pThrd->pool) { - SConnList* connList = taosHashGet((SHashObj*)pThrd->pool, conn->dstAddr, strlen(conn->dstAddr) + 1); - if (connList != NULL) connList->list->numOfConn--; + if (conn->list) { + SConnList* list = conn->list; + list->list->numOfConn--; + if (conn->status == ConnInPool) { + list->size--; } } + conn->list = NULL; pThrd->newConnCount--; @@ -1963,7 +1966,17 @@ static FORCE_INLINE void destroyCmsg(void* arg) { transFreeMsg(pMsg->msg.pCont); taosMemoryFree(pMsg); } - +static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param) { + SCliMsg* pMsg = arg; + if (pMsg == NULL) { + return; + } + if (param != NULL) { + SCliThrd* pThrd = param; + if (pThrd->destroyAhandleFp) (*pThrd->destroyAhandleFp)(pMsg->msg.info.ahandle); + } + destroyCmsg(pMsg); +} static FORCE_INLINE void destroyCmsgAndAhandle(void* param) { if (param == NULL) return; @@ -2057,7 +2070,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosThreadJoin(pThrd->thread, NULL); CLI_RELEASE_UV(pThrd->loop); taosThreadMutexDestroy(&pThrd->msgMtx); - TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsg); + TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsgWrapper, (void*)pThrd); transAsyncPoolDestroy(pThrd->asyncPool); transDQDestroy(pThrd->delayQueue, destroyCmsgAndAhandle); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index b324ca5f91..f47a688e6f 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -527,6 +527,10 @@ void uvOnSendCb(uv_write_t* req, int status) { if (!transQueueEmpty(&conn->srvMsgs)) { msg = (SSvrMsg*)transQueueGet(&conn->srvMsgs, 0); if (msg->type == Register && conn->status == ConnAcquire) { + if (conn->regArg.init) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } conn->regArg.notifyCount = 0; conn->regArg.init = 1; conn->regArg.msg = msg->msg; @@ -671,7 +675,8 @@ static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) { transFreeMsg(smsg->msg.pCont); taosMemoryFree(smsg); } -static void destroyAllConn(SWorkThrd* pThrd) { +static FORCE_INLINE void destroySmsgWrapper(void* smsg, void* param) { destroySmsg((SSvrMsg*)smsg); } +static void destroyAllConn(SWorkThrd* pThrd) { tTrace("thread %p destroy all conn ", pThrd); while (!QUEUE_IS_EMPTY(&pThrd->conn)) { queue* h = QUEUE_HEAD(&pThrd->conn); @@ -1349,6 +1354,11 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) { return; } transQueuePop(&conn->srvMsgs); + + if (conn->regArg.init) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } conn->regArg.notifyCount = 0; conn->regArg.init = 1; conn->regArg.msg = msg->msg; @@ -1394,7 +1404,7 @@ void destroyWorkThrd(SWorkThrd* pThrd) { } taosThreadJoin(pThrd->thread, NULL); SRV_RELEASE_UV(pThrd->loop); - TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsg); + TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL); transAsyncPoolDestroy(pThrd->asyncPool); uvWhiteListDestroy(pThrd->pWhiteList); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 88eb51d500..d9686d77f8 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -89,12 +89,14 @@ static int32_t taosArrayResize(SArray* pArray) { int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { if (newCap > pArray->capacity) { float factor = BOUNDARY_BIG_FACTOR; - if(newCap * pArray->elemSize > BOUNDARY_SIZE){ + if (newCap * pArray->elemSize > BOUNDARY_SIZE) { factor = BOUNDARY_SMALL_FACTOR; } + size_t tsize = (pArray->capacity * factor); while (newCap > tsize) { - tsize = (tsize * factor); + size_t newSize = (tsize * factor); + tsize = (newSize == tsize) ? (tsize + 2) : newSize; } pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index ad3c766510..e072ee2276 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -457,6 +457,7 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { if (pItem->dtype == CFG_DTYPE_STRING) { taosMemoryFree(pItem->str); } + taosMemoryFree(pItem->name); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 836fd980d0..1f3aaa3835 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -329,6 +329,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VIEW_NOT_EXIST, "view not exists in db //mnode-compact TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COMPACT_ID, "Invalid compact id") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST, "compact detail doesn't exist") // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_OFFLINE, "Dnode is offline") diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 2cc13be6ba..7a4eb09b99 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -21,6 +21,40 @@ int64_t tsRpcQueueMemoryAllowed = 0; int64_t tsRpcQueueMemoryUsed = 0; +struct STaosQueue { + STaosQnode *head; + STaosQnode *tail; + STaosQueue *next; // for queue set + STaosQset *qset; // for queue set + void *ahandle; // for queue set + FItem itemFp; + FItems itemsFp; + TdThreadMutex mutex; + int64_t memOfItems; + int32_t numOfItems; + int64_t threadId; + int64_t memLimit; + int64_t itemLimit; +}; + +struct STaosQset { + STaosQueue *head; + STaosQueue *current; + TdThreadMutex mutex; + tsem_t sem; + int32_t numOfQueues; + int32_t numOfItems; +}; + +struct STaosQall { + STaosQnode *current; + STaosQnode *start; + int32_t numOfItems; + int64_t memOfItems; + int32_t unAccessedNumOfItems; + int64_t unAccessMemOfItems; +}; + void taosSetQueueMemoryCapacity(STaosQueue *queue, int64_t cap) { queue->memLimit = cap; } void taosSetQueueCapacity(STaosQueue *queue, int64_t size) { queue->itemLimit = size; } @@ -497,6 +531,12 @@ int64_t taosQallUnAccessedMemSize(STaosQall *qall) { return qall->unAccessMemOfI void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; } +void taosQueueSetThreadId(STaosQueue* pQueue, int64_t threadId) { + pQueue->threadId = threadId; +} + +int64_t taosQueueGetThreadId(STaosQueue *pQueue) { return pQueue->threadId; } + #if 0 void taosResetQsetThread(STaosQset *qset, void *pItem) { diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 3b4975b701..7af794546b 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -20,6 +20,9 @@ #define DEFAULT_GROWTH 2 #define DEFAULT_TIGHTENING_RATIO 0.5 +#define DEFAULT_MAX_BLOOMFILTERS 4 +#define SBF_INVALID -1 +#define SBF_VALID 0 static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate); @@ -32,6 +35,8 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) { if (pSBf == NULL) { return NULL; } + pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; + pSBf->status = SBF_VALID; pSBf->numBits = 0; pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *)); if (tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO) == NULL) { @@ -45,6 +50,9 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) { } int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + if (pSBf->status == SBF_INVALID) { + return TSDB_CODE_FAILED; + } int32_t size = taosArrayGetSize(pSBf->bfArray); SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); ASSERT(pNormalBf); @@ -52,6 +60,7 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); if (pNormalBf == NULL) { + pSBf->status = SBF_INVALID; return TSDB_CODE_OUT_OF_MEMORY; } } @@ -59,6 +68,9 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le } int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + if (pSBf->status == SBF_INVALID) { + return TSDB_CODE_FAILED; + } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); int32_t size = taosArrayGetSize(pSBf->bfArray); @@ -74,6 +86,7 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); if (pNormalBf == NULL) { + pSBf->status = SBF_INVALID; return TSDB_CODE_OUT_OF_MEMORY; } } @@ -81,6 +94,9 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { } int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + if (pSBf->status == SBF_INVALID) { + return TSDB_CODE_FAILED; + } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); int32_t size = taosArrayGetSize(pSBf->bfArray); @@ -93,6 +109,10 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 } static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate) { + if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) { + return NULL; + } + SBloomFilter *pNormalBf = tBloomFilterInit(expectedEntries, errorRate); if (pNormalBf == NULL) { return NULL; @@ -128,6 +148,8 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { } if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1; if (tEncodeU64(pEncoder, pSBf->numBits) < 0) return -1; + if (tEncodeU32(pEncoder, pSBf->maxBloomFilters) < 0) return -1; + if (tEncodeI8(pEncoder, pSBf->status) < 0) return -1; return 0; } @@ -150,6 +172,8 @@ SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { } if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error; if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) goto _error; + if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) goto _error; + if (tDecodeI8(pDecoder, &pSBf->status) < 0) goto _error; return pSBf; _error: diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 3e591c7d7f..138d4bc1f4 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -417,9 +417,9 @@ _OVER: return NULL; } else { while (worker->pid <= 0) taosMsleep(10); - queue->threadId = worker->pid; - uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, - queue->threadId); + + taosQueueSetThreadId(queue, worker->pid); + uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, worker->pid); return queue; } } diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index 85f030eb03..dfd8d95c9c 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -46,7 +46,9 @@ class TDTestCase(TBase): # clusterDnodes.starttaosd(1) # time.sleep(5) autoGen.insert_data(5000, True) - tdSql.execute(f"flush database {self.db}") + self.flushDb(True) + # wait flush operation over + time.sleep(5) # sql = 'show vnodes;' # while True: diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bdccf33c32..481e111715 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -115,7 +115,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ins_topics_test.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxTopic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R +#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py @@ -228,7 +228,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py +,,n,system-test,python3 ./test.py -f 7-tmq/tmq_taosx.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_replay.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py ,,n,system-test,python3 ./test.py -f 7-tmq/tmq_offset.py @@ -238,7 +238,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db-removewal.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb-removewal.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 #,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3 @@ -1085,6 +1086,7 @@ ,,y,script,./test.sh -f tsim/parser/join_multivnode.sim ,,y,script,./test.sh -f tsim/parser/join.sim ,,y,script,./test.sh -f tsim/parser/last_cache.sim +,,y,script,./test.sh -f tsim/parser/last_both.sim ,,y,script,./test.sh -f tsim/parser/last_groupby.sim ,,y,script,./test.sh -f tsim/parser/lastrow.sim ,,y,script,./test.sh -f tsim/parser/lastrow2.sim @@ -1211,6 +1213,7 @@ ,,y,script,./test.sh -f tsim/stream/deleteState.sim ,,y,script,./test.sh -f tsim/stream/distributeInterval0.sim ,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim +,,y,script,./test.sh -f tsim/stream/distributeMultiLevelInterval0.sim ,,y,script,./test.sh -f tsim/stream/distributeSession0.sim ,,y,script,./test.sh -f tsim/stream/drop_stream.sim ,,y,script,./test.sh -f tsim/stream/event0.sim diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh index 01192763a1..9f526819c8 100644 --- a/tests/script/coverage_test.sh +++ b/tests/script/coverage_test.sh @@ -214,7 +214,7 @@ function lcovFunc { '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.c' '*/build_version.cc'\ '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' \ '*/shellAuto.c' '*/shellTire.c' '*/shellCommand.c'\ - '*/sql.c' '*/sql.y'\ + '*/sql.c' '*/sql.y' '*/smaSnapshot.c' '*/smaCommit.c'\ --branch-coverage --function-coverage -o coverage.info # generate result diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 3b3d275a07..c6bd78336c 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -113,11 +113,11 @@ echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG echo "fqdn ${HOSTNAME}" >> $TAOS_CFG echo "serverPort ${NODE}" >> $TAOS_CFG -echo "supportVnodes 1024" >> $TAOS_CFG +echo "supportVnodes 1024" >> $TAOS_CFG echo "statusInterval 1" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG -echo "debugFlag 0" >> $TAOS_CFG +echo "debugFlag 135" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "uDebugFlag 143" >> $TAOS_CFG echo "rpcDebugFlag 143" >> $TAOS_CFG @@ -143,4 +143,5 @@ echo "asyncLog 0" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG echo "telemetryReporting 0" >> $TAOS_CFG echo "querySmaOptimize 1" >> $TAOS_CFG +echo "checkpointInterval 60" >> $TAOS_CFG echo " " >> $TAOS_CFG diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index c63d6daf8a..b447a7325e 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -15,42 +15,42 @@ fi PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` while [ -n "$PID" ]; do - echo kill -9 $PID - #pkill -9 taosd - kill -9 $PID + echo kill -15 $PID + #pkill -15 taosd + kill -15 $PID echo "Killing taosd processes" if [ "$OS_TYPE" != "Darwin" ]; then fuser -k -n tcp 6030 else - lsof -nti:6030 | xargs kill -9 + lsof -nti:6030 | xargs kill -15 fi PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` done PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'` while [ -n "$PID" ]; do - echo kill -9 $PID + echo kill -15 $PID #pkill -9 taos - kill -9 $PID + kill -15 $PID echo "Killing taos processes" if [ "$OS_TYPE" != "Darwin" ]; then fuser -k -n tcp 6030 else - lsof -nti:6030 | xargs kill -9 + lsof -nti:6030 | xargs kill -15 fi PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'` done PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'` while [ -n "$PID" ]; do - echo kill -9 $PID - #pkill -9 tmq_sim - kill -9 $PID + echo kill -15 $PID + #pkill -15 tmq_sim + kill -15 $PID echo "Killing tmq_sim processes" if [ "$OS_TYPE" != "Darwin" ]; then fuser -k -n tcp 6030 else - lsof -nti:6030 | xargs kill -9 + lsof -nti:6030 | xargs kill -15 fi PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'` -done +done \ No newline at end of file diff --git a/tests/script/tsim/parser/last_both.sim b/tests/script/tsim/parser/last_both.sim new file mode 100644 index 0000000000..e01a966744 --- /dev/null +++ b/tests/script/tsim/parser/last_both.sim @@ -0,0 +1,150 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======================== dnode1 start +$db = testdb +sql drop database if exists $db +sql create database $db cachemodel 'none' minrows 10 stt_trigger 1 +sql use $db + +sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int) +sql create table tb1 using st2 tags (1); +sql create table tb2 using st2 tags (2); +sql create table tb3 using st2 tags (3); +sql create table tb4 using st2 tags (4); +sql create table tb5 using st2 tags (1); +sql create table tb6 using st2 tags (2); +sql create table tb7 using st2 tags (3); +sql create table tb8 using st2 tags (4); +sql create table tb9 using st2 tags (5); +sql create table tba using st2 tags (5); +sql create table tbb using st2 tags (5); +sql create table tbc using st2 tags (5); +sql create table tbd using st2 tags (5); +sql create table tbe using st2 tags (5); +sql create table tbf using st2 tags (5); + +sql insert into tb9 values ("2021-05-09 10:12:26.000",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.001",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.002",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.003",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.004",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.005",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.006",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.007",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.008",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.009",28, 29, '30', -1005) +sql delete from tb9 where ts = "2021-05-09 10:12:26.000" +sql flush database $db + +sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) +sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) +sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) + +sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3', -1001) +sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001) +sql insert into tb2 values ("2021-05-11 10:11:15",-6, -7, '-8', -3001) + +sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002) +sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002) +sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002) + +sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003) +sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003) +sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003) + +sql insert into tb5 values ("2021-05-09 10:12:22",19, 20, '21', -1004) +sql insert into tb6 values ("2021-05-11 10:12:23",22, 23, NULL, -2004) +sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004) +sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004) + +sql insert into tba values ("2021-05-10 10:12:27",31, 32, NULL, -2005) +sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005) +sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005) +sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL ) + +sql drop table tbf; +sql alter table st2 add column c1 int; +sql alter table st2 drop column c1; + +run tsim/parser/last_both_query.sim + +sql flush database $db +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +run tsim/parser/last_both_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +sql drop database if exists $db +sql create database $db minrows 10 stt_trigger 1 +sql use $db + +sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int) +sql create table tb1 using st2 tags (1); +sql create table tb2 using st2 tags (2); +sql create table tb3 using st2 tags (3); +sql create table tb4 using st2 tags (4); +sql create table tb5 using st2 tags (1); +sql create table tb6 using st2 tags (2); +sql create table tb7 using st2 tags (3); +sql create table tb8 using st2 tags (4); +sql create table tb9 using st2 tags (5); +sql create table tba using st2 tags (5); +sql create table tbb using st2 tags (5); +sql create table tbc using st2 tags (5); +sql create table tbd using st2 tags (5); +sql create table tbe using st2 tags (5); +sql create table tbf using st2 tags (5); + +sql insert into tb9 values ("2021-05-09 10:12:26.000",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.001",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.002",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.003",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.004",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.005",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.006",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.007",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.008",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.009",28, 29, '30', -1005) +sql delete from tb9 where ts = "2021-05-09 10:12:26.000" +sql flush database $db + +sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) +sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) +sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) + +sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3', -1001) +sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001) +sql insert into tb2 values ("2021-05-11 10:11:15",-6, -7, '-8', -3001) + +sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002) +sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002) +sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002) + +sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003) +sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003) +sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003) + +sql insert into tb5 values ("2021-05-09 10:12:22",19, 20, '21', -1004) +sql insert into tb6 values ("2021-05-11 10:12:23",22, 23, NULL, -2004) +sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004) +sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004) + +sql insert into tba values ("2021-05-10 10:12:27",31, 32, NULL, -2005) +sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005) +sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005) +sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL ) + +sql drop table tbf +sql alter database $db cachemodel 'both' +sql alter database $db cachesize 2 +sleep 11000 + +run tsim/parser/last_both_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/last_both_query.sim b/tests/script/tsim/parser/last_both_query.sim new file mode 100644 index 0000000000..5f86412199 --- /dev/null +++ b/tests/script/tsim/parser/last_both_query.sim @@ -0,0 +1,496 @@ + +sql connect + +$db = testdb +sql use $db +print "test tb1" + +sql select last(ts) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi + +sql select last(f1) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != 6 then + print $data00 + return -1 +endi + +sql select last(*) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + print expect 3, actual: $data03 + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + +sql select last(tb1.*,ts,f4) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:57.000@ then + return -1 +endi + +print "test tb2" +sql select last(ts) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi + +sql select last(f1) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != -6 then + print $data00 + return -1 +endi + +sql select last(*) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data01 != -6 then + return -1 +endi +if $data02 != -7.000000000 then + print $data02 + return -1 +endi +if $data03 != -8 then + return -1 +endi +if $data04 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi + +sql select last(tb2.*,ts,f4) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data01 != -6 then + return -1 +endi +if $data02 != -7.000000000 then + print $data02 + return -1 +endi +if $data03 != -8 then + return -1 +endi +if $data04 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi +if $data05 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi + +print "test tbd" +sql select last(*) from tbd +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:12:29.000@ then + print $data00 + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != NULL then + print $data02 + return -1 +endi +if $data03 != NULL then + return -1 +endi +if $data04 != NULL then + return -1 +endi + +print "test tbe" +sql select last(*) from tbe +if $rows != 0 then + return -1 +endi + +print "test stable" +sql select last(ts) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi + +sql select last(f1) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != 6 then + print $data00 + return -1 +endi + +sql select last(*) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 37.000000000 then + print expect 37.000000000 actual: $data02 + return -1 +endi +if $data03 != 27 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + + +sql select last(st2.*,ts,f4) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 37.000000000 then + print expect 37.000000000, acutal: $data02 + return -1 +endi +if $data03 != 27 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:57.000@ then + return -1 +endi + +sql select last(*), id from st2 group by id order by id +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 + +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 21 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data10 != @21-05-11 10:12:23.000@ then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data12 != 23.000000000 then + print $data02 + return -1 +endi +if $data13 != -8 then + return -1 +endi +if $data14 != @70-01-01 07:59:58.-04@ then + return -1 +endi +if $data15 != 2 then + return -1 +endi +if $data20 != @21-05-10 10:12:24.000@ then + return -1 +endi +if $data21 != 24 then + return -1 +endi +if $data22 != 11.000000000 then + print expect 11.000000000 actual: $data22 + return -1 +endi +if $data23 != 25 then + return -1 +endi +if $data24 != @70-01-01 07:59:57.-04@ then = + return -1 +endi +if $data25 != 3 then + return -1 +endi +if $data30 != @21-05-11 10:12:25.000@ then + return -1 +endi +if $data31 != 26 then + return -1 +endi +if $data32 != 17.000000000 then + print $data02 + return -1 +endi +if $data33 != 27 then + return -1 +endi +if $data34 != @70-01-01 07:59:56.-04@ then + return -1 +endi +if $data35 != 4 then + return -1 +endi +if $data40 != @21-05-11 10:12:29.000@ then + return -1 +endi +if $data41 != 36 then + return -1 +endi +if $data42 != 37.000000000 then + print $data02 + return -1 +endi +if $data43 != 35 then + return -1 +endi +if $data44 != @70-01-01 07:59:56.-05@ then + return -1 +endi +if $data45 != 5 then + return -1 +endi + +sql select last_row(*), id from st2 group by id order by id +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 + +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != NULL then + print $data02 + return -1 +endi +if $data03 != NULL then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data10 != @21-05-11 10:12:23.000@ then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data12 != 23.000000000 then + print $data02 + return -1 +endi +if $data13 != NULL then + return -1 +endi +if $data14 != @70-01-01 07:59:58.-04@ then + return -1 +endi +if $data15 != 2 then + return -1 +endi +if $data20 != @21-05-10 10:12:24.000@ then + return -1 +endi +if $data21 != 24 then + return -1 +endi +if $data22 != NULL then + print expect NULL actual: $data22 + return -1 +endi +if $data23 != 25 then + return -1 +endi +if $data24 != @70-01-01 07:59:57.-04@ then = + return -1 +endi +if $data25 != 3 then + return -1 +endi +if $data30 != @21-05-11 10:12:25.000@ then + return -1 +endi +if $data31 != 26 then + return -1 +endi +if $data32 != NULL then + print $data02 + return -1 +endi +if $data33 != 27 then + return -1 +endi +if $data34 != @70-01-01 07:59:56.-04@ then + return -1 +endi +if $data35 != 4 then + return -1 +endi +if $data40 != @21-05-11 10:12:29.000@ then + return -1 +endi +#if $data41 != NULL then +# return -1 +#endi +#if $data42 != NULL then +# print $data02 +# return -1 +#endi +if $data43 != NULL then + return -1 +endi +#if $data44 != NULL then +# return -1 +#endi +if $data45 != 5 then + return -1 +endi + +print "test tbn" +sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) +sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) +sql insert into tbn values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) +sql insert into tbn values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) +sql insert into tbn values ("2021-05-13 10:10:12", NULL,NULL, NULL,NULL) + +sql select last(*) from tbn; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-13 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + +sql alter table tbn add column c1 int; +sql alter table tbn drop column c1; diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim index 6cd5309590..30196e0b62 100644 --- a/tests/script/tsim/parser/last_cache_query.sim +++ b/tests/script/tsim/parser/last_cache_query.sim @@ -386,3 +386,5 @@ if $data04 != @70-01-01 07:59:57.000@ then return -1 endi +sql alter table tbn add column c1 int; +sql alter table tbn drop column c1; diff --git a/tests/script/tsim/query/query_count0.sim b/tests/script/tsim/query/query_count0.sim index b7c629e538..5b95d4fad7 100644 --- a/tests/script/tsim/query/query_count0.sim +++ b/tests/script/tsim/query/query_count0.sim @@ -9,8 +9,6 @@ print =============== create database sql create database test vgroups 1; sql use test; -sql alter local 'disableCount' '0' ; - sql create table t1(ts timestamp, a int, b int , c int, d double); sql insert into t1 values(1648791213000,0,1,1,1.0); diff --git a/tests/script/tsim/query/query_count1.sim b/tests/script/tsim/query/query_count1.sim index 0c40303e57..043b604263 100644 --- a/tests/script/tsim/query/query_count1.sim +++ b/tests/script/tsim/query/query_count1.sim @@ -9,8 +9,6 @@ print =============== create database sql create database test vgroups 4; sql use test; -sql alter local 'disableCount' '0' ; - sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); @@ -81,5 +79,27 @@ if $data22 != 4 then goto loop3 endi + +print step2 +print =============== create database +sql create database test1 vgroups 1; +sql use test1; + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +#2~INT32_MAX +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(-1); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(0); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(1); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2147483648); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(10, 0); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(10, -1); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(10, 11); + +sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2); +sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2147483647); + print query_count0 end system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/query_count_sliding0.sim b/tests/script/tsim/query/query_count_sliding0.sim index 13a6c94451..464aec6b97 100644 --- a/tests/script/tsim/query/query_count_sliding0.sim +++ b/tests/script/tsim/query/query_count_sliding0.sim @@ -9,8 +9,6 @@ print =============== create database sql create database test vgroups 1; sql use test; -sql alter local 'disableCount' '0' ; - sql create table t1(ts timestamp, a int, b int , c int, d double); sql insert into t1 values(1648791213000,0,1,1,1.0); diff --git a/tests/script/tsim/stream/distributeMultiLevelInterval0.sim b/tests/script/tsim/stream/distributeMultiLevelInterval0.sim new file mode 100644 index 0000000000..784ab7f4a5 --- /dev/null +++ b/tests/script/tsim/stream/distributeMultiLevelInterval0.sim @@ -0,0 +1,267 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +system sh/cfg.sh -n dnode1 -c streamAggCnt -v 2 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + + + +print ===== step1 +sql drop stream if exists streams1; +sql drop database if exists test; +sql create database test vgroups 4; +sql use test; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,2,2); +sql create table ts4 using st tags(4,2,2); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4 from st interval(10s); + +sleep 1000 + +sql insert into ts1 values(1648791213000,1,1,3,4.1); +sql insert into ts1 values(1648791223000,2,2,3,1.1); +sql insert into ts1 values(1648791233000,3,3,3,2.1); +sql insert into ts1 values(1648791243000,4,4,3,3.1); + +sql insert into ts2 values(1648791213000,1,5,3,4.1); +sql insert into ts2 values(1648791223000,2,6,3,1.1); +sql insert into ts2 values(1648791233000,3,7,3,2.1); +sql insert into ts2 values(1648791243000,4,8,3,3.1); + + +$loop_count = 0 +loop0: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop0 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop0 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop0 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop0 +endi + +if $data31 != 2 then + print =====data31=$data31 + goto loop0 +endi + + +sql insert into ts1 values(1648791213000,1,9,3,4.1); + +$loop_count = 0 +loop1: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop1 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop1 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop1 +endi + +if $data31 != 2 then + print =====data31=$data31 + goto loop1 +endi + +sql delete from ts2 where ts = 1648791243000 ; + +$loop_count = 0 +loop2: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop2 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop2 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop2 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop2 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop2 +endi + +sql delete from ts2 where ts = 1648791223000 ; + +$loop_count = 0 +loop3: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop3 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop3 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop3 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop3 +endi + + +sql insert into ts1 values(1648791233001,3,9,3,2.1); + +$loop_count = 0 +loop4: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop4 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop4 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop4 +endi + +if $data21 != 3 then + print =====data21=$data21 + goto loop4 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop4 +endi + +sql select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5, avg(d) from st interval(10s); + + +print ===== over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/pauseAndResume.sim b/tests/script/tsim/stream/pauseAndResume.sim index 5eb9eef010..be89d8e235 100644 --- a/tests/script/tsim/stream/pauseAndResume.sim +++ b/tests/script/tsim/stream/pauseAndResume.sim @@ -8,16 +8,19 @@ sql connect print ===== step1 sql drop stream if exists streams1; sql drop database if exists test; -sql create database test vgroups 10; +sql create database test vgroups 3; sql use test; sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); + sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt1 as select _wstart, count(*) c1, sum(a) c3 from st interval(10s); sleep 2000 +sql_error create stream stream1_same_dst into streamt1 as select _wstart, count(*) c1, sum(a) c3 from st interval(10s); + sql pause stream streams1; sql insert into ts1 values(1648791213001,1,12,3,1.0); @@ -102,6 +105,11 @@ if $data11 != 4 then goto loop1 endi +print ===== idle for 70 sec for checkpoint gen +sleep 70000 + +print ===== idle 60 sec completed , continue + print ===== step 1 over print ===== step2 @@ -113,6 +121,12 @@ sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt2 as select _wstart, count(*) c1, sum(a) c3 from t1 interval(10s); +# duplicate stream +sql_error create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt2 as select _wstart, count(*) c1, sum(a) c3 from t1 interval(10s); +sql create stream if not exists streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt2 as select _wstart, count(*) c1, sum(a) c3 from t1 interval(10s); + +sleep 1000 + sql pause stream streams2; sql insert into t1 values(1648791213001,1,12,3,1.0); @@ -237,8 +251,9 @@ print ===== step 2 over print ===== step3 sql drop stream if exists streams3; sql drop database if exists test3; -sql create database test3 vgroups 10; +sql create database test3 vgroups 3; sql use test3; + sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 0d96d0841b..37081ad1ae 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -21,6 +21,8 @@ print create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); +sleep 1000 + sql insert into t1 values(1648791213000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,3,1.0,2); $loop_count = 0 @@ -457,6 +459,8 @@ print create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); +sleep 1000 + sql insert into t1 values(1648791212000,2,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,4,1.0,2); @@ -504,6 +508,9 @@ sql create table t1(ts timestamp, a int, b int , c int, d double, id int); print create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart, count(*) c1, sum(b) c3 from t1 state_window(a); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart, count(*) c1, sum(b) c3 from t1 state_window(a); + +sleep 1000 + sql insert into t1 values(1648791212000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,2,2,3,1.0,1); sql insert into t1 values(1648791214000,3,2,4,1.0,2); @@ -557,6 +564,8 @@ print create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0 sql create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart AS startts, min(c1),count(c1) from t1 state_window(c1); +sleep 1000 + sql insert into t1 (ts, c1) values (1668073288209, 11); sql insert into t1 (ts, c1) values (1668073288210, 11); sql insert into t1 (ts, c1) values (1668073288211, 11); @@ -745,6 +754,9 @@ sql create table b (c timestamp, d int, e int , f int, g double); print create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); + +sleep 1000 + sql insert into b values(1648791213000,NULL,NULL,NULL,NULL); sql select * from streamt order by c1, c2, c3; diff --git a/tests/system-test/0-others/com_alltypedata.json b/tests/system-test/0-others/com_alltypedata.json index 0e6d8e3a07..1499ca7670 100644 --- a/tests/system-test/0-others/com_alltypedata.json +++ b/tests/system-test/0-others/com_alltypedata.json @@ -22,7 +22,7 @@ "vgroups": 2, "replica": 1, "precision": "ms", - "stt_trigger": 8, + "stt_trigger": 1, "minRows": 100, "maxRows": 4096 }, diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index c936cf1ae4..8163177a3b 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -1,11 +1,13 @@ from urllib.parse import uses_relative import taos +import taosws import sys import os import time import platform import inspect from taos.tmq import Consumer +from taos.tmq import * from pathlib import Path from util.log import * @@ -17,7 +19,7 @@ from util.dnodes import TDDnode from util.cluster import * import subprocess -BASEVERSION = "3.0.2.3" +BASEVERSION = "3.2.0.0" class TDTestCase: def caseDescription(self): f''' @@ -30,7 +32,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 4; ;use deldata; + self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 1; ;use deldata; create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); create table deldata.ct1 using deldata.stb1 tags ( 1 ); insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); @@ -104,8 +106,19 @@ class TDTestCase: print(f"{packageName} has been exists") os.system(f" cd {packagePath} && tar xvf {packageName} && cd {packageTPath} && ./install.sh -e no " ) tdDnodes.stop(1) - print(f"start taosd: rm -rf {dataPath}/* && nohup taosd -c {cPath} & ") - os.system(f"rm -rf {dataPath}/* && nohup taosd -c {cPath} & " ) + print(f"start taosd: rm -rf {dataPath}/* && nohup /usr/bin/taosd -c {cPath} & ") + os.system(f"rm -rf {dataPath}/* && nohup /usr/bin/taosd -c {cPath} & " ) + os.system(f"killall taosadapter" ) + os.system(f"cp /etc/taos/taosadapter.toml {cPath}/taosadapter.toml " ) + taosadapter_cfg = cPath + "/taosadapter.toml" + taosadapter_log_path = cPath + "/../log/" + print(f"taosadapter_cfg:{taosadapter_cfg},taosadapter_log_path:{taosadapter_log_path} ") + self.alter_string_in_file(taosadapter_cfg,"#path = \"/var/log/taos\"",f"path = \"{taosadapter_log_path}\"") + self.alter_string_in_file(taosadapter_cfg,"taosConfigDir = \"\"",f"taosConfigDir = \"{cPath}\"") + print("/usr/bin/taosadapter --version") + os.system(f" /usr/bin/taosadapter --version" ) + print(f" LD_LIBRARY_PATH=/usr/lib -c {taosadapter_cfg} 2>&1 & ") + os.system(f" LD_LIBRARY_PATH=/usr/lib /usr/bin/taosadapter -c {taosadapter_cfg} 2>&1 & " ) sleep(5) @@ -116,7 +129,24 @@ class TDTestCase: def is_list_same_as_ordered_list(self,unordered_list, ordered_list): sorted_list = sorted(unordered_list) return sorted_list == ordered_list - + + def alter_string_in_file(self,file,old_str,new_str): + """ + replace str in file + :param file + :param old_str + :param new_str + :return: + """ + file_data = "" + with open(file, "r", encoding="utf-8") as f: + for line in f: + if old_str in line: + line = line.replace(old_str,new_str) + file_data += line + with open(file,"w",encoding="utf-8") as f: + f.write(file_data) + def run(self): scriptsPath = os.path.dirname(os.path.realpath(__file__)) distro_id = distro.id() @@ -131,7 +161,7 @@ class TDTestCase: dbname = "test" stb = f"{dbname}.meters" self.installTaosd(bPath,cPath) - os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ") + # os.system(f"echo 'debugFlag 143' >> {cPath}/taos.cfg ") tableNumbers=100 recordNumbers1=100 recordNumbers2=1000 @@ -163,11 +193,46 @@ class TDTestCase: # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ') - os.system(f"sed -i 's/\/etc\/taos/{cPath}/' 0-others/tmqBasic.json ") + self.alter_string_in_file("0-others/tmqBasic.json", "/etc/taos/", cPath) # os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/tmqBasic.json -y ") os.system('LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists tmq_test_topic as select current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ') os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show topics;" ') + os.system(f" /usr/bin/taosadapter --version " ) + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) + try: + consumer.subscribe(["tmq_test_topic"]) + except TmqError: + tdLog.exit(f"subscribe error") + + while True: + message = consumer.poll(timeout=1.0) + if message: + print("message") + id = message.vgroup() + topic = message.topic() + database = message.database() + + for block in message: + nrows = block.nrows() + ncols = block.ncols() + for row in block: + print(row) + values = block.fetchall() + print(nrows, ncols) + + consumer.commit(message) + else: + print("break") + break + + consumer.close() tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") @@ -184,7 +249,8 @@ class TDTestCase: os.system("pkill taosd") # make sure all the data are saved in disk. self.checkProcessPid("taosd") - + os.system("pkill taosadapter") # make sure all the data are saved in disk. + self.checkProcessPid("taosadapter") tdLog.printNoPrefix("==========step2:update new version ") self.buildTaosd(bPath) @@ -193,6 +259,7 @@ class TDTestCase: tdsql=tdCom.newTdSql() print(tdsql) cmd = f" LD_LIBRARY_PATH=/usr/lib taos -h localhost ;" + print(os.system(cmd)) if os.system(cmd) == 0: raise Exception("failed to execute system command. cmd: %s" % cmd) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py similarity index 78% rename from tests/system-test/7-tmq/tmqVnodeTransform.py rename to tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py index c2b002ead6..a853489c3f 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py @@ -122,135 +122,7 @@ class TDTestCase: tdLog.debug(f"redistributeSql:{redistributeSql}") tdSql.query(redistributeSql) tdLog.debug("redistributeSql ok") - - def tmqCase1(self): - tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 1, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 1000, - 'batchNum': 10, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic1'] - # expectRowsList = [] - tmqCom.initConsumerTable() - - tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) - # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - # tdSql.query(queryString) - # expectRowsList.append(tdSql.getRows()) - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 - topicList = topicNameList[0] - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - tdLog.info("create ctb1") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) - - tmqCom.getStartConsumeNotifyFromTmqsim() - tmqCom.getStartCommitNotifyFromTmqsim() - - #restart dnode & remove wal - self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups(); - - tdLog.info("create ctb2") - paraDict['ctbPrefix'] = "ctbn" - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) - pInsertThread.join() - pInsertThread1.join() - - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - - if expectrowcnt / 2 > resultList[0]: - tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) - tdLog.exit("%d tmq consume rows error!"%consumerId) - - # tmqCom.checkFileContent(consumerId, queryString) - - time.sleep(10) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 1 end ...... ") - - def tmqCase2(self): - tdLog.printNoPrefix("======== test case 2: ") - paraDict = {'dbName':'dbt'} - - ntbName = "ntb" - - topicNameList = ['topic2'] - tmqCom.initConsumerTable() - - sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName) - tdLog.info("create nomal table sql: %s"%sqlString) - tdSql.execute(sqlString) - - tdLog.info("create topics from nomal table") - queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - tdSql.query("flush database %s"%(paraDict['dbName'])) - #restart dnode & remove wal - self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups(); - - sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) - tdLog.info("alter table sql: %s"%sqlString) - tdSql.error(sqlString) - expectRows = 0 - resultList = tmqCom.selectConsumeResult(expectRows) - time.sleep(1) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 2 end ...... ") - + def tmqCase3(self): tdLog.printNoPrefix("======== test case 3: ") paraDict = {'dbName': 'dbt', @@ -330,12 +202,90 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 3 end ...... ") + def tmqCaseDbname(self): + tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic4'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from database ") + queryString = "database %s "%(paraDict['dbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(1) + # restart dnode & remove wal + self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(6) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") + def run(self): - self.prepareTestEnv() - self.tmqCase1() - self.tmqCase2() self.prepareTestEnv() self.tmqCase3() + self.prepareTestEnv() + self.tmqCaseDbname() def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py new file mode 100644 index 0000000000..40879d5c66 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py @@ -0,0 +1,266 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 1000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + # tdLog.info("create ctb") + # tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + # ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + # tdLog.info("insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def restartAndRemoveWal(self): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + os.system('rm -rf ' + dataPath) + tdLog.debug("dataPath:%s"%dataPath) + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def redistributeVgroups(self): + dnodesList = [] + tdSql.query("show dnodes") + for result in tdSql.queryResult: + dnodesList.append(result[0]) + print("dnodeList:",dnodesList) + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodesList.remove(result[0]) + vnodeId = result[1] + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") + tdSql.query(redistributeSql) + tdLog.debug("redistributeSql ok") + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups(); + + tdLog.info("create ctb2") + paraDict['ctbPrefix'] = "ctbn" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 > resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName':'dbt'} + + ntbName = "ntb" + + topicNameList = ['topic2'] + tmqCom.initConsumerTable() + + sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName) + tdLog.info("create nomal table sql: %s"%sqlString) + tdSql.execute(sqlString) + + tdLog.info("create topics from nomal table") + queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query("flush database %s"%(paraDict['dbName'])) + #restart dnode & remove wal + self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups(); + + sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) + tdLog.info("alter table sql: %s"%sqlString) + tdSql.error(sqlString) + expectRows = 0 + resultList = tmqCom.selectConsumeResult(expectRows) + time.sleep(1) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1() + self.tmqCase2() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index c547385d70..2712caed3c 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -359,8 +359,74 @@ class TDTestCase: finally: consumer.close() + def consume_TS_4540_Test(self): + tdSql.execute(f'create database if not exists test') + tdSql.execute(f'use test') + tdSql.execute(f'CREATE STABLE `test`.`b` ( `time` TIMESTAMP , `task_id` NCHAR(1000) ) TAGS( `key` NCHAR(1000))') + tdSql.execute(f"insert into `test`.b1 using `test`.`b`(key) tags('1') (time, task_id) values ('2024-03-04 12:50:01.000', '32') `test`.b2 using `test`.`b`(key) tags('2') (time, task_id) values ('2024-03-04 12:50:01.000', '43') `test`.b3 using `test`.`b`(key) tags('3') (time, task_id) values ('2024-03-04 12:50:01.000', '123456')") + + tdSql.execute(f'create topic tt as select tbname,task_id,key from b') + + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = Consumer(consumer_dict) + + try: + consumer.subscribe(["tt"]) + except TmqError: + tdLog.exit(f"subscribe error") + + try: + while True: + res = consumer.poll(1) + if not res: + break + val = res.value() + if val is None: + continue + for block in val: + data = block.fetchall() + print(data) + if data != [('b1', '32', '1')] and data != [('b2', '43', '2')] and data != [('b3', '123456', '3')]: + tdLog.exit(f"index = 0 table b1 error") + + finally: + consumer.close() + + def consume_ts_4544(self): + tdSql.execute(f'create database if not exists d1') + tdSql.execute(f'use d1') + tdSql.execute(f'create table stt(ts timestamp, i int) tags(t int)') + tdSql.execute(f'insert into tt1 using stt tags(1) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into tt2 using stt tags(2) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into tt3 using stt tags(3) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into tt1 using stt tags(1) values(now+5s, 11) (now+10s, 12)') + + tdSql.execute(f'create topic topic_in as select * from stt where tbname in ("tt2")') + + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = Consumer(consumer_dict) + + try: + consumer.subscribe(["topic_in"]) + except TmqError: + tdLog.exit(f"subscribe error") + + consumer.close() + def run(self): self.consumeTest() + self.consume_ts_4544() + self.consume_TS_4540_Test() tdSql.prepare() self.checkWal1VgroupOnlyMeta() diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index b6917e5a76..847bbcf4be 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -206,6 +206,7 @@ SWords shellCommands[] = { {"show grants full;", 0, 0, NULL}, {"show grants logs;", 0, 0, NULL}, #ifdef TD_ENTERPRISE + {"show views;", 0, 0, NULL}, {"split vgroup ", 0, 0, NULL}, #endif {"insert into values(", 0, 0, NULL}, @@ -570,6 +571,7 @@ void showHelp() { split vgroup ;\n\ show compacts;\n\ show compact \n\ + show views;\n\ show create view ;"); #endif diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 23424cea98..995d3d04ec 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -75,7 +75,7 @@ bool shellIsEmptyCommand(const char *cmd) { int32_t shellRunSingleCommand(char *command) { shellCmdkilled = false; - + if (shellIsEmptyCommand(command)) { return 0; } @@ -1019,7 +1019,7 @@ void shellReadHistory() { char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1); int32_t read_size = 0; - while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) { + while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) > 0) { line[read_size - 1] = '\0'; taosMemoryFree(pHistory->hist[pHistory->hend]); pHistory->hist[pHistory->hend] = taosStrdup(line); @@ -1315,7 +1315,7 @@ int32_t shellExecute() { shellSetConn(shell.conn, runOnce); shellReadHistory(); - if(shell.args.is_bi_mode) { + if(shell.args.is_bi_mode) { // need set bi mode printf("Set BI mode is true.\n"); #ifndef WEBSOCKET diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 2c334eb67b..01619decc5 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1018,7 +1018,7 @@ int sml_escape_Test() { ASSERT(numFields == 5); ASSERT(strncmp(fields[1].name, "inode\"i,= s_used", sizeof("inode\"i,= s_used") - 1) == 0); ASSERT(strncmp(fields[2].name, "total", sizeof("total") - 1) == 0); - ASSERT(strncmp(fields[3].name, "inode\"i,= s_f\\\\ree", sizeof("inode\"i,= s_f\\\\ree") - 1) == 0); + ASSERT(strncmp(fields[3].name, "inode\"i,= s_f\\ree", sizeof("inode\"i,= s_f\\ree") - 1) == 0); ASSERT(strncmp(fields[4].name, "dev\"i,= ce", sizeof("dev\"i,= ce") - 1) == 0); TAOS_ROW row = NULL; @@ -1044,6 +1044,88 @@ int sml_escape_Test() { return code; } +// test field with end of escape +int sml_escape1_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_escape"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use db_escape"); + taos_free_result(pRes); + + const char *sql[] = { + "stab,t1\\=1 c1=3,c2=\"32fw\" 1661943970000000000", + "stab,t1=1\\ c1=3,c2=\"32fw\" 1661943980000000000", + "stab,t1=1 c1\\=3,c2=\"32fw\" 1661943990000000000", + }; + for(int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++){ + pRes = taos_schemaless_insert(taos, (char**)&sql[i], 1, TSDB_SML_LINE_PROTOCOL, 0); + int code = taos_errno(pRes); + ASSERT(code); + } + + const char *sql1[] = { + "stab\\,t1=1 c1=3,c2=\"32fw\" 1661943960000000000", + "stab\\\\,t1=1 c1=3,c2=\"32fw\" 1661943960000000000", + "stab,t1\\\\=1 c1=3,c2=\"32fw\" 1661943970000000000", + "stab,t1=1\\\\ c1=3,c2=\"32fw\" 1661943980000000000", + "stab,t1=1 c1\\\\=3,c2=\"32fw\" 1661943990000000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); + int code = taos_errno(pRes); + ASSERT(!code); + ASSERT(taos_affected_rows(pRes) == 5); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stab"); //check stable name + ASSERT(pRes); + int fieldNum = taos_field_count(pRes); + ASSERT(fieldNum == 6); + printf("fieldNum:%d\n", fieldNum); + + int numFields = taos_num_fields(pRes); + TAOS_FIELD *fields = taos_fetch_fields(pRes); + ASSERT(numFields == 6); + ASSERT(strncmp(fields[1].name, "c1", sizeof("c1") - 1) == 0); + ASSERT(strncmp(fields[2].name, "c2", sizeof("c2") - 1) == 0); + ASSERT(strncmp(fields[3].name, "c1\\", sizeof("c1\\") - 1) == 0); + ASSERT(strncmp(fields[4].name, "t1\\", sizeof("t1\\") - 1) == 0); + ASSERT(strncmp(fields[5].name, "t1", sizeof("t1") - 1) == 0); + + TAOS_ROW row = NULL; + int32_t rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t *)row[0]; + + if (ts == 1661943970000) { + ASSERT(*(double *)row[1] == 3); + ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0); + ASSERT(row[3] == NULL); + ASSERT(strncmp(row[4], "1", sizeof("1") - 1) == 0); + ASSERT(row[5] == NULL); + }else if (ts == 1661943980000) { + ASSERT(*(double *)row[1] == 3); + ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0); + ASSERT(row[3] == NULL); + ASSERT(row[4] == NULL); + ASSERT(strncmp(row[5], "1\\", sizeof("1\\") - 1) == 0); + }else if (ts == 1661943990000) { + ASSERT(row[1] == NULL); + ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0); + ASSERT(*(double *)row[3] == 3); + ASSERT(row[4] == NULL); + ASSERT(strncmp(row[5], "1", sizeof("1") - 1) == 0); + } + rowIndex++; + } + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_19221_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1775,17 +1857,14 @@ int main(int argc, char *argv[]) { ASSERT(ret); ret = sml_escape_Test(); ASSERT(!ret); + ret = sml_escape1_Test(); + ASSERT(!ret); ret = sml_ts3116_Test(); ASSERT(!ret); ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file ASSERT(!ret); ret = sml_ts3303_Test(); ASSERT(!ret); - - // for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ - // printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i]))); - // } - // int ret = 0; ret = sml_ttl_Test(); ASSERT(!ret); ret = sml_ts2164_Test(); diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 2257089f06..5012e50bab 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -574,6 +574,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.consume.excluded", "1"); if (g_conf.snapShot) { tmq_conf_set(conf, "experimental.snapshot.enable", "true");