Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/3.0_mhli
This commit is contained in:
commit
4033e2dd34
|
@ -52,12 +52,22 @@ IF(${TD_WINDOWS})
|
|||
ON
|
||||
)
|
||||
ELSE ()
|
||||
|
||||
option(
|
||||
BUILD_TEST
|
||||
"If build unit tests using googletest"
|
||||
ON
|
||||
)
|
||||
include(CheckCXXCompilerFlag)
|
||||
CHECK_CXX_COMPILER_FLAG("-std=c++13" COMPILER_SUPPORTS_CXX13)
|
||||
IF(${COMPILER_SUPPORTS_CXX13})
|
||||
add_definitions(-DCOMPILER_SUPPORTS_CXX13)
|
||||
option(
|
||||
BUILD_TEST
|
||||
"If build unit tests using googletest"
|
||||
ON
|
||||
)
|
||||
ELSE ()
|
||||
option(
|
||||
BUILD_TEST
|
||||
"If build unit tests using googletest"
|
||||
OFF
|
||||
)
|
||||
ENDIF ()
|
||||
ENDIF ()
|
||||
|
||||
option(
|
||||
|
|
|
@ -206,8 +206,8 @@ Note: InfluxDB token authorization is not supported at present. Only Basic autho
|
|||
You can use any client that supports the http protocol to access the RESTful interface address `http://<fqdn>:6041/<APIEndPoint>` to write data in OpenTSDB compatible format to TDengine.
|
||||
|
||||
```text
|
||||
/opentsdb/v1/put/json/:db
|
||||
/opentsdb/v1/put/telnet/:db
|
||||
/opentsdb/v1/put/json/<db>
|
||||
/opentsdb/v1/put/telnet/<db>
|
||||
```
|
||||
|
||||
### collectd
|
||||
|
|
|
@ -207,8 +207,8 @@ AllowWebSockets
|
|||
您可以使用任何支持 http 协议的客户端访问 Restful 接口地址 `http://<fqdn>:6041/<APIEndPoint>` 来写入 OpenTSDB 兼容格式的数据到 TDengine。EndPoint 如下:
|
||||
|
||||
```text
|
||||
/opentsdb/v1/put/json/:db
|
||||
/opentsdb/v1/put/telnet/:db
|
||||
/opentsdb/v1/put/json/<db>
|
||||
/opentsdb/v1/put/telnet/<db>
|
||||
```
|
||||
|
||||
### collectd
|
||||
|
|
|
@ -98,9 +98,10 @@ int32_t create_stream() {
|
|||
/*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/
|
||||
/*const char* sql = "select sum(k) from tu1 interval(10m)";*/
|
||||
/*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
|
||||
pRes = taos_query(pConn,
|
||||
"create stream stream1 trigger at_once into outstb as select _wstartts, sum(k) from st1 partition "
|
||||
"by tbname interval(10s) ");
|
||||
pRes = taos_query(
|
||||
pConn,
|
||||
"create stream stream1 trigger max_delay 10s into outstb as select _wstartts, sum(k) from st1 partition "
|
||||
"by tbname session(ts, 10s) ");
|
||||
if (taos_errno(pRes) != 0) {
|
||||
printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));
|
||||
return -1;
|
||||
|
|
|
@ -137,8 +137,8 @@ int32_t create_topic() {
|
|||
}
|
||||
taos_free_result(pRes);
|
||||
|
||||
pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");
|
||||
/*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/
|
||||
/*pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");*/
|
||||
pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");
|
||||
if (taos_errno(pRes) != 0) {
|
||||
printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
|
||||
return -1;
|
||||
|
@ -199,7 +199,7 @@ tmq_t* build_consumer() {
|
|||
tmq_conf_set(conf, "msg.with.table.name", "true");
|
||||
tmq_conf_set(conf, "enable.auto.commit", "true");
|
||||
|
||||
tmq_conf_set(conf, "experiment.use.snapshot", "false");
|
||||
tmq_conf_set(conf, "experimental.snapshot.enable", "true");
|
||||
|
||||
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
|
||||
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
|
||||
|
@ -225,7 +225,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
|
|||
}
|
||||
int32_t cnt = 0;
|
||||
while (running) {
|
||||
TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 0);
|
||||
TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, -1);
|
||||
if (tmqmessage) {
|
||||
cnt++;
|
||||
msg_process(tmqmessage);
|
||||
|
|
|
@ -131,10 +131,10 @@ DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...);
|
|||
DLL_EXPORT setConfRet taos_set_config(const char *config);
|
||||
DLL_EXPORT int taos_init(void);
|
||||
DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
|
||||
DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
|
||||
DLL_EXPORT void taos_close(TAOS *taos);
|
||||
DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
|
||||
DLL_EXPORT void taos_close(TAOS *taos);
|
||||
|
||||
const char *taos_data_type(int type);
|
||||
const char *taos_data_type(int type);
|
||||
|
||||
DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos);
|
||||
DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
|
||||
|
@ -164,6 +164,7 @@ DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
|
|||
DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res);
|
||||
DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result
|
||||
DLL_EXPORT void taos_free_result(TAOS_RES *res);
|
||||
DLL_EXPORT void taos_kill_query(TAOS *taos);
|
||||
DLL_EXPORT int taos_field_count(TAOS_RES *res);
|
||||
DLL_EXPORT int taos_num_fields(TAOS_RES *res);
|
||||
DLL_EXPORT int taos_affected_rows(TAOS_RES *res);
|
||||
|
|
|
@ -25,10 +25,11 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
// TODO remove it
|
||||
enum {
|
||||
TMQ_CONF__RESET_OFFSET__LATEST = -1,
|
||||
TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
|
||||
TMQ_CONF__RESET_OFFSET__NONE = -3,
|
||||
TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
|
||||
TMQ_CONF__RESET_OFFSET__LATEST = -1,
|
||||
};
|
||||
|
||||
enum {
|
||||
|
@ -39,6 +40,16 @@ enum {
|
|||
TMQ_MSG_TYPE__END_RSP,
|
||||
};
|
||||
|
||||
enum {
|
||||
STREAM_INPUT__DATA_SUBMIT = 1,
|
||||
STREAM_INPUT__DATA_BLOCK,
|
||||
STREAM_INPUT__DATA_SCAN,
|
||||
STREAM_INPUT__DATA_RETRIEVE,
|
||||
STREAM_INPUT__TRIGGER,
|
||||
STREAM_INPUT__CHECKPOINT,
|
||||
STREAM_INPUT__DROP,
|
||||
};
|
||||
|
||||
typedef enum EStreamType {
|
||||
STREAM_NORMAL = 1,
|
||||
STREAM_INVERT,
|
||||
|
@ -47,8 +58,8 @@ typedef enum EStreamType {
|
|||
STREAM_GET_ALL,
|
||||
STREAM_DELETE,
|
||||
STREAM_RETRIEVE,
|
||||
STREAM_PUSH_DATA,
|
||||
STREAM_PUSH_EMPTY,
|
||||
STREAM_PULL_DATA,
|
||||
STREAM_PULL_OVER,
|
||||
} EStreamType;
|
||||
|
||||
typedef struct {
|
||||
|
|
|
@ -224,7 +224,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
|
|||
int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n);
|
||||
|
||||
int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
|
||||
int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src);
|
||||
int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src);
|
||||
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
|
||||
SSDataBlock* createDataBlock();
|
||||
int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData);
|
||||
|
@ -232,9 +232,8 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColIn
|
|||
SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId);
|
||||
SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index);
|
||||
|
||||
void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
|
||||
int8_t needCompress);
|
||||
const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
|
||||
void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress);
|
||||
const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
|
||||
|
||||
void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag);
|
||||
// for debug
|
||||
|
|
|
@ -623,6 +623,7 @@ typedef struct {
|
|||
col_id_t colId;
|
||||
int16_t slotId;
|
||||
};
|
||||
bool output; // TODO remove it later
|
||||
|
||||
int16_t type;
|
||||
int32_t bytes;
|
||||
|
@ -1494,6 +1495,7 @@ typedef struct SSubQueryMsg {
|
|||
uint64_t queryId;
|
||||
uint64_t taskId;
|
||||
int64_t refId;
|
||||
int32_t execId;
|
||||
int8_t taskType;
|
||||
int8_t explain;
|
||||
uint32_t sqlLen; // the query sql,
|
||||
|
@ -1513,6 +1515,7 @@ typedef struct {
|
|||
uint64_t sId;
|
||||
uint64_t queryId;
|
||||
uint64_t taskId;
|
||||
int32_t execId;
|
||||
} SQueryContinueReq;
|
||||
|
||||
typedef struct {
|
||||
|
@ -1534,6 +1537,7 @@ typedef struct {
|
|||
uint64_t sId;
|
||||
uint64_t queryId;
|
||||
uint64_t taskId;
|
||||
int32_t execId;
|
||||
} SResFetchReq;
|
||||
|
||||
typedef struct {
|
||||
|
@ -1545,6 +1549,7 @@ typedef struct {
|
|||
uint64_t queryId;
|
||||
uint64_t taskId;
|
||||
int64_t refId;
|
||||
int32_t execId;
|
||||
int8_t status;
|
||||
} STaskStatus;
|
||||
|
||||
|
@ -1590,6 +1595,7 @@ typedef struct {
|
|||
uint64_t queryId;
|
||||
uint64_t taskId;
|
||||
int64_t refId;
|
||||
int32_t execId;
|
||||
} STaskCancelReq;
|
||||
|
||||
typedef struct {
|
||||
|
@ -1602,6 +1608,7 @@ typedef struct {
|
|||
uint64_t queryId;
|
||||
uint64_t taskId;
|
||||
int64_t refId;
|
||||
int32_t execId;
|
||||
} STaskDropReq;
|
||||
|
||||
typedef struct {
|
||||
|
@ -1622,6 +1629,7 @@ typedef struct {
|
|||
int8_t triggerType;
|
||||
int64_t maxDelay;
|
||||
int64_t watermark;
|
||||
int8_t igExpired;
|
||||
} SCMCreateStreamReq;
|
||||
|
||||
typedef struct {
|
||||
|
@ -2461,22 +2469,37 @@ int32_t tDecodeSMqCMCommitOffsetReq(SDecoder* decoder, SMqCMCommitOffsetReq* pRe
|
|||
|
||||
// tqOffset
|
||||
enum {
|
||||
TMQ_OFFSET__SNAPSHOT = 1,
|
||||
TMQ_OFFSET__LOG,
|
||||
TMQ_OFFSET__RESET_NONE = -3,
|
||||
TMQ_OFFSET__RESET_EARLIEAST = -2,
|
||||
TMQ_OFFSET__RESET_LATEST = -1,
|
||||
TMQ_OFFSET__LOG = 1,
|
||||
TMQ_OFFSET__SNAPSHOT_DATA = 2,
|
||||
TMQ_OFFSET__SNAPSHOT_META = 3,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
int8_t type;
|
||||
union {
|
||||
// snapshot data
|
||||
struct {
|
||||
int64_t uid;
|
||||
int64_t ts;
|
||||
};
|
||||
// log
|
||||
struct {
|
||||
int64_t version;
|
||||
};
|
||||
};
|
||||
char subKey[TSDB_SUBSCRIBE_KEY_LEN];
|
||||
} STqOffsetVal;
|
||||
|
||||
int32_t tEncodeSTqOffsetVal(SEncoder* pEncoder, const STqOffsetVal* pOffsetVal);
|
||||
int32_t tDecodeSTqOffsetVal(SDecoder* pDecoder, STqOffsetVal* pOffsetVal);
|
||||
int32_t tFormatOffset(char* buf, int32_t maxLen, const STqOffsetVal* pVal);
|
||||
bool tOffsetEqual(const STqOffsetVal* pLeft, const STqOffsetVal* pRight);
|
||||
|
||||
typedef struct {
|
||||
STqOffsetVal val;
|
||||
char subKey[TSDB_SUBSCRIBE_KEY_LEN];
|
||||
} STqOffset;
|
||||
|
||||
int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset);
|
||||
|
@ -2709,7 +2732,8 @@ typedef struct {
|
|||
uint64_t reqId;
|
||||
int64_t consumerId;
|
||||
int64_t timeout;
|
||||
int64_t currentOffset;
|
||||
// int64_t currentOffset;
|
||||
STqOffsetVal reqOffset;
|
||||
} SMqPollReq;
|
||||
|
||||
typedef struct {
|
||||
|
@ -2778,12 +2802,14 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) {
|
|||
}
|
||||
|
||||
typedef struct {
|
||||
SMqRspHead head;
|
||||
int64_t reqOffset;
|
||||
int64_t rspOffset;
|
||||
int16_t resMsgType;
|
||||
int32_t metaRspLen;
|
||||
void* metaRsp;
|
||||
SMqRspHead head;
|
||||
int64_t reqOffset;
|
||||
int64_t rspOffset;
|
||||
STqOffsetVal reqOffsetNew;
|
||||
STqOffsetVal rspOffsetNew;
|
||||
int16_t resMsgType;
|
||||
int32_t metaRspLen;
|
||||
void* metaRsp;
|
||||
} SMqMetaRsp;
|
||||
|
||||
static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp) {
|
||||
|
@ -2805,6 +2831,24 @@ static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) {
|
|||
return (void*)buf;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
SMqRspHead head;
|
||||
STqOffsetVal reqOffset;
|
||||
STqOffsetVal rspOffset;
|
||||
int32_t skipLogNum;
|
||||
int32_t blockNum;
|
||||
int8_t withTbName;
|
||||
int8_t withSchema;
|
||||
SArray* blockDataLen;
|
||||
SArray* blockData;
|
||||
SArray* blockTbName;
|
||||
SArray* blockSchema;
|
||||
} SMqDataRsp;
|
||||
|
||||
int32_t tEncodeSMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp);
|
||||
int32_t tDecodeSMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp);
|
||||
|
||||
#if 0
|
||||
typedef struct {
|
||||
SMqRspHead head;
|
||||
int64_t reqOffset;
|
||||
|
@ -2813,13 +2857,10 @@ typedef struct {
|
|||
int32_t blockNum;
|
||||
int8_t withTbName;
|
||||
int8_t withSchema;
|
||||
int8_t withTag;
|
||||
SArray* blockDataLen; // SArray<int32_t>
|
||||
SArray* blockData; // SArray<SRetrieveTableRsp*>
|
||||
SArray* blockTbName; // SArray<char*>
|
||||
SArray* blockSchema; // SArray<SSchemaWrapper>
|
||||
SArray* blockTags; // SArray<kvrow>
|
||||
SArray* blockTagSchema; // SArray<kvrow>
|
||||
SArray* blockDataLen; // SArray<int32_t>
|
||||
SArray* blockData; // SArray<SRetrieveTableRsp*>
|
||||
SArray* blockTbName; // SArray<char*>
|
||||
SArray* blockSchema; // SArray<SSchemaWrapper>
|
||||
} SMqDataBlkRsp;
|
||||
|
||||
static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) {
|
||||
|
@ -2831,7 +2872,6 @@ static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp
|
|||
if (pRsp->blockNum != 0) {
|
||||
tlen += taosEncodeFixedI8(buf, pRsp->withTbName);
|
||||
tlen += taosEncodeFixedI8(buf, pRsp->withSchema);
|
||||
tlen += taosEncodeFixedI8(buf, pRsp->withTag);
|
||||
|
||||
for (int32_t i = 0; i < pRsp->blockNum; i++) {
|
||||
int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i);
|
||||
|
@ -2861,7 +2901,6 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
|
|||
pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t));
|
||||
buf = taosDecodeFixedI8(buf, &pRsp->withTbName);
|
||||
buf = taosDecodeFixedI8(buf, &pRsp->withSchema);
|
||||
buf = taosDecodeFixedI8(buf, &pRsp->withTag);
|
||||
if (pRsp->withTbName) {
|
||||
pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*));
|
||||
}
|
||||
|
@ -2890,6 +2929,7 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
|
|||
}
|
||||
return (void*)buf;
|
||||
}
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
SMqRspHead head;
|
||||
|
|
|
@ -167,10 +167,6 @@ enum {
|
|||
|
||||
TD_NEW_MSG_SEG(TDMT_VND_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_QUERY, "query", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_QUERY_CONTINUE, "query-continue", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_QUERY_HEARTBEAT, "query-heartbeat", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_FETCH, "fetch", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TABLE, "create-table", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TABLE, "alter-table", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TABLE, "drop-table", NULL, NULL)
|
||||
|
@ -184,12 +180,9 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_DELETE, "vnode-mq-vg-delete", SMqVDeleteReq, SMqVDeleteRsp)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_MQ_COMMIT_OFFSET, "vnode-commit-offset", STqOffset, STqOffset)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TASK, "vnode-drop-task", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TOPIC, "vnode-create-topic", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TOPIC, "vnode-alter-topic", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_EXPLAIN, "vnode-explain", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqPollReq, SMqDataBlkRsp)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL)
|
||||
|
@ -206,6 +199,17 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "drop-ttl-stb", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_SCH_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_MERGE_QUERY, "merge-query", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_QUERY_CONTINUE, "query-continue", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_QUERY_HEARTBEAT, "query-heartbeat", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_FETCH, "fetch", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_CANCEL_TASK, "cancel-task", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_DROP_TASK, "drop-task", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_EXPLAIN, "explain", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_STREAM_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DROP, "stream-task-drop", NULL, NULL)
|
||||
|
@ -214,9 +218,6 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RECOVER, "stream-task-recover", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_SCH_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_MON_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_MON_MM_INFO, "monitor-minfo", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MON_VM_INFO, "monitor-vinfo", NULL, NULL)
|
||||
|
|
|
@ -192,79 +192,81 @@
|
|||
#define TK_TRIGGER 174
|
||||
#define TK_AT_ONCE 175
|
||||
#define TK_WINDOW_CLOSE 176
|
||||
#define TK_KILL 177
|
||||
#define TK_CONNECTION 178
|
||||
#define TK_TRANSACTION 179
|
||||
#define TK_BALANCE 180
|
||||
#define TK_VGROUP 181
|
||||
#define TK_MERGE 182
|
||||
#define TK_REDISTRIBUTE 183
|
||||
#define TK_SPLIT 184
|
||||
#define TK_SYNCDB 185
|
||||
#define TK_DELETE 186
|
||||
#define TK_NULL 187
|
||||
#define TK_NK_QUESTION 188
|
||||
#define TK_NK_ARROW 189
|
||||
#define TK_ROWTS 190
|
||||
#define TK_TBNAME 191
|
||||
#define TK_QSTARTTS 192
|
||||
#define TK_QENDTS 193
|
||||
#define TK_WSTARTTS 194
|
||||
#define TK_WENDTS 195
|
||||
#define TK_WDURATION 196
|
||||
#define TK_CAST 197
|
||||
#define TK_NOW 198
|
||||
#define TK_TODAY 199
|
||||
#define TK_TIMEZONE 200
|
||||
#define TK_CLIENT_VERSION 201
|
||||
#define TK_SERVER_VERSION 202
|
||||
#define TK_SERVER_STATUS 203
|
||||
#define TK_CURRENT_USER 204
|
||||
#define TK_COUNT 205
|
||||
#define TK_LAST_ROW 206
|
||||
#define TK_BETWEEN 207
|
||||
#define TK_IS 208
|
||||
#define TK_NK_LT 209
|
||||
#define TK_NK_GT 210
|
||||
#define TK_NK_LE 211
|
||||
#define TK_NK_GE 212
|
||||
#define TK_NK_NE 213
|
||||
#define TK_MATCH 214
|
||||
#define TK_NMATCH 215
|
||||
#define TK_CONTAINS 216
|
||||
#define TK_JOIN 217
|
||||
#define TK_INNER 218
|
||||
#define TK_SELECT 219
|
||||
#define TK_DISTINCT 220
|
||||
#define TK_WHERE 221
|
||||
#define TK_PARTITION 222
|
||||
#define TK_BY 223
|
||||
#define TK_SESSION 224
|
||||
#define TK_STATE_WINDOW 225
|
||||
#define TK_SLIDING 226
|
||||
#define TK_FILL 227
|
||||
#define TK_VALUE 228
|
||||
#define TK_NONE 229
|
||||
#define TK_PREV 230
|
||||
#define TK_LINEAR 231
|
||||
#define TK_NEXT 232
|
||||
#define TK_HAVING 233
|
||||
#define TK_RANGE 234
|
||||
#define TK_EVERY 235
|
||||
#define TK_ORDER 236
|
||||
#define TK_SLIMIT 237
|
||||
#define TK_SOFFSET 238
|
||||
#define TK_LIMIT 239
|
||||
#define TK_OFFSET 240
|
||||
#define TK_ASC 241
|
||||
#define TK_NULLS 242
|
||||
#define TK_ID 243
|
||||
#define TK_NK_BITNOT 244
|
||||
#define TK_INSERT 245
|
||||
#define TK_VALUES 246
|
||||
#define TK_IMPORT 247
|
||||
#define TK_NK_SEMI 248
|
||||
#define TK_FILE 249
|
||||
#define TK_IGNORE 177
|
||||
#define TK_EXPIRED 178
|
||||
#define TK_KILL 179
|
||||
#define TK_CONNECTION 180
|
||||
#define TK_TRANSACTION 181
|
||||
#define TK_BALANCE 182
|
||||
#define TK_VGROUP 183
|
||||
#define TK_MERGE 184
|
||||
#define TK_REDISTRIBUTE 185
|
||||
#define TK_SPLIT 186
|
||||
#define TK_SYNCDB 187
|
||||
#define TK_DELETE 188
|
||||
#define TK_NULL 189
|
||||
#define TK_NK_QUESTION 190
|
||||
#define TK_NK_ARROW 191
|
||||
#define TK_ROWTS 192
|
||||
#define TK_TBNAME 193
|
||||
#define TK_QSTARTTS 194
|
||||
#define TK_QENDTS 195
|
||||
#define TK_WSTARTTS 196
|
||||
#define TK_WENDTS 197
|
||||
#define TK_WDURATION 198
|
||||
#define TK_CAST 199
|
||||
#define TK_NOW 200
|
||||
#define TK_TODAY 201
|
||||
#define TK_TIMEZONE 202
|
||||
#define TK_CLIENT_VERSION 203
|
||||
#define TK_SERVER_VERSION 204
|
||||
#define TK_SERVER_STATUS 205
|
||||
#define TK_CURRENT_USER 206
|
||||
#define TK_COUNT 207
|
||||
#define TK_LAST_ROW 208
|
||||
#define TK_BETWEEN 209
|
||||
#define TK_IS 210
|
||||
#define TK_NK_LT 211
|
||||
#define TK_NK_GT 212
|
||||
#define TK_NK_LE 213
|
||||
#define TK_NK_GE 214
|
||||
#define TK_NK_NE 215
|
||||
#define TK_MATCH 216
|
||||
#define TK_NMATCH 217
|
||||
#define TK_CONTAINS 218
|
||||
#define TK_JOIN 219
|
||||
#define TK_INNER 220
|
||||
#define TK_SELECT 221
|
||||
#define TK_DISTINCT 222
|
||||
#define TK_WHERE 223
|
||||
#define TK_PARTITION 224
|
||||
#define TK_BY 225
|
||||
#define TK_SESSION 226
|
||||
#define TK_STATE_WINDOW 227
|
||||
#define TK_SLIDING 228
|
||||
#define TK_FILL 229
|
||||
#define TK_VALUE 230
|
||||
#define TK_NONE 231
|
||||
#define TK_PREV 232
|
||||
#define TK_LINEAR 233
|
||||
#define TK_NEXT 234
|
||||
#define TK_HAVING 235
|
||||
#define TK_RANGE 236
|
||||
#define TK_EVERY 237
|
||||
#define TK_ORDER 238
|
||||
#define TK_SLIMIT 239
|
||||
#define TK_SOFFSET 240
|
||||
#define TK_LIMIT 241
|
||||
#define TK_OFFSET 242
|
||||
#define TK_ASC 243
|
||||
#define TK_NULLS 244
|
||||
#define TK_ID 245
|
||||
#define TK_NK_BITNOT 246
|
||||
#define TK_INSERT 247
|
||||
#define TK_VALUES 248
|
||||
#define TK_IMPORT 249
|
||||
#define TK_NK_SEMI 250
|
||||
#define TK_FILE 251
|
||||
|
||||
#define TK_NK_SPACE 300
|
||||
#define TK_NK_COMMENT 301
|
||||
|
|
|
@ -36,14 +36,9 @@ typedef struct SReadHandle {
|
|||
void* vnode;
|
||||
void* mnd;
|
||||
SMsgCb* pMsgCb;
|
||||
bool tqReader;
|
||||
} SReadHandle;
|
||||
|
||||
enum {
|
||||
STREAM_DATA_TYPE_SUBMIT_BLOCK = 1,
|
||||
STREAM_DATA_TYPE_SSDATA_BLOCK = 2,
|
||||
STREAM_DATA_TYPE_FROM_SNAPSHOT = 3,
|
||||
};
|
||||
|
||||
typedef enum {
|
||||
OPTR_EXEC_MODEL_BATCH = 0x1,
|
||||
OPTR_EXEC_MODEL_STREAM = 0x2,
|
||||
|
@ -139,7 +134,6 @@ int32_t qKillTask(qTaskInfo_t tinfo);
|
|||
*/
|
||||
int32_t qAsyncKillTask(qTaskInfo_t tinfo);
|
||||
|
||||
|
||||
/**
|
||||
* destroy query info structure
|
||||
* @param qHandle
|
||||
|
@ -178,6 +172,8 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le
|
|||
*/
|
||||
int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts);
|
||||
|
||||
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
@ -125,6 +125,7 @@ typedef enum EFunctionType {
|
|||
FUNCTION_TYPE_BLOCK_DIST_INFO, // block distribution pseudo column function
|
||||
FUNCTION_TYPE_TO_COLUMN,
|
||||
FUNCTION_TYPE_GROUP_KEY,
|
||||
FUNCTION_TYPE_CACHE_LAST_ROW,
|
||||
|
||||
// distributed splitting functions
|
||||
FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000,
|
||||
|
|
|
@ -338,6 +338,7 @@ typedef struct SStreamOptions {
|
|||
int8_t triggerType;
|
||||
SNode* pDelay;
|
||||
SNode* pWatermark;
|
||||
bool ignoreExpired;
|
||||
} SStreamOptions;
|
||||
|
||||
typedef struct SCreateStreamStmt {
|
||||
|
|
|
@ -73,8 +73,7 @@ typedef struct SScanLogicNode {
|
|||
SNode* pTagIndexCond;
|
||||
int8_t triggerType;
|
||||
int64_t watermark;
|
||||
int16_t tsColId;
|
||||
double filesFactor;
|
||||
int8_t igExpired;
|
||||
SArray* pSmaIndexes;
|
||||
SNodeList* pGroupTags;
|
||||
bool groupSort;
|
||||
|
@ -91,6 +90,7 @@ typedef struct SAggLogicNode {
|
|||
SLogicNode node;
|
||||
SNodeList* pGroupKeys;
|
||||
SNodeList* pAggFuncs;
|
||||
bool hasLastRow;
|
||||
} SAggLogicNode;
|
||||
|
||||
typedef struct SProjectLogicNode {
|
||||
|
@ -174,7 +174,7 @@ typedef struct SWindowLogicNode {
|
|||
SNode* pStateExpr;
|
||||
int8_t triggerType;
|
||||
int64_t watermark;
|
||||
double filesFactor;
|
||||
int8_t igExpired;
|
||||
EWindowAlgorithm windowAlgo;
|
||||
} SWindowLogicNode;
|
||||
|
||||
|
@ -295,8 +295,7 @@ typedef struct STableScanPhysiNode {
|
|||
int8_t slidingUnit;
|
||||
int8_t triggerType;
|
||||
int64_t watermark;
|
||||
int16_t tsColId;
|
||||
double filesFactor;
|
||||
int8_t igExpired;
|
||||
} STableScanPhysiNode;
|
||||
|
||||
typedef STableScanPhysiNode STableSeqScanPhysiNode;
|
||||
|
@ -346,6 +345,7 @@ typedef struct SDownstreamSourceNode {
|
|||
SQueryNodeAddr addr;
|
||||
uint64_t taskId;
|
||||
uint64_t schedId;
|
||||
int32_t execId;
|
||||
} SDownstreamSourceNode;
|
||||
|
||||
typedef struct SExchangePhysiNode {
|
||||
|
@ -372,7 +372,7 @@ typedef struct SWinodwPhysiNode {
|
|||
SNode* pTsEnd; // window end timestamp
|
||||
int8_t triggerType;
|
||||
int64_t watermark;
|
||||
double filesFactor;
|
||||
int8_t igExpired;
|
||||
} SWinodwPhysiNode;
|
||||
|
||||
typedef struct SIntervalPhysiNode {
|
||||
|
|
|
@ -34,6 +34,7 @@ typedef struct SPlanContext {
|
|||
bool showRewrite;
|
||||
int8_t triggerType;
|
||||
int64_t watermark;
|
||||
int8_t igExpired;
|
||||
char* pMsg;
|
||||
int32_t msgLen;
|
||||
const char* pUser;
|
||||
|
|
|
@ -135,9 +135,11 @@ typedef struct STableMetaOutput {
|
|||
} STableMetaOutput;
|
||||
|
||||
typedef struct SDataBuf {
|
||||
int32_t msgType;
|
||||
void* pData;
|
||||
uint32_t len;
|
||||
void* handle;
|
||||
SEpSet* pEpSet;
|
||||
} SDataBuf;
|
||||
|
||||
typedef struct STargetInfo {
|
||||
|
@ -146,7 +148,7 @@ typedef struct STargetInfo {
|
|||
int32_t vgId;
|
||||
} STargetInfo;
|
||||
|
||||
typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32_t code);
|
||||
typedef int32_t (*__async_send_cb_fn_t)(void* param, SDataBuf* pMsg, int32_t code);
|
||||
typedef int32_t (*__async_exec_fn_t)(void* param);
|
||||
|
||||
typedef struct SRequestConnInfo {
|
||||
|
@ -234,13 +236,25 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
|
|||
#define NEED_CLIENT_HANDLE_ERROR(_code) \
|
||||
(NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
|
||||
NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
|
||||
#define NEED_REDIRECT_ERROR(_code) \
|
||||
((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \
|
||||
(_code) == TSDB_CODE_NODE_NOT_DEPLOYED || (_code) == TSDB_CODE_SYN_NOT_LEADER || \
|
||||
(_code) == TSDB_CODE_APP_NOT_READY || (_code) == TSDB_CODE_RPC_BROKEN_LINK)
|
||||
|
||||
#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \
|
||||
((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
|
||||
(_type) == TDMT_VND_DROP_STB)
|
||||
|
||||
#define NEED_SCHEDULER_RETRY_ERROR(_code) \
|
||||
((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \
|
||||
(_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
|
||||
#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \
|
||||
((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || \
|
||||
(_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_APP_NOT_READY)
|
||||
|
||||
#define NEED_SCHEDULER_RETRY_ERROR(_code) \
|
||||
(NEED_SCHEDULER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \
|
||||
(_code) == TSDB_CODE_SCH_TIMEOUT_ERROR || (_code) == TSDB_CODE_RPC_BROKEN_LINK)
|
||||
|
||||
|
||||
|
||||
|
||||
#define REQUEST_TOTAL_EXEC_TIMES 2
|
||||
|
||||
|
|
|
@ -69,18 +69,20 @@ typedef struct SSchdFetchParam {
|
|||
int32_t* code;
|
||||
} SSchdFetchParam;
|
||||
|
||||
typedef void (*schedulerExecCallback)(SQueryResult* pResult, void* param, int32_t code);
|
||||
typedef void (*schedulerFetchCallback)(void* pResult, void* param, int32_t code);
|
||||
typedef void (*schedulerExecFp)(SQueryResult* pResult, void* param, int32_t code);
|
||||
typedef void (*schedulerFetchFp)(void* pResult, void* param, int32_t code);
|
||||
typedef bool (*schedulerChkKillFp)(void* param);
|
||||
|
||||
typedef struct SSchedulerReq {
|
||||
bool *reqKilled;
|
||||
SRequestConnInfo *pConn;
|
||||
SArray *pNodeList;
|
||||
SQueryPlan *pDag;
|
||||
const char *sql;
|
||||
int64_t startTs;
|
||||
schedulerExecCallback fp;
|
||||
void* cbParam;
|
||||
schedulerExecFp execFp;
|
||||
void* execParam;
|
||||
schedulerChkKillFp chkKillFp;
|
||||
void* chkKillParam;
|
||||
} SSchedulerReq;
|
||||
|
||||
|
||||
|
@ -110,7 +112,7 @@ int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes)
|
|||
*/
|
||||
int32_t schedulerFetchRows(int64_t job, void **data);
|
||||
|
||||
void schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param);
|
||||
void schedulerAsyncFetchRows(int64_t job, schedulerFetchFp fp, void* param);
|
||||
|
||||
int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub);
|
||||
|
||||
|
|
|
@ -55,15 +55,6 @@ enum {
|
|||
TASK_OUTPUT_STATUS__BLOCKED,
|
||||
};
|
||||
|
||||
enum {
|
||||
STREAM_INPUT__DATA_SUBMIT = 1,
|
||||
STREAM_INPUT__DATA_BLOCK,
|
||||
STREAM_INPUT__DATA_RETRIEVE,
|
||||
STREAM_INPUT__TRIGGER,
|
||||
STREAM_INPUT__CHECKPOINT,
|
||||
STREAM_INPUT__DROP,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
int8_t type;
|
||||
} SStreamQueueItem;
|
||||
|
@ -152,10 +143,6 @@ typedef struct {
|
|||
void* executor;
|
||||
} STaskExec;
|
||||
|
||||
typedef struct {
|
||||
int32_t taskId;
|
||||
} STaskDispatcherInplace;
|
||||
|
||||
typedef struct {
|
||||
int32_t taskId;
|
||||
int32_t nodeId;
|
||||
|
@ -208,7 +195,6 @@ enum {
|
|||
|
||||
enum {
|
||||
TASK_DISPATCH__NONE = 1,
|
||||
TASK_DISPATCH__INPLACE,
|
||||
TASK_DISPATCH__FIXED,
|
||||
TASK_DISPATCH__SHUFFLE,
|
||||
};
|
||||
|
@ -260,7 +246,7 @@ struct SStreamTask {
|
|||
// exec
|
||||
STaskExec exec;
|
||||
|
||||
// TODO: merge sink and dispatch
|
||||
// TODO: unify sink and dispatch
|
||||
|
||||
// local sink
|
||||
union {
|
||||
|
@ -269,9 +255,8 @@ struct SStreamTask {
|
|||
STaskSinkFetch fetchSink;
|
||||
};
|
||||
|
||||
// dispatch
|
||||
// remote dispatcher
|
||||
union {
|
||||
STaskDispatcherInplace inplaceDispatcher;
|
||||
STaskDispatcherFixedEp fixedEpDispatcher;
|
||||
STaskDispatcherShuffle shuffleDispatcher;
|
||||
};
|
||||
|
@ -327,9 +312,8 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem
|
|||
taosWriteQitem(pTask->inputQueue->queue, pItem);
|
||||
}
|
||||
|
||||
if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0 &&
|
||||
pTask->triggerStatus == TASK_TRIGGER_STATUS__IN_ACTIVE) {
|
||||
atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE);
|
||||
if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) {
|
||||
atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE, TASK_TRIGGER_STATUS__ACTIVE);
|
||||
}
|
||||
|
||||
// TODO: back pressure
|
||||
|
|
|
@ -45,7 +45,7 @@ typedef struct SRpcHandleInfo {
|
|||
int32_t noResp; // has response or not(default 0, 0: resp, 1: no resp);
|
||||
int32_t persistHandle; // persist handle or not
|
||||
STraceId traceId;
|
||||
// int64_t traceId;
|
||||
int8_t hasEpSet;
|
||||
|
||||
// app info
|
||||
void *ahandle; // app handle set by client
|
||||
|
@ -123,7 +123,7 @@ void * rpcReallocCont(void *ptr, int32_t contLen);
|
|||
void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
|
||||
void rpcSendResponse(const SRpcMsg *pMsg);
|
||||
void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
|
||||
void rpcReleaseHandle(void *handle, int8_t type); // just release client conn to rpc instance, no close sock
|
||||
void rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc instance, no close sock
|
||||
|
||||
// These functions will not be called in the child process
|
||||
void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet);
|
||||
|
|
|
@ -86,6 +86,7 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x0102)
|
||||
#define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0103)
|
||||
#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0104)
|
||||
#define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0105)
|
||||
|
||||
//client
|
||||
#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200)
|
||||
|
|
|
@ -1,20 +0,0 @@
|
|||
[Unit]
|
||||
Description=TDengine arbitrator service
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/tarbitrator
|
||||
TimeoutStopSec=1000000s
|
||||
LimitNOFILE=infinity
|
||||
LimitNPROC=infinity
|
||||
LimitCORE=infinity
|
||||
TimeoutStartSec=0
|
||||
StandardOutput=null
|
||||
Restart=always
|
||||
StartLimitBurst=3
|
||||
StartLimitInterval=60s
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -1,88 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Modified from original source: Elastic Search
|
||||
# https://github.com/elasticsearch/elasticsearch
|
||||
# Thank you to the Elastic Search authors
|
||||
#
|
||||
# chkconfig: 2345 99 01
|
||||
#
|
||||
### BEGIN INIT INFO
|
||||
# Provides: taoscluster
|
||||
# Required-Start: $local_fs $network $syslog
|
||||
# Required-Stop: $local_fs $network $syslog
|
||||
# Default-Start: 2 3 4 5
|
||||
# Default-Stop: 0 1 6
|
||||
# Short-Description: Starts taoscluster tarbitrator
|
||||
# Description: Starts taoscluster tarbitrator, a arbitrator
|
||||
### END INIT INFO
|
||||
|
||||
set -e
|
||||
|
||||
PATH="/bin:/usr/bin:/sbin:/usr/sbin"
|
||||
NAME="taoscluster"
|
||||
USER="root"
|
||||
GROUP="root"
|
||||
DAEMON="/usr/local/taos/bin/tarbitrator"
|
||||
DAEMON_OPTS=""
|
||||
PID_FILE="/var/run/$NAME.pid"
|
||||
APPARGS=""
|
||||
|
||||
# Maximum number of open files
|
||||
MAX_OPEN_FILES=65535
|
||||
|
||||
. /lib/lsb/init-functions
|
||||
|
||||
case "$1" in
|
||||
start)
|
||||
|
||||
log_action_begin_msg "Starting tarbitrator..."
|
||||
if start-stop-daemon --test --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS &> /dev/null; then
|
||||
|
||||
touch "$PID_FILE" && chown "$USER":"$GROUP" "$PID_FILE"
|
||||
|
||||
if [ -n "$MAX_OPEN_FILES" ]; then
|
||||
ulimit -n $MAX_OPEN_FILES
|
||||
fi
|
||||
|
||||
start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS
|
||||
|
||||
log_end_msg $?
|
||||
fi
|
||||
;;
|
||||
|
||||
stop)
|
||||
log_action_begin_msg "Stopping tarbitrator..."
|
||||
set +e
|
||||
if [ -f "$PID_FILE" ]; then
|
||||
start-stop-daemon --stop --pidfile "$PID_FILE" --user "$USER" --retry=TERM/120/KILL/5 > /dev/null
|
||||
if [ $? -eq 1 ]; then
|
||||
log_action_cont_msg "TSD is not running but pid file exists, cleaning up"
|
||||
elif [ $? -eq 3 ]; then
|
||||
PID="`cat $PID_FILE`"
|
||||
log_failure_msg "Failed to stop tarbitrator (pid $PID)"
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$PID_FILE"
|
||||
else
|
||||
log_action_cont_msg "tarbitrator was not running"
|
||||
fi
|
||||
log_action_end_msg 0
|
||||
set -e
|
||||
;;
|
||||
|
||||
restart|force-reload)
|
||||
if [ -f "$PID_FILE" ]; then
|
||||
$0 stop
|
||||
sleep 1
|
||||
fi
|
||||
$0 start
|
||||
;;
|
||||
status)
|
||||
status_of_proc -p "$PID_FILE" "$DAEMON" "$NAME"
|
||||
;;
|
||||
*)
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
exit 0
|
|
@ -111,9 +111,9 @@ else
|
|||
fi
|
||||
|
||||
csudo=""
|
||||
if command -v sudo > /dev/null; then
|
||||
csudo="sudo "
|
||||
fi
|
||||
#if command -v sudo > /dev/null; then
|
||||
# csudo="sudo "
|
||||
#fi
|
||||
|
||||
function is_valid_version() {
|
||||
[ -z $1 ] && return 1 || :
|
||||
|
@ -182,14 +182,10 @@ cd "${curr_dir}"
|
|||
# 2. cmake executable file
|
||||
compile_dir="${top_dir}/debug"
|
||||
if [ -d ${compile_dir} ]; then
|
||||
${csudo}rm -rf ${compile_dir}
|
||||
rm -rf ${compile_dir}
|
||||
fi
|
||||
|
||||
if [ "$osType" != "Darwin" ]; then
|
||||
${csudo}mkdir -p ${compile_dir}
|
||||
else
|
||||
mkdir -p ${compile_dir}
|
||||
fi
|
||||
mkdir -p ${compile_dir}
|
||||
cd ${compile_dir}
|
||||
|
||||
if [[ "$allocator" == "jemalloc" ]]; then
|
||||
|
@ -255,18 +251,18 @@ if [ "$osType" != "Darwin" ]; then
|
|||
echo "====do deb package for the ubuntu system===="
|
||||
output_dir="${top_dir}/debs"
|
||||
if [ -d ${output_dir} ]; then
|
||||
${csudo}rm -rf ${output_dir}
|
||||
rm -rf ${output_dir}
|
||||
fi
|
||||
${csudo}mkdir -p ${output_dir}
|
||||
mkdir -p ${output_dir}
|
||||
cd ${script_dir}/deb
|
||||
${csudo}./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
|
||||
|
||||
if [[ "$pagMode" == "full" ]]; then
|
||||
if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then
|
||||
cd ${top_dir}/tools/taos-tools/packaging/deb
|
||||
taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}')
|
||||
[ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
|
||||
|
||||
taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}')
|
||||
${csudo}./make-taos-tools-deb.sh ${top_dir} \
|
||||
${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType}
|
||||
fi
|
||||
|
@ -280,18 +276,18 @@ if [ "$osType" != "Darwin" ]; then
|
|||
echo "====do rpm package for the centos system===="
|
||||
output_dir="${top_dir}/rpms"
|
||||
if [ -d ${output_dir} ]; then
|
||||
${csudo}rm -rf ${output_dir}
|
||||
rm -rf ${output_dir}
|
||||
fi
|
||||
${csudo}mkdir -p ${output_dir}
|
||||
mkdir -p ${output_dir}
|
||||
cd ${script_dir}/rpm
|
||||
${csudo}./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
|
||||
|
||||
if [[ "$pagMode" == "full" ]]; then
|
||||
if [ -d ${top_dir}/tools/taos-tools/packaging/rpm ]; then
|
||||
cd ${top_dir}/tools/taos-tools/packaging/rpm
|
||||
taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}' | sed -e 's/-/_/g')
|
||||
[ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
|
||||
|
||||
taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}' | sed -e 's/-/_/g')
|
||||
${csudo}./make-taos-tools-rpm.sh ${top_dir} \
|
||||
${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType}
|
||||
fi
|
||||
|
@ -306,7 +302,6 @@ if [ "$osType" != "Darwin" ]; then
|
|||
|
||||
${csudo}./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp} ${dbName}
|
||||
${csudo}./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName}
|
||||
# ${csudo}./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode}
|
||||
|
||||
else
|
||||
# only make client for Darwin
|
||||
|
|
|
@ -1,141 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# tarbitratord This shell script takes care of starting and stopping tarbitrator.
|
||||
#
|
||||
# chkconfig: 2345 99 01
|
||||
# description: tarbitrator is a arbitrator used in TDengine cluster.
|
||||
#
|
||||
#
|
||||
### BEGIN INIT INFO
|
||||
# Provides: taoscluster
|
||||
# Required-Start: $network $local_fs $remote_fs
|
||||
# Required-Stop: $network $local_fs $remote_fs
|
||||
# Short-Description: start and stop tarbitrator
|
||||
# Description: tarbitrator is a arbitrator used in TDengine cluster.
|
||||
### END INIT INFO
|
||||
|
||||
# Source init functions
|
||||
. /etc/init.d/functions
|
||||
|
||||
# Maximum number of open files
|
||||
MAX_OPEN_FILES=65535
|
||||
|
||||
# Default program options
|
||||
NAME=tarbitrator
|
||||
PROG=/usr/local/taos/bin/tarbitrator
|
||||
USER=root
|
||||
GROUP=root
|
||||
|
||||
# Default directories
|
||||
LOCK_DIR=/var/lock/subsys
|
||||
PID_DIR=/var/run/$NAME
|
||||
|
||||
# Set file names
|
||||
LOCK_FILE=$LOCK_DIR/$NAME
|
||||
PID_FILE=$PID_DIR/$NAME.pid
|
||||
|
||||
[ -e $PID_DIR ] || mkdir -p $PID_DIR
|
||||
|
||||
PROG_OPTS=""
|
||||
|
||||
start() {
|
||||
echo -n "Starting ${NAME}: "
|
||||
# check identity
|
||||
curid="`id -u -n`"
|
||||
if [ "$curid" != root ] && [ "$curid" != "$USER" ] ; then
|
||||
echo "Must be run as root or $USER, but was run as $curid"
|
||||
return 1
|
||||
fi
|
||||
# Sets the maximum number of open file descriptors allowed.
|
||||
ulimit -n $MAX_OPEN_FILES
|
||||
curulimit="`ulimit -n`"
|
||||
if [ "$curulimit" -lt $MAX_OPEN_FILES ] ; then
|
||||
echo "'ulimit -n' must be greater than or equal to $MAX_OPEN_FILES, is $curulimit"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ "`id -u -n`" == root ] ; then
|
||||
# Changes the owner of the lock, and the pid files to allow
|
||||
# non-root OpenTSDB daemons to run /usr/share/opentsdb/bin/opentsdb_restart.py.
|
||||
touch $LOCK_FILE && chown $USER:$GROUP $LOCK_FILE
|
||||
touch $PID_FILE && chown $USER:$GROUP $PID_FILE
|
||||
daemon --user $USER --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &"
|
||||
else
|
||||
# Don't have to change user.
|
||||
daemon --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &"
|
||||
fi
|
||||
retval=$?
|
||||
sleep 2
|
||||
echo
|
||||
[ $retval -eq 0 ] && (findproc > $PID_FILE && touch $LOCK_FILE)
|
||||
return $retval
|
||||
}
|
||||
|
||||
stop() {
|
||||
echo -n "Stopping ${NAME}: "
|
||||
killproc -p $PID_FILE $NAME
|
||||
retval=$?
|
||||
echo
|
||||
# Non-root users don't have enough permission to remove pid and lock files.
|
||||
# So, the opentsdb_restart.py cannot get rid of the files, and the command
|
||||
# "service opentsdb status" will complain about the existing pid file.
|
||||
# Makes the pid file empty.
|
||||
echo > $PID_FILE
|
||||
[ $retval -eq 0 ] && (rm -f $PID_FILE && rm -f $LOCK_FILE)
|
||||
return $retval
|
||||
}
|
||||
|
||||
restart() {
|
||||
stop
|
||||
start
|
||||
}
|
||||
|
||||
reload() {
|
||||
restart
|
||||
}
|
||||
|
||||
force_reload() {
|
||||
restart
|
||||
}
|
||||
|
||||
rh_status() {
|
||||
# run checks to determine if the service is running or use generic status
|
||||
status -p $PID_FILE -l $LOCK_FILE $NAME
|
||||
}
|
||||
|
||||
rh_status_q() {
|
||||
rh_status >/dev/null 2>&1
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
start)
|
||||
rh_status_q && exit 0
|
||||
$1
|
||||
;;
|
||||
stop)
|
||||
rh_status_q || exit 0
|
||||
$1
|
||||
;;
|
||||
restart)
|
||||
$1
|
||||
;;
|
||||
reload)
|
||||
rh_status_q || exit 7
|
||||
$1
|
||||
;;
|
||||
force-reload)
|
||||
force_reload
|
||||
;;
|
||||
status)
|
||||
rh_status
|
||||
;;
|
||||
condrestart|try-restart)
|
||||
rh_status_q || exit 0
|
||||
restart
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
|
||||
exit 2
|
||||
esac
|
||||
|
||||
exit $?
|
|
@ -194,7 +194,6 @@ function install_bin() {
|
|||
${csudo}rm -f ${bin_link_dir}/${serverName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${adapterName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
|
||||
${csudo}rm -f ${bin_link_dir}/tarbitrator || :
|
||||
${csudo}rm -f ${bin_link_dir}/set_core || :
|
||||
${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
|
||||
|
||||
|
@ -210,7 +209,6 @@ function install_bin() {
|
|||
[ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
|
||||
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
|
||||
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
|
||||
[ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo}ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || :
|
||||
|
||||
if [ "$verMode" == "cluster" ]; then
|
||||
${csudo}cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo}chmod 0555 ${nginx_dir}/*
|
||||
|
@ -606,28 +604,19 @@ function install_service_on_sysvinit() {
|
|||
if ((${os_type} == 1)); then
|
||||
# ${csudo}cp -f ${script_dir}/init.d/${serverName}.deb ${install_main_dir}/init.d/${serverName}
|
||||
${csudo}cp ${script_dir}/init.d/${serverName}.deb ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName}
|
||||
# ${csudo}cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord
|
||||
${csudo}cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord
|
||||
elif ((${os_type} == 2)); then
|
||||
# ${csudo}cp -f ${script_dir}/init.d/${serverName}.rpm ${install_main_dir}/init.d/${serverName}
|
||||
${csudo}cp ${script_dir}/init.d/${serverName}.rpm ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName}
|
||||
# ${csudo}cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord
|
||||
${csudo}cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord
|
||||
fi
|
||||
|
||||
if ((${initd_mod} == 1)); then
|
||||
${csudo}chkconfig --add ${serverName} || :
|
||||
${csudo}chkconfig --level 2345 ${serverName} on || :
|
||||
${csudo}chkconfig --add tarbitratord || :
|
||||
${csudo}chkconfig --level 2345 tarbitratord on || :
|
||||
elif ((${initd_mod} == 2)); then
|
||||
${csudo}insserv ${serverName} || :
|
||||
${csudo}insserv -d ${serverName} || :
|
||||
${csudo}insserv tarbitratord || :
|
||||
${csudo}insserv -d tarbitratord || :
|
||||
elif ((${initd_mod} == 3)); then
|
||||
${csudo}update-rc.d ${serverName} defaults || :
|
||||
${csudo}update-rc.d tarbitratord defaults || :
|
||||
fi
|
||||
}
|
||||
|
||||
|
@ -669,9 +658,6 @@ function install_service_on_systemd() {
|
|||
|
||||
${csudo}systemctl enable ${serverName}
|
||||
|
||||
[ -f ${script_dir}/cfg/tarbitratord.service ] &&
|
||||
${csudo}cp ${script_dir}/cfg/tarbitratord.service \
|
||||
${service_config_dir}/ || :
|
||||
${csudo}systemctl daemon-reload
|
||||
|
||||
if [ "$verMode" == "cluster" ]; then
|
||||
|
|
|
@ -1,340 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# This file is used to install database on linux systems. The operating system
|
||||
# is required to use systemd to manage services at boot
|
||||
|
||||
set -e
|
||||
#set -x
|
||||
|
||||
# -----------------------Variables definition---------------------
|
||||
script_dir=$(dirname $(readlink -f "$0"))
|
||||
|
||||
bin_link_dir="/usr/bin"
|
||||
#inc_link_dir="/usr/include"
|
||||
|
||||
#install main path
|
||||
install_main_dir="/usr/local/tarbitrator"
|
||||
|
||||
# old bin dir
|
||||
bin_dir="/usr/local/tarbitrator/bin"
|
||||
|
||||
service_config_dir="/etc/systemd/system"
|
||||
|
||||
# Color setting
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[1;32m'
|
||||
GREEN_DARK='\033[0;32m'
|
||||
GREEN_UNDERLINE='\033[4;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
csudo=""
|
||||
if command -v sudo >/dev/null; then
|
||||
csudo="sudo "
|
||||
fi
|
||||
|
||||
update_flag=0
|
||||
|
||||
initd_mod=0
|
||||
service_mod=2
|
||||
if pidof systemd &>/dev/null; then
|
||||
service_mod=0
|
||||
elif $(which service &>/dev/null); then
|
||||
service_mod=1
|
||||
service_config_dir="/etc/init.d"
|
||||
if $(which chkconfig &>/dev/null); then
|
||||
initd_mod=1
|
||||
elif $(which insserv &>/dev/null); then
|
||||
initd_mod=2
|
||||
elif $(which update-rc.d &>/dev/null); then
|
||||
initd_mod=3
|
||||
else
|
||||
service_mod=2
|
||||
fi
|
||||
else
|
||||
service_mod=2
|
||||
fi
|
||||
|
||||
# get the operating system type for using the corresponding init file
|
||||
# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification
|
||||
#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
|
||||
if [[ -e /etc/os-release ]]; then
|
||||
osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) || :
|
||||
else
|
||||
osinfo=""
|
||||
fi
|
||||
#echo "osinfo: ${osinfo}"
|
||||
os_type=0
|
||||
if echo $osinfo | grep -qwi "ubuntu"; then
|
||||
# echo "This is ubuntu system"
|
||||
os_type=1
|
||||
elif echo $osinfo | grep -qwi "debian"; then
|
||||
# echo "This is debian system"
|
||||
os_type=1
|
||||
elif echo $osinfo | grep -qwi "Kylin"; then
|
||||
# echo "This is Kylin system"
|
||||
os_type=1
|
||||
elif echo $osinfo | grep -qwi "centos"; then
|
||||
# echo "This is centos system"
|
||||
os_type=2
|
||||
elif echo $osinfo | grep -qwi "fedora"; then
|
||||
# echo "This is fedora system"
|
||||
os_type=2
|
||||
else
|
||||
echo " osinfo: ${osinfo}"
|
||||
echo " This is an officially unverified linux system,"
|
||||
echo " if there are any problems with the installation and operation, "
|
||||
echo " please feel free to contact taosdata.com for support."
|
||||
os_type=1
|
||||
fi
|
||||
|
||||
function kill_tarbitrator() {
|
||||
pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}')
|
||||
if [ -n "$pid" ]; then
|
||||
${csudo}kill -9 $pid || :
|
||||
fi
|
||||
}
|
||||
|
||||
function install_main_path() {
|
||||
#create install main dir and all sub dir
|
||||
${csudo}rm -rf ${install_main_dir} || :
|
||||
${csudo}mkdir -p ${install_main_dir}
|
||||
${csudo}mkdir -p ${install_main_dir}/bin
|
||||
#${csudo}mkdir -p ${install_main_dir}/include
|
||||
${csudo}mkdir -p ${install_main_dir}/init.d
|
||||
}
|
||||
|
||||
function install_bin() {
|
||||
# Remove links
|
||||
${csudo}rm -f ${bin_link_dir}/rmtarbitrator || :
|
||||
${csudo}rm -f ${bin_link_dir}/tarbitrator || :
|
||||
${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/*
|
||||
|
||||
#Make link
|
||||
[ -x ${install_main_dir}/bin/remove_arbi.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_arbi.sh ${bin_link_dir}/rmtarbitrator || :
|
||||
[ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo}ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || :
|
||||
}
|
||||
|
||||
function install_header() {
|
||||
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
|
||||
${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/*
|
||||
${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
|
||||
${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
|
||||
${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
|
||||
${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h
|
||||
}
|
||||
|
||||
function install_jemalloc() {
|
||||
jemalloc_dir=${script_dir}/jemalloc
|
||||
|
||||
if [ -d ${jemalloc_dir} ]; then
|
||||
${csudo}/usr/bin/install -c -d /usr/local/bin
|
||||
|
||||
if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then
|
||||
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then
|
||||
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/bin/jeprof ]; then
|
||||
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then
|
||||
${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc
|
||||
${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then
|
||||
${csudo}/usr/bin/install -c -d /usr/local/lib
|
||||
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib
|
||||
${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so
|
||||
${csudo}/usr/bin/install -c -d /usr/local/lib
|
||||
if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then
|
||||
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
|
||||
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
|
||||
${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig
|
||||
${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig
|
||||
fi
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then
|
||||
${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc
|
||||
${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then
|
||||
${csudo}/usr/bin/install -c -d /usr/local/share/man/man3
|
||||
${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3
|
||||
fi
|
||||
|
||||
if [ -d /etc/ld.so.conf.d ]; then
|
||||
echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf"
|
||||
${csudo}ldconfig
|
||||
else
|
||||
echo "/etc/ld.so.conf.d not found!"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function clean_service_on_sysvinit() {
|
||||
if pidof tarbitrator &>/dev/null; then
|
||||
${csudo}service tarbitratord stop || :
|
||||
fi
|
||||
|
||||
if ((${initd_mod} == 1)); then
|
||||
if [ -e ${service_config_dir}/tarbitratord ]; then
|
||||
${csudo}chkconfig --del tarbitratord || :
|
||||
fi
|
||||
elif ((${initd_mod} == 2)); then
|
||||
if [ -e ${service_config_dir}/tarbitratord ]; then
|
||||
${csudo}insserv -r tarbitratord || :
|
||||
fi
|
||||
elif ((${initd_mod} == 3)); then
|
||||
if [ -e ${service_config_dir}/tarbitratord ]; then
|
||||
${csudo}update-rc.d -f tarbitratord remove || :
|
||||
fi
|
||||
fi
|
||||
|
||||
${csudo}rm -f ${service_config_dir}/tarbitratord || :
|
||||
|
||||
if $(which init &>/dev/null); then
|
||||
${csudo}init q || :
|
||||
fi
|
||||
}
|
||||
|
||||
function install_service_on_sysvinit() {
|
||||
clean_service_on_sysvinit
|
||||
sleep 1
|
||||
|
||||
if ((${os_type} == 1)); then
|
||||
${csudo}cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord
|
||||
${csudo}cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord
|
||||
elif ((${os_type} == 2)); then
|
||||
${csudo}cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord
|
||||
${csudo}cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord
|
||||
fi
|
||||
|
||||
if ((${initd_mod} == 1)); then
|
||||
${csudo}chkconfig --add tarbitratord || :
|
||||
${csudo}chkconfig --level 2345 tarbitratord on || :
|
||||
elif ((${initd_mod} == 2)); then
|
||||
${csudo}insserv tarbitratord || :
|
||||
${csudo}insserv -d tarbitratord || :
|
||||
elif ((${initd_mod} == 3)); then
|
||||
${csudo}update-rc.d tarbitratord defaults || :
|
||||
fi
|
||||
}
|
||||
|
||||
function clean_service_on_systemd() {
|
||||
tarbitratord_service_config="${service_config_dir}/tarbitratord.service"
|
||||
if systemctl is-active --quiet tarbitratord; then
|
||||
echo "tarbitrator is running, stopping it..."
|
||||
${csudo}systemctl stop tarbitratord &>/dev/null || echo &>/dev/null
|
||||
fi
|
||||
${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null
|
||||
|
||||
${csudo}rm -f ${tarbitratord_service_config}
|
||||
}
|
||||
|
||||
function install_service_on_systemd() {
|
||||
clean_service_on_systemd
|
||||
|
||||
tarbitratord_service_config="${service_config_dir}/tarbitratord.service"
|
||||
|
||||
${csudo}bash -c "echo '[Unit]' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo '[Service]' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo '[Install]' >> ${tarbitratord_service_config}"
|
||||
${csudo}bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}"
|
||||
${csudo}systemctl enable tarbitratord
|
||||
}
|
||||
|
||||
function install_service() {
|
||||
if ((${service_mod} == 0)); then
|
||||
install_service_on_systemd
|
||||
elif ((${service_mod} == 1)); then
|
||||
install_service_on_sysvinit
|
||||
else
|
||||
kill_tarbitrator
|
||||
fi
|
||||
}
|
||||
|
||||
function update_TDengine() {
|
||||
# Start to update
|
||||
echo -e "${GREEN}Start to update TDengine's arbitrator ...${NC}"
|
||||
# Stop the service if running
|
||||
if pidof tarbitrator &>/dev/null; then
|
||||
if ((${service_mod} == 0)); then
|
||||
${csudo}systemctl stop tarbitratord || :
|
||||
elif ((${service_mod} == 1)); then
|
||||
${csudo}service tarbitratord stop || :
|
||||
else
|
||||
kill_tarbitrator
|
||||
fi
|
||||
sleep 1
|
||||
fi
|
||||
|
||||
install_main_path
|
||||
#install_header
|
||||
install_bin
|
||||
install_service
|
||||
install_jemalloc
|
||||
|
||||
echo
|
||||
if ((${service_mod} == 0)); then
|
||||
echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}systemctl start tarbitratord${NC}"
|
||||
elif ((${service_mod} == 1)); then
|
||||
echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}service tarbitratord start${NC}"
|
||||
else
|
||||
echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}"
|
||||
fi
|
||||
echo
|
||||
echo -e "\033[44;32;1mTDengine's arbitrator is updated successfully!${NC}"
|
||||
}
|
||||
|
||||
function install_TDengine() {
|
||||
# Start to install
|
||||
echo -e "${GREEN}Start to install TDengine's arbitrator ...${NC}"
|
||||
|
||||
install_main_path
|
||||
#install_header
|
||||
install_bin
|
||||
install_service
|
||||
install_jemalloc
|
||||
|
||||
echo
|
||||
if ((${service_mod} == 0)); then
|
||||
echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}systemctl start tarbitratord${NC}"
|
||||
elif ((${service_mod} == 1)); then
|
||||
echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}service tarbitratord start${NC}"
|
||||
else
|
||||
echo -e "${GREEN_DARK}To start arbitrator ${NC}: tarbitrator${NC}"
|
||||
fi
|
||||
|
||||
echo -e "\033[44;32;1mTDengine's arbitrator is installed successfully!${NC}"
|
||||
echo
|
||||
}
|
||||
|
||||
## ==============================Main program starts from here============================
|
||||
# Install server and client
|
||||
if [ -x ${bin_dir}/tarbitrator ]; then
|
||||
update_flag=1
|
||||
update_TDengine
|
||||
else
|
||||
install_TDengine
|
||||
fi
|
|
@ -185,7 +185,6 @@ function install_bin() {
|
|||
[ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || :
|
||||
[ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || :
|
||||
${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || :
|
||||
# ${csudo}cp -r ${binary_dir}/build/bin/tarbitrator ${install_main_dir}/bin || :
|
||||
|
||||
${csudo}cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin || :
|
||||
${csudo}cp -r ${script_dir}/remove.sh ${install_main_dir}/bin || :
|
||||
|
|
|
@ -1,71 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Generate arbitrator's tar.gz setup package for all os system
|
||||
|
||||
set -e
|
||||
#set -x
|
||||
|
||||
curr_dir=$(pwd)
|
||||
compile_dir=$1
|
||||
version=$2
|
||||
build_time=$3
|
||||
cpuType=$4
|
||||
osType=$5
|
||||
verMode=$6
|
||||
verType=$7
|
||||
pagMode=$8
|
||||
|
||||
script_dir="$(dirname $(readlink -f $0))"
|
||||
top_dir="$(readlink -f ${script_dir}/../..)"
|
||||
|
||||
productName="TDengine"
|
||||
|
||||
# create compressed install file.
|
||||
build_dir="${compile_dir}/build"
|
||||
code_dir="${top_dir}"
|
||||
release_dir="${top_dir}/release"
|
||||
|
||||
#package_name='linux'
|
||||
if [ "$verMode" == "cluster" ]; then
|
||||
install_dir="${release_dir}/${productName}-enterprise-arbitrator-${version}"
|
||||
else
|
||||
install_dir="${release_dir}/${productName}-arbitrator-${version}"
|
||||
fi
|
||||
|
||||
# Directories and files.
|
||||
bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi.sh"
|
||||
install_files="${script_dir}/install_arbi.sh"
|
||||
|
||||
#header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h"
|
||||
init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord
|
||||
init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord
|
||||
|
||||
# make directories.
|
||||
mkdir -p ${install_dir} && cp ${install_files} ${install_dir} && chmod a+x ${install_dir}/install_arbi.sh || :
|
||||
#mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc || :
|
||||
mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
|
||||
mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || :
|
||||
mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || :
|
||||
|
||||
cd ${release_dir}
|
||||
|
||||
# install_dir has been distinguishes cluster from edege, so comments this code
|
||||
pkg_name=${install_dir}-${osType}-${cpuType}
|
||||
|
||||
if [[ "$verType" == "beta" ]] || [[ "$verType" == "preRelease" ]]; then
|
||||
pkg_name=${install_dir}-${verType}-${osType}-${cpuType}
|
||||
elif [ "$verType" == "stable" ]; then
|
||||
pkg_name=${pkg_name}
|
||||
else
|
||||
echo "unknow verType, nor stabel or beta"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || :
|
||||
exitcode=$?
|
||||
if [ "$exitcode" != "0" ]; then
|
||||
echo "tar ${pkg_name}.tar.gz error !!!"
|
||||
exit $exitcode
|
||||
fi
|
||||
|
||||
cd ${curr_dir}
|
|
@ -85,7 +85,6 @@ else
|
|||
${build_dir}/bin/${clientName} \
|
||||
${taostools_bin_files} \
|
||||
${build_dir}/bin/taosadapter \
|
||||
${build_dir}/bin/tarbitrator\
|
||||
${script_dir}/remove.sh \
|
||||
${script_dir}/set_core.sh \
|
||||
${script_dir}/startPre.sh \
|
||||
|
@ -106,8 +105,6 @@ nginx_dir="${top_dir}/../enterprise/src/plugins/web"
|
|||
|
||||
init_file_deb=${script_dir}/../deb/taosd
|
||||
init_file_rpm=${script_dir}/../rpm/taosd
|
||||
init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord
|
||||
init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord
|
||||
|
||||
# make directories.
|
||||
mkdir -p ${install_dir}
|
||||
|
@ -126,10 +123,6 @@ if [ -f "${cfg_dir}/${serverName}.service" ]; then
|
|||
cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || :
|
||||
fi
|
||||
|
||||
if [ -f "${top_dir}/packaging/cfg/tarbitratord.service" ]; then
|
||||
cp ${top_dir}/packaging/cfg/tarbitratord.service ${install_dir}/cfg || :
|
||||
fi
|
||||
|
||||
if [ -f "${top_dir}/packaging/cfg/nginxd.service" ]; then
|
||||
cp ${top_dir}/packaging/cfg/nginxd.service ${install_dir}/cfg || :
|
||||
fi
|
||||
|
@ -137,8 +130,6 @@ fi
|
|||
mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
|
||||
mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb
|
||||
mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm
|
||||
mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || :
|
||||
mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || :
|
||||
|
||||
if [ $adapterName != "taosadapter" ]; then
|
||||
mv ${install_dir}/cfg/taosadapter.toml ${install_dir}/cfg/$adapterName.toml
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Script to stop the service and uninstall TDengine, but retain the config, data and log files.
|
||||
# Script to stop and uninstall the service, but retain the config, data and log files.
|
||||
|
||||
set -e
|
||||
#set -x
|
||||
|
@ -11,121 +11,216 @@ RED='\033[0;31m'
|
|||
GREEN='\033[1;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
installDir="/usr/local/taos"
|
||||
serverName="taosd"
|
||||
clientName="taos"
|
||||
uninstallScript="rmtaos"
|
||||
productName="TDengine"
|
||||
|
||||
#install main path
|
||||
install_main_dir="/usr/local/taos"
|
||||
data_link_dir="/usr/local/taos/data"
|
||||
log_link_dir="/usr/local/taos/log"
|
||||
cfg_link_dir="/usr/local/taos/cfg"
|
||||
install_main_dir=${installDir}
|
||||
data_link_dir=${installDir}/data
|
||||
log_link_dir=${installDir}/log
|
||||
cfg_link_dir=${installDir}/cfg
|
||||
bin_link_dir="/usr/bin"
|
||||
local_bin_link_dir="/usr/local/bin"
|
||||
lib_link_dir="/usr/lib"
|
||||
lib64_link_dir="/usr/lib64"
|
||||
inc_link_dir="/usr/include"
|
||||
install_nginxd_dir="/usr/local/nginxd"
|
||||
|
||||
service_config_dir="/etc/systemd/system"
|
||||
taos_service_name="taosd"
|
||||
|
||||
taos_service_name=${serverName}
|
||||
taosadapter_service_name="taosadapter"
|
||||
tarbitrator_service_name="tarbitratord"
|
||||
nginx_service_name="nginxd"
|
||||
csudo=""
|
||||
if command -v sudo > /dev/null; then
|
||||
csudo="sudo"
|
||||
if command -v sudo >/dev/null; then
|
||||
csudo="sudo "
|
||||
fi
|
||||
|
||||
initd_mod=0
|
||||
service_mod=2
|
||||
if pidof systemd &> /dev/null; then
|
||||
service_mod=0
|
||||
elif $(which service &> /dev/null); then
|
||||
service_mod=1
|
||||
service_config_dir="/etc/init.d"
|
||||
if $(which chkconfig &> /dev/null); then
|
||||
initd_mod=1
|
||||
elif $(which insserv &> /dev/null); then
|
||||
initd_mod=2
|
||||
elif $(which update-rc.d &> /dev/null); then
|
||||
initd_mod=3
|
||||
else
|
||||
service_mod=2
|
||||
fi
|
||||
else
|
||||
if pidof systemd &>/dev/null; then
|
||||
service_mod=0
|
||||
elif $(which service &>/dev/null); then
|
||||
service_mod=1
|
||||
service_config_dir="/etc/init.d"
|
||||
if $(which chkconfig &>/dev/null); then
|
||||
initd_mod=1
|
||||
elif $(which insserv &>/dev/null); then
|
||||
initd_mod=2
|
||||
elif $(which update-rc.d &>/dev/null); then
|
||||
initd_mod=3
|
||||
else
|
||||
service_mod=2
|
||||
fi
|
||||
else
|
||||
service_mod=2
|
||||
fi
|
||||
|
||||
|
||||
function kill_taosd() {
|
||||
pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}')
|
||||
function kill_taosadapter() {
|
||||
pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}')
|
||||
if [ -n "$pid" ]; then
|
||||
${csudo} kill -9 $pid || :
|
||||
${csudo}kill -9 $pid || :
|
||||
fi
|
||||
}
|
||||
|
||||
function kill_taosd() {
|
||||
pid=$(ps -ef | grep ${serverName} | grep -v "grep" | awk '{print $2}')
|
||||
if [ -n "$pid" ]; then
|
||||
${csudo}kill -9 $pid || :
|
||||
fi
|
||||
}
|
||||
|
||||
function kill_tarbitrator() {
|
||||
pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}')
|
||||
if [ -n "$pid" ]; then
|
||||
${csudo}kill -9 $pid || :
|
||||
fi
|
||||
}
|
||||
|
||||
function clean_bin() {
|
||||
# Remove link
|
||||
${csudo} rm -f ${bin_link_dir}/taos || :
|
||||
${csudo} rm -f ${bin_link_dir}/taosd || :
|
||||
${csudo} rm -f ${bin_link_dir}/create_table || :
|
||||
${csudo} rm -f ${bin_link_dir}/tmq_sim || :
|
||||
${csudo} rm -f ${bin_link_dir}/taosdemo || :
|
||||
${csudo} rm -f ${bin_link_dir}/taosdump || :
|
||||
${csudo} rm -f ${bin_link_dir}/rmtaos || :
|
||||
#${csudo} rm -f ${bin_link_dir}/set_core || :
|
||||
# Remove link
|
||||
${csudo}rm -f ${bin_link_dir}/${clientName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${serverName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/taosadapter || :
|
||||
${csudo}rm -f ${bin_link_dir}/taosBenchmark || :
|
||||
${csudo}rm -f ${bin_link_dir}/taosdemo || :
|
||||
${csudo}rm -f ${bin_link_dir}/taosdump || :
|
||||
${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
|
||||
${csudo}rm -f ${bin_link_dir}/tarbitrator || :
|
||||
${csudo}rm -f ${bin_link_dir}/set_core || :
|
||||
${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
|
||||
}
|
||||
|
||||
function clean_local_bin() {
|
||||
${csudo}rm -f ${local_bin_link_dir}/taosBenchmark || :
|
||||
${csudo}rm -f ${local_bin_link_dir}/taosdemo || :
|
||||
}
|
||||
|
||||
function clean_lib() {
|
||||
# Remove link
|
||||
${csudo} rm -f ${lib_link_dir}/libtaos.* || :
|
||||
${csudo} rm -f ${lib64_link_dir}/libtaos.* || :
|
||||
|
||||
${csudo} rm -f ${lib_link_dir}/libtdb.* || :
|
||||
${csudo} rm -f ${lib64_link_dir}/libtdb.* || :
|
||||
# Remove link
|
||||
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
|
||||
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
|
||||
#${csudo}rm -rf ${v15_java_app_dir} || :
|
||||
}
|
||||
|
||||
function clean_header() {
|
||||
# Remove link
|
||||
${csudo} rm -f ${inc_link_dir}/taos.h || :
|
||||
${csudo} rm -f ${inc_link_dir}/taosdef.h || :
|
||||
${csudo} rm -f ${inc_link_dir}/taoserror.h || :
|
||||
${csudo} rm -f ${inc_link_dir}/taosudf.h || :
|
||||
# Remove link
|
||||
${csudo}rm -f ${inc_link_dir}/taos.h || :
|
||||
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
|
||||
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
|
||||
}
|
||||
|
||||
function clean_config() {
|
||||
# Remove link
|
||||
${csudo} rm -f ${cfg_link_dir}/* || :
|
||||
# Remove link
|
||||
${csudo}rm -f ${cfg_link_dir}/* || :
|
||||
}
|
||||
|
||||
function clean_log() {
|
||||
# Remove link
|
||||
${csudo} rm -rf ${log_link_dir} || :
|
||||
# Remove link
|
||||
${csudo}rm -rf ${log_link_dir} || :
|
||||
}
|
||||
|
||||
function clean_service_on_systemd() {
|
||||
taosd_service_config="${service_config_dir}/${taos_service_name}.service"
|
||||
if systemctl is-active --quiet ${taos_service_name}; then
|
||||
echo "TDengine taosd is running, stopping it..."
|
||||
${csudo} systemctl stop ${taos_service_name} &> /dev/null || echo &> /dev/null
|
||||
taosd_service_config="${service_config_dir}/${taos_service_name}.service"
|
||||
if systemctl is-active --quiet ${taos_service_name}; then
|
||||
echo "${productName} ${serverName} is running, stopping it..."
|
||||
${csudo}systemctl stop ${taos_service_name} &>/dev/null || echo &>/dev/null
|
||||
fi
|
||||
${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null
|
||||
${csudo}rm -f ${taosd_service_config}
|
||||
|
||||
taosadapter_service_config="${service_config_dir}/taosadapter.service"
|
||||
if systemctl is-active --quiet ${taosadapter_service_name}; then
|
||||
echo "${productName} taosAdapter is running, stopping it..."
|
||||
${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null
|
||||
fi
|
||||
${csudo}systemctl disable ${taosadapter_service_name} &>/dev/null || echo &>/dev/null
|
||||
[ -f ${taosadapter_service_config} ] && ${csudo}rm -f ${taosadapter_service_config}
|
||||
|
||||
tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service"
|
||||
if systemctl is-active --quiet ${tarbitrator_service_name}; then
|
||||
echo "${productName} tarbitrator is running, stopping it..."
|
||||
${csudo}systemctl stop ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
|
||||
fi
|
||||
${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
|
||||
${csudo}rm -f ${tarbitratord_service_config}
|
||||
|
||||
if [ "$verMode" == "cluster" ]; then
|
||||
nginx_service_config="${service_config_dir}/${nginx_service_name}.service"
|
||||
if [ -d ${install_nginxd_dir} ]; then
|
||||
if systemctl is-active --quiet ${nginx_service_name}; then
|
||||
echo "Nginx for ${productName} is running, stopping it..."
|
||||
${csudo}systemctl stop ${nginx_service_name} &>/dev/null || echo &>/dev/null
|
||||
fi
|
||||
${csudo}systemctl disable ${nginx_service_name} &>/dev/null || echo &>/dev/null
|
||||
${csudo}rm -f ${nginx_service_config}
|
||||
fi
|
||||
${csudo} systemctl disable ${taos_service_name} &> /dev/null || echo &> /dev/null
|
||||
${csudo} rm -f ${taosd_service_config}
|
||||
fi
|
||||
}
|
||||
|
||||
function clean_service_on_sysvinit() {
|
||||
echo " "
|
||||
if pidof ${serverName} &>/dev/null; then
|
||||
echo "${productName} ${serverName} is running, stopping it..."
|
||||
${csudo}service ${serverName} stop || :
|
||||
fi
|
||||
|
||||
if pidof tarbitrator &>/dev/null; then
|
||||
echo "${productName} tarbitrator is running, stopping it..."
|
||||
${csudo}service tarbitratord stop || :
|
||||
fi
|
||||
|
||||
if ((${initd_mod} == 1)); then
|
||||
if [ -e ${service_config_dir}/${serverName} ]; then
|
||||
${csudo}chkconfig --del ${serverName} || :
|
||||
fi
|
||||
if [ -e ${service_config_dir}/tarbitratord ]; then
|
||||
${csudo}chkconfig --del tarbitratord || :
|
||||
fi
|
||||
elif ((${initd_mod} == 2)); then
|
||||
if [ -e ${service_config_dir}/${serverName} ]; then
|
||||
${csudo}insserv -r ${serverName} || :
|
||||
fi
|
||||
if [ -e ${service_config_dir}/tarbitratord ]; then
|
||||
${csudo}insserv -r tarbitratord || :
|
||||
fi
|
||||
elif ((${initd_mod} == 3)); then
|
||||
if [ -e ${service_config_dir}/${serverName} ]; then
|
||||
${csudo}update-rc.d -f ${serverName} remove || :
|
||||
fi
|
||||
if [ -e ${service_config_dir}/tarbitratord ]; then
|
||||
${csudo}update-rc.d -f tarbitratord remove || :
|
||||
fi
|
||||
fi
|
||||
|
||||
${csudo}rm -f ${service_config_dir}/${serverName} || :
|
||||
${csudo}rm -f ${service_config_dir}/tarbitratord || :
|
||||
|
||||
if $(which init &>/dev/null); then
|
||||
${csudo}init q || :
|
||||
fi
|
||||
}
|
||||
|
||||
function clean_service() {
|
||||
if ((${service_mod}==0)); then
|
||||
clean_service_on_systemd
|
||||
elif ((${service_mod}==1)); then
|
||||
clean_service_on_sysvinit
|
||||
else
|
||||
# must manual stop taosd
|
||||
kill_taosd
|
||||
fi
|
||||
if ((${service_mod} == 0)); then
|
||||
clean_service_on_systemd
|
||||
elif ((${service_mod} == 1)); then
|
||||
clean_service_on_sysvinit
|
||||
else
|
||||
kill_taosadapter
|
||||
kill_taosd
|
||||
kill_tarbitrator
|
||||
fi
|
||||
}
|
||||
|
||||
# Stop service and disable booting start.
|
||||
clean_service
|
||||
# Remove binary file and links
|
||||
clean_bin
|
||||
# Remove links of local bin
|
||||
clean_local_bin
|
||||
# Remove header file.
|
||||
clean_header
|
||||
# Remove lib file
|
||||
|
@ -135,15 +230,26 @@ clean_log
|
|||
# Remove link configuration file
|
||||
clean_config
|
||||
# Remove data link directory
|
||||
${csudo} rm -rf ${data_link_dir} || :
|
||||
|
||||
${csudo} rm -rf ${install_main_dir}
|
||||
${csudo}rm -rf ${data_link_dir} || :
|
||||
|
||||
${csudo}rm -rf ${install_main_dir}
|
||||
${csudo}rm -rf ${install_nginxd_dir}
|
||||
if [[ -e /etc/os-release ]]; then
|
||||
osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
|
||||
else
|
||||
osinfo=""
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}TDengine is removed successfully!${NC}"
|
||||
if echo $osinfo | grep -qwi "ubuntu"; then
|
||||
# echo "this is ubuntu system"
|
||||
${csudo}dpkg --force-all -P tdengine >/dev/null 2>&1 || :
|
||||
elif echo $osinfo | grep -qwi "debian"; then
|
||||
# echo "this is debian system"
|
||||
${csudo}dpkg --force-all -P tdengine >/dev/null 2>&1 || :
|
||||
elif echo $osinfo | grep -qwi "centos"; then
|
||||
# echo "this is centos system"
|
||||
${csudo}rpm -e --noscripts tdengine >/dev/null 2>&1 || :
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}${productName} is removed successfully!${NC}"
|
||||
echo
|
||||
|
|
|
@ -1,132 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Script to stop the service and uninstall TDengine's arbitrator
|
||||
|
||||
set -e
|
||||
#set -x
|
||||
|
||||
verMode=edge
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[1;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
#install main path
|
||||
install_main_dir="/usr/local/tarbitrator"
|
||||
bin_link_dir="/usr/bin"
|
||||
#inc_link_dir="/usr/include"
|
||||
|
||||
service_config_dir="/etc/systemd/system"
|
||||
tarbitrator_service_name="tarbitratord"
|
||||
csudo=""
|
||||
if command -v sudo > /dev/null; then
|
||||
csudo="sudo "
|
||||
fi
|
||||
|
||||
initd_mod=0
|
||||
service_mod=2
|
||||
if pidof systemd &> /dev/null; then
|
||||
service_mod=0
|
||||
elif $(which service &> /dev/null); then
|
||||
service_mod=1
|
||||
service_config_dir="/etc/init.d"
|
||||
if $(which chkconfig &> /dev/null); then
|
||||
initd_mod=1
|
||||
elif $(which insserv &> /dev/null); then
|
||||
initd_mod=2
|
||||
elif $(which update-rc.d &> /dev/null); then
|
||||
initd_mod=3
|
||||
else
|
||||
service_mod=2
|
||||
fi
|
||||
else
|
||||
service_mod=2
|
||||
fi
|
||||
|
||||
function kill_tarbitrator() {
|
||||
pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}')
|
||||
if [ -n "$pid" ]; then
|
||||
${csudo}kill -9 $pid || :
|
||||
fi
|
||||
}
|
||||
function clean_bin() {
|
||||
# Remove link
|
||||
${csudo}rm -f ${bin_link_dir}/tarbitrator || :
|
||||
}
|
||||
|
||||
function clean_header() {
|
||||
# Remove link
|
||||
${csudo}rm -f ${inc_link_dir}/taos.h || :
|
||||
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
|
||||
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
|
||||
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
|
||||
|
||||
}
|
||||
|
||||
function clean_log() {
|
||||
# Remove link
|
||||
${csudo}rm -rf /arbitrator.log || :
|
||||
}
|
||||
|
||||
function clean_service_on_systemd() {
|
||||
tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service"
|
||||
|
||||
if systemctl is-active --quiet ${tarbitrator_service_name}; then
|
||||
echo "TDengine tarbitrator is running, stopping it..."
|
||||
${csudo}systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null
|
||||
fi
|
||||
${csudo}systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null
|
||||
|
||||
${csudo}rm -f ${tarbitratord_service_config}
|
||||
}
|
||||
|
||||
function clean_service_on_sysvinit() {
|
||||
if pidof tarbitrator &> /dev/null; then
|
||||
echo "TDengine's tarbitrator is running, stopping it..."
|
||||
${csudo}service tarbitratord stop || :
|
||||
fi
|
||||
|
||||
if ((${initd_mod}==1)); then
|
||||
if [ -e ${service_config_dir}/tarbitratord ]; then
|
||||
${csudo}chkconfig --del tarbitratord || :
|
||||
fi
|
||||
elif ((${initd_mod}==2)); then
|
||||
if [ -e ${service_config_dir}/tarbitratord ]; then
|
||||
${csudo}insserv -r tarbitratord || :
|
||||
fi
|
||||
elif ((${initd_mod}==3)); then
|
||||
if [ -e ${service_config_dir}/tarbitratord ]; then
|
||||
${csudo}update-rc.d -f tarbitratord remove || :
|
||||
fi
|
||||
fi
|
||||
|
||||
${csudo}rm -f ${service_config_dir}/tarbitratord || :
|
||||
|
||||
if $(which init &> /dev/null); then
|
||||
${csudo}init q || :
|
||||
fi
|
||||
}
|
||||
|
||||
function clean_service() {
|
||||
if ((${service_mod}==0)); then
|
||||
clean_service_on_systemd
|
||||
elif ((${service_mod}==1)); then
|
||||
clean_service_on_sysvinit
|
||||
else
|
||||
# must manual stop
|
||||
kill_tarbitrator
|
||||
fi
|
||||
}
|
||||
|
||||
# Stop service and disable booting start.
|
||||
clean_service
|
||||
# Remove binary file and links
|
||||
clean_bin
|
||||
# Remove header file.
|
||||
##clean_header
|
||||
# Remove log file
|
||||
clean_log
|
||||
|
||||
${csudo}rm -rf ${install_main_dir}
|
||||
|
||||
echo -e "${GREEN}TDengine's arbitrator is removed successfully!${NC}"
|
|
@ -54,11 +54,10 @@ enum {
|
|||
RES_TYPE__TMQ_META,
|
||||
};
|
||||
|
||||
#define SHOW_VARIABLES_RESULT_COLS 2
|
||||
#define SHOW_VARIABLES_RESULT_COLS 2
|
||||
#define SHOW_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE)
|
||||
#define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
|
||||
|
||||
|
||||
#define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY)
|
||||
#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ)
|
||||
#define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META)
|
||||
|
@ -66,7 +65,7 @@ enum {
|
|||
typedef struct SAppInstInfo SAppInstInfo;
|
||||
|
||||
typedef struct {
|
||||
char* key;
|
||||
char* key;
|
||||
// statistics
|
||||
int32_t reportCnt;
|
||||
int32_t connKeyCnt;
|
||||
|
@ -118,6 +117,7 @@ struct SAppInstInfo {
|
|||
uint64_t clusterId;
|
||||
void* pTransporter;
|
||||
SAppHbMgr* pAppHbMgr;
|
||||
char* instKey;
|
||||
};
|
||||
|
||||
typedef struct SAppInfo {
|
||||
|
@ -139,7 +139,7 @@ typedef struct STscObj {
|
|||
int8_t connType;
|
||||
int32_t acctId;
|
||||
uint32_t connId;
|
||||
TAOS* id; // ref ID returned by taosAddRef
|
||||
int64_t id; // ref ID returned by taosAddRef
|
||||
TdThreadMutex mutex; // used to protect the operation on db
|
||||
int32_t numOfReqs; // number of sqlObj bound to this connection
|
||||
SAppInstInfo* pAppInfo;
|
||||
|
@ -183,7 +183,7 @@ typedef struct SRequestSendRecvBody {
|
|||
void* param;
|
||||
SDataBuf requestMsg;
|
||||
int64_t queryJob; // query job, created according to sql query DAG.
|
||||
struct SQueryPlan* pDag; // the query dag, generated according to the sql statement.
|
||||
int32_t subplanNum;
|
||||
SReqResultInfo resInfo;
|
||||
} SRequestSendRecvBody;
|
||||
|
||||
|
@ -194,7 +194,7 @@ typedef struct {
|
|||
int32_t vgId;
|
||||
SSchemaWrapper schema;
|
||||
int32_t resIter;
|
||||
SMqDataBlkRsp rsp;
|
||||
SMqDataRsp rsp;
|
||||
SReqResultInfo resInfo;
|
||||
} SMqRspObj;
|
||||
|
||||
|
@ -238,18 +238,18 @@ typedef struct SSyncQueryParam {
|
|||
void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
|
||||
void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
|
||||
|
||||
void doSetOneRowPtr(SReqResultInfo* pResultInfo);
|
||||
void setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
|
||||
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4,
|
||||
bool freeAfterUse);
|
||||
void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
|
||||
void doFreeReqResultInfo(SReqResultInfo* pResInfo);
|
||||
int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq);
|
||||
void syncCatalogFn(SMetaData* pResult, void* param, int32_t code);
|
||||
void doSetOneRowPtr(SReqResultInfo* pResultInfo);
|
||||
void setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
|
||||
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4,
|
||||
bool freeAfterUse);
|
||||
void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
|
||||
void doFreeReqResultInfo(SReqResultInfo* pResInfo);
|
||||
int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq);
|
||||
void syncCatalogFn(SMetaData* pResult, void* param, int32_t code);
|
||||
|
||||
SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly);
|
||||
TAOS_RES *taosQueryImpl(TAOS *taos, const char *sql, bool validateOnly);
|
||||
void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param, bool validateOnly);
|
||||
TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly);
|
||||
void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly);
|
||||
|
||||
static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
|
||||
SMqRspObj* msg = (SMqRspObj*)res;
|
||||
|
@ -300,6 +300,8 @@ void* createRequest(STscObj* pObj, int32_t type);
|
|||
void destroyRequest(SRequestObj* pRequest);
|
||||
SRequestObj* acquireRequest(int64_t rid);
|
||||
int32_t releaseRequest(int64_t rid);
|
||||
int32_t removeRequest(int64_t rid);
|
||||
void doDestroyRequest(void *p);
|
||||
|
||||
char* getDbOfConnection(STscObj* pObj);
|
||||
void setConnectionDB(STscObj* pTscObj, const char* db);
|
||||
|
@ -334,6 +336,8 @@ int hbHandleRsp(SClientHbBatchRsp* hbRsp);
|
|||
// cluster level
|
||||
SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key);
|
||||
void appHbMgrCleanup(void);
|
||||
void hbRemoveAppHbMrg(SAppHbMgr **pAppHbMgr);
|
||||
void closeAllRequests(SHashObj *pRequests);
|
||||
|
||||
// conn level
|
||||
int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);
|
||||
|
|
|
@ -37,10 +37,12 @@ int32_t clientConnRefPool = -1;
|
|||
static TdThreadOnce tscinit = PTHREAD_ONCE_INIT;
|
||||
volatile int32_t tscInitRes = 0;
|
||||
|
||||
static void registerRequest(SRequestObj *pRequest) {
|
||||
STscObj *pTscObj = acquireTscObj(*(int64_t *)pRequest->pTscObj->id);
|
||||
|
||||
assert(pTscObj != NULL);
|
||||
static int32_t registerRequest(SRequestObj *pRequest) {
|
||||
STscObj *pTscObj = acquireTscObj(pRequest->pTscObj->id);
|
||||
if (NULL == pTscObj) {
|
||||
terrno = TSDB_CODE_TSC_DISCONNECTED;
|
||||
return terrno;
|
||||
}
|
||||
|
||||
// connection has been released already, abort creating request.
|
||||
pRequest->self = taosAddRef(clientReqRefPool, pRequest);
|
||||
|
@ -54,8 +56,10 @@ static void registerRequest(SRequestObj *pRequest) {
|
|||
int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1);
|
||||
tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
|
||||
", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
|
||||
pRequest->self, *(int64_t *)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
|
||||
pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static void deregisterRequest(SRequestObj *pRequest) {
|
||||
|
@ -70,24 +74,23 @@ static void deregisterRequest(SRequestObj *pRequest) {
|
|||
int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
|
||||
tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
|
||||
" ms, current:%d, app current:%d",
|
||||
pRequest->self, *(int64_t *)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
|
||||
releaseTscObj(*(int64_t *)pTscObj->id);
|
||||
pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
|
||||
releaseTscObj(pTscObj->id);
|
||||
}
|
||||
|
||||
// todo close the transporter properly
|
||||
void closeTransporter(STscObj *pTscObj) {
|
||||
if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) {
|
||||
void closeTransporter(SAppInstInfo *pAppInfo) {
|
||||
if (pAppInfo == NULL || pAppInfo->pTransporter == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t *)pTscObj->id);
|
||||
rpcClose(pTscObj->pAppInfo->pTransporter);
|
||||
tscDebug("free transporter:%p in app inst %p", pAppInfo->pTransporter, pAppInfo);
|
||||
rpcClose(pAppInfo->pTransporter);
|
||||
}
|
||||
|
||||
static bool clientRpcRfp(int32_t code, tmsg_t msgType) {
|
||||
if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
|
||||
code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
|
||||
if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) {
|
||||
if (NEED_REDIRECT_ERROR(code)) {
|
||||
if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
@ -123,26 +126,46 @@ void closeAllRequests(SHashObj *pRequests) {
|
|||
while (pIter != NULL) {
|
||||
int64_t *rid = pIter;
|
||||
|
||||
releaseRequest(*rid);
|
||||
removeRequest(*rid);
|
||||
|
||||
pIter = taosHashIterate(pRequests, pIter);
|
||||
}
|
||||
}
|
||||
|
||||
void destroyAppInst(SAppInstInfo *pAppInfo) {
|
||||
tscDebug("destroy app inst mgr %p", pAppInfo);
|
||||
|
||||
taosThreadMutexLock(&appInfo.mutex);
|
||||
|
||||
hbRemoveAppHbMrg(&pAppInfo->pAppHbMgr);
|
||||
taosHashRemove(appInfo.pInstMap, pAppInfo->instKey, strlen(pAppInfo->instKey));
|
||||
|
||||
taosThreadMutexUnlock(&appInfo.mutex);
|
||||
|
||||
taosMemoryFreeClear(pAppInfo->instKey);
|
||||
closeTransporter(pAppInfo);
|
||||
|
||||
taosThreadMutexLock(&pAppInfo->qnodeMutex);
|
||||
taosArrayDestroy(pAppInfo->pQnodeList);
|
||||
taosThreadMutexUnlock(&pAppInfo->qnodeMutex);
|
||||
|
||||
taosMemoryFree(pAppInfo);
|
||||
}
|
||||
|
||||
void destroyTscObj(void *pObj) {
|
||||
STscObj *pTscObj = pObj;
|
||||
|
||||
SClientHbKey connKey = {.tscRid = *(int64_t *)pTscObj->id, .connType = pTscObj->connType};
|
||||
SClientHbKey connKey = {.tscRid = pTscObj->id, .connType = pTscObj->connType};
|
||||
hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey);
|
||||
int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
|
||||
closeAllRequests(pTscObj->pRequests);
|
||||
schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter);
|
||||
if (0 == connNum) {
|
||||
// TODO
|
||||
// closeTransporter(pTscObj);
|
||||
}
|
||||
tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t *)pTscObj->id,
|
||||
tscDebug("connObj 0x%" PRIx64 " p:%p destroyed, remain inst totalConn:%" PRId64, pTscObj->id, pTscObj,
|
||||
pTscObj->pAppInfo->numOfConns);
|
||||
|
||||
if (0 == connNum) {
|
||||
destroyAppInst(pTscObj->pAppInfo);
|
||||
}
|
||||
taosThreadMutexDestroy(&pTscObj->mutex);
|
||||
taosMemoryFreeClear(pTscObj);
|
||||
}
|
||||
|
@ -171,11 +194,12 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c
|
|||
}
|
||||
|
||||
taosThreadMutexInit(&pObj->mutex, NULL);
|
||||
pObj->id = taosMemoryMalloc(sizeof(int64_t));
|
||||
*(int64_t *)pObj->id = taosAddRef(clientConnRefPool, pObj);
|
||||
pObj->id = taosAddRef(clientConnRefPool, pObj);
|
||||
pObj->schemalessType = 1;
|
||||
|
||||
tscDebug("connObj created, 0x%" PRIx64, *(int64_t *)pObj->id);
|
||||
atomic_add_fetch_64(&pObj->pAppInfo->numOfConns, 1);
|
||||
|
||||
tscDebug("connObj created, 0x%" PRIx64 ",p:%p", pObj->id, pObj);
|
||||
return pObj;
|
||||
}
|
||||
|
||||
|
@ -205,7 +229,10 @@ void *createRequest(STscObj *pObj, int32_t type) {
|
|||
pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE;
|
||||
tsem_init(&pRequest->body.rspSem, 0, 0);
|
||||
|
||||
registerRequest(pRequest);
|
||||
if (registerRequest(pRequest)) {
|
||||
doDestroyRequest(pRequest);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return pRequest;
|
||||
}
|
||||
|
@ -227,12 +254,16 @@ void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
|
|||
}
|
||||
}
|
||||
|
||||
static void doDestroyRequest(void *p) {
|
||||
SRequestObj *acquireRequest(int64_t rid) { return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); }
|
||||
|
||||
int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, rid); }
|
||||
|
||||
int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); }
|
||||
|
||||
void doDestroyRequest(void *p) {
|
||||
assert(p != NULL);
|
||||
SRequestObj *pRequest = (SRequestObj *)p;
|
||||
|
||||
assert(RID_VALID(pRequest->self));
|
||||
|
||||
taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self));
|
||||
|
||||
if (pRequest->body.queryJob != 0) {
|
||||
|
@ -244,14 +275,15 @@ static void doDestroyRequest(void *p) {
|
|||
taosMemoryFreeClear(pRequest->pDb);
|
||||
|
||||
doFreeReqResultInfo(&pRequest->body.resInfo);
|
||||
qDestroyQueryPlan(pRequest->body.pDag);
|
||||
|
||||
taosArrayDestroy(pRequest->tableList);
|
||||
taosArrayDestroy(pRequest->dbList);
|
||||
|
||||
destroyQueryExecRes(&pRequest->body.resInfo.execRes);
|
||||
|
||||
deregisterRequest(pRequest);
|
||||
if (pRequest->self) {
|
||||
deregisterRequest(pRequest);
|
||||
}
|
||||
taosMemoryFreeClear(pRequest);
|
||||
}
|
||||
|
||||
|
@ -260,13 +292,9 @@ void destroyRequest(SRequestObj *pRequest) {
|
|||
return;
|
||||
}
|
||||
|
||||
taosRemoveRef(clientReqRefPool, pRequest->self);
|
||||
removeRequest(pRequest->self);
|
||||
}
|
||||
|
||||
SRequestObj *acquireRequest(int64_t rid) { return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); }
|
||||
|
||||
int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, rid); }
|
||||
|
||||
void taos_init_imp(void) {
|
||||
// In the APIs of other program language, taos_cleanup is not available yet.
|
||||
// So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning.
|
||||
|
|
|
@ -66,25 +66,31 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
|
|||
if (rsp->vgVersion < 0) {
|
||||
code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid);
|
||||
} else {
|
||||
SDBVgInfo vgInfo = {0};
|
||||
vgInfo.vgVersion = rsp->vgVersion;
|
||||
vgInfo.hashMethod = rsp->hashMethod;
|
||||
vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
|
||||
if (NULL == vgInfo.vgHash) {
|
||||
SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo));
|
||||
if (NULL == vgInfo) {
|
||||
return TSDB_CODE_TSC_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
vgInfo->vgVersion = rsp->vgVersion;
|
||||
vgInfo->hashMethod = rsp->hashMethod;
|
||||
vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
|
||||
if (NULL == vgInfo->vgHash) {
|
||||
taosMemoryFree(vgInfo);
|
||||
tscError("hash init[%d] failed", rsp->vgNum);
|
||||
return TSDB_CODE_TSC_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
for (int32_t j = 0; j < rsp->vgNum; ++j) {
|
||||
SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j);
|
||||
if (taosHashPut(vgInfo.vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) {
|
||||
if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) {
|
||||
tscError("hash push failed, errno:%d", errno);
|
||||
taosHashCleanup(vgInfo.vgHash);
|
||||
taosHashCleanup(vgInfo->vgHash);
|
||||
taosMemoryFree(vgInfo);
|
||||
return TSDB_CODE_TSC_OUT_OF_MEMORY;
|
||||
}
|
||||
}
|
||||
|
||||
catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, &vgInfo);
|
||||
catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, vgInfo);
|
||||
}
|
||||
|
||||
if (code) {
|
||||
|
@ -256,7 +262,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) {
|
||||
static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {
|
||||
static int32_t emptyRspNum = 0;
|
||||
if (code != 0) {
|
||||
taosMemoryFreeClear(param);
|
||||
|
@ -269,8 +275,11 @@ static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code)
|
|||
|
||||
int32_t rspNum = taosArrayGetSize(pRsp.rsps);
|
||||
|
||||
taosThreadMutexLock(&appInfo.mutex);
|
||||
|
||||
SAppInstInfo **pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
|
||||
if (pInst == NULL || NULL == *pInst) {
|
||||
taosThreadMutexUnlock(&appInfo.mutex);
|
||||
tscError("cluster not exist, key:%s", key);
|
||||
taosMemoryFreeClear(param);
|
||||
tFreeClientHbBatchRsp(&pRsp);
|
||||
|
@ -294,6 +303,8 @@ static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code)
|
|||
}
|
||||
}
|
||||
|
||||
taosThreadMutexUnlock(&appInfo.mutex);
|
||||
|
||||
tFreeClientHbBatchRsp(&pRsp);
|
||||
|
||||
return code;
|
||||
|
@ -320,7 +331,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) {
|
|||
desc.reqRid = pRequest->self;
|
||||
desc.stableQuery = pRequest->stableQuery;
|
||||
taosGetFqdn(desc.fqdn);
|
||||
desc.subPlanNum = pRequest->body.pDag ? pRequest->body.pDag->numOfSubplans : 0;
|
||||
desc.subPlanNum = pRequest->body.subplanNum;
|
||||
|
||||
if (desc.subPlanNum) {
|
||||
desc.subDesc = taosArrayInit(desc.subPlanNum, sizeof(SQuerySubDesc));
|
||||
|
@ -790,22 +801,40 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
|
|||
return pAppHbMgr;
|
||||
}
|
||||
|
||||
void hbFreeAppHbMgr(SAppHbMgr *pTarget) {
|
||||
void *pIter = taosHashIterate(pTarget->activeInfo, NULL);
|
||||
while (pIter != NULL) {
|
||||
SClientHbReq *pOneReq = pIter;
|
||||
tFreeClientHbReq(pOneReq);
|
||||
pIter = taosHashIterate(pTarget->activeInfo, pIter);
|
||||
}
|
||||
taosHashCleanup(pTarget->activeInfo);
|
||||
pTarget->activeInfo = NULL;
|
||||
|
||||
taosMemoryFree(pTarget->key);
|
||||
taosMemoryFree(pTarget);
|
||||
}
|
||||
|
||||
void hbRemoveAppHbMrg(SAppHbMgr **pAppHbMgr) {
|
||||
taosThreadMutexLock(&clientHbMgr.lock);
|
||||
int32_t mgrSize = taosArrayGetSize(clientHbMgr.appHbMgrs);
|
||||
for (int32_t i = 0; i < mgrSize; ++i) {
|
||||
SAppHbMgr *pItem = taosArrayGetP(clientHbMgr.appHbMgrs, i);
|
||||
if (pItem == *pAppHbMgr) {
|
||||
hbFreeAppHbMgr(*pAppHbMgr);
|
||||
*pAppHbMgr = NULL;
|
||||
taosArrayRemove(clientHbMgr.appHbMgrs, i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
taosThreadMutexUnlock(&clientHbMgr.lock);
|
||||
}
|
||||
|
||||
void appHbMgrCleanup(void) {
|
||||
int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
|
||||
for (int i = 0; i < sz; i++) {
|
||||
SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i);
|
||||
|
||||
void *pIter = taosHashIterate(pTarget->activeInfo, NULL);
|
||||
while (pIter != NULL) {
|
||||
SClientHbReq *pOneReq = pIter;
|
||||
tFreeClientHbReq(pOneReq);
|
||||
pIter = taosHashIterate(pTarget->activeInfo, pIter);
|
||||
}
|
||||
taosHashCleanup(pTarget->activeInfo);
|
||||
pTarget->activeInfo = NULL;
|
||||
|
||||
taosMemoryFree(pTarget->key);
|
||||
taosMemoryFree(pTarget);
|
||||
hbFreeAppHbMgr(pTarget);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -55,6 +55,18 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i
|
|||
return strdup(key);
|
||||
}
|
||||
|
||||
bool chkRequestKilled(void* param) {
|
||||
bool killed = false;
|
||||
SRequestObj* pRequest = acquireRequest((int64_t)param);
|
||||
if (NULL == pRequest || pRequest->killed) {
|
||||
killed = true;
|
||||
}
|
||||
|
||||
releaseRequest((int64_t)param);
|
||||
|
||||
return killed;
|
||||
}
|
||||
|
||||
static STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
|
||||
SAppInstInfo* pAppInfo, int connType);
|
||||
|
||||
|
@ -122,6 +134,9 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
|
|||
p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores);
|
||||
p->pAppHbMgr = appHbMgrInit(p, key);
|
||||
taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES);
|
||||
p->instKey = key;
|
||||
key = NULL;
|
||||
tscDebug("new app inst mgr %p, user:%s, ip:%s, port:%d", p, user, ip, port);
|
||||
|
||||
pInst = &p;
|
||||
}
|
||||
|
@ -609,58 +624,6 @@ _return:
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
|
||||
tsem_init(&schdRspSem, 0, 0);
|
||||
|
||||
SQueryResult res = {.code = 0, .numOfRows = 0};
|
||||
SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
|
||||
.requestId = pRequest->requestId,
|
||||
.requestObjRefId = pRequest->self};
|
||||
SSchedulerReq req = {.pConn = &conn,
|
||||
.pNodeList = pNodeList,
|
||||
.pDag = pDag,
|
||||
.sql = pRequest->sqlstr,
|
||||
.startTs = pRequest->metric.start,
|
||||
.fp = schdExecCallback,
|
||||
.cbParam = &res};
|
||||
|
||||
int32_t code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob);
|
||||
|
||||
pRequest->body.resInfo.execRes = res.res;
|
||||
|
||||
while (true) {
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
if (pRequest->body.queryJob != 0) {
|
||||
schedulerFreeJob(pRequest->body.queryJob, 0);
|
||||
}
|
||||
|
||||
pRequest->code = code;
|
||||
terrno = code;
|
||||
return pRequest->code;
|
||||
} else {
|
||||
tsem_wait(&schdRspSem);
|
||||
|
||||
if (res.code) {
|
||||
code = res.code;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_CREATE_TABLE == pRequest->type) {
|
||||
pRequest->body.resInfo.numOfRows = res.numOfRows;
|
||||
|
||||
if (pRequest->body.queryJob != 0) {
|
||||
schedulerFreeJob(pRequest->body.queryJob, 0);
|
||||
}
|
||||
}
|
||||
|
||||
pRequest->code = res.code;
|
||||
terrno = res.code;
|
||||
return pRequest->code;
|
||||
}
|
||||
|
||||
int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
|
||||
void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter;
|
||||
|
||||
|
@ -669,13 +632,14 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
|
|||
.requestId = pRequest->requestId,
|
||||
.requestObjRefId = pRequest->self};
|
||||
SSchedulerReq req = {.pConn = &conn,
|
||||
.pNodeList = pNodeList,
|
||||
.pDag = pDag,
|
||||
.sql = pRequest->sqlstr,
|
||||
.startTs = pRequest->metric.start,
|
||||
.fp = NULL,
|
||||
.cbParam = NULL,
|
||||
.reqKilled = &pRequest->killed};
|
||||
.pNodeList = pNodeList,
|
||||
.pDag = pDag,
|
||||
.sql = pRequest->sqlstr,
|
||||
.startTs = pRequest->metric.start,
|
||||
.execFp = NULL,
|
||||
.execParam = NULL,
|
||||
.chkKillFp = chkRequestKilled,
|
||||
.chkKillParam = (void*)pRequest->self};
|
||||
|
||||
int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res);
|
||||
pRequest->body.resInfo.execRes = res.res;
|
||||
|
@ -805,7 +769,8 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
|
|||
code = handleSubmitExecRes(pRequest, pRes->res, pCatalog, &epset);
|
||||
break;
|
||||
}
|
||||
case TDMT_VND_QUERY: {
|
||||
case TDMT_SCH_QUERY:
|
||||
case TDMT_SCH_MERGE_QUERY: {
|
||||
code = handleQueryExecRes(pRequest, pRes->res, pCatalog, &epset);
|
||||
break;
|
||||
}
|
||||
|
@ -874,14 +839,18 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
|
|||
}
|
||||
break;
|
||||
case QUERY_EXEC_MODE_SCHEDULE: {
|
||||
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
|
||||
code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pMnodeList);
|
||||
if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
|
||||
SArray* pNodeList = NULL;
|
||||
buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList);
|
||||
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
|
||||
SQueryPlan* pDag = NULL;
|
||||
code = getPlan(pRequest, pQuery, &pDag, pMnodeList);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
pRequest->body.subplanNum = pDag->numOfSubplans;
|
||||
if (!pRequest->validateOnly) {
|
||||
SArray* pNodeList = NULL;
|
||||
buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList);
|
||||
|
||||
code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList);
|
||||
taosArrayDestroy(pNodeList);
|
||||
code = scheduleQuery(pRequest, pDag, pNodeList);
|
||||
taosArrayDestroy(pNodeList);
|
||||
}
|
||||
}
|
||||
taosArrayDestroy(pMnodeList);
|
||||
break;
|
||||
|
@ -959,10 +928,13 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
|
|||
.pUser = pRequest->pTscObj->user};
|
||||
|
||||
SAppInstInfo* pAppInfo = getAppInfo(pRequest);
|
||||
code = qCreateQueryPlan(&cxt, &pRequest->body.pDag, pMnodeList);
|
||||
SQueryPlan* pDag = NULL;
|
||||
code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
|
||||
if (code) {
|
||||
tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
|
||||
pRequest->requestId);
|
||||
} else {
|
||||
pRequest->body.subplanNum = pDag->numOfSubplans;
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
|
||||
|
@ -973,12 +945,13 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
|
|||
.pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
|
||||
SSchedulerReq req = {.pConn = &conn,
|
||||
.pNodeList = pNodeList,
|
||||
.pDag = pRequest->body.pDag,
|
||||
.pDag = pDag,
|
||||
.sql = pRequest->sqlstr,
|
||||
.startTs = pRequest->metric.start,
|
||||
.fp = schedulerExecCb,
|
||||
.cbParam = pRequest,
|
||||
.reqKilled = &pRequest->killed};
|
||||
.execFp = schedulerExecCb,
|
||||
.execParam = pRequest,
|
||||
.chkKillFp = chkRequestKilled,
|
||||
.chkKillParam = (void*)pRequest->self};
|
||||
code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob);
|
||||
taosArrayDestroy(pNodeList);
|
||||
} else {
|
||||
|
@ -1163,7 +1136,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
|
|||
taos_close_internal(pTscObj);
|
||||
pTscObj = NULL;
|
||||
} else {
|
||||
tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, reqId:0x%" PRIx64, *(int64_t*)pTscObj->id,
|
||||
tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, reqId:0x%" PRIx64, pTscObj->id,
|
||||
pTscObj->connId, pTscObj->pAppInfo->pTransporter, pRequest->requestId);
|
||||
destroyRequest(pRequest);
|
||||
}
|
||||
|
@ -1283,10 +1256,10 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
|
|||
*/
|
||||
int32_t elapsed = pRequest->metric.rsp - pRequest->metric.start;
|
||||
if (pMsg->code == TSDB_CODE_SUCCESS) {
|
||||
tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%d ms, reqId:0x%" PRIx64, pRequest->self,
|
||||
tscDebug("0x%" PRIx64 " rsp msg:%s, code:%s rspLen:%d, elapsed:%d ms, reqId:0x%" PRIx64, pRequest->self,
|
||||
TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed / 1000, pRequest->requestId);
|
||||
} else {
|
||||
tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d, elapsed time:%d ms, reqId:0x%" PRIx64, pRequest->self,
|
||||
tscError("0x%" PRIx64 " rsp msg:%s, code:%s rspLen:%d, elapsed time:%d ms, reqId:0x%" PRIx64, pRequest->self,
|
||||
TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed / 1000, pRequest->requestId);
|
||||
}
|
||||
|
||||
|
@ -1296,7 +1269,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
|
|||
|
||||
updateTargetEpSet(pSendInfo, pTscObj, pMsg, pEpSet);
|
||||
|
||||
SDataBuf buf = {.len = pMsg->contLen, .pData = NULL, .handle = pMsg->info.handle};
|
||||
SDataBuf buf = {.msgType = pMsg->msgType, .len = pMsg->contLen, .pData = NULL, .handle = pMsg->info.handle, .pEpSet = pEpSet};
|
||||
|
||||
if (pMsg->contLen > 0) {
|
||||
buf.pData = taosMemoryCalloc(1, pMsg->contLen);
|
||||
|
@ -1326,7 +1299,9 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
|
|||
|
||||
STscObj* pObj = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY);
|
||||
if (pObj) {
|
||||
return pObj->id;
|
||||
int64_t* rid = taosMemoryCalloc(1, sizeof(int64_t));
|
||||
*rid = pObj->id;
|
||||
return (TAOS*)rid;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
|
@ -1433,7 +1408,11 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU
|
|||
}
|
||||
|
||||
SSyncQueryParam* pParam = pRequest->body.param;
|
||||
|
||||
if (NULL == pParam) {
|
||||
pParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
|
||||
tsem_init(&pParam->sem, 0, 0);
|
||||
}
|
||||
|
||||
// convert ucs4 to native multi-bytes string
|
||||
pResultInfo->convertUcs4 = convertUcs4;
|
||||
|
||||
|
@ -2001,17 +1980,26 @@ void syncCatalogFn(SMetaData* pResult, void* param, int32_t code) {
|
|||
void syncQueryFn(void* param, void* res, int32_t code) {
|
||||
SSyncQueryParam* pParam = param;
|
||||
pParam->pRequest = res;
|
||||
pParam->pRequest->code = code;
|
||||
if (pParam->pRequest) {
|
||||
pParam->pRequest->code = code;
|
||||
}
|
||||
|
||||
tsem_post(&pParam->sem);
|
||||
}
|
||||
|
||||
void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) {
|
||||
STscObj* pTscObj = acquireTscObj(*(int64_t*)taos);
|
||||
if (NULL == taos) {
|
||||
terrno = TSDB_CODE_TSC_DISCONNECTED;
|
||||
fp(param, NULL, terrno);
|
||||
return;
|
||||
}
|
||||
|
||||
int64_t rid = *(int64_t*)taos;
|
||||
STscObj* pTscObj = acquireTscObj(rid);
|
||||
if (pTscObj == NULL || sql == NULL || NULL == fp) {
|
||||
terrno = TSDB_CODE_INVALID_PARA;
|
||||
if (pTscObj) {
|
||||
releaseTscObj(*(int64_t*)taos);
|
||||
releaseTscObj(rid);
|
||||
} else {
|
||||
terrno = TSDB_CODE_TSC_DISCONNECTED;
|
||||
}
|
||||
|
@ -2023,6 +2011,7 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void*
|
|||
if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
|
||||
tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
|
||||
terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
|
||||
releaseTscObj(rid);
|
||||
|
||||
fp(param, NULL, terrno);
|
||||
return;
|
||||
|
@ -2032,6 +2021,7 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void*
|
|||
int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
terrno = code;
|
||||
releaseTscObj(rid);
|
||||
fp(param, NULL, terrno);
|
||||
return;
|
||||
}
|
||||
|
@ -2040,6 +2030,7 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void*
|
|||
pRequest->body.queryFp = fp;
|
||||
pRequest->body.param = param;
|
||||
doAsyncQuery(pRequest, false);
|
||||
releaseTscObj(rid);
|
||||
}
|
||||
|
||||
TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
|
||||
|
@ -2048,7 +2039,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
STscObj* pTscObj = acquireTscObj(*(int64_t*)taos);
|
||||
int64_t rid = *(int64_t*)taos;
|
||||
STscObj* pTscObj = acquireTscObj(rid);
|
||||
if (pTscObj == NULL || sql == NULL) {
|
||||
terrno = TSDB_CODE_TSC_DISCONNECTED;
|
||||
return NULL;
|
||||
|
@ -2058,16 +2050,16 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
|
|||
SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
|
||||
tsem_init(¶m->sem, 0, 0);
|
||||
|
||||
taosAsyncQueryImpl(taos, sql, syncQueryFn, param, validateOnly);
|
||||
taosAsyncQueryImpl((TAOS*)&rid, sql, syncQueryFn, param, validateOnly);
|
||||
tsem_wait(¶m->sem);
|
||||
|
||||
releaseTscObj(*(int64_t*)taos);
|
||||
releaseTscObj(rid);
|
||||
|
||||
return param->pRequest;
|
||||
#else
|
||||
size_t sqlLen = strlen(sql);
|
||||
if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
|
||||
releaseTscObj(*(int64_t*)taos);
|
||||
releaseTscObj(rid);
|
||||
tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
|
||||
terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
|
||||
return NULL;
|
||||
|
@ -2075,7 +2067,7 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
|
|||
|
||||
TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen, validateOnly);
|
||||
|
||||
releaseTscObj(*(int64_t*)taos);
|
||||
releaseTscObj(rid);
|
||||
|
||||
return pRes;
|
||||
#endif
|
||||
|
|
|
@ -81,16 +81,16 @@ void taos_cleanup(void) {
|
|||
taosCloseLog();
|
||||
}
|
||||
|
||||
static setConfRet taos_set_config_imp(const char *config){
|
||||
static setConfRet taos_set_config_imp(const char *config) {
|
||||
setConfRet ret = {SET_CONF_RET_SUCC, {0}};
|
||||
// TODO: need re-implementation
|
||||
return ret;
|
||||
}
|
||||
|
||||
setConfRet taos_set_config(const char *config){
|
||||
// TODO pthread_mutex_lock(&setConfMutex);
|
||||
setConfRet taos_set_config(const char *config) {
|
||||
// TODO pthread_mutex_lock(&setConfMutex);
|
||||
setConfRet ret = taos_set_config_imp(config);
|
||||
// pthread_mutex_unlock(&setConfMutex);
|
||||
// pthread_mutex_unlock(&setConfMutex);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
@ -106,7 +106,9 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
|
|||
|
||||
STscObj *pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY);
|
||||
if (pObj) {
|
||||
return pObj->id;
|
||||
int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t));
|
||||
*rid = pObj->id;
|
||||
return (TAOS*)rid;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
|
@ -118,9 +120,9 @@ void taos_close_internal(void *taos) {
|
|||
}
|
||||
|
||||
STscObj *pTscObj = (STscObj *)taos;
|
||||
tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", *(int64_t *)pTscObj->id, pTscObj->numOfReqs);
|
||||
tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
|
||||
|
||||
taosRemoveRef(clientConnRefPool, *(int64_t *)pTscObj->id);
|
||||
taosRemoveRef(clientConnRefPool, pTscObj->id);
|
||||
}
|
||||
|
||||
void taos_close(TAOS *taos) {
|
||||
|
@ -179,8 +181,6 @@ void taos_free_result(TAOS_RES *res) {
|
|||
SMqRspObj *pRsp = (SMqRspObj *)res;
|
||||
if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree);
|
||||
if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen);
|
||||
if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags);
|
||||
if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema);
|
||||
if (pRsp->rsp.withTbName) taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree);
|
||||
if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
|
||||
pRsp->resInfo.pRspMsg = NULL;
|
||||
|
@ -192,6 +192,17 @@ void taos_free_result(TAOS_RES *res) {
|
|||
}
|
||||
}
|
||||
|
||||
void taos_kill_query(TAOS *taos) {
|
||||
if (NULL == taos) {
|
||||
return;
|
||||
}
|
||||
int64_t rid = *(int64_t*)taos;
|
||||
|
||||
STscObj* pTscObj = acquireTscObj(rid);
|
||||
closeAllRequests(pTscObj->pRequests);
|
||||
releaseTscObj(rid);
|
||||
}
|
||||
|
||||
int taos_field_count(TAOS_RES *res) {
|
||||
if (res == NULL || TD_RES_TMQ_META(res)) {
|
||||
return 0;
|
||||
|
@ -895,6 +906,12 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
|
|||
}
|
||||
|
||||
int taos_load_table_info(TAOS *taos, const char *tableNameList) {
|
||||
if (NULL == taos) {
|
||||
terrno = TSDB_CODE_TSC_DISCONNECTED;
|
||||
return terrno;
|
||||
}
|
||||
|
||||
int64_t rid = *(int64_t*)taos;
|
||||
const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024; // 12MB list
|
||||
int32_t code = 0;
|
||||
SRequestObj *pRequest = NULL;
|
||||
|
@ -912,7 +929,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
|
|||
return TSDB_CODE_TSC_INVALID_OPERATION;
|
||||
}
|
||||
|
||||
STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
|
||||
STscObj *pTscObj = acquireTscObj(rid);
|
||||
if (pTscObj == NULL) {
|
||||
terrno = TSDB_CODE_TSC_DISCONNECTED;
|
||||
return terrno;
|
||||
|
@ -957,7 +974,7 @@ _return:
|
|||
taosArrayDestroy(catalogReq.pTableMeta);
|
||||
destroyRequest(pRequest);
|
||||
|
||||
releaseTscObj(*(int64_t *)taos);
|
||||
releaseTscObj(rid);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
|
@ -26,7 +26,7 @@ static void setErrno(SRequestObj* pRequest, int32_t code) {
|
|||
terrno = code;
|
||||
}
|
||||
|
||||
int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SRequestObj* pRequest = param;
|
||||
setErrno(pRequest, code);
|
||||
|
||||
|
@ -39,7 +39,7 @@ int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SRequestObj* pRequest = param;
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
taosMemoryFree(pMsg->pData);
|
||||
|
@ -67,17 +67,17 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
dstEpSet.eps[dstEpSet.inUse].fqdn);
|
||||
} else if (connectRsp.dnodeNum > 1 && !isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) {
|
||||
SEpSet* pOrig = &pTscObj->pAppInfo->mgmtEp.epSet;
|
||||
SEp* pOrigEp = &pOrig->eps[pOrig->inUse];
|
||||
SEp* pNewEp = &connectRsp.epSet.eps[connectRsp.epSet.inUse];
|
||||
tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in connRsp",
|
||||
pOrig->inUse, pOrig->numOfEps, pOrigEp->fqdn, pOrigEp->port,
|
||||
connectRsp.epSet.inUse, connectRsp.epSet.numOfEps, pNewEp->fqdn, pNewEp->port);
|
||||
SEp* pOrigEp = &pOrig->eps[pOrig->inUse];
|
||||
SEp* pNewEp = &connectRsp.epSet.eps[connectRsp.epSet.inUse];
|
||||
tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in connRsp", pOrig->inUse, pOrig->numOfEps,
|
||||
pOrigEp->fqdn, pOrigEp->port, connectRsp.epSet.inUse, connectRsp.epSet.numOfEps, pNewEp->fqdn,
|
||||
pNewEp->port);
|
||||
updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet);
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < connectRsp.epSet.numOfEps; ++i) {
|
||||
tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%" PRIx64, pRequest->requestId, i,
|
||||
connectRsp.epSet.eps[i].fqdn, connectRsp.epSet.eps[i].port, *(int64_t*)pTscObj->id);
|
||||
connectRsp.epSet.eps[i].fqdn, connectRsp.epSet.eps[i].port, pTscObj->id);
|
||||
}
|
||||
|
||||
pTscObj->connId = connectRsp.connId;
|
||||
|
@ -87,11 +87,10 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
|
||||
// update the appInstInfo
|
||||
pTscObj->pAppInfo->clusterId = connectRsp.clusterId;
|
||||
atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
|
||||
|
||||
pTscObj->connType = connectRsp.connType;
|
||||
|
||||
hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, *(int64_t*)pTscObj->id, connectRsp.clusterId, connectRsp.connType);
|
||||
hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType);
|
||||
|
||||
// pRequest->body.resInfo.pRspMsg = pMsg->pData;
|
||||
tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId,
|
||||
|
@ -117,7 +116,7 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) {
|
|||
return pMsgSendInfo;
|
||||
}
|
||||
|
||||
int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
// todo rsp with the vnode id list
|
||||
SRequestObj* pRequest = param;
|
||||
taosMemoryFree(pMsg->pData);
|
||||
|
@ -133,7 +132,7 @@ int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SRequestObj* pRequest = param;
|
||||
|
||||
if (TSDB_CODE_MND_DB_NOT_EXIST == code) {
|
||||
|
@ -212,7 +211,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t processCreateSTableRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
assert(pMsg != NULL && param != NULL);
|
||||
SRequestObj* pRequest = param;
|
||||
|
||||
|
@ -230,7 +229,7 @@ int32_t processCreateSTableRsp(void* param, const SDataBuf* pMsg, int32_t code)
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SRequestObj* pRequest = param;
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
setErrno(pRequest, code);
|
||||
|
@ -251,7 +250,7 @@ int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t processAlterStbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SRequestObj* pRequest = param;
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
setErrno(pRequest, code);
|
||||
|
@ -308,13 +307,13 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) {
|
|||
blockDataEnsureCapacity(pBlock, numOfCfg);
|
||||
|
||||
for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) {
|
||||
SVariablesInfo *pInfo = taosArrayGet(pVars, i);
|
||||
SVariablesInfo* pInfo = taosArrayGet(pVars, i);
|
||||
|
||||
char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(name, pInfo->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
|
||||
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
|
||||
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
|
||||
colDataAppend(pColInfo, i, name, false);
|
||||
|
||||
|
||||
char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(value, pInfo->value, TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
|
||||
|
@ -324,14 +323,13 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) {
|
|||
pBlock->info.rows = numOfCfg;
|
||||
|
||||
*block = pBlock;
|
||||
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) {
|
||||
SSDataBlock* pBlock = NULL;
|
||||
int32_t code = buildShowVariablesBlock(pVars, &pBlock);
|
||||
int32_t code = buildShowVariablesBlock(pVars, &pBlock);
|
||||
if (code) {
|
||||
return code;
|
||||
}
|
||||
|
@ -351,19 +349,19 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) {
|
|||
(*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS);
|
||||
|
||||
int32_t len = 0;
|
||||
blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false);
|
||||
blockEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false);
|
||||
ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
|
||||
|
||||
blockDataDestroy(pBlock);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t processShowVariablesRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SRequestObj* pRequest = param;
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
setErrno(pRequest, code);
|
||||
} else {
|
||||
SShowVariablesRsp rsp = {0};
|
||||
SShowVariablesRsp rsp = {0};
|
||||
SRetrieveTableRsp* pRes = NULL;
|
||||
code = tDeserializeSShowVariablesRsp(pMsg->pData, pMsg->len, &rsp);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
|
@ -372,7 +370,7 @@ int32_t processShowVariablesRsp(void* param, const SDataBuf* pMsg, int32_t code)
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, false);
|
||||
}
|
||||
|
||||
|
||||
tFreeSShowVariablesRsp(&rsp);
|
||||
}
|
||||
|
||||
|
@ -384,7 +382,6 @@ int32_t processShowVariablesRsp(void* param, const SDataBuf* pMsg, int32_t code)
|
|||
return code;
|
||||
}
|
||||
|
||||
|
||||
__async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
|
||||
switch (msgType) {
|
||||
case TDMT_MND_CONNECT:
|
||||
|
|
|
@ -309,7 +309,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) {
|
|||
case SCHEMA_ACTION_ADD_COLUMN: {
|
||||
int n = sprintf(result, "alter stable `%s` add column ", action->alterSTable.sTableName);
|
||||
smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes);
|
||||
TAOS_RES *res = taos_query(info->taos->id, result); // TODO async doAsyncQuery
|
||||
TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); // TODO async doAsyncQuery
|
||||
code = taos_errno(res);
|
||||
const char *errStr = taos_errstr(res);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
@ -323,7 +323,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) {
|
|||
case SCHEMA_ACTION_ADD_TAG: {
|
||||
int n = sprintf(result, "alter stable `%s` add tag ", action->alterSTable.sTableName);
|
||||
smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes);
|
||||
TAOS_RES *res = taos_query(info->taos->id, result); // TODO async doAsyncQuery
|
||||
TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); // TODO async doAsyncQuery
|
||||
code = taos_errno(res);
|
||||
const char *errStr = taos_errstr(res);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
@ -337,7 +337,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) {
|
|||
case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: {
|
||||
int n = sprintf(result, "alter stable `%s` modify column ", action->alterSTable.sTableName);
|
||||
smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes);
|
||||
TAOS_RES *res = taos_query(info->taos->id, result); // TODO async doAsyncQuery
|
||||
TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); // TODO async doAsyncQuery
|
||||
code = taos_errno(res);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res));
|
||||
|
@ -350,7 +350,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) {
|
|||
case SCHEMA_ACTION_CHANGE_TAG_SIZE: {
|
||||
int n = sprintf(result, "alter stable `%s` modify tag ", action->alterSTable.sTableName);
|
||||
smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes);
|
||||
TAOS_RES *res = taos_query(info->taos->id, result); // TODO async doAsyncQuery
|
||||
TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); // TODO async doAsyncQuery
|
||||
code = taos_errno(res);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res));
|
||||
|
@ -405,7 +405,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) {
|
|||
pos--;
|
||||
++freeBytes;
|
||||
outBytes = snprintf(pos, freeBytes, ")");
|
||||
TAOS_RES *res = taos_query(info->taos->id, result);
|
||||
TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result);
|
||||
code = taos_errno(res);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res));
|
||||
|
@ -2436,7 +2436,13 @@ static void smlInsertCallback(void *param, void *res, int32_t code) {
|
|||
*/
|
||||
|
||||
TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) {
|
||||
STscObj* pTscObj = acquireTscObj(*(int64_t*)taos);
|
||||
if (NULL == taos) {
|
||||
terrno = TSDB_CODE_TSC_DISCONNECTED;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int64_t rid = *(int64_t*)taos;
|
||||
STscObj* pTscObj = acquireTscObj(rid);
|
||||
if (NULL == pTscObj) {
|
||||
terrno = TSDB_CODE_TSC_DISCONNECTED;
|
||||
uError("SML:taos_schemaless_insert invalid taos");
|
||||
|
@ -2445,15 +2451,15 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
|
|||
|
||||
SRequestObj* request = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT);
|
||||
if(!request){
|
||||
releaseTscObj(*(int64_t*)taos);
|
||||
releaseTscObj(rid);
|
||||
uError("SML:taos_schemaless_insert error request is null");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int batchs = 0;
|
||||
pTscObj->schemalessType = 1;
|
||||
SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf};
|
||||
|
||||
int cnt = ceil(((double)numLines) / LINE_BATCH);
|
||||
Params params;
|
||||
params.request = request;
|
||||
tsem_init(¶ms.sem, 0, 0);
|
||||
|
@ -2490,7 +2496,16 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
|
|||
goto end;
|
||||
}
|
||||
|
||||
for (int i = 0; i < cnt; ++i) {
|
||||
if(protocol == TSDB_SML_JSON_PROTOCOL){
|
||||
numLines = 1;
|
||||
}else if(numLines <= 0){
|
||||
request->code = TSDB_CODE_SML_INVALID_DATA;
|
||||
smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL);
|
||||
goto end;
|
||||
}
|
||||
|
||||
batchs = ceil(((double)numLines) / LINE_BATCH);
|
||||
for (int i = 0; i < batchs; ++i) {
|
||||
SRequestObj* req = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT);
|
||||
if(!req){
|
||||
request->code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
@ -2533,6 +2548,6 @@ end:
|
|||
// ((STscObj *)taos)->schemalessType = 0;
|
||||
pTscObj->schemalessType = 1;
|
||||
uDebug("resultend:%s", request->msgBuf);
|
||||
releaseTscObj(*(int64_t*)taos);
|
||||
releaseTscObj(rid);
|
||||
return (TAOS_RES*)request;
|
||||
}
|
||||
|
|
|
@ -54,7 +54,8 @@ struct tmq_conf_t {
|
|||
int8_t autoCommit;
|
||||
int8_t resetOffset;
|
||||
int8_t withTbName;
|
||||
int8_t useSnapshot;
|
||||
int8_t spEnable;
|
||||
int32_t spBatchSize;
|
||||
uint16_t port;
|
||||
int32_t autoCommitInterval;
|
||||
char* ip;
|
||||
|
@ -126,8 +127,10 @@ typedef struct {
|
|||
// statistics
|
||||
int64_t pollCnt;
|
||||
// offset
|
||||
int64_t committedOffset;
|
||||
int64_t currentOffset;
|
||||
/*int64_t committedOffset;*/
|
||||
/*int64_t currentOffset;*/
|
||||
STqOffsetVal committedOffsetNew;
|
||||
STqOffsetVal currentOffsetNew;
|
||||
// connection info
|
||||
int32_t vgId;
|
||||
int32_t vgStatus;
|
||||
|
@ -152,8 +155,8 @@ typedef struct {
|
|||
SMqClientVg* vgHandle;
|
||||
SMqClientTopic* topicHandle;
|
||||
union {
|
||||
SMqDataBlkRsp dataRsp;
|
||||
SMqMetaRsp metaRsp;
|
||||
SMqDataRsp dataRsp;
|
||||
SMqMetaRsp metaRsp;
|
||||
};
|
||||
} SMqPollRspWrapper;
|
||||
|
||||
|
@ -179,6 +182,7 @@ typedef struct {
|
|||
tsem_t rspSem;
|
||||
} SMqPollCbParam;
|
||||
|
||||
#if 0
|
||||
typedef struct {
|
||||
tmq_t* tmq;
|
||||
int8_t async;
|
||||
|
@ -190,12 +194,13 @@ typedef struct {
|
|||
SArray* offsets;
|
||||
void* userParam;
|
||||
} SMqCommitCbParam;
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
tmq_t* tmq;
|
||||
int8_t automatic;
|
||||
int8_t async;
|
||||
int8_t freeOffsets;
|
||||
tmq_t* tmq;
|
||||
int8_t automatic;
|
||||
int8_t async;
|
||||
/*int8_t freeOffsets;*/
|
||||
int32_t waitingRspNum;
|
||||
int32_t totalRspNum;
|
||||
int32_t rspErr;
|
||||
|
@ -284,18 +289,23 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
|
|||
}
|
||||
}
|
||||
|
||||
if (strcmp(key, "experiment.use.snapshot") == 0) {
|
||||
if (strcmp(key, "experimental.snapshot.enable") == 0) {
|
||||
if (strcmp(value, "true") == 0) {
|
||||
conf->useSnapshot = true;
|
||||
conf->spEnable = true;
|
||||
return TMQ_CONF_OK;
|
||||
} else if (strcmp(value, "false") == 0) {
|
||||
conf->useSnapshot = false;
|
||||
conf->spEnable = false;
|
||||
return TMQ_CONF_OK;
|
||||
} else {
|
||||
return TMQ_CONF_INVALID;
|
||||
}
|
||||
}
|
||||
|
||||
if (strcmp(key, "experimental.snapshot.batch.size") == 0) {
|
||||
conf->spBatchSize = atoi(value);
|
||||
return TMQ_CONF_OK;
|
||||
}
|
||||
|
||||
if (strcmp(key, "td.connect.ip") == 0) {
|
||||
conf->ip = strdup(value);
|
||||
return TMQ_CONF_OK;
|
||||
|
@ -351,6 +361,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
|
|||
return sprintf(dst, "%s:%d", topicName, vg);
|
||||
}
|
||||
|
||||
#if 0
|
||||
int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
|
||||
pParam->rspErr = code;
|
||||
|
@ -371,8 +382,9 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
}
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) {
|
||||
int32_t tmqCommitCb2(void* param, SDataBuf* pBuf, int32_t code) {
|
||||
SMqCommitCbParam2* pParam = (SMqCommitCbParam2*)param;
|
||||
SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params;
|
||||
// push into array
|
||||
|
@ -413,139 +425,148 @@ int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pTopic, SMqCommitCbParamSet* pParamSet) {
|
||||
STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset));
|
||||
if (pOffset == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
pOffset->val = pVg->currentOffsetNew;
|
||||
|
||||
int32_t groupLen = strlen(tmq->groupId);
|
||||
memcpy(pOffset->subKey, tmq->groupId, groupLen);
|
||||
pOffset->subKey[groupLen] = TMQ_SEPARATOR;
|
||||
strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName);
|
||||
|
||||
int32_t len;
|
||||
int32_t code;
|
||||
tEncodeSize(tEncodeSTqOffset, pOffset, len, code);
|
||||
if (code < 0) {
|
||||
ASSERT(0);
|
||||
return -1;
|
||||
}
|
||||
void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
|
||||
if (buf == NULL) return -1;
|
||||
((SMsgHead*)buf)->vgId = htonl(pVg->vgId);
|
||||
|
||||
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
|
||||
|
||||
SEncoder encoder;
|
||||
tEncoderInit(&encoder, abuf, len);
|
||||
tEncodeSTqOffset(&encoder, pOffset);
|
||||
|
||||
// build param
|
||||
SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2));
|
||||
pParam->params = pParamSet;
|
||||
pParam->pOffset = pOffset;
|
||||
|
||||
// build send info
|
||||
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
|
||||
if (pMsgSendInfo == NULL) {
|
||||
return -1;
|
||||
}
|
||||
pMsgSendInfo->msgInfo = (SDataBuf){
|
||||
.pData = buf,
|
||||
.len = sizeof(SMsgHead) + len,
|
||||
.handle = NULL,
|
||||
};
|
||||
|
||||
tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, pVg->vgId,
|
||||
pOffset->val.version);
|
||||
|
||||
// TODO: put into cb
|
||||
pVg->committedOffsetNew = pVg->currentOffsetNew;
|
||||
|
||||
pMsgSendInfo->requestId = generateRequestId();
|
||||
pMsgSendInfo->requestObjRefId = 0;
|
||||
pMsgSendInfo->param = pParam;
|
||||
pMsgSendInfo->fp = tmqCommitCb2;
|
||||
pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET;
|
||||
// send msg
|
||||
|
||||
int64_t transporterId = 0;
|
||||
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo);
|
||||
pParamSet->waitingRspNum++;
|
||||
pParamSet->totalRspNum++;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_commit_cb* userCb, void* userParam) {
|
||||
char* topic;
|
||||
int32_t vgId;
|
||||
ASSERT(msg != NULL);
|
||||
if (TD_RES_TMQ(msg)) {
|
||||
SMqRspObj* pRspObj = (SMqRspObj*)msg;
|
||||
topic = pRspObj->topic;
|
||||
vgId = pRspObj->vgId;
|
||||
} else if (TD_RES_TMQ_META(msg)) {
|
||||
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg;
|
||||
topic = pMetaRspObj->topic;
|
||||
vgId = pMetaRspObj->vgId;
|
||||
} else {
|
||||
return TSDB_CODE_TMQ_INVALID_MSG;
|
||||
}
|
||||
|
||||
SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
|
||||
if (pParamSet == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
pParamSet->tmq = tmq;
|
||||
pParamSet->automatic = 0;
|
||||
pParamSet->async = async;
|
||||
/*pParamSet->freeOffsets = 1;*/
|
||||
pParamSet->userCb = userCb;
|
||||
pParamSet->userParam = userParam;
|
||||
tsem_init(&pParamSet->rspSem, 0, 0);
|
||||
|
||||
int32_t code = -1;
|
||||
|
||||
for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
|
||||
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
|
||||
if (strcmp(pTopic->topicName, topic) != 0) continue;
|
||||
for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
|
||||
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
|
||||
if (pVg->vgId != vgId) continue;
|
||||
|
||||
if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) {
|
||||
if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) {
|
||||
goto FAIL;
|
||||
}
|
||||
goto HANDLE_RSP;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
HANDLE_RSP:
|
||||
if (pParamSet->totalRspNum == 0) {
|
||||
tsem_destroy(&pParamSet->rspSem);
|
||||
taosMemoryFree(pParamSet);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!async) {
|
||||
tsem_wait(&pParamSet->rspSem);
|
||||
code = pParamSet->rspErr;
|
||||
tsem_destroy(&pParamSet->rspSem);
|
||||
return code;
|
||||
} else {
|
||||
code = 0;
|
||||
}
|
||||
|
||||
FAIL:
|
||||
if (code != 0 && async) {
|
||||
userCb(tmq, code, userParam);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb,
|
||||
void* userParam) {
|
||||
int32_t code = -1;
|
||||
|
||||
if (msg != NULL) {
|
||||
char* topic;
|
||||
int32_t vgId;
|
||||
if (TD_RES_TMQ(msg)) {
|
||||
SMqRspObj* pRspObj = (SMqRspObj*)msg;
|
||||
topic = pRspObj->topic;
|
||||
vgId = pRspObj->vgId;
|
||||
} else if (TD_RES_TMQ_META(msg)) {
|
||||
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg;
|
||||
topic = pMetaRspObj->topic;
|
||||
vgId = pMetaRspObj->vgId;
|
||||
} else {
|
||||
return TSDB_CODE_TMQ_INVALID_MSG;
|
||||
}
|
||||
|
||||
SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
|
||||
if (pParamSet == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
pParamSet->tmq = tmq;
|
||||
pParamSet->automatic = automatic;
|
||||
pParamSet->async = async;
|
||||
pParamSet->freeOffsets = 1;
|
||||
pParamSet->userCb = userCb;
|
||||
pParamSet->userParam = userParam;
|
||||
tsem_init(&pParamSet->rspSem, 0, 0);
|
||||
|
||||
for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
|
||||
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
|
||||
if (strcmp(pTopic->topicName, topic) == 0) {
|
||||
for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
|
||||
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
|
||||
if (pVg->vgId == vgId) {
|
||||
if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) {
|
||||
tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld",
|
||||
tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset));
|
||||
if (pOffset == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
pOffset->type = TMQ_OFFSET__LOG;
|
||||
pOffset->version = pVg->currentOffset;
|
||||
|
||||
int32_t groupLen = strlen(tmq->groupId);
|
||||
memcpy(pOffset->subKey, tmq->groupId, groupLen);
|
||||
pOffset->subKey[groupLen] = TMQ_SEPARATOR;
|
||||
strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName);
|
||||
|
||||
int32_t len;
|
||||
int32_t code;
|
||||
tEncodeSize(tEncodeSTqOffset, pOffset, len, code);
|
||||
if (code < 0) {
|
||||
ASSERT(0);
|
||||
}
|
||||
void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
|
||||
((SMsgHead*)buf)->vgId = htonl(pVg->vgId);
|
||||
|
||||
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
|
||||
|
||||
SEncoder encoder;
|
||||
tEncoderInit(&encoder, abuf, len);
|
||||
tEncodeSTqOffset(&encoder, pOffset);
|
||||
|
||||
// build param
|
||||
SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2));
|
||||
pParam->params = pParamSet;
|
||||
pParam->pOffset = pOffset;
|
||||
|
||||
// build send info
|
||||
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
|
||||
if (pMsgSendInfo == NULL) {
|
||||
// TODO
|
||||
continue;
|
||||
}
|
||||
pMsgSendInfo->msgInfo = (SDataBuf){
|
||||
.pData = buf,
|
||||
.len = sizeof(SMsgHead) + len,
|
||||
.handle = NULL,
|
||||
};
|
||||
|
||||
tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey,
|
||||
pVg->vgId, pOffset->version);
|
||||
|
||||
// TODO: put into cb
|
||||
pVg->committedOffset = pVg->currentOffset;
|
||||
|
||||
pMsgSendInfo->requestId = generateRequestId();
|
||||
pMsgSendInfo->requestObjRefId = 0;
|
||||
pMsgSendInfo->param = pParam;
|
||||
pMsgSendInfo->fp = tmqCommitCb2;
|
||||
pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET;
|
||||
// send msg
|
||||
|
||||
int64_t transporterId = 0;
|
||||
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo);
|
||||
pParamSet->waitingRspNum++;
|
||||
pParamSet->totalRspNum++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (pParamSet->totalRspNum == 0) {
|
||||
tsem_destroy(&pParamSet->rspSem);
|
||||
taosMemoryFree(pParamSet);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!async) {
|
||||
tsem_wait(&pParamSet->rspSem);
|
||||
code = pParamSet->rspErr;
|
||||
tsem_destroy(&pParamSet->rspSem);
|
||||
} else {
|
||||
code = 0;
|
||||
}
|
||||
|
||||
if (code != 0 && async) {
|
||||
if (automatic) {
|
||||
tmq->commitCb(tmq, code, tmq->commitCbUserParam);
|
||||
} else {
|
||||
userCb(tmq, code, userParam);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam);
|
||||
}
|
||||
|
||||
SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
|
||||
|
@ -556,7 +577,7 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_
|
|||
pParamSet->tmq = tmq;
|
||||
pParamSet->automatic = automatic;
|
||||
pParamSet->async = async;
|
||||
pParamSet->freeOffsets = 1;
|
||||
/*pParamSet->freeOffsets = 1;*/
|
||||
pParamSet->userCb = userCb;
|
||||
pParamSet->userParam = userParam;
|
||||
tsem_init(&pParamSet->rspSem, 0, 0);
|
||||
|
@ -572,75 +593,11 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_
|
|||
|
||||
tscDebug("consumer %ld begin commit for topic %s, vgId %d", tmq->consumerId, pTopic->topicName, pVg->vgId);
|
||||
|
||||
/*if (pVg->currentOffset < 0) {*/
|
||||
if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) {
|
||||
tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld",
|
||||
tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset);
|
||||
|
||||
continue;
|
||||
if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) {
|
||||
if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset));
|
||||
if (pOffset == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
pOffset->type = TMQ_OFFSET__LOG;
|
||||
pOffset->version = pVg->currentOffset;
|
||||
|
||||
int32_t groupLen = strlen(tmq->groupId);
|
||||
memcpy(pOffset->subKey, tmq->groupId, groupLen);
|
||||
pOffset->subKey[groupLen] = TMQ_SEPARATOR;
|
||||
strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName);
|
||||
|
||||
int32_t len;
|
||||
int32_t code;
|
||||
tEncodeSize(tEncodeSTqOffset, pOffset, len, code);
|
||||
if (code < 0) {
|
||||
ASSERT(0);
|
||||
}
|
||||
void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
|
||||
((SMsgHead*)buf)->vgId = htonl(pVg->vgId);
|
||||
|
||||
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
|
||||
|
||||
SEncoder encoder;
|
||||
tEncoderInit(&encoder, abuf, len);
|
||||
tEncodeSTqOffset(&encoder, pOffset);
|
||||
|
||||
// build param
|
||||
SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2));
|
||||
pParam->params = pParamSet;
|
||||
pParam->pOffset = pOffset;
|
||||
|
||||
// build send info
|
||||
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
|
||||
if (pMsgSendInfo == NULL) {
|
||||
// TODO
|
||||
continue;
|
||||
}
|
||||
pMsgSendInfo->msgInfo = (SDataBuf){
|
||||
.pData = buf,
|
||||
.len = sizeof(SMsgHead) + len,
|
||||
.handle = NULL,
|
||||
};
|
||||
|
||||
tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, pVg->vgId,
|
||||
pOffset->version);
|
||||
|
||||
// TODO: put into cb
|
||||
pVg->committedOffset = pVg->currentOffset;
|
||||
|
||||
pMsgSendInfo->requestId = generateRequestId();
|
||||
pMsgSendInfo->requestObjRefId = 0;
|
||||
pMsgSendInfo->param = pParam;
|
||||
pMsgSendInfo->fp = tmqCommitCb2;
|
||||
pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET;
|
||||
// send msg
|
||||
|
||||
int64_t transporterId = 0;
|
||||
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo);
|
||||
pParamSet->waitingRspNum++;
|
||||
pParamSet->totalRspNum++;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -862,7 +819,7 @@ void tmqClearUnhandleMsg(tmq_t* tmq) {
|
|||
}
|
||||
}
|
||||
|
||||
int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
|
||||
pParam->rspErr = code;
|
||||
/*tmq_t* tmq = pParam->tmq;*/
|
||||
|
@ -967,7 +924,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
|
|||
strcpy(pTmq->clientId, conf->clientId);
|
||||
strcpy(pTmq->groupId, conf->groupId);
|
||||
pTmq->withTbName = conf->withTbName;
|
||||
pTmq->useSnapshot = conf->useSnapshot;
|
||||
pTmq->useSnapshot = conf->spEnable;
|
||||
pTmq->autoCommit = conf->autoCommit;
|
||||
pTmq->autoCommitInterval = conf->autoCommitInterval;
|
||||
pTmq->commitCb = conf->commitCb;
|
||||
|
@ -1116,7 +1073,7 @@ int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
|
|||
}
|
||||
#endif
|
||||
|
||||
int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SMqPollCbParam* pParam = (SMqPollCbParam*)param;
|
||||
SMqClientVg* pVg = pParam->pVg;
|
||||
SMqClientTopic* pTopic = pParam->pTopic;
|
||||
|
@ -1173,8 +1130,11 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
pRspWrapper->topicHandle = pTopic;
|
||||
|
||||
if (rspType == TMQ_MSG_TYPE__POLL_RSP) {
|
||||
SDecoder decoder;
|
||||
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
|
||||
tDecodeSMqDataRsp(&decoder, &pRspWrapper->dataRsp);
|
||||
memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead));
|
||||
tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp);
|
||||
/*tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp);*/
|
||||
} else {
|
||||
ASSERT(rspType == TMQ_MSG_TYPE__POLL_META_RSP);
|
||||
memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead));
|
||||
|
@ -1184,7 +1144,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
taosMemoryFree(pMsg->pData);
|
||||
|
||||
tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld, type %d", tmq->consumerId, pVg->vgId,
|
||||
pRspWrapper->dataRsp.reqOffset, pRspWrapper->dataRsp.rspOffset, rspType);
|
||||
pRspWrapper->dataRsp.reqOffset.version, pRspWrapper->dataRsp.rspOffset.version, rspType);
|
||||
|
||||
taosWriteQitem(tmq->mqueue, pRspWrapper);
|
||||
tsem_post(&tmq->rspSem);
|
||||
|
@ -1226,9 +1186,11 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
|
|||
for (int32_t j = 0; j < vgNumCur; j++) {
|
||||
SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j);
|
||||
sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId);
|
||||
tscDebug("consumer %ld epoch %d vg %d vgKey is %s, offset is %ld", tmq->consumerId, epoch, pVgCur->vgId, vgKey,
|
||||
pVgCur->currentOffset);
|
||||
taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t));
|
||||
char buf[50];
|
||||
tFormatOffset(buf, 50, &pVgCur->currentOffsetNew);
|
||||
tscDebug("consumer %ld epoch %d vg %d vgKey is %s, offset is %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey,
|
||||
buf);
|
||||
taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffsetNew, sizeof(STqOffsetVal));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1247,17 +1209,17 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
|
|||
for (int32_t j = 0; j < vgNumGet; j++) {
|
||||
SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j);
|
||||
sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId);
|
||||
int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey));
|
||||
int64_t offset = tmq->resetOffsetCfg;
|
||||
STqOffsetVal* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey));
|
||||
STqOffsetVal offsetNew = {.type = tmq->resetOffsetCfg};
|
||||
if (pOffset != NULL) {
|
||||
offset = *pOffset;
|
||||
offsetNew = *pOffset;
|
||||
}
|
||||
|
||||
tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld, vgKey is %s", tmq->consumerId, epoch,
|
||||
pVgEp->vgId, offset, vgKey);
|
||||
/*tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld, vgKey is %s", tmq->consumerId, epoch,*/
|
||||
/*pVgEp->vgId, offset, vgKey);*/
|
||||
SMqClientVg clientVg = {
|
||||
.pollCnt = 0,
|
||||
.currentOffset = offset,
|
||||
.currentOffsetNew = offsetNew,
|
||||
.vgId = pVgEp->vgId,
|
||||
.epSet = pVgEp->epSet,
|
||||
.vgStatus = TMQ_VG_STATUS__IDLE,
|
||||
|
@ -1281,7 +1243,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
|
|||
return set;
|
||||
}
|
||||
|
||||
#if 1
|
||||
#if 0
|
||||
bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
|
||||
/*printf("call update ep %d\n", epoch);*/
|
||||
bool set = false;
|
||||
|
@ -1368,7 +1330,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
|
|||
}
|
||||
#endif
|
||||
|
||||
int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
|
||||
tmq_t* tmq = pParam->tmq;
|
||||
int8_t async = pParam->async;
|
||||
|
@ -1516,16 +1478,16 @@ int32_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
|
|||
#endif
|
||||
|
||||
SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) {
|
||||
int64_t reqOffset;
|
||||
if (pVg->currentOffset >= 0) {
|
||||
reqOffset = pVg->currentOffset;
|
||||
} else {
|
||||
/*if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {*/
|
||||
/*tscError("unable to poll since no committed offset but reset offset is set to none");*/
|
||||
/*return NULL;*/
|
||||
/*}*/
|
||||
reqOffset = tmq->resetOffsetCfg;
|
||||
}
|
||||
/*int64_t reqOffset;*/
|
||||
/*if (pVg->currentOffset >= 0) {*/
|
||||
/*reqOffset = pVg->currentOffset;*/
|
||||
/*} else {*/
|
||||
/*if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {*/
|
||||
/*tscError("unable to poll since no committed offset but reset offset is set to none");*/
|
||||
/*return NULL;*/
|
||||
/*}*/
|
||||
/*reqOffset = tmq->resetOffsetCfg;*/
|
||||
/*}*/
|
||||
|
||||
SMqPollReq* pReq = taosMemoryCalloc(1, sizeof(SMqPollReq));
|
||||
if (pReq == NULL) {
|
||||
|
@ -1544,7 +1506,8 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic*
|
|||
pReq->timeout = timeout;
|
||||
pReq->consumerId = tmq->consumerId;
|
||||
pReq->epoch = tmq->epoch;
|
||||
pReq->currentOffset = reqOffset;
|
||||
/*pReq->currentOffset = reqOffset;*/
|
||||
pReq->reqOffset = pVg->currentOffsetNew;
|
||||
pReq->reqId = generateRequestId();
|
||||
|
||||
pReq->useSnapshot = tmq->useSnapshot;
|
||||
|
@ -1572,7 +1535,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
|
|||
tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
|
||||
pRspObj->vgId = pWrapper->vgHandle->vgId;
|
||||
pRspObj->resIter = -1;
|
||||
memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataBlkRsp));
|
||||
memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
|
||||
|
||||
pRspObj->resInfo.totalRows = 0;
|
||||
pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
|
||||
|
@ -1584,7 +1547,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
|
|||
}
|
||||
|
||||
int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
|
||||
/*printf("call poll\n");*/
|
||||
/*tscDebug("call poll");*/
|
||||
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
|
||||
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
|
||||
for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
|
||||
|
@ -1645,8 +1608,11 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
|
|||
|
||||
int64_t transporterId = 0;
|
||||
/*printf("send poll\n");*/
|
||||
tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId,
|
||||
pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId);
|
||||
|
||||
char offsetFormatBuf[50];
|
||||
tFormatOffset(offsetFormatBuf, 50, &pVg->currentOffsetNew);
|
||||
tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %s, reqId %lu", tmq->consumerId,
|
||||
pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId);
|
||||
/*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
|
||||
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
|
||||
pVg->pollCnt++;
|
||||
|
@ -1695,7 +1661,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
|
|||
if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) {
|
||||
SMqClientVg* pVg = pollRspWrapper->vgHandle;
|
||||
/*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
|
||||
pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset;
|
||||
pVg->currentOffsetNew = pollRspWrapper->dataRsp.rspOffset;
|
||||
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
|
||||
if (pollRspWrapper->dataRsp.blockNum == 0) {
|
||||
taosFreeQitem(pollRspWrapper);
|
||||
|
@ -1717,7 +1683,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
|
|||
if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) {
|
||||
SMqClientVg* pVg = pollRspWrapper->vgHandle;
|
||||
/*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
|
||||
pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset;
|
||||
pVg->currentOffsetNew = pollRspWrapper->metaRsp.rspOffsetNew;
|
||||
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
|
||||
// build rsp
|
||||
SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper);
|
||||
|
@ -1742,6 +1708,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
|
|||
}
|
||||
|
||||
TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
|
||||
/*tscDebug("call poll1");*/
|
||||
void* rspObj;
|
||||
int64_t startTime = taosGetTimestampMs();
|
||||
|
||||
|
|
|
@ -1613,7 +1613,8 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) {
|
|||
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
|
||||
|
||||
int32_t rows = pDataBlock->info.rows;
|
||||
printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
|
||||
printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type,
|
||||
pDataBlock->info.childId, pDataBlock->info.groupId);
|
||||
for (int32_t j = 0; j < rows; j++) {
|
||||
printf("%s |", flag);
|
||||
for (int32_t k = 0; k < numOfCols; k++) {
|
||||
|
@ -1662,8 +1663,8 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
|
|||
int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock);
|
||||
int32_t rows = pDataBlock->info.rows;
|
||||
int32_t len = 0;
|
||||
len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|\n", flag,
|
||||
(int32_t)pDataBlock->info.type, pDataBlock->info.childId);
|
||||
len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id %lu|\n", flag,
|
||||
(int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
|
||||
for (int32_t j = 0; j < rows; j++) {
|
||||
len += snprintf(dumpBuf + len, size - len, "%s |", flag);
|
||||
for (int32_t k = 0; k < colNum; k++) {
|
||||
|
@ -1902,8 +1903,7 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) {
|
|||
return rname.childTableName;
|
||||
}
|
||||
|
||||
void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
|
||||
int8_t needCompress) {
|
||||
void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) {
|
||||
// todo extract method
|
||||
int32_t* actualLen = (int32_t*)data;
|
||||
data += sizeof(int32_t);
|
||||
|
@ -1913,6 +1913,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
|
|||
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
|
||||
|
||||
*((int16_t*)data) = pColInfoData->info.type;
|
||||
data += sizeof(int16_t);
|
||||
|
||||
|
@ -1960,7 +1961,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
|
|||
*groupId = pBlock->info.groupId;
|
||||
}
|
||||
|
||||
const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) {
|
||||
const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) {
|
||||
const char* pStart = pData;
|
||||
|
||||
int32_t dataLen = *(int32_t*)pStart;
|
||||
|
|
|
@ -4309,6 +4309,7 @@ int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pR
|
|||
if (tEncodeU64(&encoder, status->queryId) < 0) return -1;
|
||||
if (tEncodeU64(&encoder, status->taskId) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, status->refId) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, status->execId) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, status->status) < 0) return -1;
|
||||
}
|
||||
} else {
|
||||
|
@ -4339,6 +4340,7 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *
|
|||
if (tDecodeU64(&decoder, &status.queryId) < 0) return -1;
|
||||
if (tDecodeU64(&decoder, &status.taskId) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &status.refId) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &status.execId) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &status.status) < 0) return -1;
|
||||
taosArrayPush(pRsp->taskStatus, &status);
|
||||
}
|
||||
|
@ -4641,6 +4643,7 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
|
|||
if (tEncodeI8(&encoder, pReq->triggerType) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pReq->maxDelay) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pReq->watermark) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->igExpired) < 0) return -1;
|
||||
if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
|
||||
if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
|
||||
|
||||
|
@ -4668,6 +4671,7 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
|
|||
if (tDecodeI8(&decoder, &pReq->triggerType) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &pReq->maxDelay) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &pReq->watermark) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->igExpired) < 0) return -1;
|
||||
|
||||
if (sqlLen > 0) {
|
||||
pReq->sql = taosMemoryCalloc(1, sqlLen + 1);
|
||||
|
@ -5357,30 +5361,148 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) {
|
|||
}
|
||||
}
|
||||
|
||||
int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) {
|
||||
if (tEncodeI8(pEncoder, pOffset->type) < 0) return -1;
|
||||
if (pOffset->type == TMQ_OFFSET__SNAPSHOT) {
|
||||
if (tEncodeI64(pEncoder, pOffset->uid) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pOffset->ts) < 0) return -1;
|
||||
} else if (pOffset->type == TMQ_OFFSET__LOG) {
|
||||
if (tEncodeI64(pEncoder, pOffset->version) < 0) return -1;
|
||||
int32_t tEncodeSTqOffsetVal(SEncoder *pEncoder, const STqOffsetVal *pOffsetVal) {
|
||||
if (tEncodeI8(pEncoder, pOffsetVal->type) < 0) return -1;
|
||||
if (pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_DATA) {
|
||||
if (tEncodeI64(pEncoder, pOffsetVal->uid) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pOffsetVal->ts) < 0) return -1;
|
||||
} else if (pOffsetVal->type == TMQ_OFFSET__LOG) {
|
||||
if (tEncodeI64(pEncoder, pOffsetVal->version) < 0) return -1;
|
||||
} else if (pOffsetVal->type < 0) {
|
||||
// do nothing
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) {
|
||||
if (tDecodeI8(pDecoder, &pOffsetVal->type) < 0) return -1;
|
||||
if (pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_DATA) {
|
||||
if (tDecodeI64(pDecoder, &pOffsetVal->uid) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pOffsetVal->ts) < 0) return -1;
|
||||
} else if (pOffsetVal->type == TMQ_OFFSET__LOG) {
|
||||
if (tDecodeI64(pDecoder, &pOffsetVal->version) < 0) return -1;
|
||||
} else if (pOffsetVal->type < 0) {
|
||||
// do nothing
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#if 1
|
||||
int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
|
||||
if (pVal->type == TMQ_OFFSET__RESET_NONE) {
|
||||
snprintf(buf, maxLen, "offset(reset to none)");
|
||||
} else if (pVal->type == TMQ_OFFSET__RESET_EARLIEAST) {
|
||||
snprintf(buf, maxLen, "offset(reset to earlieast)");
|
||||
} else if (pVal->type == TMQ_OFFSET__RESET_LATEST) {
|
||||
snprintf(buf, maxLen, "offset(reset to latest)");
|
||||
} else if (pVal->type == TMQ_OFFSET__LOG) {
|
||||
snprintf(buf, maxLen, "offset(log) ver:%ld", pVal->version);
|
||||
} else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA) {
|
||||
snprintf(buf, maxLen, "offset(ss data) uid:%ld, ts:%ld", pVal->uid, pVal->ts);
|
||||
} else if (pVal->type == TMQ_OFFSET__SNAPSHOT_META) {
|
||||
snprintf(buf, maxLen, "offset(ss meta) uid:%ld, ts:%ld", pVal->uid, pVal->ts);
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
bool tOffsetEqual(const STqOffsetVal *pLeft, const STqOffsetVal *pRight) {
|
||||
if (pLeft->type == pRight->type) {
|
||||
if (pLeft->type == TMQ_OFFSET__LOG) {
|
||||
return pLeft->version == pRight->version;
|
||||
} else if (pLeft->type == TMQ_OFFSET__SNAPSHOT_DATA) {
|
||||
return pLeft->uid == pRight->uid && pLeft->ts == pRight->ts;
|
||||
} else if (pLeft->type == TMQ_OFFSET__SNAPSHOT_META) {
|
||||
ASSERT(0);
|
||||
// TODO
|
||||
return pLeft->uid == pRight->uid && pLeft->ts == pRight->ts;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) {
|
||||
if (tEncodeSTqOffsetVal(pEncoder, &pOffset->val) < 0) return -1;
|
||||
if (tEncodeCStr(pEncoder, pOffset->subKey) < 0) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) {
|
||||
if (tDecodeI8(pDecoder, &pOffset->type) < 0) return -1;
|
||||
if (pOffset->type == TMQ_OFFSET__SNAPSHOT) {
|
||||
if (tDecodeI64(pDecoder, &pOffset->uid) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pOffset->ts) < 0) return -1;
|
||||
} else if (pOffset->type == TMQ_OFFSET__LOG) {
|
||||
if (tDecodeI64(pDecoder, &pOffset->version) < 0) return -1;
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
if (tDecodeSTqOffsetVal(pDecoder, &pOffset->val) < 0) return -1;
|
||||
if (tDecodeCStrTo(pDecoder, pOffset->subKey) < 0) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
|
||||
if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1;
|
||||
if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->skipLogNum) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1;
|
||||
if (pRsp->blockNum != 0) {
|
||||
if (tEncodeI8(pEncoder, pRsp->withTbName) < 0) return -1;
|
||||
if (tEncodeI8(pEncoder, pRsp->withSchema) < 0) return -1;
|
||||
|
||||
for (int32_t i = 0; i < pRsp->blockNum; i++) {
|
||||
int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i);
|
||||
void *data = taosArrayGetP(pRsp->blockData, i);
|
||||
if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1;
|
||||
if (pRsp->withSchema) {
|
||||
SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i);
|
||||
if (tEncodeSSchemaWrapper(pEncoder, pSW) < 0) return -1;
|
||||
}
|
||||
if (pRsp->withTbName) {
|
||||
char *tbName = (char *)taosArrayGetP(pRsp->blockTbName, i);
|
||||
if (tEncodeCStr(pEncoder, tbName) < 0) return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
|
||||
if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1;
|
||||
if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->skipLogNum) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1;
|
||||
if (pRsp->blockNum != 0) {
|
||||
pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void *));
|
||||
pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t));
|
||||
if (tDecodeI8(pDecoder, &pRsp->withTbName) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, &pRsp->withSchema) < 0) return -1;
|
||||
if (pRsp->withTbName) {
|
||||
pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void *));
|
||||
}
|
||||
if (pRsp->withSchema) {
|
||||
pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void *));
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < pRsp->blockNum; i++) {
|
||||
void *data;
|
||||
uint64_t bLen;
|
||||
if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1;
|
||||
taosArrayPush(pRsp->blockData, &data);
|
||||
int32_t len = bLen;
|
||||
taosArrayPush(pRsp->blockDataLen, &len);
|
||||
|
||||
if (pRsp->withSchema) {
|
||||
SSchemaWrapper *pSW = (SSchemaWrapper *)taosMemoryCalloc(1, sizeof(SSchemaWrapper));
|
||||
if (pSW == NULL) return -1;
|
||||
if (tDecodeSSchemaWrapper(pDecoder, pSW) < 0) return -1;
|
||||
taosArrayPush(pRsp->blockSchema, &pSW);
|
||||
}
|
||||
|
||||
if (pRsp->withTbName) {
|
||||
char *tbName;
|
||||
if (tDecodeCStrAlloc(pDecoder, &tbName) < 0) return -1;
|
||||
taosArrayPush(pRsp->blockTbName, &tbName);
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -17,7 +17,6 @@
|
|||
#include "dmInt.h"
|
||||
#include "systable.h"
|
||||
|
||||
|
||||
extern SConfig *tsCfg;
|
||||
|
||||
static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) {
|
||||
|
@ -83,7 +82,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
|
|||
(*pMgmt->getVnodeLoadsFp)(&vinfo);
|
||||
req.pVloads = vinfo.pVloads;
|
||||
|
||||
SMonMloadInfo minfo = {0};
|
||||
SMonMloadInfo minfo = {0};
|
||||
(*pMgmt->getMnodeLoadsFp)(&minfo);
|
||||
req.mload = minfo.load;
|
||||
|
||||
|
@ -186,10 +185,10 @@ int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
SSDataBlock* dmBuildVariablesBlock(void) {
|
||||
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
|
||||
SSDataBlock *dmBuildVariablesBlock(void) {
|
||||
SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
|
||||
size_t size = 0;
|
||||
const SSysTableMeta* pMeta = NULL;
|
||||
const SSysTableMeta *pMeta = NULL;
|
||||
getInfosDbMeta(&pMeta, &size);
|
||||
|
||||
int32_t index = 0;
|
||||
|
@ -215,7 +214,7 @@ SSDataBlock* dmBuildVariablesBlock(void) {
|
|||
return pBlock;
|
||||
}
|
||||
|
||||
int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) {
|
||||
int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) {
|
||||
int32_t numOfCfg = taosArrayGetSize(tsCfg->array);
|
||||
int32_t numOfRows = 0;
|
||||
blockDataEnsureCapacity(pBlock, numOfCfg);
|
||||
|
@ -230,8 +229,8 @@ int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) {
|
|||
STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
|
||||
colDataAppend(pColInfo, i, name, false);
|
||||
|
||||
char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
|
||||
char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
int32_t valueLen = 0;
|
||||
cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen);
|
||||
varDataSetLen(value, valueLen);
|
||||
|
@ -241,9 +240,8 @@ int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) {
|
|||
numOfRows++;
|
||||
}
|
||||
|
||||
|
||||
pBlock->info.rows = numOfRows;
|
||||
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
@ -267,7 +265,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
SSDataBlock* pBlock = dmBuildVariablesBlock();
|
||||
SSDataBlock *pBlock = dmBuildVariablesBlock();
|
||||
|
||||
dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId);
|
||||
|
||||
|
@ -283,14 +281,14 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
char *pStart = pRsp->data;
|
||||
char *pStart = pRsp->data;
|
||||
*(int32_t *)pStart = htonl(numOfCols);
|
||||
pStart += sizeof(int32_t); // number of columns
|
||||
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
SSysTableSchema *pSchema = (SSysTableSchema *)pStart;
|
||||
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i);
|
||||
|
||||
|
||||
pSchema->bytes = htonl(pColInfo->info.bytes);
|
||||
pSchema->colId = htons(pColInfo->info.colId);
|
||||
pSchema->type = pColInfo->info.type;
|
||||
|
@ -299,7 +297,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
int32_t len = 0;
|
||||
blockCompressEncode(pBlock, pStart, &len, numOfCols, false);
|
||||
blockEncode(pBlock, pStart, &len, numOfCols, false);
|
||||
|
||||
pRsp->numOfRows = htonl(pBlock->info.rows);
|
||||
pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision
|
||||
|
|
|
@ -209,10 +209,11 @@ SArray *mmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_CONTINUE, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_FETCH, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
@ -220,7 +221,7 @@ SArray *mmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_DROP_TASK, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
|
|
@ -107,14 +107,15 @@ SArray *qmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_MON_QM_INFO, qmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
// Requests handled by VNODE
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, qmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, qmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_RSP, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, qmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, qmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_CONTINUE, qmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_FETCH, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_FETCH_RSP, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_TASK, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_CANCEL_TASK, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_DROP_TASK, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
|
||||
|
||||
code = 0;
|
||||
_OVER:
|
||||
|
|
|
@ -324,16 +324,17 @@ SArray *vmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_MON_VM_LOAD, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_CONTINUE, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_FETCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_UPDATE_TAG_VAL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_TABLE_META, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_TABLE_CFG, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_TABLES_META, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_CANCEL_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_DROP_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
@ -349,7 +350,7 @@ SArray *vmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
|
|
@ -107,13 +107,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
|
|||
const STraceId *trace = &pMsg->info.traceId;
|
||||
dGTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg);
|
||||
|
||||
int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL);
|
||||
if (code != 0) {
|
||||
if (terrno != 0) code = terrno;
|
||||
dGError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr());
|
||||
vmSendRsp(pMsg, code);
|
||||
}
|
||||
|
||||
int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); // no response here
|
||||
dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
|
|
|
@ -41,6 +41,7 @@ static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) {
|
||||
pMsg->info.hasEpSet = 1;
|
||||
SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
|
||||
int32_t contLen = tSerializeSEpSet(NULL, 0, pNewEpSet);
|
||||
|
||||
|
@ -86,7 +87,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
|
|||
return;
|
||||
case TDMT_MND_SYSTABLE_RETRIEVE_RSP:
|
||||
case TDMT_DND_SYSTABLE_RETRIEVE_RSP:
|
||||
case TDMT_VND_FETCH_RSP:
|
||||
case TDMT_SCH_FETCH_RSP:
|
||||
qWorkerProcessFetchRsp(NULL, NULL, pRpc, 0);
|
||||
return;
|
||||
case TDMT_MND_STATUS_RSP:
|
||||
|
@ -250,8 +251,8 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) {
|
|||
|
||||
static bool rpcRfp(int32_t code, tmsg_t msgType) {
|
||||
if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
|
||||
code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
|
||||
if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) {
|
||||
code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK) {
|
||||
if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
@ -264,7 +265,7 @@ int32_t dmInitClient(SDnode *pDnode) {
|
|||
SDnodeTrans *pTrans = &pDnode->trans;
|
||||
|
||||
SRpcInit rpcInit = {0};
|
||||
rpcInit.label = "DND";
|
||||
rpcInit.label = "DND-C";
|
||||
rpcInit.numOfThreads = 1;
|
||||
rpcInit.cfp = (RpcCfp)dmProcessRpcMsg;
|
||||
rpcInit.sessions = 1024;
|
||||
|
@ -298,7 +299,7 @@ int32_t dmInitServer(SDnode *pDnode) {
|
|||
SRpcInit rpcInit = {0};
|
||||
strncpy(rpcInit.localFqdn, tsLocalFqdn, strlen(tsLocalFqdn));
|
||||
rpcInit.localPort = tsServerPort;
|
||||
rpcInit.label = "DND";
|
||||
rpcInit.label = "DND-S";
|
||||
rpcInit.numOfThreads = tsNumOfRpcThreads;
|
||||
rpcInit.cfp = (RpcCfp)dmProcessRpcMsg;
|
||||
rpcInit.sessions = tsMaxShellConns;
|
||||
|
|
|
@ -58,7 +58,7 @@ static void *mndBuildTimerMsg(int32_t *pContLen) {
|
|||
|
||||
static void mndPullupTrans(SMnode *pMnode) {
|
||||
int32_t contLen = 0;
|
||||
void *pReq = mndBuildTimerMsg(&contLen);
|
||||
void * pReq = mndBuildTimerMsg(&contLen);
|
||||
if (pReq != NULL) {
|
||||
SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen};
|
||||
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
|
||||
|
@ -67,14 +67,14 @@ static void mndPullupTrans(SMnode *pMnode) {
|
|||
|
||||
static void mndTtlTimer(SMnode *pMnode) {
|
||||
int32_t contLen = 0;
|
||||
void *pReq = mndBuildTimerMsg(&contLen);
|
||||
void * pReq = mndBuildTimerMsg(&contLen);
|
||||
SRpcMsg rpcMsg = {.msgType = TDMT_MND_TTL_TIMER, .pCont = pReq, .contLen = contLen};
|
||||
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
|
||||
}
|
||||
|
||||
static void mndCalMqRebalance(SMnode *pMnode) {
|
||||
int32_t contLen = 0;
|
||||
void *pReq = mndBuildTimerMsg(&contLen);
|
||||
void * pReq = mndBuildTimerMsg(&contLen);
|
||||
if (pReq != NULL) {
|
||||
SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen};
|
||||
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
|
||||
|
@ -83,7 +83,7 @@ static void mndCalMqRebalance(SMnode *pMnode) {
|
|||
|
||||
static void mndPullupTelem(SMnode *pMnode) {
|
||||
int32_t contLen = 0;
|
||||
void *pReq = mndBuildTimerMsg(&contLen);
|
||||
void * pReq = mndBuildTimerMsg(&contLen);
|
||||
if (pReq != NULL) {
|
||||
SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen};
|
||||
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
|
||||
|
@ -395,7 +395,7 @@ void mndStop(SMnode *pMnode) {
|
|||
}
|
||||
|
||||
int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
SMnode * pMnode = pMsg->info.node;
|
||||
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
|
||||
int32_t code = 0;
|
||||
|
||||
|
@ -413,7 +413,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
do {
|
||||
char *syncNodeStr = sync2SimpleStr(pMgmt->sync);
|
||||
char * syncNodeStr = sync2SimpleStr(pMgmt->sync);
|
||||
static int64_t mndTick = 0;
|
||||
if (++mndTick % 10 == 1) {
|
||||
mTrace("vgId:%d, sync trace msg:%s, %s", syncGetVgId(pMgmt->sync), TMSG_INFO(pMsg->msgType), syncNodeStr);
|
||||
|
@ -527,6 +527,11 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
|
|||
|
||||
static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
|
||||
if (!IsReq(pMsg)) return 0;
|
||||
if (pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_SCH_MERGE_QUERY ||
|
||||
pMsg->msgType == TDMT_SCH_QUERY_CONTINUE || pMsg->msgType == TDMT_SCH_QUERY_HEARTBEAT ||
|
||||
pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) {
|
||||
return 0;
|
||||
}
|
||||
if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0;
|
||||
if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
|
||||
pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER) {
|
||||
|
@ -547,6 +552,7 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
|
|||
|
||||
int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
|
||||
pMsg->info.rsp = rpcMallocCont(contLen);
|
||||
pMsg->info.hasEpSet = 1;
|
||||
if (pMsg->info.rsp != NULL) {
|
||||
tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet);
|
||||
pMsg->info.rspLen = contLen;
|
||||
|
@ -573,7 +579,7 @@ static int32_t mndCheckMsgContent(SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
SMnode * pMnode = pMsg->info.node;
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
|
||||
MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(pMsg->msgType)];
|
||||
|
@ -626,7 +632,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
|
|||
SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) {
|
||||
if (mndAcquireRpcRef(pMnode) != 0) return -1;
|
||||
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
SSdb * pSdb = pMnode->pSdb;
|
||||
int64_t ms = taosGetTimestampMs();
|
||||
|
||||
pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc));
|
||||
|
@ -707,7 +713,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
|
|||
pGrantInfo->timeseries_used += pVgroup->numOfTimeSeries;
|
||||
tstrncpy(desc.status, "unsynced", sizeof(desc.status));
|
||||
for (int32_t i = 0; i < pVgroup->replica; ++i) {
|
||||
SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
|
||||
SVnodeGid * pVgid = &pVgroup->vnodeGid[i];
|
||||
SMonVnodeDesc *pVnDesc = &desc.vnodes[i];
|
||||
pVnDesc->dnode_id = pVgid->dnodeId;
|
||||
tstrncpy(pVnDesc->vnode_role, syncStr(pVgid->role), sizeof(pVnDesc->vnode_role));
|
||||
|
|
|
@ -19,13 +19,13 @@
|
|||
#include "qworker.h"
|
||||
|
||||
int32_t mndPreProcessQueryMsg(SRpcMsg *pMsg) {
|
||||
if (TDMT_VND_QUERY != pMsg->msgType) return 0;
|
||||
if (TDMT_SCH_QUERY != pMsg->msgType && TDMT_SCH_MERGE_QUERY != pMsg->msgType) return 0;
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
return qWorkerPreprocessQueryMsg(pMnode->pQuery, pMsg);
|
||||
}
|
||||
|
||||
void mndPostProcessQueryMsg(SRpcMsg *pMsg) {
|
||||
if (TDMT_VND_QUERY != pMsg->msgType) return;
|
||||
if (TDMT_SCH_QUERY != pMsg->msgType && TDMT_SCH_MERGE_QUERY != pMsg->msgType) return;
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
qWorkerAbortPreprocessQueryMsg(pMnode->pQuery, pMsg);
|
||||
}
|
||||
|
@ -37,19 +37,20 @@ int32_t mndProcessQueryMsg(SRpcMsg *pMsg) {
|
|||
|
||||
mTrace("msg:%p, in query queue is processing", pMsg);
|
||||
switch (pMsg->msgType) {
|
||||
case TDMT_VND_QUERY:
|
||||
case TDMT_SCH_QUERY:
|
||||
case TDMT_SCH_MERGE_QUERY:
|
||||
code = qWorkerProcessQueryMsg(&handle, pMnode->pQuery, pMsg, 0);
|
||||
break;
|
||||
case TDMT_VND_QUERY_CONTINUE:
|
||||
case TDMT_SCH_QUERY_CONTINUE:
|
||||
code = qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, pMsg, 0);
|
||||
break;
|
||||
case TDMT_VND_FETCH:
|
||||
case TDMT_SCH_FETCH:
|
||||
code = qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, pMsg, 0);
|
||||
break;
|
||||
case TDMT_VND_DROP_TASK:
|
||||
case TDMT_SCH_DROP_TASK:
|
||||
code = qWorkerProcessDropMsg(pMnode, pMnode->pQuery, pMsg, 0);
|
||||
break;
|
||||
case TDMT_VND_QUERY_HEARTBEAT:
|
||||
case TDMT_SCH_QUERY_HEARTBEAT:
|
||||
code = qWorkerProcessHbMsg(pMnode, pMnode->pQuery, pMsg, 0);
|
||||
break;
|
||||
default:
|
||||
|
@ -67,11 +68,12 @@ int32_t mndInitQuery(SMnode *pMnode) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_QUERY, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_QUERY_CONTINUE, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_FETCH, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_DROP_TASK, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_QUERY_HEARTBEAT, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_SCH_QUERY, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_SCH_MERGE_QUERY, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_SCH_QUERY_CONTINUE, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_SCH_FETCH, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_SCH_DROP_TASK, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_SCH_QUERY_HEARTBEAT, mndProcessQueryMsg);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -15,8 +15,8 @@
|
|||
|
||||
#define _DEFAULT_SOURCE
|
||||
#include "mndShow.h"
|
||||
#include "systable.h"
|
||||
#include "mndPrivilege.h"
|
||||
#include "systable.h"
|
||||
|
||||
#define SHOW_STEP_SIZE 100
|
||||
|
||||
|
@ -307,7 +307,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
int32_t len = 0;
|
||||
blockCompressEncode(pBlock, pStart, &len, pShow->pMeta->numOfColumns, false);
|
||||
blockEncode(pBlock, pStart, &len, pShow->pMeta->numOfColumns, false);
|
||||
}
|
||||
|
||||
pRsp->numOfRows = htonl(rowsRead);
|
||||
|
|
|
@ -65,7 +65,7 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) {
|
|||
}
|
||||
|
||||
int32_t qndPreprocessQueryMsg(SQnode *pQnode, SRpcMsg * pMsg) {
|
||||
if (TDMT_VND_QUERY != pMsg->msgType) {
|
||||
if (TDMT_SCH_QUERY != pMsg->msgType && TDMT_SCH_MERGE_QUERY != pMsg->msgType) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -78,28 +78,29 @@ int32_t qndProcessQueryMsg(SQnode *pQnode, int64_t ts, SRpcMsg *pMsg) {
|
|||
qTrace("message in qnode queue is processing");
|
||||
|
||||
switch (pMsg->msgType) {
|
||||
case TDMT_VND_QUERY:
|
||||
case TDMT_SCH_QUERY:
|
||||
case TDMT_SCH_MERGE_QUERY:
|
||||
code = qWorkerProcessQueryMsg(&handle, pQnode->pQuery, pMsg, ts);
|
||||
break;
|
||||
case TDMT_VND_QUERY_CONTINUE:
|
||||
case TDMT_SCH_QUERY_CONTINUE:
|
||||
code = qWorkerProcessCQueryMsg(&handle, pQnode->pQuery, pMsg, ts);
|
||||
break;
|
||||
case TDMT_VND_FETCH:
|
||||
case TDMT_SCH_FETCH:
|
||||
code = qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg, ts);
|
||||
break;
|
||||
case TDMT_VND_FETCH_RSP:
|
||||
case TDMT_SCH_FETCH_RSP:
|
||||
code = qWorkerProcessFetchRsp(pQnode, pQnode->pQuery, pMsg, ts);
|
||||
break;
|
||||
case TDMT_VND_CANCEL_TASK:
|
||||
case TDMT_SCH_CANCEL_TASK:
|
||||
code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg, ts);
|
||||
break;
|
||||
case TDMT_VND_DROP_TASK:
|
||||
case TDMT_SCH_DROP_TASK:
|
||||
code = qWorkerProcessDropMsg(pQnode, pQnode->pQuery, pMsg, ts);
|
||||
break;
|
||||
case TDMT_VND_CONSUME:
|
||||
// code = tqProcessConsumeReq(pQnode->pTq, pMsg);
|
||||
// break;
|
||||
case TDMT_VND_QUERY_HEARTBEAT:
|
||||
case TDMT_SCH_QUERY_HEARTBEAT:
|
||||
code = qWorkerProcessHbMsg(pQnode, pQnode->pQuery, pMsg, ts);
|
||||
break;
|
||||
default:
|
||||
|
|
|
@ -116,6 +116,7 @@ typedef void *tsdbReaderT;
|
|||
#define BLOCK_LOAD_TABLE_SEQ_ORDER 2
|
||||
#define BLOCK_LOAD_TABLE_RR_ORDER 3
|
||||
|
||||
int32_t tsdbSetTableId(tsdbReaderT reader, int64_t uid);
|
||||
int32_t tsdbSetTableList(tsdbReaderT reader, SArray *tableList);
|
||||
tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *tableList, uint64_t qId,
|
||||
uint64_t taskId);
|
||||
|
@ -139,19 +140,19 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle);
|
|||
|
||||
// tq
|
||||
|
||||
typedef struct STqReadHandle STqReadHandle;
|
||||
typedef struct STqReadHandle SStreamReader;
|
||||
|
||||
STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta);
|
||||
SStreamReader *tqInitSubmitMsgScanner(SMeta *pMeta);
|
||||
|
||||
void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList);
|
||||
int32_t tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
|
||||
int32_t tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
|
||||
int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
|
||||
void tqReadHandleSetColIdList(SStreamReader *pReadHandle, SArray *pColIdList);
|
||||
int32_t tqReadHandleSetTbUidList(SStreamReader *pHandle, const SArray *tbUidList);
|
||||
int32_t tqReadHandleAddTbUidList(SStreamReader *pHandle, const SArray *tbUidList);
|
||||
int32_t tqReadHandleRemoveTbUidList(SStreamReader *pHandle, const SArray *tbUidList);
|
||||
|
||||
int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
|
||||
bool tqNextDataBlock(STqReadHandle *pHandle);
|
||||
bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids);
|
||||
int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReadHandle *pHandle);
|
||||
int32_t tqReadHandleSetMsg(SStreamReader *pHandle, SSubmitReq *pMsg, int64_t ver);
|
||||
bool tqNextDataBlock(SStreamReader *pHandle);
|
||||
bool tqNextDataBlockFilterOut(SStreamReader *pHandle, SHashObj *filterOutUids);
|
||||
int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, SStreamReader *pHandle);
|
||||
|
||||
// sma
|
||||
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
|
||||
|
|
|
@ -46,6 +46,10 @@ struct SSmaEnv {
|
|||
SSmaStat *pStat;
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
int32_t smaRef;
|
||||
} SSmaMgmt;
|
||||
|
||||
#define SMA_ENV_LOCK(env) ((env)->lock)
|
||||
#define SMA_ENV_TYPE(env) ((env)->type)
|
||||
#define SMA_ENV_STAT(env) ((env)->pStat)
|
||||
|
@ -58,11 +62,12 @@ struct STSmaStat {
|
|||
|
||||
struct SRSmaStat {
|
||||
SSma *pSma;
|
||||
void *tmrHandle;
|
||||
tmr_h tmrId;
|
||||
int32_t tmrSeconds;
|
||||
int8_t triggerStat;
|
||||
int8_t runningStat;
|
||||
int64_t refId; // shared by persistence/fetch tasks
|
||||
void *tmrHandle; // for persistence task
|
||||
tmr_h tmrId; // for persistence task
|
||||
int32_t tmrSeconds; // for persistence task
|
||||
int8_t triggerStat; // for persistence task
|
||||
int8_t runningStat; // for persistence task
|
||||
SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo;
|
||||
};
|
||||
|
||||
|
@ -73,6 +78,7 @@ struct SSmaStat {
|
|||
};
|
||||
T_REF_DECLARE()
|
||||
};
|
||||
|
||||
#define SMA_TSMA_STAT(s) (&(s)->tsmaStat)
|
||||
#define SMA_RSMA_STAT(s) (&(s)->rsmaStat)
|
||||
#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash)
|
||||
|
@ -80,6 +86,7 @@ struct SSmaStat {
|
|||
#define RSMA_TMR_HANDLE(r) ((r)->tmrHandle)
|
||||
#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
|
||||
#define RSMA_RUNNING_STAT(r) (&(r)->runningStat)
|
||||
#define RSMA_REF_ID(r) ((r)->refId)
|
||||
|
||||
enum {
|
||||
TASK_TRIGGER_STAT_INIT = 0,
|
||||
|
@ -192,10 +199,18 @@ typedef struct STFInfo STFInfo;
|
|||
typedef struct STFile STFile;
|
||||
|
||||
struct STFInfo {
|
||||
// common fields
|
||||
uint32_t magic;
|
||||
uint32_t ftype;
|
||||
uint32_t fver;
|
||||
int64_t fsize;
|
||||
|
||||
// specific fields
|
||||
union {
|
||||
struct {
|
||||
int64_t applyVer[2];
|
||||
} qTaskInfo;
|
||||
};
|
||||
};
|
||||
|
||||
struct STFile {
|
||||
|
@ -230,7 +245,7 @@ int32_t tdUpdateTFileHeader(STFile *pTFile);
|
|||
void tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
|
||||
void tdCloseTFile(STFile *pTFile);
|
||||
|
||||
void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName);
|
||||
void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
@ -39,6 +39,16 @@ extern "C" {
|
|||
#define tqInfo(...) do { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0)
|
||||
#define tqDebug(...) do { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0)
|
||||
#define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
|
||||
|
||||
#define IS_META_MSG(x) ( \
|
||||
x == TDMT_VND_CREATE_STB \
|
||||
|| x == TDMT_VND_ALTER_STB \
|
||||
|| x == TDMT_VND_DROP_STB \
|
||||
|| x == TDMT_VND_CREATE_TABLE \
|
||||
|| x == TDMT_VND_ALTER_TABLE \
|
||||
|| x == TDMT_VND_DROP_TABLE \
|
||||
|| x == TDMT_VND_DROP_TTL_TABLE \
|
||||
)
|
||||
// clang-format on
|
||||
|
||||
typedef struct STqOffsetStore STqOffsetStore;
|
||||
|
@ -101,12 +111,13 @@ typedef struct {
|
|||
typedef struct {
|
||||
int8_t subType;
|
||||
|
||||
STqReadHandle* pExecReader[5];
|
||||
SStreamReader* pExecReader[5];
|
||||
union {
|
||||
STqExecCol execCol;
|
||||
STqExecTb execTb;
|
||||
STqExecDb execDb;
|
||||
};
|
||||
|
||||
} STqExecHandle;
|
||||
|
||||
typedef struct {
|
||||
|
@ -149,9 +160,9 @@ static STqMgmt tqMgmt = {0};
|
|||
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead** pHeadWithCkSum);
|
||||
|
||||
// tqExec
|
||||
int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId);
|
||||
int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId);
|
||||
int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp);
|
||||
int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId);
|
||||
int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset, int32_t workerId);
|
||||
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp);
|
||||
|
||||
// tqMeta
|
||||
int32_t tqMetaOpen(STQ* pTq);
|
||||
|
@ -172,6 +183,17 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore);
|
|||
// tqSink
|
||||
void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data);
|
||||
|
||||
static FORCE_INLINE void tqOffsetResetToData(STqOffsetVal* pOffsetVal, int64_t uid, int64_t ts) {
|
||||
pOffsetVal->type = TMQ_OFFSET__SNAPSHOT_DATA;
|
||||
pOffsetVal->uid = uid;
|
||||
pOffsetVal->ts = ts;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void tqOffsetResetToLog(STqOffsetVal* pOffsetVal, int64_t ver) {
|
||||
pOffsetVal->type = TMQ_OFFSET__LOG;
|
||||
pOffsetVal->version = ver;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
@ -27,6 +27,7 @@
|
|||
#include "tdatablock.h"
|
||||
#include "tdb.h"
|
||||
#include "tencode.h"
|
||||
#include "tref.h"
|
||||
#include "tfs.h"
|
||||
#include "tglobal.h"
|
||||
#include "tjson.h"
|
||||
|
|
|
@ -18,6 +18,9 @@
|
|||
typedef struct SSmaStat SSmaStat;
|
||||
|
||||
#define RSMA_TASK_INFO_HASH_SLOT 8
|
||||
#define SMA_MGMT_REF_NUM 1024
|
||||
|
||||
extern SSmaMgmt smaMgmt;
|
||||
|
||||
// declaration of static functions
|
||||
|
||||
|
@ -25,6 +28,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *p
|
|||
static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path);
|
||||
static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SSmaEnv **pEnv);
|
||||
static void *tdFreeTSmaStat(STSmaStat *pStat);
|
||||
static void tdDestroyRSmaStat(void *pRSmaStat);
|
||||
|
||||
// implementation
|
||||
|
||||
|
@ -128,6 +132,22 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
|
|||
if (smaType == TSDB_SMA_TYPE_ROLLUP) {
|
||||
SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat);
|
||||
pRSmaStat->pSma = (SSma *)pSma;
|
||||
|
||||
// init smaMgmt
|
||||
smaMgmt.smaRef = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat);
|
||||
if (smaMgmt.smaRef < 0) {
|
||||
smaError("init smaRef failed, num:%d", SMA_MGMT_REF_NUM);
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
int64_t refId = taosAddRef(smaMgmt.smaRef, pRSmaStat);
|
||||
if (refId < 0) {
|
||||
smaError("taosAddRef smaRef failed, since:%s", tstrerror(terrno));
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
pRSmaStat->refId = refId;
|
||||
|
||||
// init timer
|
||||
RSMA_TMR_HANDLE(pRSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA");
|
||||
if (!RSMA_TMR_HANDLE(pRSmaStat)) {
|
||||
|
@ -169,9 +189,10 @@ static void *tdFreeTSmaStat(STSmaStat *pStat) {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
static void tdDestroyRSmaStat(SRSmaStat *pStat) {
|
||||
if (pStat) {
|
||||
smaDebug("vgId:%d destroy rsma stat", SMA_VID(pStat->pSma));
|
||||
static void tdDestroyRSmaStat(void *pRSmaStat) {
|
||||
if (pRSmaStat) {
|
||||
SRSmaStat *pStat = (SRSmaStat *)pRSmaStat;
|
||||
smaDebug("vgId:%d %s:%d destroy rsma stat %p", SMA_VID(pStat->pSma), __func__, __LINE__, pRSmaStat);
|
||||
// step 1: set persistence task cancelled
|
||||
atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED);
|
||||
|
||||
|
@ -183,9 +204,11 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) {
|
|||
if (atomic_load_8(RSMA_RUNNING_STAT(pStat)) == 1) {
|
||||
while (1) {
|
||||
if (atomic_load_8(RSMA_TRIGGER_STAT(pStat)) == TASK_TRIGGER_STAT_FINISHED) {
|
||||
smaDebug("rsma, persist task finished already");
|
||||
break;
|
||||
} else {
|
||||
smaDebug("not destroyed since rsma stat in %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pStat)));
|
||||
smaDebug("rsma, persist task not finished yet since rsma stat in %" PRIi8,
|
||||
atomic_load_8(RSMA_TRIGGER_STAT(pStat)));
|
||||
}
|
||||
++nLoops;
|
||||
if (nLoops > 1000) {
|
||||
|
@ -209,7 +232,10 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) {
|
|||
nLoops = 0;
|
||||
while (1) {
|
||||
if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) {
|
||||
smaDebug("rsma, all fetch task finished already");
|
||||
break;
|
||||
} else {
|
||||
smaDebug("rsma, fetch tasks not all finished yet");
|
||||
}
|
||||
++nLoops;
|
||||
if (nLoops > 1000) {
|
||||
|
@ -225,15 +251,13 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) {
|
|||
}
|
||||
}
|
||||
|
||||
static void *tdFreeRSmaStat(SRSmaStat *pStat) {
|
||||
tdDestroyRSmaStat(pStat);
|
||||
taosMemoryFreeClear(pStat);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
|
||||
tdDestroySmaState(pSmaStat, smaType);
|
||||
taosMemoryFreeClear(pSmaStat);
|
||||
if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
|
||||
taosMemoryFreeClear(pSmaStat);
|
||||
}
|
||||
// tref used to free rsma stat
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
@ -243,17 +267,21 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
|
|||
* @param pSmaStat
|
||||
* @return int32_t
|
||||
*/
|
||||
|
||||
int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
|
||||
if (pSmaStat) {
|
||||
if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
|
||||
tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat));
|
||||
} else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
|
||||
tdDestroyRSmaStat(SMA_RSMA_STAT(pSmaStat));
|
||||
SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSmaStat);
|
||||
if (taosRemoveRef(smaMgmt.smaRef, RSMA_REF_ID(pRSmaStat)) < 0) {
|
||||
smaError("remove refId from smaRef failed, refId:0x%" PRIx64, RSMA_REF_ID(pRSmaStat));
|
||||
}
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tdLockSma(SSma *pSma) {
|
||||
|
|
|
@ -18,9 +18,13 @@
|
|||
#define RSMA_QTASKINFO_PERSIST_MS 7200000
|
||||
#define RSMA_QTASKINFO_BUFSIZE 32768
|
||||
#define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid
|
||||
typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T;
|
||||
static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
|
||||
|
||||
SSmaMgmt smaMgmt = {
|
||||
.smaRef = -1,
|
||||
};
|
||||
|
||||
typedef enum { TD_QTASK_TMP_F = 0, TD_QTASK_CUR_F } TD_QTASK_FILE_T;
|
||||
static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
|
||||
typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
|
||||
typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
|
||||
|
||||
|
@ -46,6 +50,7 @@ static int32_t tdRSmaRestoreTSDataReload(SSma *pSma);
|
|||
|
||||
struct SRSmaInfoItem {
|
||||
SRSmaInfo *pRsmaInfo;
|
||||
int64_t refId;
|
||||
void *taskInfo; // qTaskInfo_t
|
||||
tmr_h tmrId;
|
||||
int8_t level;
|
||||
|
@ -56,11 +61,14 @@ struct SRSmaInfoItem {
|
|||
|
||||
struct SRSmaInfo {
|
||||
STSchema *pTSchema;
|
||||
SSma *pSma;
|
||||
SRSmaStat *pStat;
|
||||
int64_t suid;
|
||||
SRSmaInfoItem items[TSDB_RETENTION_L2];
|
||||
};
|
||||
|
||||
#define RSMA_INFO_SMA(r) ((r)->pStat->pSma)
|
||||
#define RSMA_INFO_STAT(r) ((r)->pStat)
|
||||
|
||||
struct SRSmaQTaskInfoItem {
|
||||
int32_t len;
|
||||
int8_t type;
|
||||
|
@ -80,6 +88,10 @@ struct SRSmaQTaskInfoIter {
|
|||
int32_t nBufPos;
|
||||
};
|
||||
|
||||
static void tdRSmaQTaskInfoGetFName(int32_t vgId, int8_t ftype, char *outputName) {
|
||||
tdGetVndFileName(vgId, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName);
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) {
|
||||
return lenWithHead - RSMA_QTASKINFO_HEAD_LEN;
|
||||
}
|
||||
|
@ -99,22 +111,21 @@ static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId,
|
|||
|
||||
void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
|
||||
if (pInfo) {
|
||||
SSma *pSma = RSMA_INFO_SMA(pInfo);
|
||||
for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
|
||||
SRSmaInfoItem *pItem = &pInfo->items[i];
|
||||
if (pItem->taskInfo) {
|
||||
smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pInfo->pSma), pInfo->suid,
|
||||
pItem->tmrId, i + 1);
|
||||
smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId,
|
||||
i + 1);
|
||||
taosTmrStopA(&pItem->tmrId);
|
||||
tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pInfo->pSma), i + 1);
|
||||
tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pSma), i + 1);
|
||||
} else {
|
||||
smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo",
|
||||
SMA_VID(pInfo->pSma), pInfo->suid, i + 1);
|
||||
smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma),
|
||||
pInfo->suid, i + 1);
|
||||
}
|
||||
}
|
||||
taosMemoryFree(pInfo->pTSchema);
|
||||
taosMemoryFree(pInfo);
|
||||
} else {
|
||||
smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info since empty", SMA_VID(pInfo->pSma), pInfo->suid);
|
||||
}
|
||||
|
||||
return NULL;
|
||||
|
@ -247,6 +258,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo
|
|||
|
||||
if (param->qmsg[idx]) {
|
||||
SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
|
||||
pItem->refId = RSMA_REF_ID(pRSmaInfo->pStat);
|
||||
pItem->pRsmaInfo = pRSmaInfo;
|
||||
pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle);
|
||||
if (!pItem->taskInfo) {
|
||||
|
@ -313,7 +325,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
|
|||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
STqReadHandle *pReadHandle = tqInitSubmitMsgScanner(pMeta);
|
||||
SStreamReader *pReadHandle = tqInitSubmitMsgScanner(pMeta);
|
||||
if (!pReadHandle) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto _err;
|
||||
|
@ -332,7 +344,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
|
|||
goto _err;
|
||||
}
|
||||
pRSmaInfo->pTSchema = pTSchema;
|
||||
pRSmaInfo->pSma = pSma;
|
||||
pRSmaInfo->pStat = pStat;
|
||||
pRSmaInfo->suid = suid;
|
||||
|
||||
if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) {
|
||||
|
@ -514,7 +526,7 @@ static void tdDestroySDataBlockArray(SArray *pArray) {
|
|||
static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) {
|
||||
SArray *pResult = NULL;
|
||||
SRSmaInfo *pRSmaInfo = pItem->pRsmaInfo;
|
||||
SSma *pSma = pRSmaInfo->pSma;
|
||||
SSma *pSma = RSMA_INFO_SMA(pRSmaInfo);
|
||||
|
||||
while (1) {
|
||||
SSDataBlock *output = NULL;
|
||||
|
@ -577,34 +589,44 @@ _err:
|
|||
*/
|
||||
static void tdRSmaFetchTrigger(void *param, void *tmrId) {
|
||||
SRSmaInfoItem *pItem = param;
|
||||
SSma *pSma = pItem->pRsmaInfo->pSma;
|
||||
SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv);
|
||||
SSma *pSma = NULL;
|
||||
SRSmaStat *pStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, pItem->refId);
|
||||
if (!pStat) {
|
||||
smaDebug("rsma fetch task not start since already destroyed");
|
||||
return;
|
||||
}
|
||||
|
||||
pSma = RSMA_INFO_SMA(pItem->pRsmaInfo);
|
||||
|
||||
// if rsma trigger stat in cancelled or finished, not start fetch task anymore
|
||||
int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
|
||||
if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) {
|
||||
smaDebug("vgId:%d, level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma),
|
||||
pItem->level, pItem->pRsmaInfo->suid);
|
||||
taosReleaseRef(smaMgmt.smaRef, pItem->refId);
|
||||
smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled",
|
||||
SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
|
||||
return;
|
||||
}
|
||||
|
||||
int8_t fetchTriggerStat =
|
||||
atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
|
||||
if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) {
|
||||
smaDebug("vgId:%d, level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), pItem->level,
|
||||
pItem->pRsmaInfo->suid);
|
||||
smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
|
||||
pItem->level, pItem->pRsmaInfo->suid);
|
||||
|
||||
tdRefSmaStat(pSma, (SSmaStat *)pStat);
|
||||
|
||||
SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
|
||||
qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
|
||||
tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK);
|
||||
qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
|
||||
tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK);
|
||||
|
||||
tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
|
||||
|
||||
} else {
|
||||
smaDebug("vgId:%d, level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), pItem->level,
|
||||
pItem->pRsmaInfo->suid);
|
||||
smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
|
||||
SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
|
||||
}
|
||||
_end:
|
||||
taosReleaseRef(smaMgmt.smaRef, pItem->refId);
|
||||
}
|
||||
|
||||
static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid,
|
||||
|
@ -617,14 +639,13 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
|
|||
smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level,
|
||||
pItem->taskInfo, suid);
|
||||
|
||||
if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) { // STREAM_DATA_TYPE_SUBMIT_BLOCK
|
||||
if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) { // INPUT__DATA_SUBMIT
|
||||
smaError("vgId:%d, rsma % " PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno));
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK);
|
||||
tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_SUBMIT);
|
||||
atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
|
||||
smaDebug("vgId:%d, process rsma insert", SMA_VID(pSma));
|
||||
|
||||
SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
|
||||
SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat);
|
||||
|
@ -660,7 +681,7 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
|
||||
if (inputType == STREAM_INPUT__DATA_SUBMIT) {
|
||||
tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], suid, TSDB_RETENTION_L1);
|
||||
tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], suid, TSDB_RETENTION_L2);
|
||||
}
|
||||
|
@ -681,7 +702,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
|
||||
if (inputType == STREAM_INPUT__DATA_SUBMIT) {
|
||||
STbUidStore uidStore = {0};
|
||||
tdFetchSubmitReqSuids(pMsg, &uidStore);
|
||||
|
||||
|
@ -761,7 +782,7 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
|
|||
STFile tFile = {0};
|
||||
char qTaskInfoFName[TSDB_FILENAME_LEN];
|
||||
|
||||
tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName);
|
||||
tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_TMP_F, qTaskInfoFName);
|
||||
if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
|
@ -797,9 +818,9 @@ _err:
|
|||
|
||||
/**
|
||||
* @brief reload ts data from checkpoint
|
||||
*
|
||||
* @param pSma
|
||||
* @return int32_t
|
||||
*
|
||||
* @param pSma
|
||||
* @return int32_t
|
||||
*/
|
||||
static int32_t tdRSmaRestoreTSDataReload(SSma *pSma) {
|
||||
// TODO
|
||||
|
@ -861,7 +882,8 @@ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *
|
|||
pItem->type, terrstr(terrno));
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid, pItem->type);
|
||||
smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid,
|
||||
pItem->type);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
@ -1003,10 +1025,6 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static void tdRSmaQTaskInfoGetFName(int32_t vid, int8_t ftype, char *outputName) {
|
||||
tdGetVndFileName(vid, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName);
|
||||
}
|
||||
|
||||
static void *tdRSmaPersistExec(void *param) {
|
||||
setThreadName("rsma-task-persist");
|
||||
SRSmaStat *pRSmaStat = param;
|
||||
|
@ -1031,7 +1049,7 @@ static void *tdRSmaPersistExec(void *param) {
|
|||
for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
|
||||
qTaskInfo_t taskInfo = pRSmaInfo->items[i].taskInfo;
|
||||
if (!taskInfo) {
|
||||
smaDebug("vgId:%d, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1);
|
||||
smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -1039,31 +1057,24 @@ static void *tdRSmaPersistExec(void *param) {
|
|||
int32_t len = 0;
|
||||
int8_t type = (int8_t)(i + 1);
|
||||
if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) {
|
||||
smaError("vgId:%d, table %" PRIi64 " level %d serialize rsma task failed since %s", vid, pRSmaInfo->suid, i + 1,
|
||||
terrstr(terrno));
|
||||
smaError("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo failed since %s", vid, pRSmaInfo->suid,
|
||||
i + 1, terrstr(terrno));
|
||||
goto _err;
|
||||
}
|
||||
if (!pOutput || len <= 0) {
|
||||
smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success but no output(len %d), not persist",
|
||||
smaDebug("vgId:%d, rsma, table %" PRIi64
|
||||
" level %d serialize qTaskInfo success but no output(len %d), not persist",
|
||||
vid, pRSmaInfo->suid, i + 1, len);
|
||||
taosMemoryFreeClear(pOutput);
|
||||
continue;
|
||||
}
|
||||
|
||||
smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d, need persist", vid,
|
||||
smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo success with len %d, need persist", vid,
|
||||
pRSmaInfo->suid, i + 1, len);
|
||||
#if 0
|
||||
if (qDeserializeTaskStatus(taskInfo, pOutput, len) < 0) {
|
||||
smaError("vgId:%d, table %" PRIi64 "level %d deserialize rsma task failed since %s", vid, pRSmaInfo->suid,
|
||||
i + 1, terrstr(terrno));
|
||||
} else {
|
||||
smaDebug("vgId:%d, table %" PRIi64 " level %d deserialize rsma task success", vid, pRSmaInfo->suid, i + 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (!isFileCreated) {
|
||||
char qTaskInfoFName[TSDB_FILENAME_LEN];
|
||||
tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName);
|
||||
tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_F, qTaskInfoFName);
|
||||
tdInitTFile(&tFile, pTfs, qTaskInfoFName);
|
||||
tdCreateTFile(&tFile, pTfs, true, -1);
|
||||
|
||||
|
@ -1079,11 +1090,11 @@ static void *tdRSmaPersistExec(void *param) {
|
|||
|
||||
ASSERT(headLen <= RSMA_QTASKINFO_HEAD_LEN);
|
||||
tdAppendTFile(&tFile, (void *)&tmpBuf, headLen, &toffset);
|
||||
smaDebug("vgId:%d, table %" PRIi64 " level %d head part len:%d appended to offset:%" PRIi64, vid, pRSmaInfo->suid,
|
||||
i + 1, headLen, toffset);
|
||||
smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d head part(len:%d) appended to offset:%" PRIi64, vid,
|
||||
pRSmaInfo->suid, i + 1, headLen, toffset);
|
||||
tdAppendTFile(&tFile, pOutput, len, &toffset);
|
||||
smaDebug("vgId:%d, table %" PRIi64 " level %d body part len:%d appended to offset:%" PRIi64, vid, pRSmaInfo->suid,
|
||||
i + 1, len, toffset);
|
||||
smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d body part len:%d appended to offset:%" PRIi64, vid,
|
||||
pRSmaInfo->suid, i + 1, len, toffset);
|
||||
|
||||
taosMemoryFree(pOutput);
|
||||
}
|
||||
|
@ -1093,26 +1104,26 @@ static void *tdRSmaPersistExec(void *param) {
|
|||
_normal:
|
||||
if (isFileCreated) {
|
||||
if (tdUpdateTFileHeader(&tFile) < 0) {
|
||||
smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile),
|
||||
smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile),
|
||||
tstrerror(terrno));
|
||||
tdCloseTFile(&tFile);
|
||||
tdRemoveTFile(&tFile);
|
||||
goto _err;
|
||||
} else {
|
||||
smaDebug("vgId:%d, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile));
|
||||
smaDebug("vgId:%d, rsma, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile));
|
||||
}
|
||||
|
||||
tdCloseTFile(&tFile);
|
||||
|
||||
char newFName[TSDB_FILENAME_LEN];
|
||||
strncpy(newFName, TD_TFILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN);
|
||||
char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]);
|
||||
strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName));
|
||||
char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_F]);
|
||||
strncpy(pos, tdQTaskInfoFname[TD_QTASK_TMP_F], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName));
|
||||
if (taosRenameFile(TD_TFILE_FULL_NAME(&tFile), newFName) != 0) {
|
||||
smaError("vgId:%d, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
|
||||
smaError("vgId:%d, rsma, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
|
||||
goto _err;
|
||||
} else {
|
||||
smaDebug("vgId:%d, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
|
||||
smaDebug("vgId:%d, rsma, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
|
||||
}
|
||||
}
|
||||
goto _end;
|
||||
|
@ -1124,16 +1135,17 @@ _end:
|
|||
if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
|
||||
TASK_TRIGGER_STAT_INACTIVE,
|
||||
TASK_TRIGGER_STAT_ACTIVE)) {
|
||||
smaDebug("vgId:%d, persist task is active again", vid);
|
||||
smaDebug("vgId:%d, rsma persist task is active again", vid);
|
||||
} else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
|
||||
TASK_TRIGGER_STAT_CANCELLED,
|
||||
TASK_TRIGGER_STAT_FINISHED)) {
|
||||
smaDebug("vgId:%d, persist task is cancelled", vid);
|
||||
smaDebug("vgId:%d, rsma persist task is cancelled", vid);
|
||||
} else {
|
||||
smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
|
||||
smaWarn("vgId:%d, rsma persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
|
||||
ASSERT(0);
|
||||
}
|
||||
atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
|
||||
taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
|
||||
taosThreadExit(NULL);
|
||||
return NULL;
|
||||
}
|
||||
|
@ -1159,6 +1171,7 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
|
|||
ASSERT(0);
|
||||
}
|
||||
atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
|
||||
taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
|
||||
}
|
||||
|
||||
taosThreadAttrDestroy(&thAttr);
|
||||
|
@ -1171,7 +1184,13 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
|
|||
* @param tmrId
|
||||
*/
|
||||
static void tdRSmaPersistTrigger(void *param, void *tmrId) {
|
||||
SRSmaStat *pRSmaStat = param;
|
||||
SRSmaStat *rsmaStat = param;
|
||||
SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, rsmaStat->refId);
|
||||
|
||||
if (!pRSmaStat) {
|
||||
smaDebug("rsma persistence task not start since already destroyed");
|
||||
return;
|
||||
}
|
||||
|
||||
int8_t tmrStat =
|
||||
atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
|
||||
|
@ -1191,6 +1210,7 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
|
|||
} else {
|
||||
atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
|
||||
}
|
||||
return;
|
||||
} break;
|
||||
case TASK_TRIGGER_STAT_CANCELLED: {
|
||||
atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED);
|
||||
|
@ -1206,4 +1226,5 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
|
|||
smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat);
|
||||
} break;
|
||||
}
|
||||
taosReleaseRef(smaMgmt.smaRef, rsmaStat->refId);
|
||||
}
|
||||
|
|
|
@ -113,7 +113,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
|
|||
if (!pDataBlocks) {
|
||||
terrno = TSDB_CODE_TSMA_INVALID_PTR;
|
||||
smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is NULL", SMA_VID(pSma));
|
||||
return terrno;
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
if (taosArrayGetSize(pDataBlocks) <= 0) {
|
||||
|
@ -127,9 +127,9 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
|
|||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
SSmaEnv *pEnv = SMA_TSMA_ENV(pSma);
|
||||
SSmaStat *pStat = NULL;
|
||||
STSmaStat *pItem = NULL;
|
||||
SSmaEnv *pEnv = SMA_TSMA_ENV(pSma);
|
||||
SSmaStat *pStat = NULL;
|
||||
STSmaStat *pTsmaStat = NULL;
|
||||
|
||||
if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) {
|
||||
terrno = TSDB_CODE_TSMA_INVALID_STAT;
|
||||
|
@ -137,32 +137,43 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
|
|||
}
|
||||
|
||||
tdRefSmaStat(pSma, pStat);
|
||||
pItem = &pStat->tsmaStat;
|
||||
pTsmaStat = SMA_TSMA_STAT(pStat);
|
||||
|
||||
ASSERT(pItem);
|
||||
|
||||
if (!pItem->pTSma) {
|
||||
if (!pTsmaStat->pTSma) {
|
||||
STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid);
|
||||
if (!pTSma) {
|
||||
terrno = TSDB_CODE_TSMA_NO_INDEX_IN_META;
|
||||
smaWarn("vgId:%d, tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno));
|
||||
return TSDB_CODE_FAILED;
|
||||
smaError("vgId:%d, failed to get STSma while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),
|
||||
indexUid, tstrerror(terrno));
|
||||
goto _err;
|
||||
}
|
||||
pTsmaStat->pTSma = pTSma;
|
||||
pTsmaStat->pTSchema = metaGetTbTSchema(SMA_META(pSma), pTSma->dstTbUid, -1);
|
||||
if (!pTsmaStat->pTSchema) {
|
||||
smaError("vgId:%d, failed to get STSchema while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),
|
||||
indexUid, tstrerror(terrno));
|
||||
goto _err;
|
||||
}
|
||||
pItem->pTSma = pTSma;
|
||||
pItem->pTSchema = metaGetTbTSchema(SMA_META(pSma), pTSma->dstTbUid, -1);
|
||||
ASSERT(pItem->pTSchema); // TODO
|
||||
}
|
||||
|
||||
ASSERT(pItem->pTSma->indexUid == indexUid);
|
||||
if (pTsmaStat->pTSma->indexUid != indexUid) {
|
||||
terrno = TSDB_CODE_VND_APP_ERROR;
|
||||
smaError("vgId:%d, tsma insert for smaIndex %" PRIi64 "(!=%" PRIi64 ") failed since %s", SMA_VID(pSma), indexUid,
|
||||
pTsmaStat->pTSma->indexUid, tstrerror(terrno));
|
||||
goto _err;
|
||||
}
|
||||
|
||||
SSubmitReq *pSubmitReq = NULL;
|
||||
SSubmitReq *pSubmitReq = tdBlockToSubmit((const SArray *)msg, pTsmaStat->pTSchema, true, pTsmaStat->pTSma->dstTbUid,
|
||||
pTsmaStat->pTSma->dstTbName, pTsmaStat->pTSma->dstVgId);
|
||||
|
||||
pSubmitReq = tdBlockToSubmit((const SArray *)msg, pItem->pTSchema, true, pItem->pTSma->dstTbUid,
|
||||
pItem->pTSma->dstTbName, pItem->pTSma->dstVgId);
|
||||
if (!pSubmitReq) {
|
||||
smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),
|
||||
indexUid, tstrerror(terrno));
|
||||
goto _err;
|
||||
}
|
||||
|
||||
ASSERT(pSubmitReq); // TODO
|
||||
|
||||
ASSERT(!strncasecmp("td.tsma.rst.tb", pItem->pTSma->dstTbName, 14));
|
||||
#if 0
|
||||
ASSERT(!strncasecmp("td.tsma.rst.tb", pTsmaStat->pTSma->dstTbName, 14));
|
||||
#endif
|
||||
|
||||
SRpcMsg submitReqMsg = {
|
||||
.msgType = TDMT_VND_SUBMIT,
|
||||
|
@ -170,9 +181,15 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
|
|||
.contLen = ntohl(pSubmitReq->length),
|
||||
};
|
||||
|
||||
ASSERT(tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) == 0);
|
||||
if (tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) < 0) {
|
||||
smaError("vgId:%d, failed to put SubmitReq msg while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),
|
||||
indexUid, tstrerror(terrno));
|
||||
goto _err;
|
||||
}
|
||||
|
||||
tdUnRefSmaStat(pSma, pStat);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
_err:
|
||||
tdUnRefSmaStat(pSma, pStat);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
|
@ -141,8 +141,8 @@ int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset)
|
|||
}
|
||||
|
||||
#if 1
|
||||
smaDebug("append to file %s, offset:%" PRIi64 " + nbyte:%" PRIi64 " =%" PRIi64, TD_TFILE_FULL_NAME(pTFile), toffset,
|
||||
nbyte, toffset + nbyte);
|
||||
smaDebug("append to file %s, offset:%" PRIi64 " nbyte:%" PRIi64 " fsize:%" PRIi64, TD_TFILE_FULL_NAME(pTFile),
|
||||
toffset, nbyte, toffset + nbyte);
|
||||
#endif
|
||||
|
||||
ASSERT(pTFile->info.fsize == toffset);
|
||||
|
@ -179,8 +179,8 @@ void tdCloseTFile(STFile *pTFile) {
|
|||
}
|
||||
}
|
||||
|
||||
void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName) {
|
||||
snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vid, dname, fname);
|
||||
void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName) {
|
||||
snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vgId, dname, fname);
|
||||
}
|
||||
|
||||
int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) {
|
||||
|
|
|
@ -113,8 +113,23 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq,
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp) {
|
||||
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, pRsp);
|
||||
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp) {
|
||||
ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
|
||||
ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
|
||||
|
||||
if (pRsp->withSchema) {
|
||||
ASSERT(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum);
|
||||
} else {
|
||||
ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0);
|
||||
}
|
||||
|
||||
int32_t len;
|
||||
int32_t code;
|
||||
tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code);
|
||||
if (code < 0) {
|
||||
return -1;
|
||||
}
|
||||
int32_t tlen = sizeof(SMqRspHead) + len;
|
||||
void* buf = rpcMallocCont(tlen);
|
||||
if (buf == NULL) {
|
||||
return -1;
|
||||
|
@ -125,18 +140,26 @@ int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
|
|||
((SMqRspHead*)buf)->consumerId = pReq->consumerId;
|
||||
|
||||
void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
|
||||
tEncodeSMqDataBlkRsp(&abuf, pRsp);
|
||||
|
||||
SRpcMsg resp = {
|
||||
SEncoder encoder;
|
||||
tEncoderInit(&encoder, abuf, len);
|
||||
tEncodeSMqDataRsp(&encoder, pRsp);
|
||||
/*tEncodeSMqDataBlkRsp(&abuf, pRsp);*/
|
||||
|
||||
SRpcMsg rsp = {
|
||||
.info = pMsg->info,
|
||||
.pCont = buf,
|
||||
.contLen = tlen,
|
||||
.code = 0,
|
||||
};
|
||||
tmsgSendRsp(&resp);
|
||||
tmsgSendRsp(&rsp);
|
||||
|
||||
tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
|
||||
TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, pRsp->reqOffset, pRsp->rspOffset);
|
||||
char buf1[80];
|
||||
char buf2[80];
|
||||
tFormatOffset(buf1, 80, &pRsp->reqOffset);
|
||||
tFormatOffset(buf2, 80, &pRsp->rspOffset);
|
||||
tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %s, rspOffset: %s",
|
||||
TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -151,17 +174,17 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) {
|
|||
}
|
||||
tDecoderClear(&decoder);
|
||||
|
||||
if (offset.type == TMQ_OFFSET__SNAPSHOT) {
|
||||
if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA) {
|
||||
tqDebug("receive offset commit msg to %s on vg %d, offset(type:snapshot) uid: %ld, ts: %ld", offset.subKey,
|
||||
TD_VID(pTq->pVnode), offset.uid, offset.ts);
|
||||
} else if (offset.type == TMQ_OFFSET__LOG) {
|
||||
TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts);
|
||||
} else if (offset.val.type == TMQ_OFFSET__LOG) {
|
||||
tqDebug("receive offset commit msg to %s on vg %d, offset(type:log) version: %ld", offset.subKey,
|
||||
TD_VID(pTq->pVnode), offset.version);
|
||||
TD_VID(pTq->pVnode), offset.val.version);
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey);
|
||||
if (pOffset == NULL || pOffset->version < offset.version) {
|
||||
if (pOffset == NULL || pOffset->val.version < offset.val.version) {
|
||||
if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) {
|
||||
ASSERT(0);
|
||||
return -1;
|
||||
|
@ -171,6 +194,221 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) {
|
||||
pRsp->reqOffset = pReq->reqOffset;
|
||||
|
||||
pRsp->blockData = taosArrayInit(0, sizeof(void*));
|
||||
pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t));
|
||||
|
||||
if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
pRsp->withTbName = pReq->withTbName;
|
||||
if (pRsp->withTbName) {
|
||||
pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
|
||||
if (pRsp->blockTbName == NULL) {
|
||||
// TODO free
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
pRsp->withSchema = false;
|
||||
} else {
|
||||
pRsp->withSchema = true;
|
||||
pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
|
||||
if (pRsp->blockSchema == NULL) {
|
||||
// TODO free
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t tqInitMetaRsp(SMqMetaRsp* pRsp, const SMqPollReq* pReq) { return 0; }
|
||||
|
||||
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
||||
SMqPollReq* pReq = pMsg->pCont;
|
||||
int64_t consumerId = pReq->consumerId;
|
||||
int64_t timeout = pReq->timeout;
|
||||
int32_t reqEpoch = pReq->epoch;
|
||||
int32_t code = 0;
|
||||
STqOffsetVal reqOffset = pReq->reqOffset;
|
||||
STqOffsetVal fetchOffsetNew;
|
||||
|
||||
// 1.find handle
|
||||
char buf[80];
|
||||
tFormatOffset(buf, 80, &reqOffset);
|
||||
tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %s", consumerId, pReq->epoch,
|
||||
TD_VID(pTq->pVnode), buf);
|
||||
|
||||
STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey));
|
||||
/*ASSERT(pHandle);*/
|
||||
if (pHandle == NULL) {
|
||||
tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode),
|
||||
pReq->subKey);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// check rebalance
|
||||
if (pHandle->consumerId != consumerId) {
|
||||
tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld",
|
||||
consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// update epoch if need
|
||||
int32_t consumerEpoch = atomic_load_32(&pHandle->epoch);
|
||||
while (consumerEpoch < reqEpoch) {
|
||||
consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch);
|
||||
}
|
||||
|
||||
// 2.reset offset if needed
|
||||
if (reqOffset.type > 0) {
|
||||
fetchOffsetNew = reqOffset;
|
||||
} else {
|
||||
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey);
|
||||
if (pOffset != NULL) {
|
||||
fetchOffsetNew = pOffset->val;
|
||||
char formatBuf[50];
|
||||
tFormatOffset(formatBuf, 50, &fetchOffsetNew);
|
||||
tqDebug("tmq poll: consumer %ld, offset reset to %s", consumerId, formatBuf);
|
||||
} else {
|
||||
if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
|
||||
if (pReq->useSnapshot) {
|
||||
if (!pHandle->fetchMeta) {
|
||||
tqOffsetResetToData(&fetchOffsetNew, 0, 0);
|
||||
} else {
|
||||
// reset to meta
|
||||
ASSERT(0);
|
||||
}
|
||||
} else {
|
||||
tqOffsetResetToLog(&fetchOffsetNew, walGetFirstVer(pTq->pVnode->pWal));
|
||||
}
|
||||
} else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
|
||||
tqOffsetResetToLog(&fetchOffsetNew, walGetLastVer(pTq->pVnode->pWal));
|
||||
} else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
|
||||
tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s, reset none failed", consumerId,
|
||||
TD_VID(pTq->pVnode), pReq->subKey);
|
||||
terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3.query
|
||||
SMqDataRsp dataRsp = {0};
|
||||
tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
|
||||
|
||||
if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {
|
||||
int64_t fetchVer = fetchOffsetNew.version + 1;
|
||||
SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048);
|
||||
if (pHeadWithCkSum == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
walSetReaderCapacity(pHandle->pWalReader, 2048);
|
||||
|
||||
while (1) {
|
||||
consumerEpoch = atomic_load_32(&pHandle->epoch);
|
||||
if (consumerEpoch > reqEpoch) {
|
||||
tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d",
|
||||
consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch);
|
||||
break;
|
||||
}
|
||||
|
||||
if (tqFetchLog(pTq, pHandle, &fetchVer, &pHeadWithCkSum) < 0) {
|
||||
// TODO add push mgr
|
||||
|
||||
tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer);
|
||||
ASSERT(dataRsp.rspOffset.version >= dataRsp.reqOffset.version);
|
||||
if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
|
||||
code = -1;
|
||||
}
|
||||
goto OVER;
|
||||
}
|
||||
|
||||
SWalReadHead* pHead = &pHeadWithCkSum->head;
|
||||
|
||||
tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
|
||||
TD_VID(pTq->pVnode), fetchVer, pHead->msgType);
|
||||
|
||||
if (pHead->msgType == TDMT_VND_SUBMIT) {
|
||||
SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
|
||||
|
||||
if (tqLogScanExec(pTq, &pHandle->execHandle, pCont, &dataRsp, workerId) < 0) {
|
||||
/*ASSERT(0);*/
|
||||
}
|
||||
// TODO batch optimization:
|
||||
// TODO continue scan until meeting batch requirement
|
||||
if (dataRsp.blockNum > 0 /* threshold */) {
|
||||
tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer);
|
||||
ASSERT(dataRsp.rspOffset.version >= dataRsp.reqOffset.version);
|
||||
|
||||
if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
|
||||
code = -1;
|
||||
}
|
||||
goto OVER;
|
||||
} else {
|
||||
fetchVer++;
|
||||
}
|
||||
|
||||
} else {
|
||||
ASSERT(pHandle->fetchMeta);
|
||||
ASSERT(IS_META_MSG(pHead->msgType));
|
||||
tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
|
||||
SMqMetaRsp metaRsp = {0};
|
||||
metaRsp.reqOffset = pReq->reqOffset.version;
|
||||
metaRsp.rspOffset = fetchVer;
|
||||
metaRsp.resMsgType = pHead->msgType;
|
||||
metaRsp.metaRspLen = pHead->bodyLen;
|
||||
metaRsp.metaRsp = pHead->body;
|
||||
if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
|
||||
code = -1;
|
||||
goto OVER;
|
||||
}
|
||||
code = 0;
|
||||
goto OVER;
|
||||
}
|
||||
}
|
||||
|
||||
taosMemoryFree(pHeadWithCkSum);
|
||||
} else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) {
|
||||
// 1. set uid and ts
|
||||
// 2. get data (rebuild reader if needed)
|
||||
// 3. get new uid and ts
|
||||
|
||||
tqInfo("retrieve using snapshot req offset: uid %ld ts %ld", dataRsp.reqOffset.uid, dataRsp.reqOffset.ts);
|
||||
if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) {
|
||||
ASSERT(0);
|
||||
}
|
||||
|
||||
// 4. send rsp
|
||||
if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
|
||||
code = -1;
|
||||
}
|
||||
} else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_META) {
|
||||
ASSERT(0);
|
||||
}
|
||||
|
||||
OVER:
|
||||
// TODO wrap in destroy func
|
||||
taosArrayDestroy(dataRsp.blockDataLen);
|
||||
taosArrayDestroyP(dataRsp.blockData, (FDelete)taosMemoryFree);
|
||||
|
||||
if (dataRsp.withSchema) {
|
||||
taosArrayDestroyP(dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
|
||||
}
|
||||
|
||||
if (dataRsp.withTbName) {
|
||||
taosArrayDestroyP(dataRsp.blockTbName, (FDelete)taosMemoryFree);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
#if 0
|
||||
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
||||
SMqPollReq* pReq = pMsg->pCont;
|
||||
int64_t consumerId = pReq->consumerId;
|
||||
|
@ -185,10 +423,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
} else {
|
||||
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey);
|
||||
if (pOffset != NULL) {
|
||||
ASSERT(pOffset->type == TMQ_OFFSET__LOG);
|
||||
ASSERT(pOffset->val.type == TMQ_OFFSET__LOG);
|
||||
tqDebug("consumer %ld, restore offset of %s on vg %d, offset(type:log) version: %ld", consumerId, pReq->subKey,
|
||||
TD_VID(pTq->pVnode), pOffset->version);
|
||||
fetchOffset = pOffset->version + 1;
|
||||
TD_VID(pTq->pVnode), pOffset->val.version);
|
||||
fetchOffset = pOffset->val.version + 1;
|
||||
} else {
|
||||
if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
|
||||
fetchOffset = walGetFirstVer(pTq->pWal);
|
||||
|
@ -241,12 +479,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
if (rsp.withTbName) {
|
||||
rsp.blockTbName = taosArrayInit(0, sizeof(void*));
|
||||
}
|
||||
|
||||
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
rsp.withSchema = false;
|
||||
rsp.withTag = false;
|
||||
} else {
|
||||
rsp.withSchema = true;
|
||||
rsp.withTag = false;
|
||||
rsp.blockSchema = taosArrayInit(0, sizeof(void*));
|
||||
}
|
||||
|
||||
|
@ -302,10 +539,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
}
|
||||
} else {
|
||||
ASSERT(pHandle->fetchMeta);
|
||||
ASSERT(pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB ||
|
||||
pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE ||
|
||||
pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE ||
|
||||
pHead->msgType == TDMT_VND_DROP_TTL_TABLE);
|
||||
ASSERT(IS_META_MSG(pHead->msgType));
|
||||
tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
|
||||
SMqMetaRsp metaRsp = {0};
|
||||
metaRsp.reqOffset = pReq->currentOffset;
|
||||
|
@ -341,7 +575,7 @@ SEND_RSP:
|
|||
|
||||
rsp.rspOffset = fetchOffset;
|
||||
|
||||
if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) {
|
||||
if (tqSendDataRsp(pTq, pMsg, pReq, &rsp) < 0) {
|
||||
code = -1;
|
||||
}
|
||||
OVER:
|
||||
|
@ -359,6 +593,7 @@ OVER:
|
|||
|
||||
return code;
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) {
|
||||
SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg;
|
||||
|
@ -403,7 +638,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
|
|||
.reader = pHandle->execHandle.pExecReader[i],
|
||||
.meta = pTq->pVnode->pMeta,
|
||||
.vnode = pTq->pVnode,
|
||||
// .initTsdbReader = 1,
|
||||
.tqReader = true,
|
||||
};
|
||||
pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle);
|
||||
ASSERT(pHandle->execHandle.execCol.task[i]);
|
||||
|
@ -474,12 +709,11 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
|
|||
if (pTask->execType != TASK_EXEC__NONE) {
|
||||
// expand runners
|
||||
if (pTask->isDataScan) {
|
||||
STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
|
||||
SStreamReader* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
|
||||
SReadHandle handle = {
|
||||
.reader = pStreamReader,
|
||||
.meta = pTq->pVnode->pMeta,
|
||||
.vnode = pTq->pVnode,
|
||||
// .initTsdbReader = 1,
|
||||
};
|
||||
/*pTask->exec.inputHandle = pStreamReader;*/
|
||||
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
|
||||
|
|
|
@ -15,7 +15,7 @@
|
|||
|
||||
#include "tq.h"
|
||||
|
||||
static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRsp) {
|
||||
static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp) {
|
||||
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
|
||||
void* buf = taosMemoryCalloc(1, dataStrLen);
|
||||
if (buf == NULL) return -1;
|
||||
|
@ -29,7 +29,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs
|
|||
|
||||
// TODO enable compress
|
||||
int32_t actualLen = 0;
|
||||
blockCompressEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false);
|
||||
blockEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false);
|
||||
actualLen += sizeof(SRetrieveTableRsp);
|
||||
ASSERT(actualLen <= dataStrLen);
|
||||
taosArrayPush(pRsp->blockDataLen, &actualLen);
|
||||
|
@ -37,7 +37,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataBlkRsp* pRsp) {
|
||||
static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataRsp* pRsp) {
|
||||
SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper);
|
||||
if (pSW == NULL) {
|
||||
return -1;
|
||||
|
@ -46,10 +46,9 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerI
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId) {
|
||||
static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t workerId) {
|
||||
SMetaReader mr = {0};
|
||||
metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
|
||||
int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
|
||||
if (metaGetTableEntryByUid(&mr, uid) < 0) {
|
||||
ASSERT(0);
|
||||
return -1;
|
||||
|
@ -60,13 +59,19 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqD
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId) {
|
||||
int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset, int32_t workerId) {
|
||||
ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN);
|
||||
qTaskInfo_t task = pExec->execCol.task[workerId];
|
||||
if (qStreamScanSnapshot(task) < 0) {
|
||||
|
||||
/*if (qStreamScanSnapshot(task) < 0) {*/
|
||||
/*ASSERT(0);*/
|
||||
/*}*/
|
||||
|
||||
if (qStreamPrepareScan(task, offset.uid, offset.ts) < 0) {
|
||||
ASSERT(0);
|
||||
}
|
||||
// set version
|
||||
|
||||
int32_t rowCnt = 0;
|
||||
while (1) {
|
||||
SSDataBlock* pDataBlock = NULL;
|
||||
uint64_t ts = 0;
|
||||
|
@ -79,17 +84,38 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp
|
|||
ASSERT(taosArrayGetSize(pDataBlock->pDataBlock) != 0);
|
||||
|
||||
tqAddBlockDataToRsp(pDataBlock, pRsp);
|
||||
|
||||
if (pRsp->withTbName) {
|
||||
pRsp->withTbName = 0;
|
||||
#if 0
|
||||
int64_t uid;
|
||||
int64_t ts;
|
||||
if (qGetStreamScanStatus(task, &uid, &ts) < 0) {
|
||||
ASSERT(0);
|
||||
}
|
||||
tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
|
||||
#endif
|
||||
}
|
||||
pRsp->blockNum++;
|
||||
|
||||
rowCnt += pDataBlock->info.rows;
|
||||
if (rowCnt >= 4096) break;
|
||||
}
|
||||
int64_t uid;
|
||||
int64_t ts;
|
||||
if (qGetStreamScanStatus(task, &uid, &ts) < 0) {
|
||||
ASSERT(0);
|
||||
}
|
||||
tqOffsetResetToData(&pRsp->rspOffset, uid, ts);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId) {
|
||||
int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId) {
|
||||
if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
qTaskInfo_t task = pExec->execCol.task[workerId];
|
||||
ASSERT(task);
|
||||
qSetStreamInput(task, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
|
||||
qSetStreamInput(task, pReq, STREAM_INPUT__DATA_SUBMIT, false);
|
||||
while (1) {
|
||||
SSDataBlock* pDataBlock = NULL;
|
||||
uint64_t ts = 0;
|
||||
|
@ -102,13 +128,14 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
|
|||
|
||||
tqAddBlockDataToRsp(pDataBlock, pRsp);
|
||||
if (pRsp->withTbName) {
|
||||
tqAddTbNameToRsp(pTq, pExec, pRsp, workerId);
|
||||
int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
|
||||
tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
|
||||
}
|
||||
pRsp->blockNum++;
|
||||
}
|
||||
} else if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
|
||||
pRsp->withSchema = 1;
|
||||
STqReadHandle* pReader = pExec->pExecReader[workerId];
|
||||
SStreamReader* pReader = pExec->pExecReader[workerId];
|
||||
tqReadHandleSetMsg(pReader, pReq, 0);
|
||||
while (tqNextDataBlock(pReader)) {
|
||||
SSDataBlock block = {0};
|
||||
|
@ -118,14 +145,15 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
|
|||
}
|
||||
tqAddBlockDataToRsp(&block, pRsp);
|
||||
if (pRsp->withTbName) {
|
||||
tqAddTbNameToRsp(pTq, pExec, pRsp, workerId);
|
||||
int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
|
||||
tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
|
||||
}
|
||||
tqAddBlockSchemaToRsp(pExec, workerId, pRsp);
|
||||
pRsp->blockNum++;
|
||||
}
|
||||
} else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
|
||||
pRsp->withSchema = 1;
|
||||
STqReadHandle* pReader = pExec->pExecReader[workerId];
|
||||
SStreamReader* pReader = pExec->pExecReader[workerId];
|
||||
tqReadHandleSetMsg(pReader, pReq, 0);
|
||||
while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
|
||||
SSDataBlock block = {0};
|
||||
|
@ -135,7 +163,8 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
|
|||
}
|
||||
tqAddBlockDataToRsp(&block, pRsp);
|
||||
if (pRsp->withTbName) {
|
||||
tqAddTbNameToRsp(pTq, pExec, pRsp, workerId);
|
||||
int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
|
||||
tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
|
||||
}
|
||||
tqAddBlockSchemaToRsp(pExec, workerId, pRsp);
|
||||
pRsp->blockNum++;
|
||||
|
|
|
@ -92,8 +92,8 @@ STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey) {
|
|||
}
|
||||
|
||||
int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) {
|
||||
ASSERT(pOffset->type == TMQ_OFFSET__LOG);
|
||||
ASSERT(pOffset->version >= 0);
|
||||
/*ASSERT(pOffset->val.type == TMQ_OFFSET__LOG);*/
|
||||
/*ASSERT(pOffset->val.version >= 0);*/
|
||||
return taosHashPut(pStore->pHash, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset));
|
||||
}
|
||||
|
||||
|
|
|
@ -15,16 +15,17 @@
|
|||
|
||||
#include "tq.h"
|
||||
|
||||
#if 0
|
||||
void tqTmrRspFunc(void* param, void* tmrId) {
|
||||
STqHandle* pHandle = (STqHandle*)param;
|
||||
atomic_store_8(&pHandle->pushHandle.tmrStopped, 1);
|
||||
}
|
||||
|
||||
static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataBlkRsp* pRsp) {
|
||||
static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataRsp* pRsp) {
|
||||
SStreamDataSubmit* pSubmit = *ppSubmit;
|
||||
while (pSubmit != NULL) {
|
||||
ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1);
|
||||
if (tqDataExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) {
|
||||
if (tqLogScanExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) {
|
||||
/*ASSERT(0);*/
|
||||
}
|
||||
// update processed
|
||||
|
@ -43,7 +44,7 @@ static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubm
|
|||
}
|
||||
|
||||
int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) {
|
||||
SMqDataBlkRsp rsp = {0};
|
||||
SMqDataRsp rsp = {0};
|
||||
// 1. guard and set status executing
|
||||
int8_t execStatus = atomic_val_compare_exchange_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE,
|
||||
TASK_EXEC_STATUS__EXECUTING);
|
||||
|
@ -175,13 +176,13 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
|
|||
|
||||
taosWLockLatch(&pHandle->pushHandle.lock);
|
||||
|
||||
SMqDataBlkRsp rsp = {0};
|
||||
SMqDataRsp rsp = {0};
|
||||
rsp.reqOffset = pHandle->pushHandle.reqOffset;
|
||||
rsp.blockData = taosArrayInit(0, sizeof(void*));
|
||||
rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
|
||||
|
||||
if (msgType == TDMT_VND_SUBMIT) {
|
||||
tqDataExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId);
|
||||
tqLogScanExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId);
|
||||
} else {
|
||||
// TODO
|
||||
ASSERT(0);
|
||||
|
@ -233,6 +234,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
|
|||
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
|
||||
if (msgType == TDMT_VND_SUBMIT) {
|
||||
|
|
|
@ -44,10 +44,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead*
|
|||
} else {
|
||||
if (pHandle->fetchMeta) {
|
||||
SWalReadHead* pHead = &((*ppHeadWithCkSum)->head);
|
||||
if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB ||
|
||||
pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE ||
|
||||
pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE ||
|
||||
pHead->msgType == TDMT_VND_DROP_TTL_TABLE) {
|
||||
if (IS_META_MSG(pHead->msgType)) {
|
||||
code = walFetchBody(pHandle->pWalReader, ppHeadWithCkSum);
|
||||
|
||||
if (code < 0) {
|
||||
|
@ -76,8 +73,8 @@ END:
|
|||
return code;
|
||||
}
|
||||
|
||||
STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
|
||||
STqReadHandle* pReadHandle = taosMemoryMalloc(sizeof(STqReadHandle));
|
||||
SStreamReader* tqInitSubmitMsgScanner(SMeta* pMeta) {
|
||||
SStreamReader* pReadHandle = taosMemoryMalloc(sizeof(SStreamReader));
|
||||
if (pReadHandle == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
@ -85,15 +82,15 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
|
|||
pReadHandle->pMsg = NULL;
|
||||
pReadHandle->ver = -1;
|
||||
pReadHandle->pColIdList = NULL;
|
||||
pReadHandle->cachedSchemaVer = -1;
|
||||
pReadHandle->cachedSchemaSuid = -1;
|
||||
pReadHandle->cachedSchemaVer = 0;
|
||||
pReadHandle->cachedSchemaSuid = 0;
|
||||
pReadHandle->pSchema = NULL;
|
||||
pReadHandle->pSchemaWrapper = NULL;
|
||||
pReadHandle->tbIdHash = NULL;
|
||||
return pReadHandle;
|
||||
}
|
||||
|
||||
int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t ver) {
|
||||
int32_t tqReadHandleSetMsg(SStreamReader* pReadHandle, SSubmitReq* pMsg, int64_t ver) {
|
||||
pReadHandle->pMsg = pMsg;
|
||||
|
||||
if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1;
|
||||
|
@ -108,7 +105,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t
|
|||
return 0;
|
||||
}
|
||||
|
||||
bool tqNextDataBlock(STqReadHandle* pHandle) {
|
||||
bool tqNextDataBlock(SStreamReader* pHandle) {
|
||||
if (pHandle->pMsg == NULL) return false;
|
||||
while (1) {
|
||||
if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
|
||||
|
@ -130,7 +127,7 @@ bool tqNextDataBlock(STqReadHandle* pHandle) {
|
|||
return false;
|
||||
}
|
||||
|
||||
bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) {
|
||||
bool tqNextDataBlockFilterOut(SStreamReader* pHandle, SHashObj* filterOutUids) {
|
||||
while (1) {
|
||||
if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
|
||||
return false;
|
||||
|
@ -146,7 +143,7 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) {
|
|||
return false;
|
||||
}
|
||||
|
||||
int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) {
|
||||
int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) {
|
||||
// TODO: cache multiple schema
|
||||
int32_t sversion = htonl(pHandle->pBlock->sversion);
|
||||
if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion ||
|
||||
|
@ -231,7 +228,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) {
|
|||
tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter);
|
||||
|
||||
pBlock->info.groupId = 0;
|
||||
pBlock->info.uid = pHandle->msgIter.uid; // set the uid of table for submit block
|
||||
pBlock->info.uid = pHandle->msgIter.uid;
|
||||
pBlock->info.rows = pHandle->msgIter.numOfRows;
|
||||
|
||||
while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
|
||||
|
@ -251,14 +248,14 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) {
|
|||
}
|
||||
return 0;
|
||||
|
||||
FAIL: // todo refactor here
|
||||
// if (*ppCols) taosArrayDestroy(*ppCols);
|
||||
FAIL:
|
||||
tDeleteSSDataBlock(pBlock);
|
||||
return -1;
|
||||
}
|
||||
|
||||
void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; }
|
||||
void tqReadHandleSetColIdList(SStreamReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; }
|
||||
|
||||
int tqReadHandleSetTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) {
|
||||
int tqReadHandleSetTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
|
||||
if (pHandle->tbIdHash) {
|
||||
taosHashClear(pHandle->tbIdHash);
|
||||
}
|
||||
|
@ -277,7 +274,7 @@ int tqReadHandleSetTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int tqReadHandleAddTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) {
|
||||
int tqReadHandleAddTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
|
||||
if (pHandle->tbIdHash == NULL) {
|
||||
pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
|
||||
if (pHandle->tbIdHash == NULL) {
|
||||
|
@ -294,7 +291,7 @@ int tqReadHandleAddTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) {
|
||||
int tqReadHandleRemoveTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
|
||||
ASSERT(pHandle->tbIdHash != NULL);
|
||||
|
||||
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
|
||||
|
|
|
@ -112,9 +112,9 @@ typedef struct STsdbReadHandle {
|
|||
STimeWindow window; // the primary query time window that applies to all queries
|
||||
// SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time
|
||||
// SColumnDataAgg** pstatis;// the ptr array list to return to caller
|
||||
int32_t numOfBlocks;
|
||||
int32_t numOfBlocks;
|
||||
SSDataBlock* pResBlock;
|
||||
// SArray* pColumns; // column list, SColumnInfoData array list
|
||||
// SArray* pColumns; // column list, SColumnInfoData array list
|
||||
bool locateStart;
|
||||
int32_t outputCapacity;
|
||||
int32_t realNumOfRows;
|
||||
|
@ -223,6 +223,22 @@ int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) {
|
|||
return rows;
|
||||
}
|
||||
|
||||
static SArray* createCheckInfoFromUid(STsdbReadHandle* pTsdbReadHandle, int64_t uid) {
|
||||
SArray* pTableCheckInfo = taosArrayInit(1, sizeof(STableCheckInfo));
|
||||
if (pTableCheckInfo == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
STableCheckInfo info = {
|
||||
.tableId = uid,
|
||||
};
|
||||
info.suid = pTsdbReadHandle->suid;
|
||||
|
||||
taosArrayPush(pTableCheckInfo, &info);
|
||||
tsdbDebug("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReadHandle, info.tableId, info.lastKey,
|
||||
pTsdbReadHandle->idStr);
|
||||
return pTableCheckInfo;
|
||||
}
|
||||
|
||||
static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, SArray* pTableList) {
|
||||
size_t tableSize = taosArrayGetSize(pTableList);
|
||||
|
||||
|
@ -428,8 +444,8 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond*
|
|||
|
||||
for (int32_t i = 0; i < pCond->numOfCols; ++i) {
|
||||
SColumnInfoData colInfo = {.info = pCond->colList[i], 0};
|
||||
int32_t code = blockDataAppendColInfo(pReadHandle->pResBlock, &colInfo);
|
||||
if (code != TSDB_CODE_SUCCESS){
|
||||
int32_t code = blockDataAppendColInfo(pReadHandle->pResBlock, &colInfo);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _end;
|
||||
}
|
||||
}
|
||||
|
@ -494,9 +510,19 @@ static int32_t setCurrentSchema(SVnode* pVnode, STsdbReadHandle* pTsdbReadHandle
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList){
|
||||
int32_t tsdbSetTableId(tsdbReaderT reader, int64_t uid) {
|
||||
STsdbReadHandle* pTsdbReadHandle = reader;
|
||||
if(pTsdbReadHandle->pTableCheckInfo) taosArrayDestroy(pTsdbReadHandle->pTableCheckInfo);
|
||||
if (pTsdbReadHandle->pTableCheckInfo) taosArrayDestroy(pTsdbReadHandle->pTableCheckInfo);
|
||||
pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromUid(pTsdbReadHandle, uid);
|
||||
if (pTsdbReadHandle->pTableCheckInfo == NULL) {
|
||||
return TSDB_CODE_TDB_OUT_OF_MEMORY;
|
||||
}
|
||||
return TDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList) {
|
||||
STsdbReadHandle* pTsdbReadHandle = reader;
|
||||
if (pTsdbReadHandle->pTableCheckInfo) taosArrayDestroy(pTsdbReadHandle->pTableCheckInfo);
|
||||
pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, tableList);
|
||||
if (pTsdbReadHandle->pTableCheckInfo == NULL) {
|
||||
return TSDB_CODE_TDB_OUT_OF_MEMORY;
|
||||
|
@ -505,8 +531,8 @@ int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList){
|
|||
}
|
||||
|
||||
tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* tableList, uint64_t qId,
|
||||
uint64_t taskId) {
|
||||
if(taosArrayGetSize(tableList) == 0){
|
||||
uint64_t taskId) {
|
||||
if (taosArrayGetSize(tableList) == 0) {
|
||||
return NULL;
|
||||
}
|
||||
STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(pVnode, pCond, qId, taskId);
|
||||
|
@ -553,8 +579,7 @@ tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* t
|
|||
}
|
||||
|
||||
tsdbDebug("%p total numOfTable:%" PRIzu " in this query, table %" PRIzu " %s", pTsdbReadHandle,
|
||||
taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(tableList),
|
||||
pTsdbReadHandle->idStr);
|
||||
taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(tableList), pTsdbReadHandle->idStr);
|
||||
|
||||
return (tsdbReaderT)pTsdbReadHandle;
|
||||
}
|
||||
|
@ -1073,7 +1098,7 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio
|
|||
}
|
||||
|
||||
int64_t fid = (int64_t)(key / (daysPerFile * tsTickPerMin[precision])); // set the starting fileId
|
||||
if (fid < 0LL && llabs(fid) > INT32_MAX) { // data value overflow for INT32
|
||||
if (fid < 0LL && llabs(fid) > INT32_MAX) { // data value overflow for INT32
|
||||
fid = INT32_MIN;
|
||||
}
|
||||
|
||||
|
@ -2612,7 +2637,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo*
|
|||
tsdbGetFidKeyRange(pCfg->days, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey);
|
||||
|
||||
// current file are not overlapped with query time window, ignore remain files
|
||||
if ((win.skey > pTsdbReadHandle->window.ekey)/* || (!ascTraverse && win.ekey < pTsdbReadHandle->window.ekey)*/) {
|
||||
if ((win.skey > pTsdbReadHandle->window.ekey) /* || (!ascTraverse && win.ekey < pTsdbReadHandle->window.ekey)*/) {
|
||||
tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
|
||||
tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pTsdbReadHandle,
|
||||
pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr);
|
||||
|
@ -2886,7 +2911,7 @@ int32_t tsdbGetCtbIdList(SMeta* pMeta, int64_t suid, SArray* list) {
|
|||
*/
|
||||
int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list) {
|
||||
SMStbCursor* pCur = metaOpenStbCursor(pMeta, suid);
|
||||
if(!pCur) {
|
||||
if (!pCur) {
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
|
|
|
@ -266,6 +266,8 @@ int vnodeCommit(SVnode *pVnode) {
|
|||
ASSERT(0);
|
||||
return -1;
|
||||
}
|
||||
|
||||
pVnode->state.committed = info.state.committed;
|
||||
|
||||
// apply the commit (TODO)
|
||||
vnodeBufPoolReset(pVnode->onCommit);
|
||||
|
|
|
@ -216,7 +216,7 @@ _err:
|
|||
}
|
||||
|
||||
int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
|
||||
if (TDMT_VND_QUERY != pMsg->msgType) {
|
||||
if (TDMT_SCH_QUERY != pMsg->msgType && TDMT_SCH_MERGE_QUERY != pMsg->msgType) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -227,9 +227,10 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
|
|||
vTrace("message in vnode query queue is processing");
|
||||
SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
|
||||
switch (pMsg->msgType) {
|
||||
case TDMT_VND_QUERY:
|
||||
case TDMT_SCH_QUERY:
|
||||
case TDMT_SCH_MERGE_QUERY:
|
||||
return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg, 0);
|
||||
case TDMT_VND_QUERY_CONTINUE:
|
||||
case TDMT_SCH_QUERY_CONTINUE:
|
||||
return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg, 0);
|
||||
default:
|
||||
vError("unknown msg type:%d in query queue", pMsg->msgType);
|
||||
|
@ -243,15 +244,15 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
|
|||
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
|
||||
|
||||
switch (pMsg->msgType) {
|
||||
case TDMT_VND_FETCH:
|
||||
case TDMT_SCH_FETCH:
|
||||
return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg, 0);
|
||||
case TDMT_VND_FETCH_RSP:
|
||||
case TDMT_SCH_FETCH_RSP:
|
||||
return qWorkerProcessFetchRsp(pVnode, pVnode->pQuery, pMsg, 0);
|
||||
case TDMT_VND_CANCEL_TASK:
|
||||
case TDMT_SCH_CANCEL_TASK:
|
||||
return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg, 0);
|
||||
case TDMT_VND_DROP_TASK:
|
||||
case TDMT_SCH_DROP_TASK:
|
||||
return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg, 0);
|
||||
case TDMT_VND_QUERY_HEARTBEAT:
|
||||
case TDMT_SCH_QUERY_HEARTBEAT:
|
||||
return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg, 0);
|
||||
case TDMT_VND_TABLE_META:
|
||||
return vnodeGetTableMeta(pVnode, pMsg);
|
||||
|
@ -780,7 +781,7 @@ _exit:
|
|||
// TODO: the partial success scenario and the error case
|
||||
// TODO: refactor
|
||||
if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
|
||||
tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
|
||||
tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
|
|
@ -73,7 +73,7 @@ static int32_t vnodeSetStandBy(SVnode *pVnode) {
|
|||
vInfo("vgId:%d, set standby success", TD_VID(pVnode));
|
||||
return 0;
|
||||
} else {
|
||||
vError("vgId:%d, failed to set standby since %s", TD_VID(pVnode), terrstr());
|
||||
vError("vgId:%d, failed to set standby after leader transfer since %s", TD_VID(pVnode), terrstr());
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
@ -119,7 +119,7 @@ static int32_t vnodeProcessAlterReplicaReq(SVnode *pVnode, SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
||||
SVnode *pVnode = pInfo->ahandle;
|
||||
SVnode * pVnode = pInfo->ahandle;
|
||||
int32_t vgId = pVnode->config.vgId;
|
||||
int32_t code = 0;
|
||||
SRpcMsg *pMsg = NULL;
|
||||
|
@ -178,7 +178,7 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
|||
for (int32_t i = 0; i < newEpSet.numOfEps; ++i) {
|
||||
vGTrace("vgId:%d, msg:%p redirect:%d ep:%s:%u", vgId, pMsg, i, newEpSet.eps[i].fqdn, newEpSet.eps[i].port);
|
||||
}
|
||||
|
||||
pMsg->info.hasEpSet = 1;
|
||||
SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
|
||||
tmsgSendRedirectRsp(&rsp, &newEpSet);
|
||||
} else {
|
||||
|
@ -199,7 +199,7 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
|||
}
|
||||
|
||||
void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
||||
SVnode *pVnode = pInfo->ahandle;
|
||||
SVnode * pVnode = pInfo->ahandle;
|
||||
int32_t vgId = pVnode->config.vgId;
|
||||
int32_t code = 0;
|
||||
SRpcMsg *pMsg = NULL;
|
||||
|
@ -240,7 +240,7 @@ int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
|
|||
STraceId *trace = &pMsg->info.traceId;
|
||||
|
||||
do {
|
||||
char *syncNodeStr = sync2SimpleStr(pVnode->sync);
|
||||
char * syncNodeStr = sync2SimpleStr(pVnode->sync);
|
||||
static int64_t vndTick = 0;
|
||||
if (++vndTick % 10 == 1) {
|
||||
vGTrace("vgId:%d, sync trace msg:%s, %s", syncGetVgId(pVnode->sync), TMSG_INFO(pMsg->msgType), syncNodeStr);
|
||||
|
@ -375,7 +375,7 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon
|
|||
}
|
||||
|
||||
static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
|
||||
SVnode *pVnode = pFsm->data;
|
||||
SVnode * pVnode = pFsm->data;
|
||||
SSnapshot snapshot = {0};
|
||||
SyncIndex beginIndex = SYNC_INDEX_INVALID;
|
||||
char logBuf[256] = {0};
|
||||
|
|
|
@ -1127,12 +1127,14 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_
|
|||
}
|
||||
|
||||
*num = taosHashGetSize(pCtg->userCache);
|
||||
if (*num > 0) {
|
||||
*users = taosMemoryCalloc(*num, sizeof(SUserAuthVersion));
|
||||
if (NULL == *users) {
|
||||
ctgError("calloc %d userAuthVersion failed", *num);
|
||||
CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY);
|
||||
}
|
||||
if (*num <= 0) {
|
||||
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
|
||||
}
|
||||
|
||||
*users = taosMemoryCalloc(*num, sizeof(SUserAuthVersion));
|
||||
if (NULL == *users) {
|
||||
ctgError("calloc %d userAuthVersion failed", *num);
|
||||
CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
uint32_t i = 0;
|
||||
|
@ -1144,6 +1146,11 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_
|
|||
(*users)[i].user[len] = 0;
|
||||
(*users)[i].version = pAuth->version;
|
||||
++i;
|
||||
if (i >= *num) {
|
||||
taosHashCancelIterate(pCtg->userCache, pAuth);
|
||||
break;
|
||||
}
|
||||
|
||||
pAuth = taosHashIterate(pCtg->userCache, pAuth);
|
||||
}
|
||||
|
||||
|
|
|
@ -825,6 +825,9 @@ _return:
|
|||
qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(code));
|
||||
|
||||
pJob->jobResCode = code;
|
||||
|
||||
//taosSsleep(2);
|
||||
//qDebug("QID:0x%" PRIx64 " ctg after sleep", pJob->queryId);
|
||||
|
||||
taosAsyncExec(ctgCallUserCb, pJob, NULL);
|
||||
|
||||
|
|
|
@ -1083,7 +1083,7 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si
|
|||
|
||||
CTG_LOCK(CTG_WRITE, &slot->lock);
|
||||
if (NULL == slot->meta) {
|
||||
qError("empty meta slot, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type);
|
||||
qDebug("empty meta slot, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type);
|
||||
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
|
||||
}
|
||||
|
||||
|
@ -1536,8 +1536,6 @@ void ctgClearAllInstance(void) {
|
|||
|
||||
pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
|
||||
}
|
||||
|
||||
taosHashClear(gCtgMgmt.pCluster);
|
||||
}
|
||||
|
||||
void ctgFreeAllInstance(void) {
|
||||
|
@ -1566,27 +1564,27 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
|
|||
SCatalog* pCtg = msg->pCtg;
|
||||
|
||||
if (NULL == dbInfo->vgHash) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
goto _return;
|
||||
}
|
||||
|
||||
if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) {
|
||||
ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d",
|
||||
dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash));
|
||||
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
|
||||
CTG_ERR_JRET(TSDB_CODE_APP_ERROR);
|
||||
}
|
||||
|
||||
bool newAdded = false;
|
||||
SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable};
|
||||
|
||||
SCtgDBCache *dbCache = NULL;
|
||||
CTG_ERR_RET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache));
|
||||
CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache));
|
||||
if (NULL == dbCache) {
|
||||
ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%"PRIx64, dbFName, msg->dbId);
|
||||
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
|
||||
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
|
||||
}
|
||||
|
||||
SCtgVgCache *vgCache = &dbCache->vgCache;
|
||||
CTG_ERR_RET(ctgWLockVgInfo(msg->pCtg, dbCache));
|
||||
CTG_ERR_JRET(ctgWLockVgInfo(msg->pCtg, dbCache));
|
||||
|
||||
if (vgCache->vgInfo) {
|
||||
SDBVgInfo *vgInfo = vgCache->vgInfo;
|
||||
|
@ -1595,14 +1593,14 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
|
|||
ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion);
|
||||
ctgWUnlockVgInfo(dbCache);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
goto _return;
|
||||
}
|
||||
|
||||
if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) {
|
||||
ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable);
|
||||
ctgWUnlockVgInfo(dbCache);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
goto _return;
|
||||
}
|
||||
|
||||
ctgFreeVgInfo(vgInfo);
|
||||
|
|
|
@ -19,7 +19,7 @@
|
|||
#include "catalogInt.h"
|
||||
|
||||
extern SCatalogMgmt gCtgMgmt;
|
||||
SCtgDebug gCTGDebug = {.lockEnable = true, .apiEnable = true};
|
||||
SCtgDebug gCTGDebug = {0};
|
||||
|
||||
void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) {
|
||||
ASSERT(*(int32_t*)param == 1);
|
||||
|
|
|
@ -241,7 +241,7 @@ int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize,
|
|||
}
|
||||
|
||||
|
||||
int32_t ctgHandleMsgCallback(void *param, const SDataBuf *pMsg, int32_t rspCode) {
|
||||
int32_t ctgHandleMsgCallback(void *param, SDataBuf *pMsg, int32_t rspCode) {
|
||||
SCtgTaskCallbackParam* cbParam = (SCtgTaskCallbackParam*)param;
|
||||
int32_t code = 0;
|
||||
|
||||
|
|
|
@ -36,7 +36,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe
|
|||
(*pRsp)->numOfCols = htonl(numOfCols);
|
||||
|
||||
int32_t len = 0;
|
||||
blockCompressEncode(pBlock, (*pRsp)->data, &len, numOfCols, false);
|
||||
blockEncode(pBlock, (*pRsp)->data, &len, numOfCols, false);
|
||||
ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
|
||||
|
||||
blockDataDestroy(pBlock);
|
||||
|
|
|
@ -13,11 +13,11 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "tdatablock.h"
|
||||
#include "commandInt.h"
|
||||
#include "plannodes.h"
|
||||
#include "query.h"
|
||||
#include "tcommon.h"
|
||||
#include "tdatablock.h"
|
||||
|
||||
int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes);
|
||||
int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level);
|
||||
|
@ -216,7 +216,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo
|
|||
SExplainResNode *pResNode = NULL;
|
||||
FOREACH(node, pPhysiChildren) {
|
||||
QRY_ERR_RET(qExplainGenerateResNode((SPhysiNode *)node, group, &pResNode));
|
||||
QRY_ERR_RET(nodesListAppend(*pChildren, (SNode*)pResNode));
|
||||
QRY_ERR_RET(nodesListAppend(*pChildren, (SNode *)pResNode));
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
@ -232,14 +232,14 @@ int32_t qExplainGenerateResNodeExecInfo(SArray **pExecInfo, SExplainGroup *group
|
|||
SExplainRsp *rsp = NULL;
|
||||
for (int32_t i = 0; i < group->nodeNum; ++i) {
|
||||
rsp = taosArrayGet(group->nodeExecInfo, i);
|
||||
/*
|
||||
if (group->physiPlanExecIdx >= rsp->numOfPlans) {
|
||||
qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
/*
|
||||
if (group->physiPlanExecIdx >= rsp->numOfPlans) {
|
||||
qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
|
||||
taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx);
|
||||
*/
|
||||
taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx);
|
||||
*/
|
||||
taosArrayPush(*pExecInfo, rsp->subplanInfo);
|
||||
}
|
||||
|
||||
|
@ -426,23 +426,23 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
|
|||
if (EXPLAIN_MODE_ANALYZE == ctx->mode) {
|
||||
EXPLAIN_ROW_NEW(level + 1, "I/O: ");
|
||||
|
||||
int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo);
|
||||
int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo);
|
||||
struct STableScanAnalyzeInfo info = {0};
|
||||
|
||||
int32_t maxIndex = 0;
|
||||
int32_t totalRows = 0;
|
||||
for(int32_t i = 0; i < nodeNum; ++i) {
|
||||
for (int32_t i = 0; i < nodeNum; ++i) {
|
||||
SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, i);
|
||||
STableScanAnalyzeInfo *pScanInfo = (STableScanAnalyzeInfo *)execInfo->verboseInfo;
|
||||
|
||||
info.totalBlocks += pScanInfo->totalBlocks;
|
||||
info.loadBlocks += pScanInfo->loadBlocks;
|
||||
info.totalRows += pScanInfo->totalRows;
|
||||
info.skipBlocks += pScanInfo->skipBlocks;
|
||||
info.filterTime += pScanInfo->filterTime;
|
||||
info.loadBlockStatis += pScanInfo->loadBlockStatis;
|
||||
info.totalBlocks += pScanInfo->totalBlocks;
|
||||
info.loadBlocks += pScanInfo->loadBlocks;
|
||||
info.totalRows += pScanInfo->totalRows;
|
||||
info.skipBlocks += pScanInfo->skipBlocks;
|
||||
info.filterTime += pScanInfo->filterTime;
|
||||
info.loadBlockStatis += pScanInfo->loadBlockStatis;
|
||||
info.totalCheckedRows += pScanInfo->totalCheckedRows;
|
||||
info.filterOutBlocks += pScanInfo->filterOutBlocks;
|
||||
info.filterOutBlocks += pScanInfo->filterOutBlocks;
|
||||
|
||||
if (pScanInfo->totalRows > totalRows) {
|
||||
totalRows = pScanInfo->totalRows;
|
||||
|
@ -468,13 +468,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
|
|||
|
||||
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
|
||||
|
||||
//Rows out: Avg 4166.7 rows x 24 workers. Max 4187 rows (seg7) with 0.220 ms to first row, 1.738 ms to end, start offset by 1.470 ms.
|
||||
// Rows out: Avg 4166.7 rows x 24 workers. Max 4187 rows (seg7) with 0.220 ms to first row, 1.738 ms to end,
|
||||
// start offset by 1.470 ms.
|
||||
SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, maxIndex);
|
||||
STableScanAnalyzeInfo *p1 = (STableScanAnalyzeInfo *)execInfo->verboseInfo;
|
||||
|
||||
EXPLAIN_ROW_NEW(level + 1, " ");
|
||||
EXPLAIN_ROW_APPEND("max_row_task=%d, total_rows:%" PRId64 ", ep:%s (cost=%.3f..%.3f)", maxIndex, p1->totalRows, "tbd",
|
||||
execInfo->startupCost, execInfo->totalCost);
|
||||
EXPLAIN_ROW_APPEND("max_row_task=%d, total_rows:%" PRId64 ", ep:%s (cost=%.3f..%.3f)", maxIndex, p1->totalRows,
|
||||
"tbd", execInfo->startupCost, execInfo->totalCost);
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
|
||||
EXPLAIN_ROW_END();
|
||||
|
||||
|
@ -752,7 +753,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
|
|||
EXPLAIN_ROW_NEW(level + 1, "Sort Key: ");
|
||||
if (pResNode->pExecInfo) {
|
||||
for (int32_t i = 0; i < LIST_LENGTH(pSortNode->pSortKeys); ++i) {
|
||||
SOrderByExprNode *ptn = (SOrderByExprNode*)nodesListGetNode(pSortNode->pSortKeys, i);
|
||||
SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pSortNode->pSortKeys, i);
|
||||
EXPLAIN_ROW_APPEND("%s ", nodesGetNameFromColumnNode(ptn->pExpr));
|
||||
}
|
||||
}
|
||||
|
@ -907,16 +908,16 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
|
|||
EXPLAIN_ROW_END();
|
||||
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
|
||||
if (pFillNode->pValues) {
|
||||
SNodeListNode *pValues = (SNodeListNode*)pFillNode->pValues;
|
||||
SNodeListNode *pValues = (SNodeListNode *)pFillNode->pValues;
|
||||
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_VALUE_FORMAT);
|
||||
SNode* tNode = NULL;
|
||||
SNode *tNode = NULL;
|
||||
int32_t i = 0;
|
||||
FOREACH(tNode, pValues->pNodeList) {
|
||||
if (i) {
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
|
||||
}
|
||||
SValueNode* tValue = (SValueNode*)tNode;
|
||||
char *value = nodesGetStrValueFromNode(tValue);
|
||||
SValueNode *tValue = (SValueNode *)tNode;
|
||||
char *value = nodesGetStrValueFromNode(tValue);
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, value);
|
||||
taosMemoryFree(value);
|
||||
++i;
|
||||
|
@ -926,8 +927,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
|
|||
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
|
||||
}
|
||||
|
||||
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pFillNode->timeRange.skey,
|
||||
pFillNode->timeRange.ekey);
|
||||
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pFillNode->timeRange.skey, pFillNode->timeRange.ekey);
|
||||
EXPLAIN_ROW_END();
|
||||
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
|
||||
|
||||
|
@ -1070,13 +1070,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
|
|||
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
|
||||
EXPLAIN_ROW_END();
|
||||
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
|
||||
|
||||
|
||||
if (EXPLAIN_MODE_ANALYZE == ctx->mode) {
|
||||
// sort key
|
||||
EXPLAIN_ROW_NEW(level + 1, "Merge Key: ");
|
||||
if (pResNode->pExecInfo) {
|
||||
for (int32_t i = 0; i < LIST_LENGTH(pMergeNode->pMergeKeys); ++i) {
|
||||
SOrderByExprNode *ptn = (SOrderByExprNode*)nodesListGetNode(pMergeNode->pMergeKeys, i);
|
||||
SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pMergeNode->pMergeKeys, i);
|
||||
EXPLAIN_ROW_APPEND("%s ", nodesGetNameFromColumnNode(ptn->pExpr));
|
||||
}
|
||||
}
|
||||
|
@ -1115,7 +1115,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
|
|||
|
||||
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGE_KEYS_FORMAT);
|
||||
for (int32_t i = 0; i < LIST_LENGTH(pMergeNode->pMergeKeys); ++i) {
|
||||
SOrderByExprNode *ptn = (SOrderByExprNode*)nodesListGetNode(pMergeNode->pMergeKeys, i);
|
||||
SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pMergeNode->pMergeKeys, i);
|
||||
EXPLAIN_ROW_APPEND("%s ", nodesGetNameFromColumnNode(ptn->pExpr));
|
||||
}
|
||||
EXPLAIN_ROW_END();
|
||||
|
@ -1130,7 +1130,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
|
|||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
default:
|
||||
qError("not supported physical node type %d", pNode->type);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
|
@ -1190,12 +1190,12 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
|
|||
QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
|
||||
}
|
||||
|
||||
SSDataBlock *pBlock = createDataBlock();
|
||||
SSDataBlock *pBlock = createDataBlock();
|
||||
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1);
|
||||
blockDataAppendColInfo(pBlock, &infoData);
|
||||
blockDataEnsureCapacity(pBlock, rowNum);
|
||||
|
||||
SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
|
||||
SColumnInfoData *pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
|
||||
|
||||
char buf[1024] = {0};
|
||||
for (int32_t i = 0; i < rowNum; ++i) {
|
||||
|
@ -1219,7 +1219,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
|
|||
rsp->numOfRows = htonl(rowNum);
|
||||
|
||||
int32_t len = 0;
|
||||
blockCompressEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0);
|
||||
blockEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0);
|
||||
ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
|
||||
|
||||
rsp->compLen = htonl(len);
|
||||
|
|
|
@ -25,37 +25,36 @@
|
|||
#define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \
|
||||
do { \
|
||||
assert(sizeof(_uid) == sizeof(uint64_t)); \
|
||||
*(uint64_t *)(_k) = (_uid); \
|
||||
*(uint64_t*)(_k) = (_uid); \
|
||||
memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \
|
||||
} while (0)
|
||||
|
||||
#define SET_RES_EXT_WINDOW_KEY(_k, _ori, _len, _uid, _buf) \
|
||||
do { \
|
||||
assert(sizeof(_uid) == sizeof(uint64_t)); \
|
||||
*(void **)(_k) = (_buf); \
|
||||
*(uint64_t *)((_k) + POINTER_BYTES) = (_uid); \
|
||||
memcpy((_k) + POINTER_BYTES + sizeof(uint64_t), (_ori), (_len)); \
|
||||
#define SET_RES_EXT_WINDOW_KEY(_k, _ori, _len, _uid, _buf) \
|
||||
do { \
|
||||
assert(sizeof(_uid) == sizeof(uint64_t)); \
|
||||
*(void**)(_k) = (_buf); \
|
||||
*(uint64_t*)((_k) + POINTER_BYTES) = (_uid); \
|
||||
memcpy((_k) + POINTER_BYTES + sizeof(uint64_t), (_ori), (_len)); \
|
||||
} while (0)
|
||||
|
||||
|
||||
#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))
|
||||
#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))
|
||||
#define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES)
|
||||
|
||||
#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str)
|
||||
#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str)
|
||||
|
||||
typedef struct SGroupResInfo {
|
||||
int32_t index;
|
||||
SArray* pRows; // SArray<SResKeyPos>
|
||||
SArray* pRows; // SArray<SResKeyPos>
|
||||
} SGroupResInfo;
|
||||
|
||||
typedef struct SResultRow {
|
||||
int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer
|
||||
int32_t offset:29; // row index in buffer page
|
||||
bool startInterp; // the time window start timestamp has done the interpolation already.
|
||||
bool endInterp; // the time window end timestamp has done the interpolation already.
|
||||
bool closed; // this result status: closed or opened
|
||||
uint32_t numOfRows; // number of rows of current time window
|
||||
STimeWindow win;
|
||||
int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer
|
||||
int32_t offset : 29; // row index in buffer page
|
||||
bool startInterp; // the time window start timestamp has done the interpolation already.
|
||||
bool endInterp; // the time window end timestamp has done the interpolation already.
|
||||
bool closed; // this result status: closed or opened
|
||||
uint32_t numOfRows; // number of rows of current time window
|
||||
STimeWindow win;
|
||||
struct SResultRowEntryInfo pEntryInfo[]; // For each result column, there is a resultInfo
|
||||
} SResultRow;
|
||||
|
||||
|
@ -66,57 +65,58 @@ typedef struct SResultRowPosition {
|
|||
|
||||
typedef struct SResKeyPos {
|
||||
SResultRowPosition pos;
|
||||
uint64_t groupId;
|
||||
char key[];
|
||||
uint64_t groupId;
|
||||
char key[];
|
||||
} SResKeyPos;
|
||||
|
||||
typedef struct SResultRowInfo {
|
||||
int32_t size; // number of result set
|
||||
int32_t size; // number of result set
|
||||
SResultRowPosition cur;
|
||||
SList* openWindow;
|
||||
SList* openWindow;
|
||||
} SResultRowInfo;
|
||||
|
||||
struct SqlFunctionCtx;
|
||||
|
||||
size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput);
|
||||
void initResultRowInfo(SResultRowInfo* pResultRowInfo);
|
||||
void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo);
|
||||
size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput);
|
||||
void initResultRowInfo(SResultRowInfo* pResultRowInfo);
|
||||
void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo);
|
||||
|
||||
void closeAllResultRows(SResultRowInfo* pResultRowInfo);
|
||||
void closeAllResultRows(SResultRowInfo* pResultRowInfo);
|
||||
|
||||
void initResultRow(SResultRow *pResultRow);
|
||||
void closeResultRow(SResultRow* pResultRow);
|
||||
bool isResultRowClosed(SResultRow* pResultRow);
|
||||
void initResultRow(SResultRow* pResultRow);
|
||||
void closeResultRow(SResultRow* pResultRow);
|
||||
bool isResultRowClosed(SResultRow* pResultRow);
|
||||
|
||||
struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset);
|
||||
|
||||
static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) {
|
||||
SFilePage* bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId);
|
||||
static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) {
|
||||
SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId);
|
||||
SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset);
|
||||
return pRow;
|
||||
}
|
||||
|
||||
void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order);
|
||||
void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);
|
||||
void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order);
|
||||
void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);
|
||||
|
||||
void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo);
|
||||
bool hasDataInGroupInfo(SGroupResInfo* pGroupResInfo);
|
||||
void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo);
|
||||
bool hasDataInGroupInfo(SGroupResInfo* pGroupResInfo);
|
||||
|
||||
int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
|
||||
|
||||
SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode);
|
||||
|
||||
EDealRes doTranslateTagExpr(SNode** pNode, void* pContext);
|
||||
int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo);
|
||||
SArray* createSortInfo(SNodeList* pNodeList);
|
||||
SArray* extractPartitionColInfo(SNodeList* pNodeList);
|
||||
SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type);
|
||||
int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo);
|
||||
SArray* createSortInfo(SNodeList* pNodeList);
|
||||
SArray* extractPartitionColInfo(SNodeList* pNodeList);
|
||||
SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
|
||||
int32_t type);
|
||||
|
||||
SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs);
|
||||
|
||||
SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset);
|
||||
void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn);
|
||||
void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow);
|
||||
void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn);
|
||||
void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow);
|
||||
|
||||
SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode);
|
||||
SColumn extractColumnFromColumnNode(SColumnNode* pColNode);
|
||||
|
|
|
@ -248,6 +248,11 @@ typedef struct SSampleExecInfo {
|
|||
uint32_t seed; // random seed value
|
||||
} SSampleExecInfo;
|
||||
|
||||
enum {
|
||||
TABLE_SCAN__TABLE_ORDER = 1,
|
||||
TABLE_SCAN__BLOCK_ORDER = 2,
|
||||
};
|
||||
|
||||
typedef struct STableScanInfo {
|
||||
void* dataReader;
|
||||
SReadHandle readHandle;
|
||||
|
@ -272,13 +277,16 @@ typedef struct STableScanInfo {
|
|||
int32_t curTWinIdx;
|
||||
|
||||
int32_t currentGroupId;
|
||||
int32_t currentTable;
|
||||
uint64_t queryId; // todo remove it
|
||||
uint64_t taskId; // todo remove it
|
||||
|
||||
struct {
|
||||
uint64_t uid;
|
||||
int64_t t;
|
||||
} scanStatus;
|
||||
int64_t ts;
|
||||
} lastStatus;
|
||||
|
||||
int8_t scanMode;
|
||||
} STableScanInfo;
|
||||
|
||||
typedef struct STagScanInfo {
|
||||
|
@ -418,6 +426,7 @@ typedef struct SIntervalAggOperatorInfo {
|
|||
STimeWindowAggSupp twAggSup;
|
||||
bool invertible;
|
||||
SArray* pPrevValues; // SArray<SGroupKeys> used to keep the previous not null value for interpolation.
|
||||
bool ignoreCloseWindow;
|
||||
} SIntervalAggOperatorInfo;
|
||||
|
||||
typedef struct SStreamFinalIntervalOperatorInfo {
|
||||
|
@ -439,6 +448,7 @@ typedef struct SStreamFinalIntervalOperatorInfo {
|
|||
SArray* pPullWins; // SPullWindowInfo
|
||||
int32_t pullIndex;
|
||||
SSDataBlock* pPullDataRes;
|
||||
bool ignoreCloseWindow;
|
||||
} SStreamFinalIntervalOperatorInfo;
|
||||
|
||||
typedef struct SAggOperatorInfo {
|
||||
|
@ -576,6 +586,7 @@ typedef struct SStreamSessionAggOperatorInfo {
|
|||
SArray* pChildren; // cache for children's result; final stream operator
|
||||
SPhysiNode* pPhyNode; // create new child
|
||||
bool isFinal;
|
||||
bool ignoreCloseWindow;
|
||||
} SStreamSessionAggOperatorInfo;
|
||||
|
||||
typedef struct STimeSliceOperatorInfo {
|
||||
|
@ -619,6 +630,7 @@ typedef struct SStreamStateAggOperatorInfo {
|
|||
void* pDelIterator;
|
||||
SArray* pScanWindow;
|
||||
SArray* pChildren; // cache for children's result;
|
||||
bool ignoreCloseWindow;
|
||||
} SStreamStateAggOperatorInfo;
|
||||
|
||||
typedef struct SSortedMergeOperatorInfo {
|
||||
|
@ -709,6 +721,7 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput);
|
|||
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
|
||||
void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId);
|
||||
|
||||
int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts);
|
||||
int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts);
|
||||
|
||||
SSDataBlock* loadNextDataBlock(void* param);
|
||||
|
|
|
@ -69,14 +69,16 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) {
|
|||
|
||||
// data format:
|
||||
// +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+
|
||||
// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
|
||||
// | | (4 bytes) |(8 bytes) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | actual size | |
|
||||
// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2
|
||||
// length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes)
|
||||
// |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | |
|
||||
// actual size | |
|
||||
// +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+
|
||||
// The length of bitmap is decided by number of rows of this data block, and the length of each column data is
|
||||
// recorded in the first segment, next to the struct header
|
||||
static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {
|
||||
int32_t numOfCols = 0;
|
||||
SNode* pNode;
|
||||
SNode* pNode;
|
||||
FOREACH(pNode, pHandle->pSchema->pSlots) {
|
||||
SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode;
|
||||
if (pSlotDesc->output) {
|
||||
|
@ -90,12 +92,12 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
|
|||
pEntry->dataLen = 0;
|
||||
|
||||
pBuf->useSize = sizeof(SDataCacheEntry);
|
||||
blockCompressEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed);
|
||||
blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed);
|
||||
|
||||
pBuf->useSize += pEntry->dataLen;
|
||||
|
||||
atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
|
||||
atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
|
||||
|
||||
atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
|
||||
atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
|
||||
}
|
||||
|
||||
static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) {
|
||||
|
@ -187,8 +189,8 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
|
|||
pOutput->numOfCols = pEntry->numOfCols;
|
||||
pOutput->compressed = pEntry->compressed;
|
||||
|
||||
atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen);
|
||||
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
|
||||
atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen);
|
||||
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
|
||||
|
||||
taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent
|
||||
pOutput->bufStatus = updateStatus(pDispatcher);
|
||||
|
@ -198,7 +200,6 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
|
|||
pOutput->precision = pDispatcher->pSchema->precision;
|
||||
taosThreadMutexUnlock(&pDispatcher->mutex);
|
||||
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
|
@ -13,10 +13,10 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "os.h"
|
||||
#include "index.h"
|
||||
#include "function.h"
|
||||
#include "functionMgt.h"
|
||||
#include "index.h"
|
||||
#include "os.h"
|
||||
#include "tdatablock.h"
|
||||
#include "thash.h"
|
||||
#include "tmsg.h"
|
||||
|
@ -25,45 +25,41 @@
|
|||
#include "executorimpl.h"
|
||||
#include "tcompression.h"
|
||||
|
||||
void initResultRowInfo(SResultRowInfo *pResultRowInfo) {
|
||||
pResultRowInfo->size = 0;
|
||||
void initResultRowInfo(SResultRowInfo* pResultRowInfo) {
|
||||
pResultRowInfo->size = 0;
|
||||
pResultRowInfo->cur.pageId = -1;
|
||||
}
|
||||
|
||||
void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) {
|
||||
void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo) {
|
||||
if (pResultRowInfo == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
for(int32_t i = 0; i < pResultRowInfo->size; ++i) {
|
||||
// if (pResultRowInfo->pResult[i]) {
|
||||
// taosMemoryFreeClear(pResultRowInfo->pResult[i]->key);
|
||||
// }
|
||||
for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
|
||||
// if (pResultRowInfo->pResult[i]) {
|
||||
// taosMemoryFreeClear(pResultRowInfo->pResult[i]->key);
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
void closeAllResultRows(SResultRowInfo *pResultRowInfo) {
|
||||
// do nothing
|
||||
void closeAllResultRows(SResultRowInfo* pResultRowInfo) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
bool isResultRowClosed(SResultRow* pRow) {
|
||||
return (pRow->closed == true);
|
||||
}
|
||||
bool isResultRowClosed(SResultRow* pRow) { return (pRow->closed == true); }
|
||||
|
||||
void closeResultRow(SResultRow* pResultRow) {
|
||||
pResultRow->closed = true;
|
||||
}
|
||||
void closeResultRow(SResultRow* pResultRow) { pResultRow->closed = true; }
|
||||
|
||||
// TODO refactor: use macro
|
||||
SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset) {
|
||||
assert(index >= 0 && offset != NULL);
|
||||
return (SResultRowEntryInfo*)((char*) pRow->pEntryInfo + offset[index]);
|
||||
return (SResultRowEntryInfo*)((char*)pRow->pEntryInfo + offset[index]);
|
||||
}
|
||||
|
||||
size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
|
||||
int32_t rowSize = (numOfOutput * sizeof(SResultRowEntryInfo)) + sizeof(SResultRow);
|
||||
|
||||
for(int32_t i = 0; i < numOfOutput; ++i) {
|
||||
for (int32_t i = 0; i < numOfOutput; ++i) {
|
||||
rowSize += pCtx[i].resDataInfo.interBufSize;
|
||||
}
|
||||
|
||||
|
@ -74,31 +70,29 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) {
|
|||
assert(pGroupResInfo != NULL);
|
||||
|
||||
taosArrayDestroy(pGroupResInfo->pRows);
|
||||
pGroupResInfo->pRows = NULL;
|
||||
pGroupResInfo->index = 0;
|
||||
pGroupResInfo->pRows = NULL;
|
||||
pGroupResInfo->index = 0;
|
||||
}
|
||||
|
||||
static int32_t resultrowComparAsc(const void* p1, const void* p2) {
|
||||
SResKeyPos* pp1 = *(SResKeyPos**) p1;
|
||||
SResKeyPos* pp2 = *(SResKeyPos**) p2;
|
||||
SResKeyPos* pp1 = *(SResKeyPos**)p1;
|
||||
SResKeyPos* pp2 = *(SResKeyPos**)p2;
|
||||
|
||||
if (pp1->groupId == pp2->groupId) {
|
||||
int64_t pts1 = *(int64_t*) pp1->key;
|
||||
int64_t pts2 = *(int64_t*) pp2->key;
|
||||
int64_t pts1 = *(int64_t*)pp1->key;
|
||||
int64_t pts2 = *(int64_t*)pp2->key;
|
||||
|
||||
if (pts1 == pts2) {
|
||||
return 0;
|
||||
} else {
|
||||
return pts1 < pts2? -1:1;
|
||||
return pts1 < pts2 ? -1 : 1;
|
||||
}
|
||||
} else {
|
||||
return pp1->groupId < pp2->groupId? -1:1;
|
||||
return pp1->groupId < pp2->groupId ? -1 : 1;
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t resultrowComparDesc(const void* p1, const void* p2) {
|
||||
return resultrowComparAsc(p2, p1);
|
||||
}
|
||||
static int32_t resultrowComparDesc(const void* p1, const void* p2) { return resultrowComparAsc(p2, p1); }
|
||||
|
||||
void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order) {
|
||||
if (pGroupResInfo->pRows != NULL) {
|
||||
|
@ -110,20 +104,20 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int
|
|||
pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES);
|
||||
|
||||
size_t keyLen = 0;
|
||||
while((pData = taosHashIterate(pHashmap, pData)) != NULL) {
|
||||
while ((pData = taosHashIterate(pHashmap, pData)) != NULL) {
|
||||
void* key = taosHashGetKey(pData, &keyLen);
|
||||
|
||||
SResKeyPos* p = taosMemoryMalloc(keyLen + sizeof(SResultRowPosition));
|
||||
|
||||
p->groupId = *(uint64_t*) key;
|
||||
p->pos = *(SResultRowPosition*) pData;
|
||||
p->groupId = *(uint64_t*)key;
|
||||
p->pos = *(SResultRowPosition*)pData;
|
||||
memcpy(p->key, (char*)key + sizeof(uint64_t), keyLen - sizeof(uint64_t));
|
||||
|
||||
taosArrayPush(pGroupResInfo->pRows, &p);
|
||||
}
|
||||
|
||||
if (order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC) {
|
||||
__compar_fn_t fn = (order == TSDB_ORDER_ASC)? resultrowComparAsc:resultrowComparDesc;
|
||||
__compar_fn_t fn = (order == TSDB_ORDER_ASC) ? resultrowComparAsc : resultrowComparDesc;
|
||||
qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn);
|
||||
}
|
||||
|
||||
|
@ -155,7 +149,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
return (int32_t) taosArrayGetSize(pGroupResInfo->pRows);
|
||||
return (int32_t)taosArrayGetSize(pGroupResInfo->pRows);
|
||||
}
|
||||
|
||||
SArray* createSortInfo(SNodeList* pNodeList) {
|
||||
|
@ -194,12 +188,13 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) {
|
|||
pBlock->info.type = STREAM_INVALID;
|
||||
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i);
|
||||
// if (!pDescNode->output) { // todo disable it temporarily
|
||||
// continue;
|
||||
// }
|
||||
SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i);
|
||||
/*if (!pDescNode->output) { // todo disable it temporarily*/
|
||||
/*continue;*/
|
||||
/*}*/
|
||||
|
||||
SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId);
|
||||
SColumnInfoData idata =
|
||||
createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId);
|
||||
idata.info.scale = pDescNode->dataType.scale;
|
||||
idata.info.precision = pDescNode->dataType.precision;
|
||||
|
||||
|
@ -211,10 +206,10 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) {
|
|||
|
||||
EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
|
||||
SMetaReader* mr = (SMetaReader*)pContext;
|
||||
if(nodeType(*pNode) == QUERY_NODE_COLUMN){
|
||||
if (nodeType(*pNode) == QUERY_NODE_COLUMN) {
|
||||
SColumnNode* pSColumnNode = *(SColumnNode**)pNode;
|
||||
|
||||
SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE);
|
||||
SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
|
||||
if (NULL == res) {
|
||||
return DEAL_RES_ERROR;
|
||||
}
|
||||
|
@ -227,8 +222,8 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
|
|||
const char* p = metaGetTableTagVal(&mr->me, pSColumnNode->node.resType.type, &tagVal);
|
||||
if (p == NULL) {
|
||||
res->node.resType.type = TSDB_DATA_TYPE_NULL;
|
||||
}else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) {
|
||||
int32_t len = ((const STag*)p) -> len;
|
||||
} else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) {
|
||||
int32_t len = ((const STag*)p)->len;
|
||||
res->datum.p = taosMemoryCalloc(len + 1, 1);
|
||||
memcpy(res->datum.p, p, len);
|
||||
} else if (IS_VAR_DATA_TYPE(pSColumnNode->node.resType.type)) {
|
||||
|
@ -240,10 +235,10 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
|
|||
}
|
||||
nodesDestroyNode(*pNode);
|
||||
*pNode = (SNode*)res;
|
||||
}else if (nodeType(*pNode) == QUERY_NODE_FUNCTION){
|
||||
SFunctionNode * pFuncNode = *(SFunctionNode**)pNode;
|
||||
if(pFuncNode->funcType == FUNCTION_TYPE_TBNAME){
|
||||
SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE);
|
||||
} else if (nodeType(*pNode) == QUERY_NODE_FUNCTION) {
|
||||
SFunctionNode* pFuncNode = *(SFunctionNode**)pNode;
|
||||
if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) {
|
||||
SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
|
||||
if (NULL == res) {
|
||||
return DEAL_RES_ERROR;
|
||||
}
|
||||
|
@ -263,12 +258,12 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
|
|||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
||||
static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){
|
||||
SMetaReader mr = {0};
|
||||
static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) {
|
||||
SMetaReader mr = {0};
|
||||
metaReaderInit(&mr, metaHandle, 0);
|
||||
metaGetTableEntryByUid(&mr, info->uid);
|
||||
|
||||
SNode *pTagCondTmp = nodesCloneNode(pTagCond);
|
||||
SNode* pTagCondTmp = nodesCloneNode(pTagCond);
|
||||
|
||||
nodesRewriteExprPostOrder(&pTagCondTmp, doTranslateTagExpr, &mr);
|
||||
metaReaderClear(&mr);
|
||||
|
@ -281,7 +276,7 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){
|
|||
}
|
||||
|
||||
ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
|
||||
SValueNode *pValue = (SValueNode *)pNew;
|
||||
SValueNode* pValue = (SValueNode*)pNew;
|
||||
|
||||
ASSERT(pValue->node.resType.type == TSDB_DATA_TYPE_BOOL);
|
||||
bool result = pValue->datum.b;
|
||||
|
@ -292,12 +287,12 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){
|
|||
int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo));
|
||||
if(pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
|
||||
if (pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
||||
uint64_t tableUid = pScanNode->uid;
|
||||
|
||||
pListInfo->suid = pScanNode->suid;
|
||||
|
||||
|
||||
SNode* pTagCond = (SNode*)pListInfo->pTagCond;
|
||||
SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond;
|
||||
if (pScanNode->tableType == TSDB_SUPER_TABLE) {
|
||||
|
@ -306,7 +301,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
|
|||
.metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
|
||||
|
||||
SArray* res = taosArrayInit(8, sizeof(uint64_t));
|
||||
//code = doFilterTag(pTagIndexCond, &metaArg, res);
|
||||
// code = doFilterTag(pTagIndexCond, &metaArg, res);
|
||||
code = TSDB_CODE_INDEX_REBUILDING;
|
||||
if (code == TSDB_CODE_INDEX_REBUILDING) {
|
||||
code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList);
|
||||
|
@ -328,33 +323,33 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
|
|||
code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList);
|
||||
}
|
||||
|
||||
if(pTagCond){
|
||||
if (pTagCond) {
|
||||
int32_t i = 0;
|
||||
while(i < taosArrayGetSize(pListInfo->pTableList)) {
|
||||
while (i < taosArrayGetSize(pListInfo->pTableList)) {
|
||||
STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i);
|
||||
bool isOk = isTableOk(info, pTagCond, metaHandle);
|
||||
if(!isOk){
|
||||
bool isOk = isTableOk(info, pTagCond, metaHandle);
|
||||
if (!isOk) {
|
||||
taosArrayRemove(pListInfo->pTableList, i);
|
||||
continue;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
}else { // Create one table group.
|
||||
} else { // Create one table group.
|
||||
STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0};
|
||||
taosArrayPush(pListInfo->pTableList, &info);
|
||||
}
|
||||
pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES);
|
||||
if(pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
|
||||
if (pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
||||
//put into list as default group, remove it if grouping sorting is required later
|
||||
// put into list as default group, remove it if grouping sorting is required later
|
||||
taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
SArray* extractPartitionColInfo(SNodeList* pNodeList) {
|
||||
if(!pNodeList) {
|
||||
if (!pNodeList) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
@ -383,7 +378,6 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) {
|
|||
return pList;
|
||||
}
|
||||
|
||||
|
||||
SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
|
||||
int32_t type) {
|
||||
size_t numOfCols = LIST_LENGTH(pNodeList);
|
||||
|
@ -399,7 +393,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
|
|||
|
||||
SColMatchInfo c = {0};
|
||||
c.output = true;
|
||||
c.colId = pColNode->colId;
|
||||
c.colId = pColNode->colId;
|
||||
c.srcSlotId = pColNode->slotId;
|
||||
c.matchType = type;
|
||||
c.targetSlotId = pNode->slotId;
|
||||
|
@ -675,8 +669,8 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
|
|||
}
|
||||
|
||||
for (int32_t i = 1; i < numOfOutput; ++i) {
|
||||
(*rowEntryInfoOffset)[i] =
|
||||
(int32_t)((*rowEntryInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pFuncCtx[i - 1].resDataInfo.interBufSize);
|
||||
(*rowEntryInfoOffset)[i] = (int32_t)((*rowEntryInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) +
|
||||
pFuncCtx[i - 1].resDataInfo.interBufSize);
|
||||
}
|
||||
|
||||
setSelectValueColumnInfo(pFuncCtx, numOfOutput);
|
||||
|
|
|
@ -44,12 +44,12 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
|
|||
// prevent setting a different type of block
|
||||
pInfo->blockType = type;
|
||||
|
||||
if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
|
||||
if (type == STREAM_INPUT__DATA_SUBMIT) {
|
||||
if (tqReadHandleSetMsg(pInfo->streamBlockReader, input, 0) < 0) {
|
||||
qError("submit msg messed up when initing stream block, %s" PRIx64, id);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
} else if (type == STREAM_DATA_TYPE_SSDATA_BLOCK) {
|
||||
} else if (type == STREAM_INPUT__DATA_BLOCK) {
|
||||
for (int32_t i = 0; i < numOfBlocks; ++i) {
|
||||
SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i];
|
||||
|
||||
|
@ -60,9 +60,9 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
|
|||
taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock);
|
||||
taosArrayPush(pInfo->pBlockLists, &p);
|
||||
}
|
||||
} else if (type == STREAM_DATA_TYPE_FROM_SNAPSHOT) {
|
||||
} else if (type == STREAM_INPUT__DATA_SCAN) {
|
||||
// do nothing
|
||||
ASSERT(pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT);
|
||||
ASSERT(pInfo->blockType == STREAM_INPUT__DATA_SCAN);
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
|
@ -76,7 +76,7 @@ int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) {
|
|||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
|
||||
return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_DATA_TYPE_FROM_SNAPSHOT, 0, NULL);
|
||||
return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_INPUT__DATA_SCAN, 0, NULL);
|
||||
}
|
||||
|
||||
int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) {
|
||||
|
|
|
@ -222,7 +222,7 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) {
|
|||
}
|
||||
|
||||
int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) {
|
||||
SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*) tinfo;
|
||||
SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo;
|
||||
|
||||
if (pTaskInfo == NULL || pInput == NULL || len == 0) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
|
@ -231,11 +231,20 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le
|
|||
return decodeOperator(pTaskInfo->pRoot, pInput, len);
|
||||
}
|
||||
|
||||
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
|
||||
|
||||
int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo;
|
||||
if (uid == 0) {
|
||||
STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0);
|
||||
uid = pTableInfo->uid;
|
||||
ts = INT64_MIN;
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
return doPrepareScan(pTaskInfo->pRoot, uid, ts);
|
||||
}
|
||||
|
||||
int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
|
||||
|
||||
return doGetScanStatus(pTaskInfo->pRoot, uid, ts);
|
||||
}
|
||||
|
|
|
@ -1033,7 +1033,7 @@ static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSData
|
|||
SqlFunctionCtx* pCtx = pTableScanInfo->pCtx;
|
||||
uint32_t status = BLK_DATA_NOT_LOAD;
|
||||
|
||||
int32_t numOfOutput = 0;//pTableScanInfo->numOfOutput;
|
||||
int32_t numOfOutput = 0; // pTableScanInfo->numOfOutput;
|
||||
for (int32_t i = 0; i < numOfOutput; ++i) {
|
||||
int32_t functionId = pCtx[i].functionId;
|
||||
int32_t colId = pTableScanInfo->pExpr[i].base.pParam[0].pCol->colId;
|
||||
|
@ -1348,7 +1348,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR
|
|||
SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i);
|
||||
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
|
||||
// it is a reserved column for scalar function, and no data in this column yet.
|
||||
if (pDst->pData == NULL) {
|
||||
if (pDst->pData == NULL || pSrc->pData == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -1933,7 +1933,7 @@ typedef struct SFetchRspHandleWrapper {
|
|||
int32_t sourceIndex;
|
||||
} SFetchRspHandleWrapper;
|
||||
|
||||
int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param;
|
||||
|
||||
SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId);
|
||||
|
@ -2010,13 +2010,14 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
|
|||
|
||||
ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY);
|
||||
|
||||
qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, GET_TASKID(pTaskInfo),
|
||||
pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources);
|
||||
qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, GET_TASKID(pTaskInfo),
|
||||
pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId, sourceIndex, totalSources);
|
||||
|
||||
pMsg->header.vgId = htonl(pSource->addr.nodeId);
|
||||
pMsg->sId = htobe64(pSource->schedId);
|
||||
pMsg->taskId = htobe64(pSource->taskId);
|
||||
pMsg->queryId = htobe64(pTaskInfo->id.queryId);
|
||||
pMsg->execId = htonl(pSource->execId);
|
||||
|
||||
// send the fetch remote task result reques
|
||||
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
|
||||
|
@ -2034,7 +2035,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
|
|||
pMsgSendInfo->param = pWrapper;
|
||||
pMsgSendInfo->msgInfo.pData = pMsg;
|
||||
pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq);
|
||||
pMsgSendInfo->msgType = TDMT_VND_FETCH;
|
||||
pMsgSendInfo->msgType = TDMT_SCH_FETCH;
|
||||
pMsgSendInfo->fp = loadRemoteDataCallback;
|
||||
|
||||
int64_t transporterId = 0;
|
||||
|
@ -2048,7 +2049,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo
|
|||
if (pColList == NULL) { // data from other sources
|
||||
blockDataCleanup(pRes);
|
||||
// blockDataEnsureCapacity(pRes, numOfRows);
|
||||
blockCompressDecode(pRes, numOfOutput, numOfRows, pData);
|
||||
blockDecode(pRes, numOfOutput, numOfRows, pData);
|
||||
} else { // extract data according to pColList
|
||||
ASSERT(numOfOutput == taosArrayGetSize(pColList));
|
||||
char* pStart = pData;
|
||||
|
@ -2072,7 +2073,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo
|
|||
blockDataAppendColInfo(pBlock, &idata);
|
||||
}
|
||||
|
||||
blockCompressDecode(pBlock, numOfCols, numOfRows, pStart);
|
||||
blockDecode(pBlock, numOfCols, numOfRows, pStart);
|
||||
blockDataEnsureCapacity(pRes, numOfRows);
|
||||
|
||||
// data from mnode
|
||||
|
@ -2145,9 +2146,9 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
|
|||
SSDataBlock* pRes = pExchangeInfo->pResult;
|
||||
SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
|
||||
if (pRsp->numOfRows == 0) {
|
||||
qDebug("%s vgId:%d, taskId:0x%" PRIx64 " index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
|
||||
qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
|
||||
", completed:%d try next %d/%" PRIzu,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, i, pDataInfo->totalRows,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows,
|
||||
pExchangeInfo->loadInfo.totalRows, completed + 1, i + 1, totalSources);
|
||||
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
|
||||
completed += 1;
|
||||
|
@ -2165,17 +2166,17 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
|
|||
}
|
||||
|
||||
if (pRsp->completed == 1) {
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d"
|
||||
" index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64
|
||||
", completed:%d try next %d/%" PRIzu,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, i, pRes->info.rows, pDataInfo->totalRows,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows, pDataInfo->totalRows,
|
||||
pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources);
|
||||
completed += 1;
|
||||
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
|
||||
} else {
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64
|
||||
", totalBytes:%" PRIu64,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pLoadInfo->totalRows,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pLoadInfo->totalRows,
|
||||
pLoadInfo->totalSize);
|
||||
}
|
||||
|
||||
|
@ -2249,8 +2250,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
|
|||
SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
|
||||
|
||||
if (pDataInfo->code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s vgId:%d, taskID:0x%" PRIx64 " error happens, code:%s", GET_TASKID(pTaskInfo), pSource->addr.nodeId,
|
||||
pSource->taskId, tstrerror(pDataInfo->code));
|
||||
qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), pSource->addr.nodeId,
|
||||
pSource->taskId, pSource->execId, tstrerror(pDataInfo->code));
|
||||
pOperator->pTaskInfo->code = pDataInfo->code;
|
||||
return NULL;
|
||||
}
|
||||
|
@ -2258,9 +2259,9 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
|
|||
SRetrieveTableRsp* pRsp = pDataInfo->pRsp;
|
||||
SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
|
||||
if (pRsp->numOfRows == 0) {
|
||||
qDebug("%s vgId:%d, taskID:0x%" PRIx64 " %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
|
||||
qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
|
||||
" try next",
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1,
|
||||
pDataInfo->totalRows, pLoadInfo->totalRows);
|
||||
|
||||
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
|
||||
|
@ -2276,17 +2277,17 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
|
|||
pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL);
|
||||
|
||||
if (pRsp->completed == 1) {
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64
|
||||
", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pDataInfo->totalRows,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pDataInfo->totalRows,
|
||||
pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources);
|
||||
|
||||
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
|
||||
pExchangeInfo->current += 1;
|
||||
} else {
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64
|
||||
", totalBytes:%" PRIu64,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pLoadInfo->totalRows,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pLoadInfo->totalRows,
|
||||
pLoadInfo->totalSize);
|
||||
}
|
||||
|
||||
|
@ -2378,7 +2379,7 @@ static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo*
|
|||
}
|
||||
|
||||
for (int32_t i = 0; i < numOfSources; ++i) {
|
||||
SNodeListNode* pNode = (SNodeListNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i);
|
||||
SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i);
|
||||
taosArrayPush(pInfo->pSources, pNode);
|
||||
}
|
||||
|
||||
|
@ -2821,13 +2822,51 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
|
|||
}
|
||||
}
|
||||
|
||||
int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
|
||||
int32_t type = pOperator->operatorType;
|
||||
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
|
||||
SStreamBlockScanInfo* pScanInfo = pOperator->info;
|
||||
pScanInfo->blockType = STREAM_INPUT__DATA_SCAN;
|
||||
|
||||
STableScanInfo* pInfo = pScanInfo->pSnapshotReadOp->info;
|
||||
|
||||
/*if (pSnapShotScanInfo->dataReader == NULL) {*/
|
||||
/*pSnapShotScanInfo->dataReader = tsdbReaderOpen(pHandle->vnode, &pSTInfo->cond, tableList, 0, 0);*/
|
||||
/*pSnapShotScanInfo->scanMode = TABLE_SCAN__TABLE_ORDER;*/
|
||||
/*}*/
|
||||
|
||||
if (pInfo->lastStatus.uid != uid || pInfo->lastStatus.ts != ts) {
|
||||
tsdbSetTableId(pInfo->dataReader, uid);
|
||||
int64_t oldSkey = pInfo->cond.twindows[0].skey;
|
||||
pInfo->cond.twindows[0].skey = ts;
|
||||
tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0);
|
||||
pInfo->cond.twindows[0].skey = oldSkey;
|
||||
pInfo->scanTimes = 0;
|
||||
pInfo->curTWinIdx = 0;
|
||||
}
|
||||
|
||||
} else {
|
||||
if (pOperator->numOfDownstream == 1) {
|
||||
return doPrepareScan(pOperator->pDownstream[0], uid, ts);
|
||||
} else if (pOperator->numOfDownstream == 0) {
|
||||
qError("failed to find stream scan operator to set the input data block");
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
} else {
|
||||
qError("join not supported for stream block scan");
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts) {
|
||||
int32_t type = pOperator->operatorType;
|
||||
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
|
||||
SStreamBlockScanInfo* pScanInfo = pOperator->info;
|
||||
STableScanInfo* pSnapShotScanInfo = pScanInfo->pSnapshotReadOp->info;
|
||||
*uid = pSnapShotScanInfo->scanStatus.uid;
|
||||
*ts = pSnapShotScanInfo->scanStatus.t;
|
||||
STableScanInfo* pSnapShotScanInfo = pScanInfo->pSnapshotReadOp->info;
|
||||
*uid = pSnapShotScanInfo->lastStatus.uid;
|
||||
*ts = pSnapShotScanInfo->lastStatus.ts;
|
||||
} else {
|
||||
if (pOperator->pDownstream[0] == NULL) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
|
@ -2952,7 +2991,7 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len
|
|||
*length = 0;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
*result = (char*)taosMemoryCalloc(1, totalSize);
|
||||
if (*result == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
@ -3916,60 +3955,59 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum){
|
||||
static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum) {
|
||||
taosArrayClear(pTableListInfo->pGroupList);
|
||||
SArray *sortSupport = taosArrayInit(groupNum, sizeof(uint64_t));
|
||||
if(sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY;
|
||||
SArray* sortSupport = taosArrayInit(groupNum, sizeof(uint64_t));
|
||||
if (sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY;
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) {
|
||||
STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
|
||||
uint64_t* groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t));
|
||||
uint64_t* groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t));
|
||||
|
||||
int32_t index = taosArraySearchIdx(sortSupport, groupId, compareUint64Val, TD_EQ);
|
||||
if (index == -1){
|
||||
void *p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT);
|
||||
SArray *tGroup = taosArrayInit(8, sizeof(STableKeyInfo));
|
||||
if(tGroup == NULL) {
|
||||
if (index == -1) {
|
||||
void* p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT);
|
||||
SArray* tGroup = taosArrayInit(8, sizeof(STableKeyInfo));
|
||||
if (tGroup == NULL) {
|
||||
taosArrayDestroy(sortSupport);
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
if(taosArrayPush(tGroup, info) == NULL){
|
||||
if (taosArrayPush(tGroup, info) == NULL) {
|
||||
qError("taos push info array error");
|
||||
taosArrayDestroy(sortSupport);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
if(p == NULL){
|
||||
if(taosArrayPush(sortSupport, groupId) != NULL){
|
||||
if (p == NULL) {
|
||||
if (taosArrayPush(sortSupport, groupId) != NULL) {
|
||||
qError("taos push support array error");
|
||||
taosArrayDestroy(sortSupport);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
if(taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL){
|
||||
if (taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL) {
|
||||
qError("taos push group array error");
|
||||
taosArrayDestroy(sortSupport);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
}else{
|
||||
} else {
|
||||
int32_t pos = TARRAY_ELEM_IDX(sortSupport, p);
|
||||
if(taosArrayInsert(sortSupport, pos, groupId) == NULL){
|
||||
if (taosArrayInsert(sortSupport, pos, groupId) == NULL) {
|
||||
qError("taos insert support array error");
|
||||
taosArrayDestroy(sortSupport);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
if(taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL){
|
||||
if (taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL) {
|
||||
qError("taos insert group array error");
|
||||
taosArrayDestroy(sortSupport);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
}
|
||||
}else{
|
||||
} else {
|
||||
SArray* tGroup = (SArray*)taosArrayGetP(pTableListInfo->pGroupList, index);
|
||||
if(taosArrayPush(tGroup, info) == NULL){
|
||||
if (taosArrayPush(tGroup, info) == NULL) {
|
||||
qError("taos push uid array error");
|
||||
taosArrayDestroy(sortSupport);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
taosArrayDestroy(sortSupport);
|
||||
return TDB_CODE_SUCCESS;
|
||||
|
@ -3987,9 +4025,9 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
|
|||
int32_t keyLen = 0;
|
||||
void* keyBuf = NULL;
|
||||
|
||||
SNode* node;
|
||||
SNode* node;
|
||||
FOREACH(node, group) {
|
||||
SExprNode *pExpr = (SExprNode *)node;
|
||||
SExprNode* pExpr = (SExprNode*)node;
|
||||
keyLen += pExpr->resType.bytes;
|
||||
}
|
||||
|
||||
|
@ -4008,15 +4046,15 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
|
|||
metaReaderInit(&mr, pHandle->meta, 0);
|
||||
metaGetTableEntryByUid(&mr, info->uid);
|
||||
|
||||
SNodeList *groupNew = nodesCloneList(group);
|
||||
SNodeList* groupNew = nodesCloneList(group);
|
||||
|
||||
nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr);
|
||||
char* isNull = (char*)keyBuf;
|
||||
char* pStart = (char*)keyBuf + nullFlagSize;
|
||||
|
||||
SNode* pNode;
|
||||
SNode* pNode;
|
||||
int32_t index = 0;
|
||||
FOREACH(pNode, groupNew){
|
||||
FOREACH(pNode, groupNew) {
|
||||
SNode* pNew = NULL;
|
||||
int32_t code = scalarCalculateConstants(pNode, &pNew);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
|
@ -4028,15 +4066,15 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
|
|||
}
|
||||
|
||||
ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
|
||||
SValueNode *pValue = (SValueNode *)pNew;
|
||||
SValueNode* pValue = (SValueNode*)pNew;
|
||||
|
||||
if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL) {
|
||||
isNull[index++] = 1;
|
||||
continue;
|
||||
} else {
|
||||
isNull[index++] = 0;
|
||||
char* data = nodesGetValueFromNode(pValue);
|
||||
if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON){
|
||||
char* data = nodesGetValueFromNode(pValue);
|
||||
if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) {
|
||||
int32_t len = getJsonValueLen(data);
|
||||
memcpy(pStart, data, len);
|
||||
pStart += len;
|
||||
|
@ -4049,7 +4087,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
|
|||
}
|
||||
}
|
||||
}
|
||||
int32_t len = (int32_t)(pStart - (char*)keyBuf);
|
||||
int32_t len = (int32_t)(pStart - (char*)keyBuf);
|
||||
uint64_t groupId = calcGroupId(keyBuf, len);
|
||||
taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t));
|
||||
info->groupId = groupId;
|
||||
|
@ -4060,7 +4098,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
|
|||
}
|
||||
taosMemoryFree(keyBuf);
|
||||
|
||||
if(pTableListInfo->needSortTableByGroupId){
|
||||
if (pTableListInfo->needSortTableByGroupId) {
|
||||
return sortTableGroup(pTableListInfo, groupNum);
|
||||
}
|
||||
|
||||
|
@ -4068,7 +4106,8 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
|
|||
}
|
||||
|
||||
SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
|
||||
uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, const char* pUser) {
|
||||
uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo,
|
||||
const char* pUser) {
|
||||
int32_t type = nodeType(pPhyNode);
|
||||
|
||||
if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) {
|
||||
|
@ -4076,7 +4115,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
|
|||
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
|
||||
|
||||
int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
|
||||
if(code){
|
||||
if (code) {
|
||||
pTaskInfo->code = code;
|
||||
return NULL;
|
||||
}
|
||||
|
@ -4094,7 +4133,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
|
|||
} else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
|
||||
STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
|
||||
int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
|
||||
if(code){
|
||||
if (code) {
|
||||
return NULL;
|
||||
}
|
||||
code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
|
||||
|
@ -4103,7 +4142,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
|
|||
return NULL;
|
||||
}
|
||||
|
||||
SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
|
||||
SOperatorInfo* pOperator =
|
||||
createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
|
||||
|
||||
STableScanInfo* pScanInfo = pOperator->info;
|
||||
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
|
||||
|
@ -4123,7 +4163,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
|
|||
createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
|
||||
}
|
||||
|
||||
SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId);
|
||||
SOperatorInfo* pOperator =
|
||||
createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId);
|
||||
return pOperator;
|
||||
|
||||
} else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
|
||||
|
@ -4621,8 +4662,8 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
|
|||
(*pTaskInfo)->sql = sql;
|
||||
(*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond;
|
||||
(*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond;
|
||||
(*pTaskInfo)->pRoot =
|
||||
createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->user);
|
||||
(*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId,
|
||||
&(*pTaskInfo)->tableqinfoList, pPlan->user);
|
||||
|
||||
if (NULL == (*pTaskInfo)->pRoot) {
|
||||
code = (*pTaskInfo)->code;
|
||||
|
@ -4640,8 +4681,8 @@ _complete:
|
|||
static void doDestroyTableList(STableListInfo* pTableqinfoList) {
|
||||
taosArrayDestroy(pTableqinfoList->pTableList);
|
||||
taosHashCleanup(pTableqinfoList->map);
|
||||
if(pTableqinfoList->needSortTableByGroupId){
|
||||
for(int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++){
|
||||
if (pTableqinfoList->needSortTableByGroupId) {
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++) {
|
||||
SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i);
|
||||
taosArrayDestroy(tmp);
|
||||
}
|
||||
|
|
|
@ -392,6 +392,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
|
|||
binfo.capacity = binfo.rows;
|
||||
blockDataEnsureCapacity(pBlock, binfo.rows);
|
||||
pBlock->info = binfo;
|
||||
ASSERT(binfo.uid != 0);
|
||||
|
||||
uint32_t status = 0;
|
||||
int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status);
|
||||
|
@ -416,9 +417,10 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
|
|||
pOperator->cost.totalCost = pTableScanInfo->readRecorder.elapsedTime;
|
||||
|
||||
// todo refactor
|
||||
pTableScanInfo->scanStatus.uid = pBlock->info.uid;
|
||||
pTableScanInfo->scanStatus.t = pBlock->info.window.ekey;
|
||||
pTableScanInfo->lastStatus.uid = pBlock->info.uid;
|
||||
pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;
|
||||
|
||||
ASSERT(pBlock->info.uid != 0);
|
||||
return pBlock;
|
||||
}
|
||||
return NULL;
|
||||
|
@ -438,6 +440,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
|
|||
while (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) {
|
||||
SSDataBlock* p = doTableScanImpl(pOperator);
|
||||
if (p != NULL) {
|
||||
ASSERT(p->info.uid != 0);
|
||||
return p;
|
||||
}
|
||||
pTableScanInfo->curTWinIdx += 1;
|
||||
|
@ -513,6 +516,26 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
|
|||
STableScanInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
||||
// if scan table by table
|
||||
if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) {
|
||||
while (1) {
|
||||
SSDataBlock* result = doTableScanGroup(pOperator);
|
||||
if (result) {
|
||||
return result;
|
||||
}
|
||||
// if no data, switch to next table and continue scan
|
||||
pInfo->currentTable++;
|
||||
if (pInfo->currentTable >= taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList)) {
|
||||
return NULL;
|
||||
}
|
||||
STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, pInfo->currentTable);
|
||||
tsdbSetTableId(pInfo->dataReader, pTableInfo->uid);
|
||||
tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0);
|
||||
pInfo->scanTimes = 0;
|
||||
pInfo->curTWinIdx = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (pInfo->currentGroupId == -1) {
|
||||
pInfo->currentGroupId++;
|
||||
if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) {
|
||||
|
@ -968,7 +991,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
|
||||
size_t total = taosArrayGetSize(pInfo->pBlockLists);
|
||||
// TODO: refactor
|
||||
if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) {
|
||||
if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) {
|
||||
if (pInfo->validBlockIndex >= total) {
|
||||
/*doClearBufferedBlocks(pInfo);*/
|
||||
pOperator->status = OP_EXEC_DONE;
|
||||
|
@ -979,7 +1002,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
|
||||
blockDataUpdateTsWindow(pBlock, 0);
|
||||
if (pBlock->info.type == STREAM_RETRIEVE) {
|
||||
pInfo->blockType = STREAM_DATA_TYPE_SUBMIT_BLOCK;
|
||||
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE;
|
||||
copyDataBlock(pInfo->pPullDataRes, pBlock);
|
||||
pInfo->pullDataResIndex = 0;
|
||||
|
@ -987,7 +1010,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
|
||||
}
|
||||
return pBlock;
|
||||
} else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
|
||||
} else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) {
|
||||
if (pInfo->scanMode == STREAM_SCAN_FROM_RES) {
|
||||
blockDataDestroy(pInfo->pUpdateRes);
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
|
||||
|
@ -1002,7 +1025,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex);
|
||||
if (pSDB != NULL) {
|
||||
getUpdateDataBlock(pInfo, true, pSDB, NULL);
|
||||
pSDB->info.type = STREAM_PUSH_DATA;
|
||||
pSDB->info.type = STREAM_PULL_DATA;
|
||||
return pSDB;
|
||||
}
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
|
||||
|
@ -1139,7 +1162,9 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
|
||||
return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
|
||||
|
||||
} else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) {
|
||||
} else if (pInfo->blockType == STREAM_INPUT__DATA_SCAN) {
|
||||
// check reader last status
|
||||
// if not match, reset status
|
||||
SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp);
|
||||
return pResult && pResult->info.rows > 0 ? pResult : NULL;
|
||||
|
||||
|
@ -1186,7 +1211,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
|
|||
|
||||
int16_t colId = id->colId;
|
||||
taosArrayPush(pColIds, &colId);
|
||||
if (id->colId == pTableScanNode->tsColId) {
|
||||
if (id->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
|
||||
pInfo->primaryTsIndex = id->targetSlotId;
|
||||
}
|
||||
}
|
||||
|
@ -1205,6 +1230,13 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
|
|||
if (pHandle) {
|
||||
SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId);
|
||||
STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info;
|
||||
|
||||
SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, 0);
|
||||
if (pHandle->tqReader) {
|
||||
pSTInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
|
||||
pSTInfo->dataReader = tsdbReaderOpen(pHandle->vnode, &pSTInfo->cond, tableList, 0, 0);
|
||||
}
|
||||
|
||||
if (pSTInfo->interval.interval > 0) {
|
||||
pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, pTwSup->waterMark);
|
||||
} else {
|
||||
|
@ -1219,7 +1251,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
|
|||
pInfo->tableUid = pScanPhyNode->uid;
|
||||
|
||||
// set the extract column id to streamHandle
|
||||
tqReadHandleSetColIdList((STqReadHandle*)pHandle->reader, pColIds);
|
||||
tqReadHandleSetColIdList((SStreamReader*)pHandle->reader, pColIds);
|
||||
SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList);
|
||||
int32_t code = tqReadHandleSetTbUidList(pHandle->reader, tableIdList);
|
||||
if (code != 0) {
|
||||
|
@ -1328,7 +1360,7 @@ static void getDBNameFromCondition(SNode* pCondition, const char* dbName) {
|
|||
nodesWalkExpr(pCondition, getDBNameFromConditionWalker, (char*)dbName);
|
||||
}
|
||||
|
||||
static int32_t loadSysTableCallback(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
static int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SOperatorInfo* operator=(SOperatorInfo*) param;
|
||||
SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*)operator->info;
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
|
|
|
@ -34,7 +34,7 @@ typedef struct SWinRes {
|
|||
|
||||
typedef struct SPullWindowInfo {
|
||||
STimeWindow window;
|
||||
uint64_t groupId;
|
||||
uint64_t groupId;
|
||||
} SPullWindowInfo;
|
||||
|
||||
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
|
||||
|
@ -695,11 +695,11 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num
|
|||
}
|
||||
|
||||
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
|
||||
if (pBlock == NULL){
|
||||
if (pBlock == NULL) {
|
||||
qDebug("======printDataBlock Block is Null");
|
||||
return;
|
||||
}
|
||||
char *pBuf = NULL;
|
||||
char* pBuf = NULL;
|
||||
qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
|
||||
taosMemoryFree(pBuf);
|
||||
}
|
||||
|
@ -813,6 +813,13 @@ static void removeResults(SArray* pWins, SArray* pUpdated) {
|
|||
}
|
||||
}
|
||||
|
||||
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) {
|
||||
ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
|
||||
return pSup->maxTs != INT64_MIN && ts < pSup->maxTs - pSup->waterMark;
|
||||
}
|
||||
|
||||
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); }
|
||||
|
||||
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
|
||||
int32_t scanFlag, SArray* pUpdated) {
|
||||
SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
|
||||
|
@ -830,15 +837,15 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
|
|||
|
||||
STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
|
||||
pInfo->interval.precision, &pInfo->win);
|
||||
int32_t ret = TSDB_CODE_SUCCESS;
|
||||
if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
|
||||
ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
|
||||
numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
|
||||
if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
|
||||
longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
|
||||
pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
|
||||
if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
|
||||
longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
|
||||
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
|
||||
if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
|
||||
saveResultRow(pResult, tableGroupId, pUpdated);
|
||||
}
|
||||
}
|
||||
|
@ -864,9 +871,11 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
|
|||
doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
|
||||
}
|
||||
|
||||
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
|
||||
doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
|
||||
pBlock->info.rows, numOfOutput, pInfo->order);
|
||||
if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
|
||||
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
|
||||
doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
|
||||
pBlock->info.rows, numOfOutput, pInfo->order);
|
||||
}
|
||||
|
||||
doCloseWindow(pResultRowInfo, pInfo, pResult);
|
||||
|
||||
|
@ -877,6 +886,12 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
|
|||
if (startPos < 0) {
|
||||
break;
|
||||
}
|
||||
if (pInfo->ignoreCloseWindow && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
|
||||
ekey = ascScan ? nextWin.ekey : nextWin.skey;
|
||||
forwardRows =
|
||||
getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
|
||||
continue;
|
||||
}
|
||||
|
||||
// null data, failed to allocate more memory buffer
|
||||
int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
|
||||
|
@ -885,10 +900,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
|
|||
longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
|
||||
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
|
||||
saveResultRow(pResult, tableGroupId, pUpdated);
|
||||
}
|
||||
if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
|
||||
saveResultRow(pResult, tableGroupId, pUpdated);
|
||||
}
|
||||
|
||||
ekey = ascScan ? nextWin.ekey : nextWin.skey;
|
||||
|
@ -970,9 +983,8 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
|
|||
getTableScanInfo(pOperator, &pInfo->order, &scanFlag);
|
||||
|
||||
if (pInfo->scalarSupp.pExprInfo != NULL) {
|
||||
SExprSupp* pExprSup =& pInfo->scalarSupp;
|
||||
projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx,
|
||||
pExprSup->numOfExprs, NULL);
|
||||
SExprSupp* pExprSup = &pInfo->scalarSupp;
|
||||
projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
|
||||
}
|
||||
|
||||
// the pDataBlock are always the same one, no need to call this again
|
||||
|
@ -1234,6 +1246,9 @@ void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprS
|
|||
pCtx[i].fpSet.init(&pCtx[i], pResInfo);
|
||||
}
|
||||
}
|
||||
SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
|
||||
setBufPageDirty(bufPage, true);
|
||||
releaseBufPage(pResultBuf, bufPage);
|
||||
}
|
||||
|
||||
bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId,
|
||||
|
@ -1255,10 +1270,10 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval*
|
|||
TSKEY* tsCols = (TSKEY*)pTsCol->pData;
|
||||
uint64_t* pGpDatas = NULL;
|
||||
if (pBlock->info.type == STREAM_RETRIEVE) {
|
||||
SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, 2);
|
||||
pGpDatas = (uint64_t*)pGpCol->pData;
|
||||
SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, 2);
|
||||
pGpDatas = (uint64_t*)pGpCol->pData;
|
||||
}
|
||||
int32_t step = 0;
|
||||
int32_t step = 0;
|
||||
for (int32_t i = 0; i < pBlock->info.rows; i += step) {
|
||||
SResultRowInfo dumyInfo;
|
||||
dumyInfo.cur.pageId = -1;
|
||||
|
@ -1292,13 +1307,8 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) {
|
||||
ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
|
||||
return pSup->maxTs != INT64_MIN && pWin->ekey < pSup->maxTs - pSup->waterMark;
|
||||
}
|
||||
|
||||
static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
|
||||
SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins) {
|
||||
static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
|
||||
SHashObj* pPullDataMap, SArray* closeWins) {
|
||||
void* pIte = NULL;
|
||||
size_t keyLen = 0;
|
||||
while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
|
||||
|
@ -1309,15 +1319,18 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
|
|||
SResultRowInfo dumyInfo;
|
||||
dumyInfo.cur.pageId = -1;
|
||||
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL);
|
||||
SWinRes winRe = {.ts = win.skey, .groupId = groupId,};
|
||||
SWinRes winRe = {
|
||||
.ts = win.skey,
|
||||
.groupId = groupId,
|
||||
};
|
||||
void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinRes));
|
||||
if (isCloseWindow(&win, pSup)) {
|
||||
if (chIds && pPullDataMap) {
|
||||
SArray* chAy = *(SArray**) chIds;
|
||||
SArray* chAy = *(SArray**)chIds;
|
||||
int32_t size = taosArrayGetSize(chAy);
|
||||
qInfo("======window %ld wait child size:%d", win.skey ,size);
|
||||
qInfo("======window %ld wait child size:%d", win.skey, size);
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
qInfo("======window %ld wait chid id:%d", win.skey ,*(int32_t*)taosArrayGet(chAy, i));
|
||||
qInfo("======window %ld wait chid id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i));
|
||||
}
|
||||
continue;
|
||||
} else if (pPullDataMap) {
|
||||
|
@ -1341,7 +1354,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
|
|||
static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) {
|
||||
int32_t size = taosArrayGetSize(pChildren);
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i);
|
||||
SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i);
|
||||
SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
|
||||
ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);
|
||||
pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
|
||||
|
@ -1411,7 +1424,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
|
|||
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
|
||||
|
||||
pOperator->status = OP_RES_TO_RETURN;
|
||||
|
||||
printDataBlock(pInfo->binfo.pRes, "single interval");
|
||||
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
|
||||
}
|
||||
|
||||
|
@ -1431,7 +1444,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
|
|||
SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param;
|
||||
cleanupBasicInfo(&pInfo->binfo);
|
||||
cleanupAggSup(&pInfo->aggSup);
|
||||
//it should be empty.
|
||||
// it should be empty.
|
||||
taosHashCleanup(pInfo->pPullDataMap);
|
||||
taosArrayDestroy(pInfo->pPullWins);
|
||||
blockDataDestroy(pInfo->pPullDataRes);
|
||||
|
@ -1509,23 +1522,25 @@ void increaseTs(SqlFunctionCtx* pCtx) {
|
|||
|
||||
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
|
||||
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
|
||||
STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream) {
|
||||
STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode,
|
||||
SExecTaskInfo* pTaskInfo, bool isStream) {
|
||||
SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
|
||||
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
|
||||
if (pInfo == NULL || pOperator == NULL) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
pInfo->win = pTaskInfo->window;
|
||||
pInfo->order = TSDB_ORDER_ASC;
|
||||
pInfo->interval = *pInterval;
|
||||
pInfo->win = pTaskInfo->window;
|
||||
pInfo->order = TSDB_ORDER_ASC;
|
||||
pInfo->interval = *pInterval;
|
||||
pInfo->execModel = pTaskInfo->execModel;
|
||||
pInfo->twAggSup = *pTwAggSupp;
|
||||
pInfo->twAggSup = *pTwAggSupp;
|
||||
pInfo->ignoreCloseWindow = false;
|
||||
|
||||
if (pPhyNode->window.pExprs != NULL) {
|
||||
int32_t numOfScalar = 0;
|
||||
SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
|
||||
int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
|
||||
int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
}
|
||||
|
@ -2236,10 +2251,10 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
|
|||
return p1 == NULL;
|
||||
}
|
||||
|
||||
int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols,
|
||||
int32_t startPos, TSKEY eKey, STimeWindow* pNextWin) {
|
||||
int32_t forwardRows = getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos,
|
||||
eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
|
||||
int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,
|
||||
STimeWindow* pNextWin) {
|
||||
int32_t forwardRows =
|
||||
getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
|
||||
int32_t prevEndPos = forwardRows - 1 + startPos;
|
||||
return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);
|
||||
}
|
||||
|
@ -2276,9 +2291,20 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
|
|||
STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
|
||||
pInfo->interval.precision, NULL);
|
||||
while (1) {
|
||||
if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && pInfo->pChildren) {
|
||||
bool ignore = true;
|
||||
SWinRes winRes = {.ts = nextWin.skey, .groupId = tableGroupId,};
|
||||
bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
|
||||
if (pInfo->ignoreCloseWindow && isClosed) {
|
||||
startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
|
||||
if (startPos < 0) {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) {
|
||||
bool ignore = true;
|
||||
SWinRes winRes = {
|
||||
.ts = nextWin.skey,
|
||||
.groupId = tableGroupId,
|
||||
};
|
||||
void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes));
|
||||
if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) {
|
||||
SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId};
|
||||
|
@ -2289,18 +2315,18 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
|
|||
int32_t index = -1;
|
||||
SArray* chArray = NULL;
|
||||
if (chIds) {
|
||||
chArray = *(void**) chIds;
|
||||
chArray = *(void**)chIds;
|
||||
int32_t chId = getChildIndex(pSDataBlock);
|
||||
index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
|
||||
}
|
||||
if (index != -1 && pSDataBlock->info.type == STREAM_PUSH_DATA) {
|
||||
if (index != -1 && pSDataBlock->info.type == STREAM_PULL_DATA) {
|
||||
taosArrayRemove(chArray, index);
|
||||
if (taosArrayGetSize(chArray) == 0) {
|
||||
// pull data is over
|
||||
taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes));
|
||||
}
|
||||
}
|
||||
if ( index == -1 || pSDataBlock->info.type == STREAM_PUSH_DATA) {
|
||||
if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {
|
||||
ignore = false;
|
||||
}
|
||||
}
|
||||
|
@ -2385,17 +2411,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB
|
|||
if (size - (*pIndex) == 0) {
|
||||
return;
|
||||
}
|
||||
blockDataEnsureCapacity(pBlock, size - (*pIndex) );
|
||||
blockDataEnsureCapacity(pBlock, size - (*pIndex));
|
||||
ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock));
|
||||
for (; (*pIndex) < size; (*pIndex)++) {
|
||||
SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex) );
|
||||
SColumnInfoData* pStartTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 0);
|
||||
SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));
|
||||
SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 0);
|
||||
colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
|
||||
|
||||
SColumnInfoData* pEndTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 1);
|
||||
SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 1);
|
||||
colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
|
||||
|
||||
SColumnInfoData* pGroupId = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 2);
|
||||
SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 2);
|
||||
colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
|
||||
pBlock->info.rows++;
|
||||
}
|
||||
|
@ -2406,17 +2432,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB
|
|||
blockDataUpdateTsWindow(pBlock, 0);
|
||||
}
|
||||
|
||||
void processPushEmpty(SSDataBlock* pBlock, SHashObj* pMap) {
|
||||
void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
|
||||
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, 0);
|
||||
TSKEY* tsData = (TSKEY*)pStartCol->pData;
|
||||
TSKEY* tsData = (TSKEY*)pStartCol->pData;
|
||||
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, 2);
|
||||
uint64_t* groupIdData = (uint64_t*)pGroupCol->pData;
|
||||
int32_t chId = getChildIndex(pBlock);
|
||||
uint64_t* groupIdData = (uint64_t*)pGroupCol->pData;
|
||||
int32_t chId = getChildIndex(pBlock);
|
||||
for (int32_t i = 0; i < pBlock->info.rows; i++) {
|
||||
SWinRes winRes = {.ts = tsData[i], .groupId = groupIdData[i]};
|
||||
void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes));
|
||||
void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes));
|
||||
if (chIds) {
|
||||
SArray* chArray = *(SArray**) chIds;
|
||||
SArray* chArray = *(SArray**)chIds;
|
||||
int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
|
||||
if (index != -1) {
|
||||
taosArrayRemove(chArray, index);
|
||||
|
@ -2484,17 +2510,18 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
|
|||
printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv");
|
||||
maxTs = TMAX(maxTs, pBlock->info.window.ekey);
|
||||
|
||||
if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PUSH_DATA || pBlock->info.type == STREAM_INVALID) {
|
||||
if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA ||
|
||||
pBlock->info.type == STREAM_INVALID) {
|
||||
pInfo->binfo.pRes->info.type = pBlock->info.type;
|
||||
} else if (pBlock->info.type == STREAM_CLEAR) {
|
||||
SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
|
||||
doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pInfo->primaryTsIndex, pOperator->exprSupp.numOfExprs,
|
||||
pBlock, pUpWins);
|
||||
if (IS_FINAL_OP(pInfo)) {
|
||||
int32_t childIndex = getChildIndex(pBlock);
|
||||
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
|
||||
int32_t childIndex = getChildIndex(pBlock);
|
||||
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
|
||||
SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
|
||||
SExprSupp* pChildSup = &pChildOp->exprSupp;
|
||||
SExprSupp* pChildSup = &pChildOp->exprSupp;
|
||||
|
||||
doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildInfo->primaryTsIndex,
|
||||
pChildSup->numOfExprs, pBlock, NULL);
|
||||
|
@ -2513,16 +2540,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
|
|||
continue;
|
||||
} else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
|
||||
SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
|
||||
doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs,
|
||||
pBlock, pUpWins);
|
||||
doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
|
||||
removeResults(pUpWins, pUpdated);
|
||||
taosArrayDestroy(pUpWins);
|
||||
if (taosArrayGetSize(pUpdated) > 0) {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
} else if (pBlock->info.type == STREAM_PUSH_EMPTY && IS_FINAL_OP(pInfo)) {
|
||||
processPushEmpty(pBlock, pInfo->pPullDataMap);
|
||||
} else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) {
|
||||
processPullOver(pBlock, pInfo->pPullDataMap);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -2552,8 +2578,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
|
|||
|
||||
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
|
||||
if (IS_FINAL_OP(pInfo)) {
|
||||
closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup,
|
||||
&pInfo->interval, pInfo->pPullDataMap, pUpdated);
|
||||
closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap,
|
||||
pUpdated);
|
||||
closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
|
||||
}
|
||||
|
||||
|
@ -2684,6 +2710,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
|
|||
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
|
||||
pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
|
||||
pInfo->pPullDataRes = createPullDataBlock();
|
||||
pInfo->ignoreCloseWindow = false;
|
||||
|
||||
pOperator->operatorType = pPhyNode->type;
|
||||
pOperator->blocking = true;
|
||||
|
@ -2830,6 +2857,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
|
|||
pInfo->pChildren = NULL;
|
||||
pInfo->isFinal = false;
|
||||
pInfo->pPhyNode = pPhyNode;
|
||||
pInfo->ignoreCloseWindow = false;
|
||||
|
||||
pOperator->name = "StreamSessionWindowAggOperator";
|
||||
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
|
||||
|
@ -3007,6 +3035,9 @@ static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStre
|
|||
updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false);
|
||||
doApplyFunctions(pTaskInfo, pSup->pCtx, &pCurWin->win, pTimeWindowData, startIndex, winRows, tsCols,
|
||||
pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC);
|
||||
SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pCurWin->pos.pageId);
|
||||
setBufPageDirty(bufPage, true);
|
||||
releaseBufPage(pAggSup->pResultBuf, bufPage);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
@ -3063,7 +3094,13 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex,
|
|||
pWinInfo->isOutput = false;
|
||||
}
|
||||
taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
|
||||
SFilePage* tmpPage = getBufPage(pInfo->streamAggSup.pResultBuf, pWinInfo->pos.pageId);
|
||||
releaseBufPage(pInfo->streamAggSup.pResultBuf, tmpPage);
|
||||
}
|
||||
SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pCurWin->pos.pageId);
|
||||
ASSERT(num > 0);
|
||||
setBufPageDirty(bufPage, true);
|
||||
releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
|
||||
}
|
||||
|
||||
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
|
||||
|
@ -3083,22 +3120,23 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
|
|||
SResultRow* pResult = NULL;
|
||||
int32_t winRows = 0;
|
||||
|
||||
if (pSDataBlock->pDataBlock != NULL) {
|
||||
SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
|
||||
startTsCols = (int64_t*)pStartTsCol->pData;
|
||||
SColumnInfoData* pEndTsCol = NULL;
|
||||
if (hasEndTs) {
|
||||
pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);
|
||||
} else {
|
||||
pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
|
||||
}
|
||||
endTsCols = (int64_t*)pEndTsCol->pData;
|
||||
ASSERT(pSDataBlock->pDataBlock);
|
||||
SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
|
||||
startTsCols = (int64_t*)pStartTsCol->pData;
|
||||
SColumnInfoData* pEndTsCol = NULL;
|
||||
if (hasEndTs) {
|
||||
pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);
|
||||
} else {
|
||||
return;
|
||||
pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
|
||||
}
|
||||
endTsCols = (int64_t*)pEndTsCol->pData;
|
||||
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
for (int32_t i = 0; i < pSDataBlock->info.rows;) {
|
||||
if (pInfo->ignoreCloseWindow && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
int32_t winIndex = 0;
|
||||
SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex);
|
||||
winRows =
|
||||
|
@ -3136,6 +3174,10 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup,
|
|||
getSessionTimeWindow(pAggSup, tsCols[i], INT64_MIN, pBlock->info.groupId, gap, &winIndex);
|
||||
step = updateSessionWindowInfo(pCurWin, tsCols, NULL, pBlock->info.rows, i, gap, NULL);
|
||||
ASSERT(isInWindow(pCurWin, tsCols[i], gap));
|
||||
if (pCurWin->pos.pageId == -1) {
|
||||
// window has been closed.
|
||||
continue;
|
||||
}
|
||||
doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput);
|
||||
if (result) {
|
||||
taosArrayPush(result, pCurWin);
|
||||
|
@ -3205,17 +3247,24 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin
|
|||
index = 0;
|
||||
}
|
||||
for (int32_t k = index; k < chWinSize; k++) {
|
||||
SResultWindowInfo* pcw = taosArrayGet(pChWins, k);
|
||||
if (pParentWin->win.skey <= pcw->win.skey && pcw->win.ekey <= pParentWin->win.ekey) {
|
||||
SResultWindowInfo* pChWin = taosArrayGet(pChWins, k);
|
||||
if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) {
|
||||
SResultRow* pChResult = NULL;
|
||||
setWindowOutputBuf(pcw, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
|
||||
setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
|
||||
pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
|
||||
compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo);
|
||||
SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
|
||||
releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage);
|
||||
continue;
|
||||
} else if (!pChWin->isClosed) {
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId);
|
||||
ASSERT(size > 0);
|
||||
setBufPageDirty(bufPage, true);
|
||||
releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -3223,7 +3272,8 @@ typedef SResultWindowInfo* (*__get_win_info_)(void*);
|
|||
SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; }
|
||||
SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
|
||||
|
||||
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn) {
|
||||
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup,
|
||||
SArray* pClosed, __get_win_info_ fn, bool delete) {
|
||||
// Todo(liuyao) save window to tdb
|
||||
void** pIte = NULL;
|
||||
size_t keyLen = 0;
|
||||
|
@ -3234,13 +3284,21 @@ int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArra
|
|||
for (int32_t i = 0; i < size; i++) {
|
||||
void* pWin = taosArrayGet(pWins, i);
|
||||
SResultWindowInfo* pSeWin = fn(pWin);
|
||||
if (pSeWin->win.ekey < pTwSup->maxTs - pTwSup->waterMark) {
|
||||
if (isCloseWindow(&pSeWin->win, pTwSup)) {
|
||||
if (!pSeWin->isClosed) {
|
||||
pSeWin->isClosed = true;
|
||||
if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
|
||||
if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {
|
||||
int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
pSeWin->isOutput = true;
|
||||
}
|
||||
if (delete) {
|
||||
taosArrayRemove(pWins, i);
|
||||
i--;
|
||||
size = taosArrayGetSize(pWins);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
@ -3250,6 +3308,16 @@ int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArra
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs, bool delete) {
|
||||
int32_t size = taosArrayGetSize(pChildren);
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i);
|
||||
SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
|
||||
pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
|
||||
closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL, getResWinForSession, delete);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) {
|
||||
void** pIte = NULL;
|
||||
while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
|
||||
|
@ -3277,14 +3345,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
|
|||
} else if (pOperator->status == OP_RES_TO_RETURN) {
|
||||
doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session");
|
||||
printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "Final Session" : "Single Session");
|
||||
return pInfo->pDelRes;
|
||||
}
|
||||
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
|
||||
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
|
||||
doSetOperatorCompleted(pOperator);
|
||||
}
|
||||
printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session");
|
||||
printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "Final Session" : "Single Session");
|
||||
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
|
||||
}
|
||||
|
||||
|
@ -3297,6 +3365,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
|
|||
if (pBlock == NULL) {
|
||||
break;
|
||||
}
|
||||
printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "Final Session Recv" : "Single Session Recv");
|
||||
|
||||
if (pBlock->info.type == STREAM_CLEAR) {
|
||||
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
|
||||
|
@ -3343,7 +3412,9 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
|
|||
// restore the value
|
||||
pOperator->status = OP_RES_TO_RETURN;
|
||||
|
||||
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession);
|
||||
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
|
||||
getResWinForSession, pInfo->ignoreCloseWindow);
|
||||
closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow);
|
||||
copyUpdateResult(pStUpdated, pUpdated);
|
||||
taosHashCleanup(pStUpdated);
|
||||
|
||||
|
@ -3352,11 +3423,11 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
|
|||
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
|
||||
doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session");
|
||||
printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "Final Session" : "Single Session");
|
||||
return pInfo->pDelRes;
|
||||
}
|
||||
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
|
||||
printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session");
|
||||
printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "Final Session" : "Single Session");
|
||||
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
|
||||
}
|
||||
|
||||
|
@ -3395,10 +3466,11 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
|
|||
} else if (pOperator->status == OP_RES_TO_RETURN) {
|
||||
doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, "Semi Session");
|
||||
return pInfo->pDelRes;
|
||||
}
|
||||
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
|
||||
if (pInfo->binfo.pRes->info.rows == 0) {
|
||||
if (pBInfo->pRes->info.rows == 0) {
|
||||
pOperator->status = OP_EXEC_DONE;
|
||||
if (pInfo->pUpdateRes->info.rows == 0) {
|
||||
// semi interval operator clear disk buffer
|
||||
|
@ -3407,9 +3479,11 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
|
|||
}
|
||||
// process the rest of the data
|
||||
pOperator->status = OP_OPENED;
|
||||
printDataBlock(pInfo->pUpdateRes, "Semi Session");
|
||||
return pInfo->pUpdateRes;
|
||||
}
|
||||
return pInfo->binfo.pRes;
|
||||
printDataBlock(pBInfo->pRes, "Semi Session");
|
||||
return pBInfo->pRes;
|
||||
}
|
||||
|
||||
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
|
||||
|
@ -3453,21 +3527,24 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
|
|||
finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
|
||||
pSup->rowEntryInfoOffset);
|
||||
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
|
||||
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
|
||||
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
|
||||
doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, "Semi Session");
|
||||
return pInfo->pDelRes;
|
||||
}
|
||||
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
|
||||
if (pInfo->binfo.pRes->info.rows == 0) {
|
||||
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
|
||||
if (pBInfo->pRes->info.rows == 0) {
|
||||
pOperator->status = OP_EXEC_DONE;
|
||||
if (pInfo->pUpdateRes->info.rows == 0) {
|
||||
return NULL;
|
||||
}
|
||||
// process the rest of the data
|
||||
pOperator->status = OP_OPENED;
|
||||
printDataBlock(pInfo->pUpdateRes, "Semi Session");
|
||||
return pInfo->pUpdateRes;
|
||||
}
|
||||
printDataBlock(pBInfo->pRes, "Semi Session");
|
||||
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
|
||||
}
|
||||
|
||||
|
@ -3745,6 +3822,10 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
|
||||
for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
|
||||
if (pInfo->ignoreCloseWindow && isOverdue(tsCols[i], &pInfo->twAggSup)) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
char* pKeyData = colDataGetData(pKeyColInfo, i);
|
||||
int32_t winIndex = 0;
|
||||
bool allEqual = true;
|
||||
|
@ -3821,7 +3902,9 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
|
|||
// restore the value
|
||||
pOperator->status = OP_RES_TO_RETURN;
|
||||
|
||||
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState);
|
||||
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
|
||||
getResWinForState, pInfo->ignoreCloseWindow);
|
||||
closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow);
|
||||
copyUpdateResult(pSeUpdated, pUpdated);
|
||||
taosHashCleanup(pSeUpdated);
|
||||
|
||||
|
@ -3895,6 +3978,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
|
|||
pInfo->pDelRes->info.type = STREAM_DELETE;
|
||||
blockDataEnsureCapacity(pInfo->pDelRes, 64);
|
||||
pInfo->pChildren = NULL;
|
||||
pInfo->ignoreCloseWindow = false;
|
||||
|
||||
pOperator->name = "StreamStateAggOperator";
|
||||
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
|
||||
|
@ -4150,16 +4234,16 @@ _error:
|
|||
// merge interval operator
|
||||
typedef struct SMergeIntervalAggOperatorInfo {
|
||||
SIntervalAggOperatorInfo intervalAggOperatorInfo;
|
||||
SList* groupIntervals;
|
||||
SListIter groupIntervalsIter;
|
||||
bool hasGroupId;
|
||||
uint64_t groupId;
|
||||
SSDataBlock* prefetchedBlock;
|
||||
bool inputBlocksFinished;
|
||||
SList* groupIntervals;
|
||||
SListIter groupIntervalsIter;
|
||||
bool hasGroupId;
|
||||
uint64_t groupId;
|
||||
SSDataBlock* prefetchedBlock;
|
||||
bool inputBlocksFinished;
|
||||
} SMergeIntervalAggOperatorInfo;
|
||||
|
||||
typedef struct SGroupTimeWindow {
|
||||
uint64_t groupId;
|
||||
uint64_t groupId;
|
||||
STimeWindow window;
|
||||
} SGroupTimeWindow;
|
||||
|
||||
|
@ -4169,7 +4253,8 @@ void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) {
|
|||
destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput);
|
||||
}
|
||||
|
||||
static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win, SSDataBlock* pResultBlock) {
|
||||
static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win,
|
||||
SSDataBlock* pResultBlock) {
|
||||
SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
|
||||
SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;
|
||||
SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
|
||||
|
@ -4202,7 +4287,7 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t
|
|||
SListNode* listNode = NULL;
|
||||
while ((listNode = tdListNext(&iter)) != NULL) {
|
||||
SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data;
|
||||
if (prevGrpWin->groupId != tableGroupId ) {
|
||||
if (prevGrpWin->groupId != tableGroupId) {
|
||||
continue;
|
||||
}
|
||||
STimeWindow* prevWin = &prevGrpWin->window;
|
||||
|
@ -4445,4 +4530,4 @@ _error:
|
|||
taosMemoryFreeClear(pOperator);
|
||||
pTaskInfo->code = code;
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -39,6 +39,21 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char
|
|||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName);
|
||||
}
|
||||
|
||||
void static addDbPrecisonParam(SNodeList** pList, uint8_t precision) {
|
||||
SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
|
||||
pVal->literal = NULL;
|
||||
pVal->isDuration = false;
|
||||
pVal->translate = true;
|
||||
pVal->notReserved = true;
|
||||
pVal->node.resType.type = TSDB_DATA_TYPE_TINYINT;
|
||||
pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TINYINT].bytes;
|
||||
pVal->node.resType.precision = precision;
|
||||
pVal->datum.i = (int64_t)precision;
|
||||
pVal->typeData = (int64_t)precision;
|
||||
|
||||
nodesListMakeAppend(pList, (SNode*)pVal);
|
||||
}
|
||||
|
||||
// There is only one parameter of numeric type, and the return type is parameter type
|
||||
static int32_t translateInOutNum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
|
||||
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
|
||||
|
@ -220,8 +235,20 @@ static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t l
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t translateNowToday(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
|
||||
// pseudo column do not need to check parameters
|
||||
|
||||
//add database precision as param
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
|
||||
|
||||
pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP};
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
|
||||
// pseudo column do not need to check parameters
|
||||
|
||||
pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP};
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
@ -990,6 +1017,10 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
|
|||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
//add database precision as param
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
|
||||
|
||||
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
@ -1446,6 +1477,10 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int
|
|||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
//add database precision as param
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
|
||||
|
||||
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
@ -1462,6 +1497,10 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
|
|||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
//add database precision as param
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
|
||||
|
||||
pFunc->node.resType =
|
||||
(SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP};
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
@ -1486,6 +1525,10 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
|
|||
}
|
||||
}
|
||||
|
||||
//add database precision as param
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
|
||||
|
||||
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
@ -1670,7 +1713,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "leastsquares",
|
||||
.type = FUNCTION_TYPE_LEASTSQUARES,
|
||||
.classification = FUNC_MGT_AGG_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_STREAM_FUNC,
|
||||
.translateFunc = translateLeastSQR,
|
||||
.getEnvFunc = getLeastSQRFuncEnv,
|
||||
.initFunc = leastSQRFunctionSetup,
|
||||
|
@ -1912,6 +1955,19 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
.name = "last_row",
|
||||
.type = FUNCTION_TYPE_LAST_ROW,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
|
||||
.translateFunc = translateFirstLast,
|
||||
.getEnvFunc = getFirstLastFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
.processFunc = lastFunction,
|
||||
.finalizeFunc = firstLastFinalize,
|
||||
.pPartialFunc = "_last_partial",
|
||||
.pMergeFunc = "_last_merge",
|
||||
.combineFunc = lastCombine,
|
||||
},
|
||||
{
|
||||
.name = "_cache_last_row",
|
||||
.type = FUNCTION_TYPE_CACHE_LAST_ROW,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
|
||||
.translateFunc = translateLastRow,
|
||||
.getEnvFunc = getMinmaxFuncEnv,
|
||||
.initFunc = minmaxFunctionSetup,
|
||||
|
@ -2440,7 +2496,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
.name = "now",
|
||||
.type = FUNCTION_TYPE_NOW,
|
||||
.classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC,
|
||||
.translateFunc = translateTimePseudoColumn,
|
||||
.translateFunc = translateNowToday,
|
||||
.getEnvFunc = NULL,
|
||||
.initFunc = NULL,
|
||||
.sprocessFunc = nowFunction,
|
||||
|
@ -2450,7 +2506,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
.name = "today",
|
||||
.type = FUNCTION_TYPE_TODAY,
|
||||
.classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC,
|
||||
.translateFunc = translateTimePseudoColumn,
|
||||
.translateFunc = translateNowToday,
|
||||
.getEnvFunc = NULL,
|
||||
.initFunc = NULL,
|
||||
.sprocessFunc = todayFunction,
|
||||
|
|
|
@ -4283,7 +4283,7 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) {
|
|||
SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
|
||||
|
||||
// TODO: process timeUnit for different db precisions
|
||||
int32_t timeUnit = 1000;
|
||||
int32_t timeUnit = 1;
|
||||
if (pCtx->numOfParams == 5) { // TODO: param number incorrect
|
||||
timeUnit = pCtx->param[3].param.i;
|
||||
}
|
||||
|
@ -5508,7 +5508,7 @@ int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0;
|
||||
|
||||
SRateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
double result = doCalcRate(pInfo, 1000);
|
||||
double result = doCalcRate(pInfo, (double)TSDB_TICK_PER_SECOND(pCtx->param[1].param.i));
|
||||
colDataAppend(pCol, pBlock->info.rows, (const char*)&result, pResInfo->isNullRes);
|
||||
|
||||
return pResInfo->numOfRes;
|
||||
|
@ -5525,7 +5525,7 @@ int32_t groupKeyFunction(SqlFunctionCtx* pCtx) {
|
|||
|
||||
int32_t startIndex = pInput->startRowIndex;
|
||||
|
||||
//escape rest of data blocks to avoid first entry be overwritten.
|
||||
//escape rest of data blocks to avoid first entry to be overwritten.
|
||||
if (pInfo->hasResult) {
|
||||
goto _group_key_over;
|
||||
}
|
||||
|
|
|
@ -548,8 +548,8 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
|
|||
}
|
||||
static bool udfdRpcRfp(int32_t code, tmsg_t msgType) {
|
||||
if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
|
||||
code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
|
||||
if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) {
|
||||
code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK) {
|
||||
if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
|
|
@ -350,9 +350,9 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
|
|||
CLONE_NODE_FIELD(pTagIndexCond);
|
||||
COPY_SCALAR_FIELD(triggerType);
|
||||
COPY_SCALAR_FIELD(watermark);
|
||||
COPY_SCALAR_FIELD(tsColId);
|
||||
COPY_SCALAR_FIELD(filesFactor);
|
||||
COPY_SCALAR_FIELD(igExpired);
|
||||
CLONE_NODE_LIST_FIELD(pGroupTags);
|
||||
COPY_SCALAR_FIELD(groupSort);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
@ -421,7 +421,7 @@ static int32_t logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* p
|
|||
CLONE_NODE_FIELD(pStateExpr);
|
||||
COPY_SCALAR_FIELD(triggerType);
|
||||
COPY_SCALAR_FIELD(watermark);
|
||||
COPY_SCALAR_FIELD(filesFactor);
|
||||
COPY_SCALAR_FIELD(igExpired);
|
||||
COPY_SCALAR_FIELD(windowAlgo);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
@ -511,8 +511,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy
|
|||
COPY_SCALAR_FIELD(slidingUnit);
|
||||
COPY_SCALAR_FIELD(triggerType);
|
||||
COPY_SCALAR_FIELD(watermark);
|
||||
COPY_SCALAR_FIELD(tsColId);
|
||||
COPY_SCALAR_FIELD(filesFactor);
|
||||
COPY_SCALAR_FIELD(igExpired);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
@ -532,7 +531,7 @@ static int32_t physiWindowCopy(const SWinodwPhysiNode* pSrc, SWinodwPhysiNode* p
|
|||
CLONE_NODE_FIELD(pTsEnd);
|
||||
COPY_SCALAR_FIELD(triggerType);
|
||||
COPY_SCALAR_FIELD(watermark);
|
||||
COPY_SCALAR_FIELD(filesFactor);
|
||||
COPY_SCALAR_FIELD(igExpired);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
@ -582,6 +581,7 @@ static int32_t downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstre
|
|||
COPY_OBJECT_FIELD(addr, sizeof(SQueryNodeAddr));
|
||||
COPY_SCALAR_FIELD(taskId);
|
||||
COPY_SCALAR_FIELD(schedId);
|
||||
COPY_SCALAR_FIELD(execId);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
|
@ -1426,12 +1426,11 @@ static const char* jkTableScanPhysiPlanDynamicScanFuncs = "DynamicScanFuncs";
|
|||
static const char* jkTableScanPhysiPlanInterval = "Interval";
|
||||
static const char* jkTableScanPhysiPlanOffset = "Offset";
|
||||
static const char* jkTableScanPhysiPlanSliding = "Sliding";
|
||||
static const char* jkTableScanPhysiPlanIntervalUnit = "intervalUnit";
|
||||
static const char* jkTableScanPhysiPlanSlidingUnit = "slidingUnit";
|
||||
static const char* jkTableScanPhysiPlanTriggerType = "triggerType";
|
||||
static const char* jkTableScanPhysiPlanWatermark = "watermark";
|
||||
static const char* jkTableScanPhysiPlanTsColId = "tsColId";
|
||||
static const char* jkTableScanPhysiPlanFilesFactor = "FilesFactor";
|
||||
static const char* jkTableScanPhysiPlanIntervalUnit = "IntervalUnit";
|
||||
static const char* jkTableScanPhysiPlanSlidingUnit = "SlidingUnit";
|
||||
static const char* jkTableScanPhysiPlanTriggerType = "TriggerType";
|
||||
static const char* jkTableScanPhysiPlanWatermark = "Watermark";
|
||||
static const char* jkTableScanPhysiPlanIgnoreExpired = "IgnoreExpired";
|
||||
static const char* jkTableScanPhysiPlanGroupTags = "GroupTags";
|
||||
static const char* jkTableScanPhysiPlanGroupSort = "GroupSort";
|
||||
|
||||
|
@ -1482,10 +1481,7 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
|
|||
code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanWatermark, pNode->watermark);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanTsColId, pNode->tsColId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddDoubleToObject(pJson, jkTableScanPhysiPlanFilesFactor, pNode->filesFactor);
|
||||
code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanIgnoreExpired, pNode->igExpired);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = nodeListToJson(pJson, jkTableScanPhysiPlanGroupTags, pNode->pGroupTags);
|
||||
|
@ -1517,37 +1513,34 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
|
|||
code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanRatio, &pNode->ratio);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanDataRequired, pNode->dataRequired, code);
|
||||
code = tjsonGetIntValue(pJson, jkTableScanPhysiPlanDataRequired, &pNode->dataRequired);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = jsonToNodeList(pJson, jkTableScanPhysiPlanDynamicScanFuncs, &pNode->pDynamicScanFuncs);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanInterval, pNode->interval, code);
|
||||
code = tjsonGetBigIntValue(pJson, jkTableScanPhysiPlanInterval, &pNode->interval);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanOffset, pNode->offset, code);
|
||||
code = tjsonGetBigIntValue(pJson, jkTableScanPhysiPlanOffset, &pNode->offset);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSliding, pNode->sliding, code);
|
||||
code = tjsonGetBigIntValue(pJson, jkTableScanPhysiPlanSliding, &pNode->sliding);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanIntervalUnit, pNode->intervalUnit, code);
|
||||
code = tjsonGetTinyIntValue(pJson, jkTableScanPhysiPlanIntervalUnit, &pNode->intervalUnit);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSlidingUnit, pNode->slidingUnit, code);
|
||||
code = tjsonGetTinyIntValue(pJson, jkTableScanPhysiPlanSlidingUnit, &pNode->slidingUnit);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanTriggerType, pNode->triggerType, code);
|
||||
code = tjsonGetTinyIntValue(pJson, jkTableScanPhysiPlanTriggerType, &pNode->triggerType);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanWatermark, pNode->watermark, code);
|
||||
code = tjsonGetBigIntValue(pJson, jkTableScanPhysiPlanWatermark, &pNode->watermark);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanTsColId, pNode->tsColId, code);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanFilesFactor, &pNode->filesFactor);
|
||||
code = tjsonGetTinyIntValue(pJson, jkTableScanPhysiPlanIgnoreExpired, &pNode->igExpired);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = jsonToNodeList(pJson, jkTableScanPhysiPlanGroupTags, &pNode->pGroupTags);
|
||||
|
@ -1826,7 +1819,7 @@ static const char* jkWindowPhysiPlanTsPk = "TsPk";
|
|||
static const char* jkWindowPhysiPlanTsEnd = "TsEnd";
|
||||
static const char* jkWindowPhysiPlanTriggerType = "TriggerType";
|
||||
static const char* jkWindowPhysiPlanWatermark = "Watermark";
|
||||
static const char* jkWindowPhysiPlanFilesFactor = "FilesFactor";
|
||||
static const char* jkWindowPhysiPlanIgnoreExpired = "IgnoreExpired";
|
||||
|
||||
static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj;
|
||||
|
@ -1851,7 +1844,7 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) {
|
|||
code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanWatermark, pNode->watermark);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddDoubleToObject(pJson, jkWindowPhysiPlanFilesFactor, pNode->filesFactor);
|
||||
code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanIgnoreExpired, pNode->igExpired);
|
||||
}
|
||||
|
||||
return code;
|
||||
|
@ -1874,15 +1867,13 @@ static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) {
|
|||
code = jsonToNodeObject(pJson, jkWindowPhysiPlanTsEnd, (SNode**)&pNode->pTsEnd);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonGetNumberValue(pJson, jkWindowPhysiPlanTriggerType, pNode->triggerType, code);
|
||||
;
|
||||
code = tjsonGetTinyIntValue(pJson, jkWindowPhysiPlanTriggerType, &pNode->triggerType);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonGetNumberValue(pJson, jkWindowPhysiPlanWatermark, pNode->watermark, code);
|
||||
;
|
||||
code = tjsonGetBigIntValue(pJson, jkWindowPhysiPlanWatermark, &pNode->watermark);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetDoubleValue(pJson, jkWindowPhysiPlanFilesFactor, &pNode->filesFactor);
|
||||
code = tjsonGetTinyIntValue(pJson, jkWindowPhysiPlanIgnoreExpired, &pNode->igExpired);
|
||||
}
|
||||
|
||||
return code;
|
||||
|
@ -3430,6 +3421,7 @@ static int32_t jsonToSlotDescNode(const SJson* pJson, void* pObj) {
|
|||
static const char* jkDownstreamSourceAddr = "Addr";
|
||||
static const char* jkDownstreamSourceTaskId = "TaskId";
|
||||
static const char* jkDownstreamSourceSchedId = "SchedId";
|
||||
static const char* jkDownstreamSourceExecId = "ExecId";
|
||||
|
||||
static int32_t downstreamSourceNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const SDownstreamSourceNode* pNode = (const SDownstreamSourceNode*)pObj;
|
||||
|
@ -3441,6 +3433,9 @@ static int32_t downstreamSourceNodeToJson(const void* pObj, SJson* pJson) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceSchedId, pNode->schedId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceExecId, pNode->execId);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
@ -3455,6 +3450,9 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceSchedId, &pNode->schedId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetIntValue(pJson, jkDownstreamSourceExecId, &pNode->execId);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
|
@ -1728,7 +1728,6 @@ static EDealRes classifyConditionImpl(SNode* pNode, void* pContext) {
|
|||
} else {
|
||||
pCxt->hasOtherCol = true;
|
||||
}
|
||||
return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END;
|
||||
}
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
|
|
@ -488,6 +488,7 @@ stream_options(A) ::= stream_options(B) TRIGGER AT_ONCE.
|
|||
stream_options(A) ::= stream_options(B) TRIGGER WINDOW_CLOSE. { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; A = B; }
|
||||
stream_options(A) ::= stream_options(B) TRIGGER MAX_DELAY duration_literal(C). { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)B)->pDelay = releaseRawExprNode(pCxt, C); A = B; }
|
||||
stream_options(A) ::= stream_options(B) WATERMARK duration_literal(C). { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; }
|
||||
stream_options(A) ::= stream_options(B) IGNORE EXPIRED. { ((SStreamOptions*)B)->ignoreExpired = true; A = B; }
|
||||
|
||||
/************************************************ kill connection/query ***********************************************/
|
||||
cmd ::= KILL CONNECTION NK_INTEGER(A). { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &A); }
|
||||
|
@ -848,14 +849,10 @@ set_quantifier_opt(A) ::= ALL.
|
|||
|
||||
%type select_list { SNodeList* }
|
||||
%destructor select_list { nodesDestroyList($$); }
|
||||
select_list(A) ::= NK_STAR. { A = NULL; }
|
||||
select_list(A) ::= select_sublist(B). { A = B; }
|
||||
|
||||
%type select_sublist { SNodeList* }
|
||||
%destructor select_sublist { nodesDestroyList($$); }
|
||||
select_sublist(A) ::= select_item(B). { A = createNodeList(pCxt, B); }
|
||||
select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { A = addNodeToList(pCxt, B, C); }
|
||||
select_list(A) ::= select_item(B). { A = createNodeList(pCxt, B); }
|
||||
select_list(A) ::= select_list(B) NK_COMMA select_item(C). { A = addNodeToList(pCxt, B, C); }
|
||||
|
||||
select_item(A) ::= NK_STAR(B). { A = createColumnNode(pCxt, NULL, &B); }
|
||||
select_item(A) ::= common_expression(B). { A = releaseRawExprNode(pCxt, B); }
|
||||
select_item(A) ::= common_expression(B) column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); }
|
||||
select_item(A) ::= common_expression(B) AS column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); }
|
||||
|
|
|
@ -1302,6 +1302,74 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataBlocks* pDataBlock, int maxRows,
|
||||
int32_t* numOfRows) {
|
||||
STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta);
|
||||
int32_t extendedRowSize = getExtendedRowSize(pDataBlock);
|
||||
CHECK_CODE(initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo));
|
||||
|
||||
(*numOfRows) = 0;
|
||||
char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \"
|
||||
char* pLine = NULL;
|
||||
int64_t readLen = 0;
|
||||
while ((readLen = taosGetLineFile(fp, &pLine)) != -1) {
|
||||
if (('\r' == pLine[readLen - 1]) || ('\n' == pLine[readLen - 1])) {
|
||||
pLine[--readLen] = '\0';
|
||||
}
|
||||
|
||||
if (readLen == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) {
|
||||
int32_t tSize;
|
||||
CHECK_CODE(allocateMemIfNeed(pDataBlock, extendedRowSize, &tSize));
|
||||
ASSERT(tSize >= maxRows);
|
||||
maxRows = tSize;
|
||||
}
|
||||
|
||||
strtolower(pLine, pLine);
|
||||
char* pRawSql = pCxt->pSql;
|
||||
pCxt->pSql = pLine;
|
||||
bool gotRow = false;
|
||||
CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &gotRow, tmpTokenBuf));
|
||||
if (gotRow) {
|
||||
pDataBlock->size += extendedRowSize; // len;
|
||||
(*numOfRows)++;
|
||||
}
|
||||
pCxt->pSql = pRawSql;
|
||||
}
|
||||
|
||||
if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
|
||||
return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL);
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STableDataBlocks* dataBuf) {
|
||||
char filePathStr[TSDB_FILENAME_LEN] = {0};
|
||||
strncpy(filePathStr, filePath.z, filePath.n);
|
||||
TdFilePtr fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM);
|
||||
if (NULL == fp) {
|
||||
return TAOS_SYSTEM_ERROR(errno);
|
||||
}
|
||||
|
||||
int32_t maxNumOfRows;
|
||||
CHECK_CODE(allocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows));
|
||||
|
||||
int32_t numOfRows = 0;
|
||||
CHECK_CODE(parseCsvFile(pCxt, fp, dataBuf, maxNumOfRows, &numOfRows));
|
||||
|
||||
SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData);
|
||||
if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf, numOfRows)) {
|
||||
return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than 32767");
|
||||
}
|
||||
|
||||
dataBuf->numOfTables = 1;
|
||||
pCxt->totalNum += numOfRows;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
void destroyCreateSubTbReq(SVCreateTbReq* pReq) {
|
||||
taosMemoryFreeClear(pReq->name);
|
||||
taosMemoryFreeClear(pReq->ctb.pTag);
|
||||
|
@ -1421,7 +1489,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
|
|||
if (0 == sToken.n || (TK_NK_STRING != sToken.type && TK_NK_ID != sToken.type)) {
|
||||
return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z);
|
||||
}
|
||||
// todo
|
||||
CHECK_CODE(parseDataFromFile(pCxt, sToken, dataBuf));
|
||||
pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT;
|
||||
|
||||
tbNum++;
|
||||
|
@ -1770,7 +1838,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN
|
|||
}
|
||||
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
SSchema* pSchema = pDataBlock->pTableMeta->schema;
|
||||
SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta);
|
||||
|
||||
bool isJson = false;
|
||||
STag* pTag = NULL;
|
||||
|
|
|
@ -84,8 +84,10 @@ static SKeyword keywordTable[] = {
|
|||
{"DURATION", TK_DURATION},
|
||||
{"ENABLE", TK_ENABLE},
|
||||
{"EXISTS", TK_EXISTS},
|
||||
{"EXPIRED", TK_EXPIRED},
|
||||
{"EXPLAIN", TK_EXPLAIN},
|
||||
{"EVERY", TK_EVERY},
|
||||
{"FILE", TK_FILE},
|
||||
{"FILL", TK_FILL},
|
||||
{"FIRST", TK_FIRST},
|
||||
{"FLOAT", TK_FLOAT},
|
||||
|
@ -98,6 +100,7 @@ static SKeyword keywordTable[] = {
|
|||
{"GROUP", TK_GROUP},
|
||||
{"HAVING", TK_HAVING},
|
||||
{"IF", TK_IF},
|
||||
{"IGNORE", TK_IGNORE},
|
||||
{"IMPORT", TK_IMPORT},
|
||||
{"IN", TK_IN},
|
||||
{"INDEX", TK_INDEX},
|
||||
|
@ -289,7 +292,6 @@ static SKeyword keywordTable[] = {
|
|||
// {"END", TK_END},
|
||||
// {"FAIL", TK_FAIL},
|
||||
// {"FOR", TK_FOR},
|
||||
// {"IGNORE", TK_IGNORE},
|
||||
// {"IMMEDIATE", TK_IMMEDIATE},
|
||||
// {"INITIALLY", TK_INITIALLY},
|
||||
// {"INSTEAD", TK_INSTEAD},
|
||||
|
|
|
@ -783,6 +783,13 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
|
|||
if (pVal->placeholderNo > 0 || pVal->isNull) {
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
if (TSDB_DATA_TYPE_NULL == pVal->node.resType.type) {
|
||||
// TODO
|
||||
//pVal->node.resType = targetDt;
|
||||
pVal->translate = true;
|
||||
pVal->isNull = true;
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
if (pVal->isDuration) {
|
||||
if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, precision) !=
|
||||
TSDB_CODE_SUCCESS) {
|
||||
|
@ -1984,34 +1991,38 @@ static int32_t createMultiResFuncsFromStar(STranslateContext* pCxt, SFunctionNod
|
|||
}
|
||||
|
||||
static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
|
||||
if (NULL == pSelect->pProjectionList) { // select * ...
|
||||
return createAllColumns(pCxt, &pSelect->pProjectionList);
|
||||
} else {
|
||||
SNode* pNode = NULL;
|
||||
WHERE_EACH(pNode, pSelect->pProjectionList) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
if (isMultiResFunc(pNode)) {
|
||||
SNodeList* pFuncs = NULL;
|
||||
code = createMultiResFuncsFromStar(pCxt, (SFunctionNode*)pNode, &pFuncs);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
INSERT_LIST(pSelect->pProjectionList, pFuncs);
|
||||
ERASE_NODE(pSelect->pProjectionList);
|
||||
continue;
|
||||
}
|
||||
} else if (isTableStar(pNode)) {
|
||||
SNodeList* pCols = NULL;
|
||||
code = createTableAllCols(pCxt, (SColumnNode*)pNode, &pCols);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
INSERT_LIST(pSelect->pProjectionList, pCols);
|
||||
ERASE_NODE(pSelect->pProjectionList);
|
||||
continue;
|
||||
}
|
||||
SNode* pNode = NULL;
|
||||
WHERE_EACH(pNode, pSelect->pProjectionList) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
if (isStar(pNode)) {
|
||||
SNodeList* pCols = NULL;
|
||||
code = createAllColumns(pCxt, &pCols);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
INSERT_LIST(pSelect->pProjectionList, pCols);
|
||||
ERASE_NODE(pSelect->pProjectionList);
|
||||
continue;
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS != code) {
|
||||
return code;
|
||||
} else if (isMultiResFunc(pNode)) {
|
||||
SNodeList* pFuncs = NULL;
|
||||
code = createMultiResFuncsFromStar(pCxt, (SFunctionNode*)pNode, &pFuncs);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
INSERT_LIST(pSelect->pProjectionList, pFuncs);
|
||||
ERASE_NODE(pSelect->pProjectionList);
|
||||
continue;
|
||||
}
|
||||
} else if (isTableStar(pNode)) {
|
||||
SNodeList* pCols = NULL;
|
||||
code = createTableAllCols(pCxt, (SColumnNode*)pNode, &pCols);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
INSERT_LIST(pSelect->pProjectionList, pCols);
|
||||
ERASE_NODE(pSelect->pProjectionList);
|
||||
continue;
|
||||
}
|
||||
WHERE_NEXT;
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS != code) {
|
||||
return code;
|
||||
}
|
||||
WHERE_NEXT;
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
@ -4204,6 +4215,7 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt*
|
|||
pReq->triggerType = pStmt->pOptions->triggerType;
|
||||
pReq->maxDelay = (NULL != pStmt->pOptions->pDelay ? ((SValueNode*)pStmt->pOptions->pDelay)->datum.i : 0);
|
||||
pReq->watermark = (NULL != pStmt->pOptions->pWatermark ? ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i : 0);
|
||||
pReq->igExpired = pStmt->pOptions->ignoreExpired;
|
||||
}
|
||||
|
||||
return code;
|
||||
|
@ -4794,6 +4806,15 @@ static const char* getSysTableName(ENodeType type) {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
static SNode* createStarCol() {
|
||||
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
|
||||
if (NULL == pCol) {
|
||||
return NULL;
|
||||
}
|
||||
strcpy(pCol->colName, "*");
|
||||
return (SNode*)pCol;
|
||||
}
|
||||
|
||||
static int32_t createSimpleSelectStmt(const char* pDb, const char* pTable, SSelectStmt** pStmt) {
|
||||
SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT);
|
||||
if (NULL == pSelect) {
|
||||
|
@ -4811,6 +4832,11 @@ static int32_t createSimpleSelectStmt(const char* pDb, const char* pTable, SSele
|
|||
strcpy(pRealTable->table.tableAlias, pTable);
|
||||
pSelect->pFromTable = (SNode*)pRealTable;
|
||||
|
||||
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSelect->pProjectionList, createStarCol())) {
|
||||
nodesDestroyNode((SNode*)pSelect);
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
*pStmt = pSelect;
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
@ -4959,6 +4985,7 @@ static int32_t rewriteShowTableDist(STranslateContext* pCxt, SQuery* pQuery) {
|
|||
SSelectStmt* pStmt = NULL;
|
||||
int32_t code = createSelectStmtForShowTableDist((SShowTableDistributedStmt*)pQuery->pRoot, &pStmt);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
NODES_DESTORY_LIST(pStmt->pProjectionList);
|
||||
code = nodesListMakeStrictAppend(&pStmt->pProjectionList, createBlockDistFunc());
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
|
@ -5335,7 +5362,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau
|
|||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto end;
|
||||
}
|
||||
} else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) {
|
||||
} else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL && !pVal->isNull) {
|
||||
char* tmpVal = nodesGetValueFromNode(pVal);
|
||||
STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
|
||||
if (IS_VAR_DATA_TYPE(pTagSchema->type)) {
|
||||
|
@ -5602,8 +5629,8 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
|
|||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
if (DEAL_RES_ERROR ==
|
||||
translateValueImpl(pCxt, pStmt->pVal, schemaToDataType(pTableMeta->tableInfo.precision, pSchema))) {
|
||||
SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema);
|
||||
if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt)) {
|
||||
return pCxt->errCode;
|
||||
}
|
||||
|
||||
|
@ -5612,7 +5639,8 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
|
|||
}
|
||||
|
||||
pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type);
|
||||
if (pStmt->pVal->node.resType.type == TSDB_DATA_TYPE_JSON) {
|
||||
if (targetDt.type == TSDB_DATA_TYPE_JSON) {
|
||||
pReq->isNull = 0;
|
||||
if (pStmt->pVal->literal &&
|
||||
strlen(pStmt->pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
|
||||
return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pStmt->pVal->literal);
|
||||
|
@ -6011,7 +6039,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
|
|||
case QUERY_NODE_EXPLAIN_STMT:
|
||||
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
|
||||
pQuery->haveResultSet = true;
|
||||
pQuery->msgType = TDMT_VND_QUERY;
|
||||
pQuery->msgType = TDMT_SCH_QUERY;
|
||||
break;
|
||||
case QUERY_NODE_DELETE_STMT:
|
||||
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -526,20 +526,22 @@ TEST_F(ParserInitialCTest, createStream) {
|
|||
memset(&expect, 0, sizeof(SCMCreateStreamReq));
|
||||
};
|
||||
|
||||
auto setCreateStreamReqFunc =
|
||||
[&](const char* pStream, const char* pSrcDb, const char* pSql, const char* pDstStb = nullptr, int8_t igExists = 0,
|
||||
int8_t triggerType = STREAM_TRIGGER_AT_ONCE, int64_t maxDelay = 0, int64_t watermark = 0) {
|
||||
snprintf(expect.name, sizeof(expect.name), "0.%s", pStream);
|
||||
snprintf(expect.sourceDB, sizeof(expect.sourceDB), "0.%s", pSrcDb);
|
||||
if (NULL != pDstStb) {
|
||||
snprintf(expect.targetStbFullName, sizeof(expect.targetStbFullName), "0.test.%s", pDstStb);
|
||||
}
|
||||
expect.igExists = igExists;
|
||||
expect.sql = strdup(pSql);
|
||||
expect.triggerType = triggerType;
|
||||
expect.maxDelay = maxDelay;
|
||||
expect.watermark = watermark;
|
||||
};
|
||||
auto setCreateStreamReqFunc = [&](const char* pStream, const char* pSrcDb, const char* pSql,
|
||||
const char* pDstStb = nullptr, int8_t igExists = 0,
|
||||
int8_t triggerType = STREAM_TRIGGER_AT_ONCE, int64_t maxDelay = 0,
|
||||
int64_t watermark = 0, int8_t igExpired = 0) {
|
||||
snprintf(expect.name, sizeof(expect.name), "0.%s", pStream);
|
||||
snprintf(expect.sourceDB, sizeof(expect.sourceDB), "0.%s", pSrcDb);
|
||||
if (NULL != pDstStb) {
|
||||
snprintf(expect.targetStbFullName, sizeof(expect.targetStbFullName), "0.test.%s", pDstStb);
|
||||
}
|
||||
expect.igExists = igExists;
|
||||
expect.sql = strdup(pSql);
|
||||
expect.triggerType = triggerType;
|
||||
expect.maxDelay = maxDelay;
|
||||
expect.watermark = watermark;
|
||||
expect.igExpired = igExpired;
|
||||
};
|
||||
|
||||
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
|
||||
ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_STREAM_STMT);
|
||||
|
@ -555,6 +557,7 @@ TEST_F(ParserInitialCTest, createStream) {
|
|||
ASSERT_EQ(req.triggerType, expect.triggerType);
|
||||
ASSERT_EQ(req.maxDelay, expect.maxDelay);
|
||||
ASSERT_EQ(req.watermark, expect.watermark);
|
||||
ASSERT_EQ(req.igExpired, expect.igExpired);
|
||||
tFreeSCMCreateStreamReq(&req);
|
||||
});
|
||||
|
||||
|
@ -571,9 +574,10 @@ TEST_F(ParserInitialCTest, createStream) {
|
|||
clearCreateStreamReq();
|
||||
|
||||
setCreateStreamReqFunc(
|
||||
"s1", "test", "create stream if not exists s1 trigger max_delay 20s watermark 10s into st1 as select * from t1",
|
||||
"st1", 1, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND);
|
||||
run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s INTO st1 AS SELECT * FROM t1");
|
||||
"s1", "test",
|
||||
"create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired into st1 as select * from t1",
|
||||
"st1", 1, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND, 1);
|
||||
run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED INTO st1 AS SELECT * FROM t1");
|
||||
clearCreateStreamReq();
|
||||
}
|
||||
|
||||
|
|
|
@ -154,16 +154,12 @@ static int32_t createSelectRootLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p
|
|||
return createRootLogicNode(pCxt, pSelect, pSelect->precision, (FCreateLogicNode)func, pRoot);
|
||||
}
|
||||
|
||||
static EScanType getScanType(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNodeList* pScanPseudoCols,
|
||||
SNodeList* pScanCols, int8_t tableType) {
|
||||
static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols, SNodeList* pScanCols,
|
||||
int8_t tableType) {
|
||||
if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) {
|
||||
return SCAN_TYPE_STREAM;
|
||||
}
|
||||
|
||||
if (pSelect->hasLastRowFunc) {
|
||||
return SCAN_TYPE_LAST_ROW;
|
||||
}
|
||||
|
||||
if (NULL == pScanCols) {
|
||||
// select count(*) from t
|
||||
return NULL == pScanPseudoCols
|
||||
|
@ -279,7 +275,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
|
|||
code = rewriteExprsForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM);
|
||||
}
|
||||
|
||||
pScan->scanType = getScanType(pCxt, pSelect, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType);
|
||||
pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType);
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = addPrimaryKeyCol(pScan->tableId, &pScan->pScanCols);
|
||||
|
@ -474,6 +470,8 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
|
|||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
pAgg->hasLastRow = pSelect->hasLastRowFunc;
|
||||
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
||||
// set grouyp keys, agg funcs and having conditions
|
||||
|
@ -616,10 +614,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm
|
|||
if (pCxt->pPlanCxt->streamQuery) {
|
||||
pWindow->triggerType = pCxt->pPlanCxt->triggerType;
|
||||
pWindow->watermark = pCxt->pPlanCxt->watermark;
|
||||
}
|
||||
|
||||
if (pCxt->pPlanCxt->rSmaQuery) {
|
||||
/*pWindow->filesFactor = pCxt->pPlanCxt->filesFactor;*/
|
||||
pWindow->igExpired = pCxt->pPlanCxt->igExpired;
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
|
|
|
@ -227,8 +227,7 @@ static void scanPathOptSetScanWin(SScanLogicNode* pScan) {
|
|||
pScan->slidingUnit = ((SWindowLogicNode*)pParent)->slidingUnit;
|
||||
pScan->triggerType = ((SWindowLogicNode*)pParent)->triggerType;
|
||||
pScan->watermark = ((SWindowLogicNode*)pParent)->watermark;
|
||||
pScan->tsColId = ((SColumnNode*)((SWindowLogicNode*)pParent)->pTspk)->colId;
|
||||
pScan->filesFactor = ((SWindowLogicNode*)pParent)->filesFactor;
|
||||
pScan->igExpired = ((SWindowLogicNode*)pParent)->igExpired;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -587,11 +586,160 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAgg) {
|
||||
// todo
|
||||
typedef struct SPartAggCondContext {
|
||||
SAggLogicNode* pAgg;
|
||||
bool hasAggFunc;
|
||||
} SPartAggCondContext;
|
||||
|
||||
static EDealRes partAggCondHasAggFuncImpl(SNode* pNode, void* pContext) {
|
||||
SPartAggCondContext* pCxt = pContext;
|
||||
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
|
||||
SNode* pAggFunc = NULL;
|
||||
FOREACH(pAggFunc, pCxt->pAgg->pAggFuncs) {
|
||||
if (strcmp(((SColumnNode*)pNode)->colName, ((SFunctionNode*)pAggFunc)->node.aliasName) == 0) {
|
||||
pCxt->hasAggFunc = true;
|
||||
return DEAL_RES_END;
|
||||
}
|
||||
}
|
||||
}
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
||||
static int32_t partitionAggCondHasAggFunc(SAggLogicNode* pAgg, SNode* pCond) {
|
||||
SPartAggCondContext cxt = {.pAgg = pAgg, .hasAggFunc = false};
|
||||
nodesWalkExpr(pCond, partAggCondHasAggFuncImpl, &cxt);
|
||||
return cxt.hasAggFunc;
|
||||
}
|
||||
|
||||
static int32_t partitionAggCondConj(SAggLogicNode* pAgg, SNode** ppAggFuncCond, SNode** ppGroupKeyCond) {
|
||||
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pAgg->node.pConditions;
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
||||
SNodeList* pAggFuncConds = NULL;
|
||||
SNodeList* pGroupKeyConds = NULL;
|
||||
SNode* pCond = NULL;
|
||||
FOREACH(pCond, pLogicCond->pParameterList) {
|
||||
if (partitionAggCondHasAggFunc(pAgg, pCond)) {
|
||||
code = nodesListMakeAppend(&pAggFuncConds, nodesCloneNode(pCond));
|
||||
} else {
|
||||
code = nodesListMakeAppend(&pGroupKeyConds, nodesCloneNode(pCond));
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS != code) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
SNode* pTempAggFuncCond = NULL;
|
||||
SNode* pTempGroupKeyCond = NULL;
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = nodesMergeConds(&pTempAggFuncCond, &pAggFuncConds);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = nodesMergeConds(&pTempGroupKeyCond, &pGroupKeyConds);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
*ppAggFuncCond = pTempAggFuncCond;
|
||||
*ppGroupKeyCond = pTempGroupKeyCond;
|
||||
} else {
|
||||
nodesDestroyList(pAggFuncConds);
|
||||
nodesDestroyList(pGroupKeyConds);
|
||||
nodesDestroyNode(pTempAggFuncCond);
|
||||
nodesDestroyNode(pTempGroupKeyCond);
|
||||
}
|
||||
pAgg->node.pConditions = NULL;
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t partitionAggCond(SAggLogicNode* pAgg, SNode** ppAggFunCond, SNode** ppGroupKeyCond) {
|
||||
SNode* pAggNodeCond = pAgg->node.pConditions;
|
||||
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pAggNodeCond) &&
|
||||
LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pAggNodeCond))->condType) {
|
||||
return partitionAggCondConj(pAgg, ppAggFunCond, ppGroupKeyCond);
|
||||
}
|
||||
if (partitionAggCondHasAggFunc(pAgg, pAggNodeCond)) {
|
||||
*ppAggFunCond = pAggNodeCond;
|
||||
} else {
|
||||
*ppGroupKeyCond = pAggNodeCond;
|
||||
}
|
||||
pAgg->node.pConditions = NULL;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t pushCondToAggCond(SOptimizeContext* pCxt, SAggLogicNode* pAgg, SNode** pAggFuncCond) {
|
||||
pushDownCondOptAppendCond(&pAgg->node.pConditions, pAggFuncCond);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
typedef struct SRewriteAggGroupKeyCondContext{
|
||||
SAggLogicNode *pAgg;
|
||||
int32_t errCode;
|
||||
} SRewriteAggGroupKeyCondContext;
|
||||
|
||||
static EDealRes rewriteAggGroupKeyCondForPushDownImpl(SNode** pNode, void* pContext) {
|
||||
SRewriteAggGroupKeyCondContext* pCxt = pContext;
|
||||
SAggLogicNode* pAgg = pCxt->pAgg;
|
||||
if (QUERY_NODE_COLUMN == nodeType(*pNode)) {
|
||||
SNode* pGroupKey = NULL;
|
||||
FOREACH(pGroupKey, pAgg->pGroupKeys) {
|
||||
SNode* pGroup = NULL;
|
||||
FOREACH(pGroup, ((SGroupingSetNode*)pGroupKey)->pParameterList) {
|
||||
if (0 == strcmp(((SExprNode*)pGroup)->aliasName, ((SColumnNode*)(*pNode))->colName)) {
|
||||
SNode* pExpr = nodesCloneNode(pGroup);
|
||||
if (pExpr == NULL) {
|
||||
pCxt->errCode = terrno;
|
||||
return DEAL_RES_ERROR;
|
||||
}
|
||||
nodesDestroyNode(*pNode);
|
||||
*pNode = pExpr;
|
||||
}
|
||||
}
|
||||
}
|
||||
return DEAL_RES_IGNORE_CHILD;
|
||||
}
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
||||
static int32_t rewriteAggGroupKeyCondForPushDown(SOptimizeContext* pCxt, SAggLogicNode* pAgg, SNode* pGroupKeyCond) {
|
||||
SRewriteAggGroupKeyCondContext cxt = {.pAgg = pAgg, .errCode = TSDB_CODE_SUCCESS};
|
||||
nodesRewriteExpr(&pGroupKeyCond, rewriteAggGroupKeyCondForPushDownImpl, &cxt);
|
||||
return cxt.errCode;
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAgg) {
|
||||
if (NULL == pAgg->node.pConditions ||
|
||||
OPTIMIZE_FLAG_TEST_MASK(pAgg->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
//TODO: remove it after full implementation of pushing down to child
|
||||
if (1 != LIST_LENGTH(pAgg->node.pChildren) ||
|
||||
QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pAgg->node.pChildren, 0))) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SNode* pAggFuncCond = NULL;
|
||||
SNode* pGroupKeyCond = NULL;
|
||||
int32_t code = partitionAggCond(pAgg, &pAggFuncCond, &pGroupKeyCond);
|
||||
if (TSDB_CODE_SUCCESS == code && NULL != pAggFuncCond) {
|
||||
code = pushCondToAggCond(pCxt, pAgg, &pAggFuncCond);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeyCond) {
|
||||
code = rewriteAggGroupKeyCondForPushDown(pCxt, pAgg, pGroupKeyCond);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeyCond) {
|
||||
SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0);
|
||||
code = pushDownCondOptPushCondToChild(pCxt, pChild, &pGroupKeyCond);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
OPTIMIZE_FLAG_SET_MASK(pAgg->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE);
|
||||
pCxt->optimized = true;
|
||||
} else {
|
||||
nodesDestroyNode(pGroupKeyCond);
|
||||
nodesDestroyNode(pAggFuncCond);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
switch (nodeType(pLogicNode)) {
|
||||
|
@ -1622,6 +1770,46 @@ static int32_t rewriteUniqueOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog
|
|||
return rewriteUniqueOptimizeImpl(pCxt, pLogicSubplan, pIndef);
|
||||
}
|
||||
|
||||
static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
|
||||
if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || !(((SAggLogicNode*)pNode)->hasLastRow) ||
|
||||
NULL != ((SAggLogicNode*)pNode)->pGroupKeys || 1 != LIST_LENGTH(pNode->pChildren) ||
|
||||
QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0)) ||
|
||||
NULL != ((SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0))->node.pConditions) {
|
||||
return false;
|
||||
}
|
||||
|
||||
SNode* pFunc = NULL;
|
||||
FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) {
|
||||
if (FUNCTION_TYPE_LAST_ROW != ((SFunctionNode*)pFunc)->funcType) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
|
||||
SAggLogicNode* pAgg = (SAggLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, lastRowScanOptMayBeOptimized);
|
||||
|
||||
if (NULL == pAgg) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SNode* pNode = NULL;
|
||||
FOREACH(pNode, pAgg->pAggFuncs) {
|
||||
SFunctionNode* pFunc = (SFunctionNode*)pNode;
|
||||
int32_t len = snprintf(pFunc->functionName, sizeof(pFunc->functionName), "_cache_last_row");
|
||||
pFunc->functionName[len] = '\0';
|
||||
fmGetFuncInfo(pFunc, NULL, 0);
|
||||
}
|
||||
pAgg->hasLastRow = false;
|
||||
|
||||
((SScanLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))->scanType = SCAN_TYPE_LAST_ROW;
|
||||
|
||||
pCxt->optimized = true;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
// merge projects
|
||||
static bool mergeProjectsMayBeOptimized(SLogicNode* pNode) {
|
||||
if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren)) {
|
||||
|
@ -1668,8 +1856,8 @@ static EDealRes mergeProjectionsExpr(SNode** pNode, void* pContext) {
|
|||
|
||||
static int32_t mergeProjectsOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pSelfNode) {
|
||||
SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pSelfNode->pChildren, 0);
|
||||
SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild, .errCode = TSDB_CODE_SUCCESS};
|
||||
|
||||
SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild, .errCode = TSDB_CODE_SUCCESS};
|
||||
nodesRewriteExprs(((SProjectLogicNode*)pSelfNode)->pProjections, mergeProjectionsExpr, &cxt);
|
||||
int32_t code = cxt.errCode;
|
||||
|
||||
|
@ -1710,7 +1898,8 @@ static const SOptimizeRule optimizeRuleSet[] = {
|
|||
{.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize},
|
||||
{.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize},
|
||||
{.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize},
|
||||
{.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize}
|
||||
{.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize},
|
||||
{.pName = "LastRowScan", .optimizeFunc = lastRowScanOptimize}
|
||||
};
|
||||
// clang-format on
|
||||
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue