Merge branch '3.0' into cpwu/3.0
This commit is contained in:
commit
fa08389784
|
@ -55,8 +55,13 @@ extern int32_t tMsgDict[];
|
|||
|
||||
#define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8)
|
||||
#define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff)
|
||||
#define TMSG_INFO(TYPE) tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)]
|
||||
#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))
|
||||
#define TMSG_INFO(TYPE) \
|
||||
((TYPE) >= 0 && \
|
||||
((TYPE) < TDMT_DND_MAX_MSG | (TYPE) < TDMT_MND_MAX_MSG | (TYPE) < TDMT_VND_MAX_MSG | (TYPE) < TDMT_SCH_MAX_MSG | \
|
||||
(TYPE) < TDMT_STREAM_MAX_MSG | (TYPE) < TDMT_MON_MAX_MSG | (TYPE) < TDMT_SYNC_MAX_MSG)) \
|
||||
? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \
|
||||
: 0
|
||||
#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))
|
||||
|
||||
typedef uint16_t tmsg_t;
|
||||
|
||||
|
|
|
@ -82,6 +82,7 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "net-test", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "config-dnode", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_DND_SYSTABLE_RETRIEVE, "dnode-retrieve", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_MND_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL)
|
||||
|
@ -164,6 +165,7 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_MND_SPLIT_VGROUP, "split-vgroup", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_SHOW_VARIABLES, "show-variables", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_SERVER_VERSION, "server-version", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_VND_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)
|
||||
|
@ -198,6 +200,7 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "drop-ttl-stb", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_SCH_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL)
|
||||
|
@ -209,6 +212,7 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_SCH_DROP_TASK, "drop-task", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_EXPLAIN, "explain", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_STREAM_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
|
||||
|
@ -217,6 +221,7 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RECOVER, "stream-task-recover", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_MON_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_MON_MM_INFO, "monitor-minfo", NULL, NULL)
|
||||
|
@ -227,6 +232,7 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_MON_VM_LOAD, "monitor-vload", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MON_MM_LOAD, "monitor-mload", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MON_QM_LOAD, "monitor-qload", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_SYNC_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL)
|
||||
|
@ -251,6 +257,7 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_SYNC_LEADER_TRANSFER, "sync-leader-transfer", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SYNC_SET_MNODE_STANDBY, "set-mnode-standby", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SYNC_SET_VNODE_STANDBY, "set-vnode-standby", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
|
||||
|
||||
#if defined(TD_MSG_NUMBER_)
|
||||
TDMT_MAX
|
||||
|
|
|
@ -204,65 +204,65 @@
|
|||
#define TK_SPLIT 186
|
||||
#define TK_SYNCDB 187
|
||||
#define TK_DELETE 188
|
||||
#define TK_NULL 189
|
||||
#define TK_NK_QUESTION 190
|
||||
#define TK_NK_ARROW 191
|
||||
#define TK_ROWTS 192
|
||||
#define TK_TBNAME 193
|
||||
#define TK_QSTARTTS 194
|
||||
#define TK_QENDTS 195
|
||||
#define TK_WSTARTTS 196
|
||||
#define TK_WENDTS 197
|
||||
#define TK_WDURATION 198
|
||||
#define TK_CAST 199
|
||||
#define TK_NOW 200
|
||||
#define TK_TODAY 201
|
||||
#define TK_TIMEZONE 202
|
||||
#define TK_CLIENT_VERSION 203
|
||||
#define TK_SERVER_VERSION 204
|
||||
#define TK_SERVER_STATUS 205
|
||||
#define TK_CURRENT_USER 206
|
||||
#define TK_COUNT 207
|
||||
#define TK_LAST_ROW 208
|
||||
#define TK_BETWEEN 209
|
||||
#define TK_IS 210
|
||||
#define TK_NK_LT 211
|
||||
#define TK_NK_GT 212
|
||||
#define TK_NK_LE 213
|
||||
#define TK_NK_GE 214
|
||||
#define TK_NK_NE 215
|
||||
#define TK_MATCH 216
|
||||
#define TK_NMATCH 217
|
||||
#define TK_CONTAINS 218
|
||||
#define TK_JOIN 219
|
||||
#define TK_INNER 220
|
||||
#define TK_SELECT 221
|
||||
#define TK_DISTINCT 222
|
||||
#define TK_WHERE 223
|
||||
#define TK_PARTITION 224
|
||||
#define TK_BY 225
|
||||
#define TK_SESSION 226
|
||||
#define TK_STATE_WINDOW 227
|
||||
#define TK_SLIDING 228
|
||||
#define TK_FILL 229
|
||||
#define TK_VALUE 230
|
||||
#define TK_NONE 231
|
||||
#define TK_PREV 232
|
||||
#define TK_LINEAR 233
|
||||
#define TK_NEXT 234
|
||||
#define TK_HAVING 235
|
||||
#define TK_RANGE 236
|
||||
#define TK_EVERY 237
|
||||
#define TK_ORDER 238
|
||||
#define TK_SLIMIT 239
|
||||
#define TK_SOFFSET 240
|
||||
#define TK_LIMIT 241
|
||||
#define TK_OFFSET 242
|
||||
#define TK_ASC 243
|
||||
#define TK_NULLS 244
|
||||
#define TK_ID 245
|
||||
#define TK_NK_BITNOT 246
|
||||
#define TK_INSERT 247
|
||||
#define TK_INSERT 189
|
||||
#define TK_NULL 190
|
||||
#define TK_NK_QUESTION 191
|
||||
#define TK_NK_ARROW 192
|
||||
#define TK_ROWTS 193
|
||||
#define TK_TBNAME 194
|
||||
#define TK_QSTARTTS 195
|
||||
#define TK_QENDTS 196
|
||||
#define TK_WSTARTTS 197
|
||||
#define TK_WENDTS 198
|
||||
#define TK_WDURATION 199
|
||||
#define TK_CAST 200
|
||||
#define TK_NOW 201
|
||||
#define TK_TODAY 202
|
||||
#define TK_TIMEZONE 203
|
||||
#define TK_CLIENT_VERSION 204
|
||||
#define TK_SERVER_VERSION 205
|
||||
#define TK_SERVER_STATUS 206
|
||||
#define TK_CURRENT_USER 207
|
||||
#define TK_COUNT 208
|
||||
#define TK_LAST_ROW 209
|
||||
#define TK_BETWEEN 210
|
||||
#define TK_IS 211
|
||||
#define TK_NK_LT 212
|
||||
#define TK_NK_GT 213
|
||||
#define TK_NK_LE 214
|
||||
#define TK_NK_GE 215
|
||||
#define TK_NK_NE 216
|
||||
#define TK_MATCH 217
|
||||
#define TK_NMATCH 218
|
||||
#define TK_CONTAINS 219
|
||||
#define TK_JOIN 220
|
||||
#define TK_INNER 221
|
||||
#define TK_SELECT 222
|
||||
#define TK_DISTINCT 223
|
||||
#define TK_WHERE 224
|
||||
#define TK_PARTITION 225
|
||||
#define TK_BY 226
|
||||
#define TK_SESSION 227
|
||||
#define TK_STATE_WINDOW 228
|
||||
#define TK_SLIDING 229
|
||||
#define TK_FILL 230
|
||||
#define TK_VALUE 231
|
||||
#define TK_NONE 232
|
||||
#define TK_PREV 233
|
||||
#define TK_LINEAR 234
|
||||
#define TK_NEXT 235
|
||||
#define TK_HAVING 236
|
||||
#define TK_RANGE 237
|
||||
#define TK_EVERY 238
|
||||
#define TK_ORDER 239
|
||||
#define TK_SLIMIT 240
|
||||
#define TK_SOFFSET 241
|
||||
#define TK_LIMIT 242
|
||||
#define TK_OFFSET 243
|
||||
#define TK_ASC 244
|
||||
#define TK_NULLS 245
|
||||
#define TK_ID 246
|
||||
#define TK_NK_BITNOT 247
|
||||
#define TK_VALUES 248
|
||||
#define TK_IMPORT 249
|
||||
#define TK_NK_SEMI 250
|
||||
|
|
|
@ -208,7 +208,7 @@ int32_t doFilterTag(const SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* re
|
|||
* destory index env
|
||||
*
|
||||
*/
|
||||
void indexCleanUp();
|
||||
void indexCleanup();
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
@ -194,6 +194,7 @@ typedef enum ENodeType {
|
|||
QUERY_NODE_KILL_QUERY_STMT,
|
||||
QUERY_NODE_KILL_TRANSACTION_STMT,
|
||||
QUERY_NODE_DELETE_STMT,
|
||||
QUERY_NODE_INSERT_STMT,
|
||||
QUERY_NODE_QUERY,
|
||||
|
||||
// logic plan node
|
||||
|
@ -247,6 +248,7 @@ typedef enum ENodeType {
|
|||
QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC,
|
||||
QUERY_NODE_PHYSICAL_PLAN_DISPATCH,
|
||||
QUERY_NODE_PHYSICAL_PLAN_INSERT,
|
||||
QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT,
|
||||
QUERY_NODE_PHYSICAL_PLAN_DELETE,
|
||||
QUERY_NODE_PHYSICAL_SUBPLAN,
|
||||
QUERY_NODE_PHYSICAL_PLAN
|
||||
|
|
|
@ -82,6 +82,7 @@ typedef struct SScanLogicNode {
|
|||
typedef struct SJoinLogicNode {
|
||||
SLogicNode node;
|
||||
EJoinType joinType;
|
||||
SNode* pMergeCondition;
|
||||
SNode* pOnConditions;
|
||||
bool isSingleTableJoin;
|
||||
} SJoinLogicNode;
|
||||
|
@ -130,6 +131,7 @@ typedef struct SVnodeModifyLogicNode {
|
|||
int8_t tableType; // table type
|
||||
char tableFName[TSDB_TABLE_FNAME_LEN];
|
||||
STimeWindow deleteTimeRange;
|
||||
SVgroupsInfo* pVgroupList;
|
||||
} SVnodeModifyLogicNode;
|
||||
|
||||
typedef struct SExchangeLogicNode {
|
||||
|
@ -329,6 +331,7 @@ typedef struct SInterpFuncPhysiNode {
|
|||
typedef struct SJoinPhysiNode {
|
||||
SPhysiNode node;
|
||||
EJoinType joinType;
|
||||
SNode* pMergeCondition;
|
||||
SNode* pOnConditions;
|
||||
SNodeList* pTargets;
|
||||
} SJoinPhysiNode;
|
||||
|
@ -454,6 +457,15 @@ typedef struct SDataInserterNode {
|
|||
char* pData;
|
||||
} SDataInserterNode;
|
||||
|
||||
typedef struct SQueryInserterNode {
|
||||
SDataSinkNode sink;
|
||||
uint64_t tableId;
|
||||
int8_t tableType; // table type
|
||||
char tableFName[TSDB_TABLE_FNAME_LEN];
|
||||
int32_t vgId;
|
||||
SEpSet epSet;
|
||||
} SQueryInserterNode;
|
||||
|
||||
typedef struct SDataDeleterNode {
|
||||
SDataSinkNode sink;
|
||||
uint64_t tableId;
|
||||
|
|
|
@ -302,6 +302,14 @@ typedef struct SDeleteStmt {
|
|||
bool deleteZeroRows;
|
||||
} SDeleteStmt;
|
||||
|
||||
typedef struct SInsertStmt {
|
||||
ENodeType type; // QUERY_NODE_INSERT_STMT
|
||||
SNode* pTable;
|
||||
SNodeList* pCols;
|
||||
SNode* pQuery;
|
||||
uint8_t precision;
|
||||
} SInsertStmt;
|
||||
|
||||
typedef enum {
|
||||
PAYLOAD_TYPE_KV = 0,
|
||||
PAYLOAD_TYPE_RAW = 1,
|
||||
|
@ -376,6 +384,7 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit
|
|||
typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType;
|
||||
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
|
||||
SNodeList** pCols);
|
||||
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols);
|
||||
|
||||
typedef bool (*FFuncClassifier)(int32_t funcId);
|
||||
int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs);
|
||||
|
|
|
@ -56,7 +56,7 @@ typedef struct SParseContext {
|
|||
} SParseContext;
|
||||
|
||||
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
|
||||
bool qIsInsertSql(const char* pStr, size_t length);
|
||||
bool qIsInsertValuesSql(const char* pStr, size_t length);
|
||||
|
||||
// for async mode
|
||||
int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq);
|
||||
|
|
|
@ -29,12 +29,13 @@ extern "C" {
|
|||
|
||||
typedef enum {
|
||||
JOB_TASK_STATUS_NULL = 0,
|
||||
JOB_TASK_STATUS_NOT_START = 1,
|
||||
JOB_TASK_STATUS_EXECUTING,
|
||||
JOB_TASK_STATUS_PARTIAL_SUCCEED,
|
||||
JOB_TASK_STATUS_SUCCEED,
|
||||
JOB_TASK_STATUS_FAILED,
|
||||
JOB_TASK_STATUS_DROPPING,
|
||||
JOB_TASK_STATUS_INIT,
|
||||
JOB_TASK_STATUS_EXEC,
|
||||
JOB_TASK_STATUS_PART_SUCC,
|
||||
JOB_TASK_STATUS_SUCC,
|
||||
JOB_TASK_STATUS_FAIL,
|
||||
JOB_TASK_STATUS_DROP,
|
||||
JOB_TASK_STATUS_MAX,
|
||||
} EJobTaskType;
|
||||
|
||||
typedef enum {
|
||||
|
@ -59,10 +60,6 @@ typedef struct STableComInfo {
|
|||
int32_t rowSize; // row size of the schema
|
||||
} STableComInfo;
|
||||
|
||||
typedef struct SQueryExecRes {
|
||||
int32_t msgType;
|
||||
void* res;
|
||||
} SQueryExecRes;
|
||||
|
||||
typedef struct SIndexMeta {
|
||||
#if defined(WINDOWS) || defined(_TD_DARWIN_64)
|
||||
|
@ -71,6 +68,13 @@ typedef struct SIndexMeta {
|
|||
|
||||
} SIndexMeta;
|
||||
|
||||
typedef struct SExecResult {
|
||||
int32_t code;
|
||||
uint64_t numOfRows;
|
||||
int32_t msgType;
|
||||
void* res;
|
||||
} SExecResult;
|
||||
|
||||
typedef struct STbVerInfo {
|
||||
char tbFName[TSDB_TABLE_FNAME_LEN];
|
||||
int32_t sversion;
|
||||
|
@ -210,7 +214,7 @@ char* jobTaskStatusStr(int32_t status);
|
|||
|
||||
SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name);
|
||||
|
||||
void destroyQueryExecRes(SQueryExecRes* pRes);
|
||||
void destroyQueryExecRes(SExecResult* pRes);
|
||||
int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len);
|
||||
char* parseTagDatatoJson(void* p);
|
||||
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst);
|
||||
|
|
|
@ -53,12 +53,6 @@ typedef struct SQueryProfileSummary {
|
|||
uint64_t resultSize; // generated result size in Kb.
|
||||
} SQueryProfileSummary;
|
||||
|
||||
typedef struct SQueryResult {
|
||||
int32_t code;
|
||||
uint64_t numOfRows;
|
||||
SQueryExecRes res;
|
||||
} SQueryResult;
|
||||
|
||||
typedef struct STaskInfo {
|
||||
SQueryNodeAddr addr;
|
||||
SSubQueryMsg *msg;
|
||||
|
@ -69,50 +63,34 @@ typedef struct SSchdFetchParam {
|
|||
int32_t* code;
|
||||
} SSchdFetchParam;
|
||||
|
||||
typedef void (*schedulerExecFp)(SQueryResult* pResult, void* param, int32_t code);
|
||||
typedef void (*schedulerExecFp)(SExecResult* pResult, void* param, int32_t code);
|
||||
typedef void (*schedulerFetchFp)(void* pResult, void* param, int32_t code);
|
||||
typedef bool (*schedulerChkKillFp)(void* param);
|
||||
|
||||
typedef struct SSchedulerReq {
|
||||
bool syncReq;
|
||||
SRequestConnInfo *pConn;
|
||||
SArray *pNodeList;
|
||||
SQueryPlan *pDag;
|
||||
const char *sql;
|
||||
int64_t startTs;
|
||||
schedulerExecFp execFp;
|
||||
void* execParam;
|
||||
schedulerFetchFp fetchFp;
|
||||
void* cbParam;
|
||||
schedulerChkKillFp chkKillFp;
|
||||
void* chkKillParam;
|
||||
SExecResult* pExecRes;
|
||||
void** pFetchRes;
|
||||
} SSchedulerReq;
|
||||
|
||||
|
||||
int32_t schedulerInit(SSchedulerCfg *cfg);
|
||||
|
||||
/**
|
||||
* Process the query job, generated according to the query physical plan.
|
||||
* This is a synchronized API, and is also thread-safety.
|
||||
* @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr
|
||||
* @return
|
||||
*/
|
||||
int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes);
|
||||
int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJob);
|
||||
|
||||
/**
|
||||
* Process the query job, generated according to the query physical plan.
|
||||
* This is a asynchronized API, and is also thread-safety.
|
||||
* @param pNodeList Qnode/Vnode address list, element is SQueryNodeAddr
|
||||
* @return
|
||||
*/
|
||||
int32_t schedulerAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob);
|
||||
int32_t schedulerFetchRows(int64_t jobId, SSchedulerReq *pReq);
|
||||
|
||||
/**
|
||||
* Fetch query result from the remote query executor
|
||||
* @param pJob
|
||||
* @param data
|
||||
* @return
|
||||
*/
|
||||
int32_t schedulerFetchRows(int64_t job, void **data);
|
||||
|
||||
void schedulerAsyncFetchRows(int64_t job, schedulerFetchFp fp, void* param);
|
||||
void schedulerFetchRowsA(int64_t job, schedulerFetchFp fp, void* param);
|
||||
|
||||
int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub);
|
||||
|
||||
|
@ -134,7 +112,7 @@ void schedulerFreeJob(int64_t* job, int32_t errCode);
|
|||
|
||||
void schedulerDestroy(void);
|
||||
|
||||
void schdExecCallback(SQueryResult* pResult, void* param, int32_t code);
|
||||
void schdExecCallback(SExecResult* pResult, void* param, int32_t code);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
@ -26,7 +26,7 @@ extern "C" {
|
|||
|
||||
extern bool gRaftDetailLog;
|
||||
|
||||
#define SYNC_MAX_BATCH_SIZE 100
|
||||
#define SYNC_MAX_BATCH_SIZE 500
|
||||
#define SYNC_INDEX_BEGIN 0
|
||||
#define SYNC_INDEX_INVALID -1
|
||||
#define SYNC_TERM_INVALID 0xFFFFFFFFFFFFFFFF
|
||||
|
|
|
@ -32,7 +32,7 @@ extern "C" {
|
|||
void *taosMemoryMalloc(int32_t size);
|
||||
void *taosMemoryCalloc(int32_t num, int32_t size);
|
||||
void *taosMemoryRealloc(void *ptr, int32_t size);
|
||||
void *taosMemoryStrDup(void *ptr);
|
||||
void *taosMemoryStrDup(const char *ptr);
|
||||
void taosMemoryFree(void *ptr);
|
||||
int32_t taosMemorySize(void *ptr);
|
||||
void taosPrintBackTrace();
|
||||
|
|
|
@ -389,10 +389,10 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_QRY_TASK_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x0719)
|
||||
#define TSDB_CODE_QRY_JOB_FREED TAOS_DEF_ERROR_CODE(0, 0x071A)
|
||||
#define TSDB_CODE_QRY_TASK_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x071B)
|
||||
//json
|
||||
#define TSDB_CODE_QRY_JSON_IN_ERROR TAOS_DEF_ERROR_CODE(0, 0x071C)
|
||||
#define TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x071D)
|
||||
#define TSDB_CODE_QRY_JSON_IN_GROUP_ERROR TAOS_DEF_ERROR_CODE(0, 0x071E)
|
||||
#define TSDB_CODE_QRY_JOB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x071F)
|
||||
|
||||
// grant
|
||||
#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800)
|
||||
|
|
|
@ -156,7 +156,7 @@ typedef struct SResultColumn {
|
|||
} SResultColumn;
|
||||
|
||||
typedef struct SReqResultInfo {
|
||||
SQueryExecRes execRes;
|
||||
SExecResult execRes;
|
||||
const char* pRspMsg;
|
||||
const char* pData;
|
||||
TAOS_FIELD* fields; // todo, column names are not needed.
|
||||
|
|
|
@ -279,7 +279,6 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
|
|||
}
|
||||
|
||||
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
|
||||
// pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows);
|
||||
}
|
||||
|
||||
int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
|
||||
|
@ -628,22 +627,26 @@ _return:
|
|||
int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
|
||||
void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter;
|
||||
|
||||
SQueryResult res = {0};
|
||||
SExecResult res = {0};
|
||||
SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
|
||||
.requestId = pRequest->requestId,
|
||||
.requestObjRefId = pRequest->self};
|
||||
SSchedulerReq req = {.pConn = &conn,
|
||||
.pNodeList = pNodeList,
|
||||
.pDag = pDag,
|
||||
.sql = pRequest->sqlstr,
|
||||
.startTs = pRequest->metric.start,
|
||||
.execFp = NULL,
|
||||
.execParam = NULL,
|
||||
.chkKillFp = chkRequestKilled,
|
||||
.chkKillParam = (void*)pRequest->self};
|
||||
SSchedulerReq req = {
|
||||
.syncReq = true,
|
||||
.pConn = &conn,
|
||||
.pNodeList = pNodeList,
|
||||
.pDag = pDag,
|
||||
.sql = pRequest->sqlstr,
|
||||
.startTs = pRequest->metric.start,
|
||||
.execFp = NULL,
|
||||
.cbParam = NULL,
|
||||
.chkKillFp = chkRequestKilled,
|
||||
.chkKillParam = (void*)pRequest->self,
|
||||
.pExecRes = &res,
|
||||
};
|
||||
|
||||
int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res);
|
||||
pRequest->body.resInfo.execRes = res.res;
|
||||
int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob);
|
||||
memcpy(&pRequest->body.resInfo.execRes, &res, sizeof(res));
|
||||
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
schedulerFreeJob(&pRequest->body.queryJob, 0);
|
||||
|
@ -754,7 +757,7 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
|
|||
}
|
||||
|
||||
SEpSet epset = getEpSet_s(&pAppInfo->mgmtEp);
|
||||
SQueryExecRes* pRes = &pRequest->body.resInfo.execRes;
|
||||
SExecResult* pRes = &pRequest->body.resInfo.execRes;
|
||||
|
||||
switch (pRes->msgType) {
|
||||
case TDMT_VND_ALTER_TABLE:
|
||||
|
@ -780,10 +783,10 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
|
|||
return code;
|
||||
}
|
||||
|
||||
void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
|
||||
void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
|
||||
SRequestObj* pRequest = (SRequestObj*)param;
|
||||
pRequest->code = code;
|
||||
pRequest->body.resInfo.execRes = pResult->res;
|
||||
memcpy(&pRequest->body.resInfo.execRes, pResult, sizeof(*pResult));
|
||||
|
||||
if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type ||
|
||||
TDMT_VND_CREATE_TABLE == pRequest->type) {
|
||||
|
@ -940,16 +943,20 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
|
|||
|
||||
SRequestConnInfo conn = {
|
||||
.pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
|
||||
SSchedulerReq req = {.pConn = &conn,
|
||||
.pNodeList = pNodeList,
|
||||
.pDag = pDag,
|
||||
.sql = pRequest->sqlstr,
|
||||
.startTs = pRequest->metric.start,
|
||||
.execFp = schedulerExecCb,
|
||||
.execParam = pRequest,
|
||||
.chkKillFp = chkRequestKilled,
|
||||
.chkKillParam = (void*)pRequest->self};
|
||||
code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob);
|
||||
SSchedulerReq req = {
|
||||
.syncReq = false,
|
||||
.pConn = &conn,
|
||||
.pNodeList = pNodeList,
|
||||
.pDag = pDag,
|
||||
.sql = pRequest->sqlstr,
|
||||
.startTs = pRequest->metric.start,
|
||||
.execFp = schedulerExecCb,
|
||||
.cbParam = pRequest,
|
||||
.chkKillFp = chkRequestKilled,
|
||||
.chkKillParam = (void*)pRequest->self,
|
||||
.pExecRes = NULL,
|
||||
};
|
||||
code = schedulerExecJob(&req, &pRequest->body.queryJob);
|
||||
taosArrayDestroy(pNodeList);
|
||||
} else {
|
||||
tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
|
||||
|
@ -1388,7 +1395,11 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
|
|||
}
|
||||
|
||||
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
|
||||
pRequest->code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData);
|
||||
SSchedulerReq req = {
|
||||
.syncReq = true,
|
||||
.pFetchRes = (void**)&pResInfo->pData,
|
||||
};
|
||||
pRequest->code = schedulerFetchRows(pRequest->body.queryJob, &req);
|
||||
if (pRequest->code != TSDB_CODE_SUCCESS) {
|
||||
pResultInfo->numOfRows = 0;
|
||||
return NULL;
|
||||
|
|
|
@ -665,8 +665,6 @@ static void destorySqlParseWrapper(SqlParseWrapper *pWrapper) {
|
|||
}
|
||||
|
||||
void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
|
||||
tscDebug("enter meta callback, code %s", tstrerror(code));
|
||||
|
||||
SqlParseWrapper *pWrapper = (SqlParseWrapper *)param;
|
||||
SQuery * pQuery = pWrapper->pQuery;
|
||||
SRequestObj * pRequest = pWrapper->pRequest;
|
||||
|
@ -686,10 +684,11 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
|
|||
TSWAP(pRequest->tableList, (pQuery)->pTableList);
|
||||
|
||||
destorySqlParseWrapper(pWrapper);
|
||||
|
||||
tscDebug("0x%"PRIx64" analysis semantics completed, start async query, reqId:0x%"PRIx64, pRequest->self, pRequest->requestId);
|
||||
launchAsyncQuery(pRequest, pQuery, pResultMeta);
|
||||
} else {
|
||||
destorySqlParseWrapper(pWrapper);
|
||||
tscDebug("error happens, code:%d", code);
|
||||
if (NEED_CLIENT_HANDLE_ERROR(code)) {
|
||||
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64,
|
||||
pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId);
|
||||
|
@ -859,7 +858,12 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
|
|||
}
|
||||
}
|
||||
|
||||
schedulerAsyncFetchRows(pRequest->body.queryJob, fetchCallback, pRequest);
|
||||
SSchedulerReq req = {
|
||||
.syncReq = false,
|
||||
.fetchFp = fetchCallback,
|
||||
.cbParam = pRequest,
|
||||
};
|
||||
schedulerFetchRows(pRequest->body.queryJob, &req);
|
||||
}
|
||||
|
||||
void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
|
||||
|
|
|
@ -266,7 +266,7 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
|||
}
|
||||
|
||||
if (pRequest->body.queryFp != NULL) {
|
||||
SQueryExecRes* pRes = &pRequest->body.resInfo.execRes;
|
||||
SExecResult* pRes = &pRequest->body.resInfo.execRes;
|
||||
|
||||
if (code == TSDB_CODE_SUCCESS) {
|
||||
SCatalog* pCatalog = NULL;
|
||||
|
|
|
@ -324,9 +324,9 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
|
|||
}
|
||||
|
||||
int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks* pDataBlock, STableDataBlocks** newBlock, uint64_t uid) {
|
||||
SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
|
||||
SVgroupInfo vgInfo = {0};
|
||||
SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter,
|
||||
SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
|
||||
SVgroupInfo vgInfo = {0};
|
||||
SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter,
|
||||
.requestId = pStmt->exec.pRequest->requestId,
|
||||
.requestObjRefId = pStmt->exec.pRequest->self,
|
||||
.mgmtEps = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp)};
|
||||
|
@ -391,13 +391,12 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
|
|||
STMT_RET(stmtCleanBindInfo(pStmt));
|
||||
}
|
||||
|
||||
STableMeta* pTableMeta = NULL;
|
||||
SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter,
|
||||
STableMeta* pTableMeta = NULL;
|
||||
SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter,
|
||||
.requestId = pStmt->exec.pRequest->requestId,
|
||||
.requestObjRefId = pStmt->exec.pRequest->self,
|
||||
.mgmtEps = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp)};
|
||||
int32_t code =
|
||||
catalogGetTableMeta(pStmt->pCatalog, &conn, &pStmt->bInfo.sname, &pTableMeta);
|
||||
int32_t code = catalogGetTableMeta(pStmt->pCatalog, &conn, &pStmt->bInfo.sname, &pTableMeta);
|
||||
if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) {
|
||||
STMT_ERR_RET(stmtCleanBindInfo(pStmt));
|
||||
|
||||
|
@ -849,7 +848,7 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) {
|
|||
if (pStmt->sql.type) {
|
||||
*insert = (STMT_TYPE_INSERT == pStmt->sql.type || STMT_TYPE_MULTI_INSERT == pStmt->sql.type);
|
||||
} else {
|
||||
*insert = qIsInsertSql(pStmt->sql.sqlStr, 0);
|
||||
*insert = qIsInsertValuesSql(pStmt->sql.sqlStr, pStmt->sql.sqlLen);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
@ -861,7 +860,7 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
|
|||
if (STMT_TYPE_QUERY == pStmt->sql.type) {
|
||||
STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
|
||||
}
|
||||
|
||||
|
||||
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
|
||||
|
||||
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
|
||||
|
@ -893,7 +892,7 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
|
|||
if (STMT_TYPE_QUERY == pStmt->sql.type) {
|
||||
STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
|
||||
}
|
||||
|
||||
|
||||
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
|
||||
|
||||
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
|
||||
|
@ -919,7 +918,6 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
|
@ -952,13 +950,13 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int stmtGetParam(TAOS_STMT *stmt, int idx, int *type, int *bytes) {
|
||||
int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
|
||||
STscStmt* pStmt = (STscStmt*)stmt;
|
||||
|
||||
if (STMT_TYPE_QUERY == pStmt->sql.type) {
|
||||
STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
|
||||
}
|
||||
|
||||
|
||||
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
|
||||
|
||||
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
|
||||
|
@ -979,8 +977,8 @@ int stmtGetParam(TAOS_STMT *stmt, int idx, int *type, int *bytes) {
|
|||
STMT_ERR_RET(stmtParseSql(pStmt));
|
||||
}
|
||||
|
||||
int32_t nums = 0;
|
||||
TAOS_FIELD_E *pField = NULL;
|
||||
int32_t nums = 0;
|
||||
TAOS_FIELD_E* pField = NULL;
|
||||
STMT_ERR_RET(stmtFetchColFields(stmt, &nums, &pField));
|
||||
if (idx >= nums) {
|
||||
tscError("idx %d is too big", idx);
|
||||
|
|
|
@ -32,8 +32,7 @@ SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) {
|
|||
if (pVnode && num < size) {
|
||||
int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
|
||||
// dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
|
||||
pVnodes[num] = (*ppVnode);
|
||||
num++;
|
||||
pVnodes[num++] = (*ppVnode);
|
||||
pIter = taosHashIterate(pMgmt->hash, pIter);
|
||||
} else {
|
||||
taosHashCancelIterate(pMgmt->hash, pIter);
|
||||
|
|
|
@ -88,7 +88,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
|
|||
while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
|
||||
while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
|
||||
while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
|
||||
dTrace("vgId:%d, vnode-fetch queue is empty", pVnode->vgId);
|
||||
dTrace("vgId:%d, vnode queue is empty", pVnode->vgId);
|
||||
|
||||
vmFreeQueue(pMgmt, pVnode);
|
||||
vnodeClose(pVnode->pImpl);
|
||||
|
@ -140,7 +140,7 @@ static void *vmOpenVnodeInThread(void *param) {
|
|||
}
|
||||
|
||||
static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
|
||||
pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
|
||||
pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
|
||||
if (pMgmt->hash == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
dError("failed to init vnode hash since %s", terrstr());
|
||||
|
@ -156,7 +156,8 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
|
|||
|
||||
pMgmt->state.totalVnodes = numOfVnodes;
|
||||
|
||||
int32_t threadNum = 1;
|
||||
int32_t threadNum = tsNumOfCores / 2;
|
||||
if (threadNum < 1) threadNum = 1;
|
||||
int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
|
||||
|
||||
SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread));
|
||||
|
|
|
@ -107,35 +107,13 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
|
|||
const STraceId *trace = &pMsg->info.traceId;
|
||||
dGTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg);
|
||||
|
||||
int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); // no response here
|
||||
int32_t code = vnodeProcessSyncMsg(pVnode->pImpl, pMsg, NULL); // no response here
|
||||
dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
}
|
||||
}
|
||||
|
||||
static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
||||
SVnodeObj *pVnode = pInfo->ahandle;
|
||||
SRpcMsg *pMsg = NULL;
|
||||
|
||||
for (int32_t i = 0; i < numOfMsgs; ++i) {
|
||||
if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
dGTrace("vgId:%d, msg:%p get from vnode-merge queue", pVnode->vgId, pMsg);
|
||||
|
||||
int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
|
||||
if (code != 0) {
|
||||
if (terrno != 0) code = terrno;
|
||||
dGError("vgId:%d, msg:%p failed to merge since %s", pVnode->vgId, pMsg, terrstr());
|
||||
vmSendRsp(pMsg, code);
|
||||
}
|
||||
|
||||
dGTrace("msg:%p, is freed, code:0x%x", pMsg, code);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) {
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
SMsgHead *pHead = pMsg->pCont;
|
||||
|
@ -207,7 +185,11 @@ int32_t vmPutMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
|
||||
int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
|
||||
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
|
||||
if (pMsg == NULL) return -1;
|
||||
if (pMsg == NULL) {
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
pRpc->pCont = NULL;
|
||||
return -1;
|
||||
}
|
||||
|
||||
SMsgHead *pHead = pRpc->pCont;
|
||||
dTrace("vgId:%d, msg:%p is created, type:%s", pHead->vgId, pMsg, TMSG_INFO(pRpc->msgType));
|
||||
|
@ -215,7 +197,16 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
|
|||
pHead->contLen = htonl(pHead->contLen);
|
||||
pHead->vgId = htonl(pHead->vgId);
|
||||
memcpy(pMsg, pRpc, sizeof(SRpcMsg));
|
||||
return vmPutMsgToQueue(pMgmt, pMsg, qtype);
|
||||
|
||||
int32_t code = vmPutMsgToQueue(pMgmt, pMsg, qtype);
|
||||
if (code != 0) {
|
||||
dTrace("msg:%p, is freed", pMsg);
|
||||
taosFreeQitem(pMsg);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
pRpc->pCont = NULL;
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "dmMgmt.h"
|
||||
#include "dmNodes.h"
|
||||
#include "index.h"
|
||||
#include "qworker.h"
|
||||
|
||||
static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
|
||||
|
@ -212,6 +213,8 @@ void dmCleanupDnode(SDnode *pDnode) {
|
|||
dmCleanupClient(pDnode);
|
||||
dmCleanupServer(pDnode);
|
||||
dmClearVars(pDnode);
|
||||
rpcCleanup();
|
||||
indexCleanup();
|
||||
dDebug("dnode is closed, ptr:%p", pDnode);
|
||||
}
|
||||
|
||||
|
|
|
@ -71,9 +71,9 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
|
||||
SDnodeTrans * pTrans = &pDnode->trans;
|
||||
SDnodeTrans *pTrans = &pDnode->trans;
|
||||
int32_t code = -1;
|
||||
SRpcMsg * pMsg = NULL;
|
||||
SRpcMsg *pMsg = NULL;
|
||||
SMgmtWrapper *pWrapper = NULL;
|
||||
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)];
|
||||
|
||||
|
@ -185,6 +185,7 @@ _OVER:
|
|||
taosFreeQitem(pMsg);
|
||||
}
|
||||
rpcFreeCont(pRpc->pCont);
|
||||
pRpc->pCont = NULL;
|
||||
}
|
||||
|
||||
dmReleaseWrapper(pWrapper);
|
||||
|
@ -195,11 +196,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
|
|||
|
||||
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
|
||||
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
|
||||
SArray * pArray = (*pWrapper->func.getHandlesFp)();
|
||||
SArray *pArray = (*pWrapper->func.getHandlesFp)();
|
||||
if (pArray == NULL) return -1;
|
||||
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
|
||||
SMgmtHandle * pMgmt = taosArrayGet(pArray, i);
|
||||
SMgmtHandle *pMgmt = taosArrayGet(pArray, i);
|
||||
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
|
||||
if (pMgmt->needCheckVgId) {
|
||||
pHandle->needCheckVgId = pMgmt->needCheckVgId;
|
||||
|
|
|
@ -1373,9 +1373,9 @@ char *buildRetension(SArray *pRetension) {
|
|||
static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables,
|
||||
bool sysDb, ESdbStatus objStatus, bool sysinfo) {
|
||||
int32_t cols = 0;
|
||||
int32_t bytes = pShow->pMeta->pSchemas[cols].bytes;
|
||||
char *buf = taosMemoryMalloc(bytes);
|
||||
|
||||
int32_t bytes = pShow->pMeta->pSchemas[cols].bytes;
|
||||
char *buf = taosMemoryMalloc(bytes);
|
||||
const char *name = mndGetDbStr(pDb->name);
|
||||
if (name != NULL) {
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(buf, name, bytes);
|
||||
|
@ -1383,11 +1383,11 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
|
|||
STR_WITH_MAXSIZE_TO_VARSTR(buf, "NULL", bytes);
|
||||
}
|
||||
|
||||
char *status = "ready";
|
||||
if (objStatus == SDB_STATUS_CREATING) status = "creating";
|
||||
if (objStatus == SDB_STATUS_DROPPING) status = "dropping";
|
||||
char statusB[24] = {0};
|
||||
STR_WITH_SIZE_TO_VARSTR(statusB, status, strlen(status));
|
||||
char *statusStr = "ready";
|
||||
if (objStatus == SDB_STATUS_CREATING) statusStr = "creating";
|
||||
if (objStatus == SDB_STATUS_DROPPING) statusStr = "dropping";
|
||||
char statusVstr[24] = {0};
|
||||
STR_WITH_SIZE_TO_VARSTR(statusVstr, statusStr, strlen(statusStr));
|
||||
|
||||
if (sysDb || !sysinfo) {
|
||||
for (int32_t i = 0; i < pShow->numOfColumns; ++i) {
|
||||
|
@ -1397,7 +1397,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
|
|||
} else if (i == 3) {
|
||||
colDataAppend(pColInfo, rows, (const char *)&numOfTables, false);
|
||||
} else if (i == 20) {
|
||||
colDataAppend(pColInfo, rows, statusB, false);
|
||||
colDataAppend(pColInfo, rows, statusVstr, false);
|
||||
} else {
|
||||
colDataAppendNULL(pColInfo, rows);
|
||||
}
|
||||
|
@ -1405,7 +1405,6 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
|
|||
} else {
|
||||
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, rows, buf, false);
|
||||
taosMemoryFree(buf);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, rows, (const char *)&pDb->createdTime, false);
|
||||
|
@ -1419,30 +1418,29 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
|
|||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false);
|
||||
|
||||
const char *src = pDb->cfg.strict ? "strict" : "no_strict";
|
||||
char strict[24] = {0};
|
||||
STR_WITH_SIZE_TO_VARSTR(strict, src, strlen(src));
|
||||
const char *strictStr = pDb->cfg.strict ? "strict" : "no_strict";
|
||||
char strictVstr[24] = {0};
|
||||
STR_WITH_SIZE_TO_VARSTR(strictVstr, strictStr, strlen(strictStr));
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, rows, (const char *)strict, false);
|
||||
colDataAppend(pColInfo, rows, (const char *)strictVstr, false);
|
||||
|
||||
char tmp[128] = {0};
|
||||
int32_t len = 0;
|
||||
len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%dm", pDb->cfg.daysPerFile);
|
||||
varDataSetLen(tmp, len);
|
||||
char durationVstr[128] = {0};
|
||||
int32_t len = sprintf(&durationVstr[VARSTR_HEADER_SIZE], "%dm", pDb->cfg.daysPerFile);
|
||||
varDataSetLen(durationVstr, len);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, rows, (const char *)tmp, false);
|
||||
colDataAppend(pColInfo, rows, (const char *)durationVstr, false);
|
||||
|
||||
char keepVstr[128] = {0};
|
||||
if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) {
|
||||
len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%dm,%dm,%dm", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2,
|
||||
len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%dm,%dm,%dm", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2,
|
||||
pDb->cfg.daysToKeep0);
|
||||
} else {
|
||||
len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%dm,%dm,%dm", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1,
|
||||
len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%dm,%dm,%dm", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1,
|
||||
pDb->cfg.daysToKeep2);
|
||||
}
|
||||
|
||||
varDataSetLen(tmp, len);
|
||||
varDataSetLen(keepVstr, len);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, rows, (const char *)tmp, false);
|
||||
colDataAppend(pColInfo, rows, (const char *)keepVstr, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.buffer, false);
|
||||
|
@ -1469,68 +1467,49 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
|
|||
colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
|
||||
STR_WITH_SIZE_TO_VARSTR(strict, src, strlen(src));
|
||||
#if 0
|
||||
char cacheModel[24] = {0};
|
||||
bool null = false;
|
||||
if (pDb->cfg.cacheLastRow == 0) {
|
||||
STR_TO_VARSTR(cacheModel, "no_cache");
|
||||
} else if (pDb->cfg.cacheLastRow == 1) {
|
||||
STR_TO_VARSTR(cacheModel, "last_row_cache")
|
||||
} else {
|
||||
null = true;
|
||||
}
|
||||
colDataAppend(pColInfo, rows, cacheModel, null);
|
||||
#endif
|
||||
colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastRow, false);
|
||||
|
||||
char *prec = NULL;
|
||||
const char *precStr = NULL;
|
||||
switch (pDb->cfg.precision) {
|
||||
case TSDB_TIME_PRECISION_MILLI:
|
||||
prec = TSDB_TIME_PRECISION_MILLI_STR;
|
||||
precStr = TSDB_TIME_PRECISION_MILLI_STR;
|
||||
break;
|
||||
case TSDB_TIME_PRECISION_MICRO:
|
||||
prec = TSDB_TIME_PRECISION_MICRO_STR;
|
||||
precStr = TSDB_TIME_PRECISION_MICRO_STR;
|
||||
break;
|
||||
case TSDB_TIME_PRECISION_NANO:
|
||||
prec = TSDB_TIME_PRECISION_NANO_STR;
|
||||
precStr = TSDB_TIME_PRECISION_NANO_STR;
|
||||
break;
|
||||
default:
|
||||
prec = "none";
|
||||
precStr = "none";
|
||||
break;
|
||||
}
|
||||
|
||||
char t[10] = {0};
|
||||
STR_WITH_SIZE_TO_VARSTR(t, prec, 2);
|
||||
char precVstr[10] = {0};
|
||||
STR_WITH_SIZE_TO_VARSTR(precVstr, precStr, 2);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, rows, (const char *)t, false);
|
||||
colDataAppend(pColInfo, rows, (const char *)precVstr, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, rows, (const char *)statusB, false);
|
||||
|
||||
// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
// colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false);
|
||||
|
||||
char *p = buildRetension(pDb->cfg.pRetensions);
|
||||
colDataAppend(pColInfo, rows, (const char *)statusVstr, false);
|
||||
|
||||
char *rentensionVstr = buildRetension(pDb->cfg.pRetensions);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
|
||||
if (p == NULL) {
|
||||
if (rentensionVstr == NULL) {
|
||||
colDataAppendNULL(pColInfo, rows);
|
||||
} else {
|
||||
colDataAppend(pColInfo, rows, (const char *)p, false);
|
||||
taosMemoryFree(p);
|
||||
colDataAppend(pColInfo, rows, (const char *)rentensionVstr, false);
|
||||
taosMemoryFree(rentensionVstr);
|
||||
}
|
||||
}
|
||||
|
||||
taosMemoryFree(buf);
|
||||
}
|
||||
|
||||
static void setInformationSchemaDbCfg(SDbObj *pDbObj) {
|
||||
ASSERT(pDbObj != NULL);
|
||||
strncpy(pDbObj->name, TSDB_INFORMATION_SCHEMA_DB, tListLen(pDbObj->name));
|
||||
|
||||
tstrncpy(pDbObj->name, TSDB_INFORMATION_SCHEMA_DB, tListLen(pDbObj->name));
|
||||
pDbObj->createdTime = 0;
|
||||
pDbObj->cfg.numOfVgroups = 0;
|
||||
pDbObj->cfg.strict = 1;
|
||||
|
@ -1539,9 +1518,7 @@ static void setInformationSchemaDbCfg(SDbObj *pDbObj) {
|
|||
}
|
||||
|
||||
static void setPerfSchemaDbCfg(SDbObj *pDbObj) {
|
||||
ASSERT(pDbObj != NULL);
|
||||
strncpy(pDbObj->name, TSDB_PERFORMANCE_SCHEMA_DB, tListLen(pDbObj->name));
|
||||
|
||||
tstrncpy(pDbObj->name, TSDB_PERFORMANCE_SCHEMA_DB, tListLen(pDbObj->name));
|
||||
pDbObj->createdTime = 0;
|
||||
pDbObj->cfg.numOfVgroups = 0;
|
||||
pDbObj->cfg.strict = 1;
|
||||
|
@ -1585,14 +1562,11 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
|
|||
|
||||
while (numOfRows < rowsCapacity) {
|
||||
pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus);
|
||||
if (pShow->pIter == NULL) {
|
||||
break;
|
||||
}
|
||||
if (pShow->pIter == NULL) break;
|
||||
|
||||
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) {
|
||||
int32_t numOfTables = 0;
|
||||
sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL);
|
||||
|
||||
dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus, sysinfo);
|
||||
numOfRows++;
|
||||
}
|
||||
|
|
|
@ -274,15 +274,14 @@ static void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps) {
|
|||
|
||||
SDnodeEp dnodeEp = {0};
|
||||
dnodeEp.id = pDnode->id;
|
||||
dnodeEp.isMnode = 0;
|
||||
dnodeEp.ep.port = pDnode->port;
|
||||
memcpy(dnodeEp.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
|
||||
sdbRelease(pSdb, pDnode);
|
||||
|
||||
dnodeEp.isMnode = 0;
|
||||
if (mndIsMnode(pMnode, pDnode->id)) {
|
||||
dnodeEp.isMnode = 1;
|
||||
}
|
||||
|
||||
sdbRelease(pSdb, pDnode);
|
||||
taosArrayPush(pDnodeEps, &dnodeEp);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -519,6 +519,8 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
|
|||
}
|
||||
}
|
||||
|
||||
syncNodeRelease(pSyncNode);
|
||||
|
||||
if (code != 0) {
|
||||
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
|
||||
}
|
||||
|
|
|
@ -739,9 +739,12 @@ int64_t mndGetVgroupMemory(SMnode *pMnode, SDbObj *pDbInput, SVgObj *pVgroup) {
|
|||
pDb = mndAcquireDb(pMnode, pVgroup->dbName);
|
||||
}
|
||||
|
||||
int64_t vgroupMemroy = (int64_t)pDb->cfg.buffer * 1024 * 1024 + (int64_t)pDb->cfg.pages * pDb->cfg.pageSize * 1024;
|
||||
if (pDb->cfg.cacheLastRow > 0) {
|
||||
vgroupMemroy += (int64_t)pDb->cfg.lastRowMem * 1024 * 1024;
|
||||
int64_t vgroupMemroy = 0;
|
||||
if (pDb != NULL) {
|
||||
vgroupMemroy = (int64_t)pDb->cfg.buffer * 1024 * 1024 + (int64_t)pDb->cfg.pages * pDb->cfg.pageSize * 1024;
|
||||
if (pDb->cfg.cacheLastRow > 0) {
|
||||
vgroupMemroy += (int64_t)pDb->cfg.lastRowMem * 1024 * 1024;
|
||||
}
|
||||
}
|
||||
|
||||
if (pDbInput == NULL) {
|
||||
|
|
|
@ -131,7 +131,7 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) {
|
|||
hashType = TSDB_DATA_TYPE_BINARY;
|
||||
}
|
||||
|
||||
SHashObj *hash = taosHashInit(64, taosGetDefaultHashFunction(hashType), true, HASH_NO_LOCK);
|
||||
SHashObj *hash = taosHashInit(64, taosGetDefaultHashFunction(hashType), true, HASH_ENTRY_LOCK);
|
||||
if (hash == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
|
|
|
@ -52,10 +52,10 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs);
|
|||
void vnodeDestroy(const char *path, STfs *pTfs);
|
||||
SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb);
|
||||
void vnodeClose(SVnode *pVnode);
|
||||
int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg);
|
||||
int32_t vnodePreProcessReq(SVnode *pVnode, SRpcMsg *pMsg);
|
||||
int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp);
|
||||
int32_t vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
|
||||
int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
|
||||
int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
|
||||
int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg);
|
||||
int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg);
|
||||
int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo);
|
||||
|
|
|
@ -240,7 +240,7 @@ struct SVnode {
|
|||
SSink* pSink;
|
||||
tsem_t canCommit;
|
||||
int64_t sync;
|
||||
int32_t syncCount;
|
||||
int32_t blockCount;
|
||||
tsem_t syncSem;
|
||||
SQHandle* pQuery;
|
||||
};
|
||||
|
|
|
@ -85,6 +85,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) {
|
|||
void tqOffsetClose(STqOffsetStore* pStore) {
|
||||
tqOffsetSnapshot(pStore);
|
||||
taosHashCleanup(pStore->pHash);
|
||||
taosMemoryFree(pStore);
|
||||
}
|
||||
|
||||
STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey) {
|
||||
|
|
|
@ -79,7 +79,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
|
|||
SJson *pNodeRetentions = tjsonCreateArray();
|
||||
tjsonAddItemToObject(pJson, "retentions", pNodeRetentions);
|
||||
for (int32_t i = 0; i < nRetention; ++i) {
|
||||
SJson *pNodeRetention = tjsonCreateObject();
|
||||
SJson * pNodeRetention = tjsonCreateObject();
|
||||
const SRetention *pRetention = pCfg->tsdbCfg.retentions + i;
|
||||
tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq);
|
||||
tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit);
|
||||
|
@ -118,45 +118,45 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
|
|||
|
||||
int32_t code;
|
||||
tjsonGetNumberValue(pJson, "vgId", pCfg->vgId, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
if (tjsonGetStringValue(pJson, "dbname", pCfg->dbname) < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "dbId", pCfg->dbId, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "szPage", pCfg->szPage, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "szCache", pCfg->szCache, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "isTsma", pCfg->isTsma, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "isRsma", pCfg->isRsma, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2, code);
|
||||
if(code < 0) return -1;
|
||||
SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
|
||||
if (code < 0) return -1;
|
||||
SJson * pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
|
||||
int32_t nRetention = tjsonGetArraySize(pNodeRetentions);
|
||||
if (nRetention > TSDB_RETENTION_MAX) {
|
||||
nRetention = TSDB_RETENTION_MAX;
|
||||
|
@ -170,30 +170,30 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
|
|||
tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code);
|
||||
}
|
||||
tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
|
||||
tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
|
||||
SJson *pNodeInfoArr = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
|
||||
int arraySize = tjsonGetArraySize(pNodeInfoArr);
|
||||
|
|
|
@ -81,7 +81,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
|
|||
pVnode->state.applied = info.state.committed;
|
||||
pVnode->pTfs = pTfs;
|
||||
pVnode->msgCb = msgCb;
|
||||
pVnode->syncCount = 0;
|
||||
pVnode->blockCount = 0;
|
||||
|
||||
tsem_init(&pVnode->syncSem, 0, 0);
|
||||
tsem_init(&(pVnode->canCommit), 0, 1);
|
||||
|
|
|
@ -27,10 +27,10 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
|
|||
SMetaReader mer1 = {0};
|
||||
SMetaReader mer2 = {0};
|
||||
char tableFName[TSDB_TABLE_FNAME_LEN];
|
||||
SRpcMsg rpcMsg;
|
||||
SRpcMsg rpcMsg = {0};
|
||||
int32_t code = 0;
|
||||
int32_t rspLen = 0;
|
||||
void *pRsp = NULL;
|
||||
void * pRsp = NULL;
|
||||
SSchemaWrapper schema = {0};
|
||||
SSchemaWrapper schemaTag = {0};
|
||||
|
||||
|
@ -111,6 +111,7 @@ _exit:
|
|||
rpcMsg.pCont = pRsp;
|
||||
rpcMsg.contLen = rspLen;
|
||||
rpcMsg.code = code;
|
||||
rpcMsg.msgType = pMsg->msgType;
|
||||
|
||||
if (code) {
|
||||
qError("get table %s meta failed cause of %s", infoReq.tbName, tstrerror(code));
|
||||
|
@ -130,10 +131,10 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg) {
|
|||
SMetaReader mer1 = {0};
|
||||
SMetaReader mer2 = {0};
|
||||
char tableFName[TSDB_TABLE_FNAME_LEN];
|
||||
SRpcMsg rpcMsg;
|
||||
SRpcMsg rpcMsg = {0};
|
||||
int32_t code = 0;
|
||||
int32_t rspLen = 0;
|
||||
void *pRsp = NULL;
|
||||
void * pRsp = NULL;
|
||||
SSchemaWrapper schema = {0};
|
||||
SSchemaWrapper schemaTag = {0};
|
||||
|
||||
|
@ -220,6 +221,7 @@ _exit:
|
|||
rpcMsg.pCont = pRsp;
|
||||
rpcMsg.contLen = rspLen;
|
||||
rpcMsg.code = code;
|
||||
rpcMsg.msgType = pMsg->msgType;
|
||||
|
||||
if (code) {
|
||||
qError("get table %s cfg failed cause of %s", cfgReq.tbName, tstrerror(code));
|
||||
|
@ -260,7 +262,7 @@ void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) {
|
|||
}
|
||||
|
||||
// wrapper of tsdb read interface
|
||||
tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo* tableList, uint64_t qId,
|
||||
tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *tableList, uint64_t qId,
|
||||
void *pMemRef) {
|
||||
#if 0
|
||||
return tsdbQueryCacheLastT(pVnode->pTsdb, pCond, groupList, qId, pMemRef);
|
||||
|
|
|
@ -28,7 +28,7 @@ static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, vo
|
|||
static int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRpcMsg *pRsp);
|
||||
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
|
||||
|
||||
int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) {
|
||||
int32_t vnodePreProcessReq(SVnode *pVnode, SRpcMsg *pMsg) {
|
||||
int32_t code = 0;
|
||||
SDecoder dc = {0};
|
||||
|
||||
|
|
|
@ -25,12 +25,12 @@ static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; }
|
|||
static inline void vnodeAccumBlockMsg(SVnode *pVnode, tmsg_t type) {
|
||||
if (!vnodeIsMsgBlock(type)) return;
|
||||
|
||||
int32_t count = atomic_add_fetch_32(&pVnode->syncCount, 1);
|
||||
int32_t count = atomic_add_fetch_32(&pVnode->blockCount, 1);
|
||||
vTrace("vgId:%d, accum block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
|
||||
}
|
||||
|
||||
static inline void vnodeWaitBlockMsg(SVnode *pVnode) {
|
||||
int32_t count = atomic_load_32(&pVnode->syncCount);
|
||||
int32_t count = atomic_load_32(&pVnode->blockCount);
|
||||
if (count <= 0) return;
|
||||
|
||||
vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count);
|
||||
|
@ -40,10 +40,10 @@ static inline void vnodeWaitBlockMsg(SVnode *pVnode) {
|
|||
static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) {
|
||||
if (!vnodeIsMsgBlock(type)) return;
|
||||
|
||||
int32_t count = atomic_load_32(&pVnode->syncCount);
|
||||
int32_t count = atomic_load_32(&pVnode->blockCount);
|
||||
if (count <= 0) return;
|
||||
|
||||
count = atomic_sub_fetch_32(&pVnode->syncCount, 1);
|
||||
count = atomic_sub_fetch_32(&pVnode->blockCount, 1);
|
||||
vTrace("vgId:%d, post block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
|
||||
if (count <= 0) {
|
||||
tsem_post(&pVnode->syncSem);
|
||||
|
@ -84,8 +84,10 @@ static int32_t vnodeProcessAlterReplicaReq(SVnode *pVnode, SRpcMsg *pMsg) {
|
|||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
return TSDB_CODE_INVALID_MSG;
|
||||
}
|
||||
STraceId *trace = &pMsg->info.traceId;
|
||||
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
vGTrace("vgId:%d, start to alter vnode replica to %d, handle:%p", TD_VID(pVnode), req.replica, pMsg->info.handle);
|
||||
|
||||
SSyncCfg cfg = {.replicaNum = req.replica, .myIndex = req.selfIndex};
|
||||
for (int32_t r = 0; r < req.replica; ++r) {
|
||||
SNodeInfo *pNode = &cfg.nodeInfo[r];
|
||||
|
@ -126,32 +128,23 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
|||
|
||||
for (int32_t m = 0; m < numOfMsgs; m++) {
|
||||
if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
|
||||
STraceId *trace = &pMsg->info.traceId;
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
vGTrace("vgId:%d, msg:%p get from vnode-write queue handle:%p", vgId, pMsg, pMsg->info.handle);
|
||||
|
||||
if (pMsg->msgType == TDMT_VND_ALTER_REPLICA) {
|
||||
code = vnodeProcessAlterReplicaReq(pVnode, pMsg);
|
||||
code = vnodePreProcessReq(pVnode, pMsg);
|
||||
if (code != 0) {
|
||||
vError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, terrstr());
|
||||
} else {
|
||||
code = vnodePreprocessReq(pVnode, pMsg);
|
||||
if (code != 0) {
|
||||
vError("vgId:%d, failed to pre-process msg:%p since %s", vgId, pMsg, terrstr());
|
||||
if (pMsg->msgType == TDMT_VND_ALTER_REPLICA) {
|
||||
code = vnodeProcessAlterReplicaReq(pVnode, pMsg);
|
||||
} else {
|
||||
code = syncPropose(pVnode->sync, pMsg, vnodeIsMsgWeak(pMsg->msgType));
|
||||
if (code == 1) {
|
||||
do {
|
||||
static int32_t cnt = 0;
|
||||
if (cnt++ % 1000 == 1) {
|
||||
vInfo("vgId:%d, msg:%p apply right now, apply index:%ld, msgtype:%s,%d", vgId, pMsg,
|
||||
pMsg->info.conn.applyIndex, TMSG_INFO(pMsg->msgType), pMsg->msgType);
|
||||
}
|
||||
} while (0);
|
||||
|
||||
if (code > 0) {
|
||||
SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info};
|
||||
if (vnodeProcessWriteReq(pVnode, pMsg, pMsg->info.conn.applyIndex, &rsp) < 0) {
|
||||
rsp.code = terrno;
|
||||
vInfo("vgId:%d, msg:%p failed to apply right now since %s", vgId, pMsg, terrstr());
|
||||
vError("vgId:%d, msg:%p failed to apply right now since %s", vgId, pMsg, terrstr());
|
||||
}
|
||||
|
||||
if (rsp.info.handle != NULL) {
|
||||
tmsgSendRsp(&rsp);
|
||||
}
|
||||
|
@ -161,33 +154,27 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
|||
|
||||
if (code == 0) {
|
||||
vnodeAccumBlockMsg(pVnode, pMsg->msgType);
|
||||
} else if (code == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) {
|
||||
SEpSet newEpSet = {0};
|
||||
syncGetRetryEpSet(pVnode->sync, &newEpSet);
|
||||
|
||||
/*
|
||||
syncGetEpSet(pVnode->sync, &newEpSet);
|
||||
SEp *pEp = &newEpSet.eps[newEpSet.inUse];
|
||||
if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) {
|
||||
newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps;
|
||||
}
|
||||
*/
|
||||
|
||||
vGTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", vgId, pMsg, newEpSet.numOfEps,
|
||||
newEpSet.inUse);
|
||||
for (int32_t i = 0; i < newEpSet.numOfEps; ++i) {
|
||||
vGTrace("vgId:%d, msg:%p redirect:%d ep:%s:%u", vgId, pMsg, i, newEpSet.eps[i].fqdn, newEpSet.eps[i].port);
|
||||
}
|
||||
pMsg->info.hasEpSet = 1;
|
||||
SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
|
||||
tmsgSendRedirectRsp(&rsp, &newEpSet);
|
||||
} else {
|
||||
if (code != 1) {
|
||||
} else if (code < 0) {
|
||||
if (terrno == TSDB_CODE_SYN_NOT_LEADER) {
|
||||
SEpSet newEpSet = {0};
|
||||
syncGetRetryEpSet(pVnode->sync, &newEpSet);
|
||||
vGTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", vgId, pMsg, newEpSet.numOfEps,
|
||||
newEpSet.inUse);
|
||||
for (int32_t i = 0; i < newEpSet.numOfEps; ++i) {
|
||||
vGTrace("vgId:%d, msg:%p redirect:%d ep:%s:%u", vgId, pMsg, i, newEpSet.eps[i].fqdn, newEpSet.eps[i].port);
|
||||
}
|
||||
pMsg->info.hasEpSet = 1;
|
||||
SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
|
||||
tmsgSendRedirectRsp(&rsp, &newEpSet);
|
||||
} else {
|
||||
if (terrno != 0) code = terrno;
|
||||
vError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", vgId, pMsg, tstrerror(code), code);
|
||||
SRpcMsg rsp = {.code = code, .info = pMsg->info};
|
||||
tmsgSendRsp(&rsp);
|
||||
if (rsp.info.handle != NULL) {
|
||||
tmsgSendRsp(&rsp);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
}
|
||||
|
||||
vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code);
|
||||
|
@ -206,7 +193,7 @@ void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
|||
|
||||
for (int32_t i = 0; i < numOfMsgs; ++i) {
|
||||
if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
|
||||
STraceId *trace = &pMsg->info.traceId;
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
vGTrace("vgId:%d, msg:%p get from vnode-apply queue, type:%s handle:%p", vgId, pMsg, TMSG_INFO(pMsg->msgType),
|
||||
pMsg->info.handle);
|
||||
|
||||
|
@ -229,172 +216,150 @@ void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
|||
}
|
||||
}
|
||||
|
||||
int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
|
||||
int32_t ret = 0;
|
||||
int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
|
||||
int32_t code = 0;
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
|
||||
if (syncEnvIsStart()) {
|
||||
SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync);
|
||||
assert(pSyncNode != NULL);
|
||||
|
||||
SMsgHead *pHead = pMsg->pCont;
|
||||
STraceId *trace = &pMsg->info.traceId;
|
||||
|
||||
do {
|
||||
char *syncNodeStr = sync2SimpleStr(pVnode->sync);
|
||||
static int64_t vndTick = 0;
|
||||
if (++vndTick % 10 == 1) {
|
||||
vGTrace("vgId:%d, sync trace msg:%s, %s", syncGetVgId(pVnode->sync), TMSG_INFO(pMsg->msgType), syncNodeStr);
|
||||
}
|
||||
if (gRaftDetailLog) {
|
||||
char logBuf[512] = {0};
|
||||
snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType,
|
||||
syncNodeStr);
|
||||
syncRpcMsgLog2(logBuf, pMsg);
|
||||
}
|
||||
taosMemoryFree(syncNodeStr);
|
||||
} while (0);
|
||||
|
||||
SRpcMsg *pRpcMsg = pMsg;
|
||||
|
||||
// ToDo: ugly! use function pointer
|
||||
// use different strategy
|
||||
if (syncNodeStrategy(pSyncNode) == SYNC_STRATEGY_NO_SNAPSHOT) {
|
||||
if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) {
|
||||
SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
|
||||
syncTimeoutDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_PING) {
|
||||
SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnPingCb(pSyncNode, pSyncMsg);
|
||||
syncPingDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) {
|
||||
SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
|
||||
syncPingReplyDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
|
||||
SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL);
|
||||
syncClientRequestDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) {
|
||||
SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
|
||||
syncRequestVoteDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) {
|
||||
SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
|
||||
syncRequestVoteReplyDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) {
|
||||
SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg);
|
||||
syncAppendEntriesDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) {
|
||||
SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg);
|
||||
syncAppendEntriesReplyDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_SET_VNODE_STANDBY) {
|
||||
ret = vnodeSetStandBy(pVnode);
|
||||
if (ret != 0 && terrno != 0) ret = terrno;
|
||||
SRpcMsg rsp = {.code = ret, .info = pMsg->info};
|
||||
tmsgSendRsp(&rsp);
|
||||
} else {
|
||||
vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType);
|
||||
ret = -1;
|
||||
}
|
||||
|
||||
} else {
|
||||
// use wal first strategy
|
||||
|
||||
if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) {
|
||||
SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
|
||||
syncTimeoutDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_PING) {
|
||||
SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnPingCb(pSyncNode, pSyncMsg);
|
||||
syncPingDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) {
|
||||
SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
|
||||
syncPingReplyDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
|
||||
SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL);
|
||||
syncClientRequestDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST_BATCH) {
|
||||
SyncClientRequestBatch *pSyncMsg = syncClientRequestBatchFromRpcMsg(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnClientRequestBatchCb(pSyncNode, pSyncMsg);
|
||||
syncClientRequestBatchDestroyDeep(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) {
|
||||
SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
|
||||
syncRequestVoteDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) {
|
||||
SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
|
||||
syncRequestVoteReplyDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_BATCH) {
|
||||
SyncAppendEntriesBatch *pSyncMsg = syncAppendEntriesBatchFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnAppendEntriesSnapshot2Cb(pSyncNode, pSyncMsg);
|
||||
syncAppendEntriesBatchDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) {
|
||||
SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
ret = syncNodeOnAppendEntriesReplySnapshot2Cb(pSyncNode, pSyncMsg);
|
||||
syncAppendEntriesReplyDestroy(pSyncMsg);
|
||||
|
||||
} else if (pRpcMsg->msgType == TDMT_SYNC_SET_VNODE_STANDBY) {
|
||||
ret = vnodeSetStandBy(pVnode);
|
||||
if (ret != 0 && terrno != 0) ret = terrno;
|
||||
SRpcMsg rsp = {.code = ret, .info = pMsg->info};
|
||||
tmsgSendRsp(&rsp);
|
||||
} else {
|
||||
vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType);
|
||||
ret = -1;
|
||||
}
|
||||
}
|
||||
|
||||
syncNodeRelease(pSyncNode);
|
||||
} else {
|
||||
vError("==vnodeProcessSyncReq== error syncEnv stop");
|
||||
ret = -1;
|
||||
if (!syncEnvIsStart()) {
|
||||
vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId);
|
||||
terrno = TSDB_CODE_APP_ERROR;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (ret != 0 && terrno == 0) {
|
||||
SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync);
|
||||
if (pSyncNode == NULL) {
|
||||
vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId);
|
||||
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
|
||||
return -1;
|
||||
}
|
||||
|
||||
#if 1
|
||||
char *syncNodeStr = sync2SimpleStr(pVnode->sync);
|
||||
static int64_t vndTick = 0;
|
||||
if (++vndTick % 10 == 1) {
|
||||
vGTrace("vgId:%d, sync trace msg:%s, %s", syncGetVgId(pVnode->sync), TMSG_INFO(pMsg->msgType), syncNodeStr);
|
||||
}
|
||||
if (gRaftDetailLog) {
|
||||
char logBuf[512] = {0};
|
||||
snprintf(logBuf, sizeof(logBuf), "vnode process syncmsg, msgType:%d, syncNode:%s", pMsg->msgType, syncNodeStr);
|
||||
syncRpcMsgLog2(logBuf, pMsg);
|
||||
}
|
||||
taosMemoryFree(syncNodeStr);
|
||||
#endif
|
||||
|
||||
if (syncNodeStrategy(pSyncNode) == SYNC_STRATEGY_NO_SNAPSHOT) {
|
||||
if (pMsg->msgType == TDMT_SYNC_TIMEOUT) {
|
||||
SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
|
||||
syncTimeoutDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_PING) {
|
||||
SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnPingCb(pSyncNode, pSyncMsg);
|
||||
syncPingDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) {
|
||||
SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
|
||||
syncPingReplyDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
|
||||
SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL);
|
||||
syncClientRequestDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) {
|
||||
SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
|
||||
syncRequestVoteDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) {
|
||||
SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
|
||||
syncRequestVoteReplyDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) {
|
||||
SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg);
|
||||
syncAppendEntriesDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) {
|
||||
SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg);
|
||||
syncAppendEntriesReplyDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_SET_VNODE_STANDBY) {
|
||||
code = vnodeSetStandBy(pVnode);
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
SRpcMsg rsp = {.code = code, .info = pMsg->info};
|
||||
tmsgSendRsp(&rsp);
|
||||
} else {
|
||||
vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg->msgType);
|
||||
code = -1;
|
||||
}
|
||||
} else {
|
||||
// use wal first strategy
|
||||
if (pMsg->msgType == TDMT_SYNC_TIMEOUT) {
|
||||
SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
|
||||
syncTimeoutDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_PING) {
|
||||
SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnPingCb(pSyncNode, pSyncMsg);
|
||||
syncPingDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) {
|
||||
SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
|
||||
syncPingReplyDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
|
||||
SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL);
|
||||
syncClientRequestDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST_BATCH) {
|
||||
SyncClientRequestBatch *pSyncMsg = syncClientRequestBatchFromRpcMsg(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnClientRequestBatchCb(pSyncNode, pSyncMsg);
|
||||
syncClientRequestBatchDestroyDeep(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) {
|
||||
SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
|
||||
syncRequestVoteDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) {
|
||||
SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
|
||||
syncRequestVoteReplyDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_BATCH) {
|
||||
SyncAppendEntriesBatch *pSyncMsg = syncAppendEntriesBatchFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnAppendEntriesSnapshot2Cb(pSyncNode, pSyncMsg);
|
||||
syncAppendEntriesBatchDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) {
|
||||
SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg);
|
||||
ASSERT(pSyncMsg != NULL);
|
||||
code = syncNodeOnAppendEntriesReplySnapshot2Cb(pSyncNode, pSyncMsg);
|
||||
syncAppendEntriesReplyDestroy(pSyncMsg);
|
||||
} else if (pMsg->msgType == TDMT_SYNC_SET_VNODE_STANDBY) {
|
||||
code = vnodeSetStandBy(pVnode);
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
SRpcMsg rsp = {.code = code, .info = pMsg->info};
|
||||
tmsgSendRsp(&rsp);
|
||||
} else {
|
||||
vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg->msgType);
|
||||
code = -1;
|
||||
}
|
||||
}
|
||||
|
||||
syncNodeRelease(pSyncNode);
|
||||
if (code != 0 && terrno == 0) {
|
||||
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
|
||||
}
|
||||
return ret;
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
|
||||
|
@ -427,7 +392,7 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon
|
|||
syncGetAndDelRespRpc(pVnode->sync, cbMeta.newCfgSeqNum, &rpcMsg.info);
|
||||
rpcMsg.info.conn.applyIndex = cbMeta.index;
|
||||
|
||||
STraceId *trace = (STraceId *)&pMsg->info.traceId;
|
||||
const STraceId *trace = (STraceId *)&pMsg->info.traceId;
|
||||
vGTrace("vgId:%d, alter vnode replica is confirmed, type:%s contLen:%d seq:%" PRIu64 " handle:%p", TD_VID(pVnode),
|
||||
TMSG_INFO(pMsg->msgType), pMsg->contLen, cbMeta.seqNum, rpcMsg.info.handle);
|
||||
if (rpcMsg.info.handle != NULL) {
|
||||
|
@ -444,9 +409,8 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c
|
|||
char logBuf[256] = {0};
|
||||
|
||||
snprintf(logBuf, sizeof(logBuf),
|
||||
"==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n",
|
||||
pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state),
|
||||
beginIndex);
|
||||
"commitCb execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n", pFsm,
|
||||
cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex);
|
||||
syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
|
||||
|
||||
SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen};
|
||||
|
@ -459,16 +423,15 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c
|
|||
|
||||
static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
|
||||
char logBuf[256] = {0};
|
||||
snprintf(logBuf, sizeof(logBuf),
|
||||
"==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index,
|
||||
cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
|
||||
snprintf(logBuf, sizeof(logBuf), "preCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm,
|
||||
cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
|
||||
syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
|
||||
}
|
||||
|
||||
static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
|
||||
char logBuf[256] = {0};
|
||||
snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
|
||||
pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
|
||||
snprintf(logBuf, sizeof(logBuf), "rollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm,
|
||||
cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
|
||||
syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
|
||||
}
|
||||
|
||||
|
|
|
@ -548,19 +548,17 @@ static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) {
|
|||
}
|
||||
|
||||
static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** pOutput) {
|
||||
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
|
||||
SSDataBlock* pBlock = createDataBlock();
|
||||
if (NULL == pBlock) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
pBlock->pDataBlock = taosArrayInit(LIST_LENGTH(pProjects), sizeof(SColumnInfoData));
|
||||
|
||||
SNode* pProj = NULL;
|
||||
FOREACH(pProj, pProjects) {
|
||||
SColumnInfoData infoData = {0};
|
||||
infoData.info.type = ((SExprNode*)pProj)->resType.type;
|
||||
infoData.info.bytes = ((SExprNode*)pProj)->resType.bytes;
|
||||
taosArrayPush(pBlock->pDataBlock, &infoData);
|
||||
blockDataAppendColInfo(pBlock, &infoData);
|
||||
}
|
||||
*pOutput = pBlock;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
|
@ -51,6 +51,13 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int
|
|||
|
||||
#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData ? 1 : 0)
|
||||
|
||||
#define START_TS_COLUMN_INDEX 0
|
||||
#define END_TS_COLUMN_INDEX 1
|
||||
#define UID_COLUMN_INDEX 2
|
||||
#define GROUPID_COLUMN_INDEX UID_COLUMN_INDEX
|
||||
#define DELETE_GROUPID_COLUMN_INDEX 2
|
||||
|
||||
|
||||
enum {
|
||||
// when this task starts to execute, this status will set
|
||||
TASK_NOT_COMPLETED = 0x1u,
|
||||
|
@ -233,7 +240,7 @@ typedef struct SColMatchInfo {
|
|||
int32_t srcSlotId; // source slot id
|
||||
int32_t colId;
|
||||
int32_t targetSlotId;
|
||||
bool output;
|
||||
bool output; // todo remove this?
|
||||
bool reserved;
|
||||
int32_t matchType; // determinate the source according to col id or slot id
|
||||
} SColMatchInfo;
|
||||
|
@ -364,6 +371,8 @@ typedef struct SStreamBlockScanInfo {
|
|||
int32_t scanWinIndex; // for state operator
|
||||
int32_t pullDataResIndex;
|
||||
SSDataBlock* pPullDataRes; // pull data SSDataBlock
|
||||
SSDataBlock* pDeleteDataRes; // delete data SSDataBlock
|
||||
int32_t deleteDataIndex;
|
||||
} SStreamBlockScanInfo;
|
||||
|
||||
typedef struct SSysTableScanInfo {
|
||||
|
@ -429,6 +438,10 @@ typedef struct SIntervalAggOperatorInfo {
|
|||
bool invertible;
|
||||
SArray* pPrevValues; // SArray<SGroupKeys> used to keep the previous not null value for interpolation.
|
||||
bool ignoreExpiredData;
|
||||
SArray* pRecycledPages;
|
||||
SArray* pDelWins; // SWinRes
|
||||
int32_t delIndex;
|
||||
SSDataBlock* pDelRes;
|
||||
} SIntervalAggOperatorInfo;
|
||||
|
||||
typedef struct SStreamFinalIntervalOperatorInfo {
|
||||
|
@ -451,6 +464,10 @@ typedef struct SStreamFinalIntervalOperatorInfo {
|
|||
int32_t pullIndex;
|
||||
SSDataBlock* pPullDataRes;
|
||||
bool ignoreExpiredData;
|
||||
SArray* pRecycledPages;
|
||||
SArray* pDelWins; // SWinRes
|
||||
int32_t delIndex;
|
||||
SSDataBlock* pDelRes;
|
||||
} SStreamFinalIntervalOperatorInfo;
|
||||
|
||||
typedef struct SAggOperatorInfo {
|
||||
|
@ -680,7 +697,7 @@ typedef struct SJoinOperatorInfo {
|
|||
SSDataBlock *pRight;
|
||||
int32_t rightPos;
|
||||
SColumnInfo rightCol;
|
||||
SNode *pOnCondition;
|
||||
SNode *pCondAfterMerge;
|
||||
} SJoinOperatorInfo;
|
||||
|
||||
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
|
||||
|
|
|
@ -0,0 +1,254 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "dataSinkInt.h"
|
||||
#include "dataSinkMgt.h"
|
||||
#include "executorimpl.h"
|
||||
#include "planner.h"
|
||||
#include "tcompression.h"
|
||||
#include "tdatablock.h"
|
||||
#include "tglobal.h"
|
||||
#include "tqueue.h"
|
||||
|
||||
extern SDataSinkStat gDataSinkStat;
|
||||
|
||||
typedef struct SDataInserterBuf {
|
||||
int32_t useSize;
|
||||
int32_t allocSize;
|
||||
char* pData;
|
||||
} SDataInserterBuf;
|
||||
|
||||
typedef struct SDataCacheEntry {
|
||||
int32_t dataLen;
|
||||
int32_t numOfRows;
|
||||
int32_t numOfCols;
|
||||
int8_t compressed;
|
||||
char data[];
|
||||
} SDataCacheEntry;
|
||||
|
||||
typedef struct SDataInserterHandle {
|
||||
SDataSinkHandle sink;
|
||||
SDataSinkManager* pManager;
|
||||
SDataBlockDescNode* pSchema;
|
||||
SDataDeleterNode* pDeleter;
|
||||
SDeleterParam* pParam;
|
||||
STaosQueue* pDataBlocks;
|
||||
SDataInserterBuf nextOutput;
|
||||
int32_t status;
|
||||
bool queryEnd;
|
||||
uint64_t useconds;
|
||||
uint64_t cachedSize;
|
||||
TdThreadMutex mutex;
|
||||
} SDataInserterHandle;
|
||||
|
||||
static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) {
|
||||
if (tsCompressColData < 0 || 0 == pData->info.rows) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (int32_t col = 0; col < numOfCols; ++col) {
|
||||
SColumnInfoData* pColRes = taosArrayGet(pData->pDataBlock, col);
|
||||
int32_t colSize = pColRes->info.bytes * pData->info.rows;
|
||||
if (NEEDTO_COMPRESS_QUERY(colSize)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static void toDataCacheEntry(SDataInserterHandle* pHandle, const SInputData* pInput, SDataInserterBuf* pBuf) {
|
||||
int32_t numOfCols = LIST_LENGTH(pHandle->pSchema->pSlots);
|
||||
|
||||
SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData;
|
||||
pEntry->compressed = 0;
|
||||
pEntry->numOfRows = pInput->pData->info.rows;
|
||||
pEntry->numOfCols = taosArrayGetSize(pInput->pData->pDataBlock);
|
||||
pEntry->dataLen = sizeof(SDeleterRes);
|
||||
|
||||
ASSERT(1 == pEntry->numOfRows);
|
||||
ASSERT(1 == pEntry->numOfCols);
|
||||
|
||||
pBuf->useSize = sizeof(SDataCacheEntry);
|
||||
|
||||
SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0);
|
||||
|
||||
SDeleterRes* pRes = (SDeleterRes*)pEntry->data;
|
||||
pRes->suid = pHandle->pParam->suid;
|
||||
pRes->uidList = pHandle->pParam->pUidList;
|
||||
pRes->skey = pHandle->pDeleter->deleteTimeRange.skey;
|
||||
pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey;
|
||||
pRes->affectedRows = *(int64_t*)pColRes->pData;
|
||||
|
||||
pBuf->useSize += pEntry->dataLen;
|
||||
|
||||
atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
|
||||
atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
|
||||
}
|
||||
|
||||
static bool allocBuf(SDataInserterHandle* pDeleter, const SInputData* pInput, SDataInserterBuf* pBuf) {
|
||||
uint32_t capacity = pDeleter->pManager->cfg.maxDataBlockNumPerQuery;
|
||||
if (taosQueueItemSize(pDeleter->pDataBlocks) > capacity) {
|
||||
qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity,
|
||||
taosQueueItemSize(pDeleter->pDataBlocks));
|
||||
return false;
|
||||
}
|
||||
|
||||
pBuf->allocSize = sizeof(SDataCacheEntry) + sizeof(SDeleterRes);
|
||||
|
||||
pBuf->pData = taosMemoryMalloc(pBuf->allocSize);
|
||||
if (pBuf->pData == NULL) {
|
||||
qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno));
|
||||
}
|
||||
|
||||
return NULL != pBuf->pData;
|
||||
}
|
||||
|
||||
static int32_t updateStatus(SDataInserterHandle* pDeleter) {
|
||||
taosThreadMutexLock(&pDeleter->mutex);
|
||||
int32_t blockNums = taosQueueItemSize(pDeleter->pDataBlocks);
|
||||
int32_t status =
|
||||
(0 == blockNums ? DS_BUF_EMPTY
|
||||
: (blockNums < pDeleter->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL));
|
||||
pDeleter->status = status;
|
||||
taosThreadMutexUnlock(&pDeleter->mutex);
|
||||
return status;
|
||||
}
|
||||
|
||||
static int32_t getStatus(SDataInserterHandle* pDeleter) {
|
||||
taosThreadMutexLock(&pDeleter->mutex);
|
||||
int32_t status = pDeleter->status;
|
||||
taosThreadMutexUnlock(&pDeleter->mutex);
|
||||
return status;
|
||||
}
|
||||
|
||||
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
|
||||
SDataInserterHandle* pDeleter = (SDataInserterHandle*)pHandle;
|
||||
SDataInserterBuf* pBuf = taosAllocateQitem(sizeof(SDataInserterBuf), DEF_QITEM);
|
||||
if (NULL == pBuf || !allocBuf(pDeleter, pInput, pBuf)) {
|
||||
return TSDB_CODE_QRY_OUT_OF_MEMORY;
|
||||
}
|
||||
toDataCacheEntry(pDeleter, pInput, pBuf);
|
||||
taosWriteQitem(pDeleter->pDataBlocks, pBuf);
|
||||
*pContinue = (DS_BUF_LOW == updateStatus(pDeleter) ? true : false);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
|
||||
SDataInserterHandle* pDeleter = (SDataInserterHandle*)pHandle;
|
||||
taosThreadMutexLock(&pDeleter->mutex);
|
||||
pDeleter->queryEnd = true;
|
||||
pDeleter->useconds = useconds;
|
||||
taosThreadMutexUnlock(&pDeleter->mutex);
|
||||
}
|
||||
|
||||
static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd) {
|
||||
SDataInserterHandle* pDeleter = (SDataInserterHandle*)pHandle;
|
||||
if (taosQueueEmpty(pDeleter->pDataBlocks)) {
|
||||
*pQueryEnd = pDeleter->queryEnd;
|
||||
*pLen = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
SDataInserterBuf* pBuf = NULL;
|
||||
taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
|
||||
memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataInserterBuf));
|
||||
taosFreeQitem(pBuf);
|
||||
*pLen = ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->dataLen;
|
||||
*pQueryEnd = pDeleter->queryEnd;
|
||||
qDebug("got data len %d, row num %d in sink", *pLen, ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows);
|
||||
}
|
||||
|
||||
static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
|
||||
SDataInserterHandle* pDeleter = (SDataInserterHandle*)pHandle;
|
||||
if (NULL == pDeleter->nextOutput.pData) {
|
||||
assert(pDeleter->queryEnd);
|
||||
pOutput->useconds = pDeleter->useconds;
|
||||
pOutput->precision = pDeleter->pSchema->precision;
|
||||
pOutput->bufStatus = DS_BUF_EMPTY;
|
||||
pOutput->queryEnd = pDeleter->queryEnd;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDeleter->nextOutput.pData);
|
||||
memcpy(pOutput->pData, pEntry->data, pEntry->dataLen);
|
||||
pOutput->numOfRows = pEntry->numOfRows;
|
||||
pOutput->numOfCols = pEntry->numOfCols;
|
||||
pOutput->compressed = pEntry->compressed;
|
||||
|
||||
atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen);
|
||||
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
|
||||
|
||||
taosMemoryFreeClear(pDeleter->nextOutput.pData); // todo persistent
|
||||
pOutput->bufStatus = updateStatus(pDeleter);
|
||||
taosThreadMutexLock(&pDeleter->mutex);
|
||||
pOutput->queryEnd = pDeleter->queryEnd;
|
||||
pOutput->useconds = pDeleter->useconds;
|
||||
pOutput->precision = pDeleter->pSchema->precision;
|
||||
taosThreadMutexUnlock(&pDeleter->mutex);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
|
||||
SDataInserterHandle* pDeleter = (SDataInserterHandle*)pHandle;
|
||||
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDeleter->cachedSize);
|
||||
taosMemoryFreeClear(pDeleter->nextOutput.pData);
|
||||
while (!taosQueueEmpty(pDeleter->pDataBlocks)) {
|
||||
SDataInserterBuf* pBuf = NULL;
|
||||
taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
|
||||
taosMemoryFreeClear(pBuf->pData);
|
||||
taosFreeQitem(pBuf);
|
||||
}
|
||||
taosCloseQueue(pDeleter->pDataBlocks);
|
||||
taosThreadMutexDestroy(&pDeleter->mutex);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) {
|
||||
SDataInserterHandle* pDispatcher = (SDataInserterHandle*)pHandle;
|
||||
|
||||
*size = atomic_load_64(&pDispatcher->cachedSize);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void *pParam) {
|
||||
SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle));
|
||||
if (NULL == inserter) {
|
||||
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
|
||||
return TSDB_CODE_QRY_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
SDataDeleterNode* pDeleterNode = (SDataDeleterNode *)pDataSink;
|
||||
inserter->sink.fPut = putDataBlock;
|
||||
inserter->sink.fEndPut = endPut;
|
||||
inserter->sink.fGetLen = getDataLength;
|
||||
inserter->sink.fGetData = getDataBlock;
|
||||
inserter->sink.fDestroy = destroyDataSinker;
|
||||
inserter->sink.fGetCacheSize = getCacheSize;
|
||||
inserter->pManager = pManager;
|
||||
inserter->pDeleter = pDeleterNode;
|
||||
inserter->pSchema = pDataSink->pInputDataBlockDesc;
|
||||
inserter->pParam = pParam;
|
||||
inserter->status = DS_BUF_EMPTY;
|
||||
inserter->queryEnd = false;
|
||||
inserter->pDataBlocks = taosOpenQueue();
|
||||
taosThreadMutexInit(&inserter->mutex, NULL);
|
||||
if (NULL == inserter->pDataBlocks) {
|
||||
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
|
||||
return TSDB_CODE_QRY_OUT_OF_MEMORY;
|
||||
}
|
||||
*pHandle = inserter;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
|
@ -700,7 +700,7 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray
|
|||
while (i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) {
|
||||
SColumnInfoData* p = taosArrayGet(pCols, i);
|
||||
SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, j);
|
||||
if (!outputEveryColumn && !pmInfo->output) {
|
||||
if (!outputEveryColumn && pmInfo->reserved) {
|
||||
j++;
|
||||
continue;
|
||||
}
|
||||
|
|
|
@ -27,6 +27,10 @@ static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT;
|
|||
int32_t exchangeObjRefPool = -1;
|
||||
|
||||
static void initRefPool() { exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo); }
|
||||
static void cleanupRefPool() {
|
||||
int32_t ref = atomic_val_compare_exchange_32(&exchangeObjRefPool, exchangeObjRefPool, 0);
|
||||
taosCloseRef(ref);
|
||||
}
|
||||
|
||||
int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan,
|
||||
qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model) {
|
||||
|
@ -34,7 +38,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
|
|||
SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo;
|
||||
|
||||
taosThreadOnce(&initPoolOnce, initRefPool);
|
||||
|
||||
atexit(cleanupRefPool);
|
||||
int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
|
|
|
@ -53,13 +53,28 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
|
|||
pOperator->info = pInfo;
|
||||
pOperator->pTaskInfo = pTaskInfo;
|
||||
|
||||
SNode* pOnCondition = pJoinNode->pOnConditions;
|
||||
if (nodeType(pOnCondition) == QUERY_NODE_OPERATOR) {
|
||||
SOperatorNode* pNode = (SOperatorNode*)pOnCondition;
|
||||
SNode* pMergeCondition = pJoinNode->pMergeCondition;
|
||||
if (nodeType(pMergeCondition) == QUERY_NODE_OPERATOR) {
|
||||
SOperatorNode* pNode = (SOperatorNode*)pMergeCondition;
|
||||
setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft);
|
||||
setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight);
|
||||
} else if (nodeType(pOnCondition) == QUERY_NODE_LOGIC_CONDITION) {
|
||||
extractTimeCondition(pInfo, (SLogicConditionNode*)pOnCondition);
|
||||
} else {
|
||||
ASSERT(false);
|
||||
}
|
||||
|
||||
if (pJoinNode->pOnConditions != NULL && pJoinNode->node.pConditions != NULL) {
|
||||
pInfo->pCondAfterMerge = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
|
||||
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pInfo->pCondAfterMerge);
|
||||
pLogicCond->pParameterList = nodesMakeList();
|
||||
nodesListMakeAppend(&pLogicCond->pParameterList, nodesCloneNode(pJoinNode->pOnConditions));
|
||||
nodesListMakeAppend(&pLogicCond->pParameterList, nodesCloneNode(pJoinNode->node.pConditions));
|
||||
pLogicCond->condType = LOGIC_COND_TYPE_AND;
|
||||
} else if (pJoinNode->pOnConditions != NULL) {
|
||||
pInfo->pCondAfterMerge = nodesCloneNode(pJoinNode->pOnConditions);
|
||||
} else if (pJoinNode->node.pConditions != NULL) {
|
||||
pInfo->pCondAfterMerge = nodesCloneNode(pJoinNode->node.pConditions);
|
||||
} else {
|
||||
pInfo->pCondAfterMerge = NULL;
|
||||
}
|
||||
|
||||
pOperator->fpSet =
|
||||
|
@ -88,15 +103,12 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) {
|
|||
|
||||
void destroyMergeJoinOperator(void* param, int32_t numOfOutput) {
|
||||
SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param;
|
||||
nodesDestroyNode(pJoinOperator->pCondAfterMerge);
|
||||
}
|
||||
|
||||
SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
|
||||
static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) {
|
||||
SJoinOperatorInfo* pJoinInfo = pOperator->info;
|
||||
|
||||
SSDataBlock* pRes = pJoinInfo->pRes;
|
||||
blockDataCleanup(pRes);
|
||||
blockDataEnsureCapacity(pRes, 4096);
|
||||
|
||||
int32_t nrows = 0;
|
||||
|
||||
while (1) {
|
||||
|
@ -181,7 +193,28 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
|
|||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
|
||||
SJoinOperatorInfo* pJoinInfo = pOperator->info;
|
||||
|
||||
SSDataBlock* pRes = pJoinInfo->pRes;
|
||||
blockDataCleanup(pRes);
|
||||
blockDataEnsureCapacity(pRes, 4096);
|
||||
while (true) {
|
||||
int32_t numOfRowsBefore = pRes->info.rows;
|
||||
doMergeJoinImpl(pOperator, pRes);
|
||||
int32_t numOfNewRows = pRes->info.rows - numOfRowsBefore;
|
||||
if (numOfNewRows == 0) {
|
||||
break;
|
||||
}
|
||||
if (pJoinInfo->pCondAfterMerge != NULL) {
|
||||
doFilter(pJoinInfo->pCondAfterMerge, pRes);
|
||||
}
|
||||
if (pRes->info.rows >= pOperator->resultInfo.threshold) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return (pRes->info.rows > 0) ? pRes : NULL;
|
||||
}
|
||||
|
||||
|
|
|
@ -807,18 +807,38 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
|
|||
return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
|
||||
}
|
||||
|
||||
static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) {
|
||||
uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t));
|
||||
if (groupId) {
|
||||
return *groupId;
|
||||
}
|
||||
return 0;
|
||||
/* Todo(liuyao) for partition by column
|
||||
recordNewGroupKeys(pTableScanInfo->pGroupCols, pTableScanInfo->pGroupColVals, pBlock, rowId);
|
||||
int32_t len = buildGroupKeys(pTableScanInfo->keyBuf, pTableScanInfo->pGroupColVals);
|
||||
uint64_t resId = 0;
|
||||
uint64_t* groupId = taosHashGet(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len);
|
||||
if (groupId) {
|
||||
return *groupId;
|
||||
} else if (len != 0) {
|
||||
resId = calcGroupId(pTableScanInfo->keyBuf, len);
|
||||
taosHashPut(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len, &resId, sizeof(uint64_t));
|
||||
}
|
||||
return resId;
|
||||
*/
|
||||
}
|
||||
|
||||
static void setGroupId(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
|
||||
ASSERT(rowIndex < pBlock->info.rows);
|
||||
switch (pBlock->info.type)
|
||||
{
|
||||
case STREAM_DELETE_DATA:
|
||||
case STREAM_RETRIEVE: {
|
||||
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
|
||||
uint64_t* groupCol = (uint64_t*)pColInfo->pData;
|
||||
pInfo->groupId = groupCol[rowIndex];
|
||||
}
|
||||
break;
|
||||
case STREAM_DELETE_DATA:
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
@ -840,14 +860,14 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3
|
|||
int64_t gap = pInfo->sessionSup.gap;
|
||||
int32_t winIndex = 0;
|
||||
SResultWindowInfo* pCurWin =
|
||||
getSessionTimeWindow(pAggSup, tsCols[(*pRowIndex)], INT64_MIN, pSDB->info.groupId, gap, &winIndex);
|
||||
getSessionTimeWindow(pAggSup, tsCols[*pRowIndex], INT64_MIN, pSDB->info.groupId, gap, &winIndex);
|
||||
win = pCurWin->win;
|
||||
(*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, (*pRowIndex), gap, NULL);
|
||||
(*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, *pRowIndex, gap, NULL);
|
||||
} else {
|
||||
win =
|
||||
getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL);
|
||||
setGroupId(pInfo, pSDB, 2, *pRowIndex);
|
||||
(*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL,
|
||||
getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, NULL);
|
||||
setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex);
|
||||
(*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL,
|
||||
TSDB_ORDER_ASC);
|
||||
}
|
||||
needRead = true;
|
||||
|
@ -891,27 +911,6 @@ static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRow
|
|||
dest->info.rows++;
|
||||
}
|
||||
|
||||
static uint64_t getGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t rowId) {
|
||||
uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
|
||||
if (groupId) {
|
||||
return *groupId;
|
||||
}
|
||||
return 0;
|
||||
/* Todo(liuyao) for partition by column
|
||||
recordNewGroupKeys(pTableScanInfo->pGroupCols, pTableScanInfo->pGroupColVals, pBlock, rowId);
|
||||
int32_t len = buildGroupKeys(pTableScanInfo->keyBuf, pTableScanInfo->pGroupColVals);
|
||||
uint64_t resId = 0;
|
||||
uint64_t* groupId = taosHashGet(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len);
|
||||
if (groupId) {
|
||||
return *groupId;
|
||||
} else if (len != 0) {
|
||||
resId = calcGroupId(pTableScanInfo->keyBuf, len);
|
||||
taosHashPut(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len, &resId, sizeof(uint64_t));
|
||||
}
|
||||
return resId;
|
||||
*/
|
||||
}
|
||||
|
||||
static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
|
||||
while (1) {
|
||||
SSDataBlock* pResult = NULL;
|
||||
|
@ -935,7 +934,7 @@ static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, i
|
|||
SSDataBlock* pBlock = createOneDataBlock(pResult, true);
|
||||
blockDataCleanup(pResult);
|
||||
for (int32_t i = 0; i < pBlock->info.rows; i++) {
|
||||
uint64_t id = getGroupId(pInfo->pOperatorDumy, pBlock, i);
|
||||
uint64_t id = getGroupId(pInfo->pOperatorDumy, pBlock->info.uid);
|
||||
if (id == pInfo->groupId) {
|
||||
copyOneRow(pResult, pBlock, i);
|
||||
}
|
||||
|
@ -944,6 +943,40 @@ static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, i
|
|||
*/
|
||||
}
|
||||
|
||||
static void copyDeleteDataBlock(SStreamBlockScanInfo* pInfo, SSDataBlock* pDelBlock, SOperatorInfo* pOperator, SSDataBlock* pUpdateRes) {
|
||||
if (pDelBlock->info.rows == 0) {
|
||||
return;
|
||||
}
|
||||
blockDataCleanup(pUpdateRes);
|
||||
blockDataEnsureCapacity(pUpdateRes, 64);
|
||||
ASSERT(taosArrayGetSize(pDelBlock->pDataBlock) >= 3);
|
||||
SColumnInfoData* pStartTsCol = taosArrayGet(pDelBlock->pDataBlock, START_TS_COLUMN_INDEX);
|
||||
TSKEY* startData = (TSKEY*)pStartTsCol->pData;
|
||||
SColumnInfoData* pEndTsCol = taosArrayGet(pDelBlock->pDataBlock, END_TS_COLUMN_INDEX);
|
||||
TSKEY* endData = (TSKEY*)pEndTsCol->pData;
|
||||
SColumnInfoData* pGpCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX);
|
||||
uint64_t* uidCol = (uint64_t*)pGpCol->pData;
|
||||
|
||||
SColumnInfoData* pDestTsCol = taosArrayGet(pUpdateRes->pDataBlock, START_TS_COLUMN_INDEX);
|
||||
SColumnInfoData* pDestGpCol = taosArrayGet(pUpdateRes->pDataBlock, DELETE_GROUPID_COLUMN_INDEX);
|
||||
for (int32_t i = pInfo->deleteDataIndex ; i < pDelBlock->info.rows &&
|
||||
i < pDelBlock->info.capacity - (endData[i] - startData[i])/pInfo->interval.interval - 1; i++) {
|
||||
uint64_t groupId = getGroupId(pOperator, uidCol[i]);
|
||||
for (TSKEY startTs = startData[i]; startTs <= endData[i]; ) {
|
||||
colDataAppend(pDestTsCol, pUpdateRes->info.rows, (const char*)&startTs, false);
|
||||
colDataAppend(pDestGpCol, pUpdateRes->info.rows, (const char*)&groupId, false);
|
||||
pUpdateRes->info.rows++;
|
||||
startTs = taosTimeAdd(startTs, pInfo->interval.interval, pInfo->interval.intervalUnit, pInfo->interval.precision);
|
||||
}
|
||||
pInfo->deleteDataIndex++;
|
||||
}
|
||||
|
||||
if (pInfo->deleteDataIndex > 0 && pInfo->deleteDataIndex == pDelBlock->info.rows) {
|
||||
blockDataCleanup(pDelBlock);
|
||||
pInfo->deleteDataIndex = 0;
|
||||
}
|
||||
}
|
||||
|
||||
static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) {
|
||||
blockDataCleanup(pUpdateBlock);
|
||||
int32_t size = taosArrayGetSize(pInfo->tsArray);
|
||||
|
@ -953,11 +986,11 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa
|
|||
blockDataEnsureCapacity(pUpdateBlock, size);
|
||||
|
||||
int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex);
|
||||
pInfo->groupId = getGroupId(pInfo->pSnapshotReadOp, pBlock, rowId);
|
||||
pInfo->groupId = getGroupId(pInfo->pSnapshotReadOp, pBlock->info.uid);
|
||||
int32_t i = 0;
|
||||
for (; i < size; i++) {
|
||||
rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, i + pInfo->tsArrayIndex);
|
||||
uint64_t id = getGroupId(pInfo->pSnapshotReadOp, pBlock, rowId);
|
||||
uint64_t id = getGroupId(pInfo->pSnapshotReadOp, pBlock->info.uid);
|
||||
if (pInfo->groupId != id) {
|
||||
break;
|
||||
}
|
||||
|
@ -974,28 +1007,32 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa
|
|||
if (size > 0 && pInfo->tsArrayIndex == size) {
|
||||
taosArrayClear(pInfo->tsArray);
|
||||
}
|
||||
|
||||
if (size == 0) {
|
||||
copyDeleteDataBlock(pInfo, pInfo->pDeleteDataRes, pInfo->pSnapshotReadOp, pUpdateBlock);
|
||||
}
|
||||
}
|
||||
|
||||
static void getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool invertible, SSDataBlock* pBlock,
|
||||
SSDataBlock* pUpdateBlock) {
|
||||
static void checkUpdateData(SStreamBlockScanInfo* pInfo, bool invertible, SSDataBlock* pBlock,
|
||||
bool out) {
|
||||
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
|
||||
ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP);
|
||||
TSKEY* ts = (TSKEY*)pColDataInfo->pData;
|
||||
for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) {
|
||||
if (updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, ts[rowId])) {
|
||||
if (updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, ts[rowId]) && out) {
|
||||
taosArrayPush(pInfo->tsArray, &rowId);
|
||||
}
|
||||
}
|
||||
if (!pUpdateBlock) {
|
||||
taosArrayClear(pInfo->tsArray);
|
||||
return;
|
||||
}
|
||||
|
||||
static void setBlockGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t uidColIndex) {
|
||||
ASSERT(taosArrayGetSize(pBlock->pDataBlock) >= 3);
|
||||
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, uidColIndex);
|
||||
uint64_t* uidCol = (uint64_t*)pColDataInfo->pData;
|
||||
ASSERT(pBlock->info.rows > 0);
|
||||
for (int32_t i = 0 ; i < pBlock->info.rows; i++) {
|
||||
uidCol[i] = getGroupId(pOperator, uidCol[i]);
|
||||
}
|
||||
setUpdateData(pInfo, pBlock, pUpdateBlock);
|
||||
// Todo(liuyao) get from tsdb
|
||||
// SSDataBlock* p = createOneDataBlock(pBlock, true);
|
||||
// p->info.type = STREAM_INVERT;
|
||||
// taosArrayClear(pInfo->tsArray);
|
||||
// return p;
|
||||
}
|
||||
|
||||
static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
||||
|
@ -1020,13 +1057,29 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
int32_t current = pInfo->validBlockIndex++;
|
||||
SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
|
||||
blockDataUpdateTsWindow(pBlock, 0);
|
||||
if (pBlock->info.type == STREAM_RETRIEVE) {
|
||||
switch (pBlock->info.type) {
|
||||
case STREAM_RETRIEVE:{
|
||||
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE;
|
||||
copyDataBlock(pInfo->pPullDataRes, pBlock);
|
||||
pInfo->pullDataResIndex = 0;
|
||||
prepareDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex);
|
||||
prepareDataScan(pInfo, pInfo->pPullDataRes, START_TS_COLUMN_INDEX, &pInfo->pullDataResIndex);
|
||||
updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
|
||||
}
|
||||
break;
|
||||
case STREAM_DELETE_DATA: {
|
||||
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
|
||||
copyDataBlock(pInfo->pDeleteDataRes, pBlock);
|
||||
copyDeleteDataBlock(pInfo, pInfo->pDeleteDataRes, pInfo->pSnapshotReadOp, pInfo->pUpdateRes);
|
||||
pInfo->updateResIndex = 0;
|
||||
prepareDataScan(pInfo, pInfo->pUpdateRes, START_TS_COLUMN_INDEX, &pInfo->updateResIndex);
|
||||
pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA;
|
||||
return pInfo->pUpdateRes;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return pBlock;
|
||||
} else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) {
|
||||
|
@ -1043,39 +1096,33 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
} else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RETRIEVE) {
|
||||
SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex);
|
||||
if (pSDB != NULL) {
|
||||
getUpdateDataBlock(pInfo, true, pSDB, NULL);
|
||||
checkUpdateData(pInfo, true, pSDB, false);
|
||||
pSDB->info.type = STREAM_PULL_DATA;
|
||||
return pSDB;
|
||||
}
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
|
||||
} else {
|
||||
if (isStateWindow(pInfo)) {
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
|
||||
pInfo->updateResIndex = pInfo->pUpdateRes->info.rows;
|
||||
if (!prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex)) {
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
|
||||
}
|
||||
} else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) {
|
||||
SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
|
||||
if (pSDB) {
|
||||
pSDB->info.type = STREAM_NORMAL;
|
||||
checkUpdateData(pInfo, true, pSDB, false);
|
||||
return pSDB;
|
||||
}
|
||||
if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) {
|
||||
SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
|
||||
if (pSDB == NULL) {
|
||||
setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes);
|
||||
if (pInfo->pUpdateRes->info.rows > 0) {
|
||||
if (!isStateWindow(pInfo)) {
|
||||
// Todo(liuyao) mybe can delete this.
|
||||
bool test = prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
|
||||
ASSERT(test == false);
|
||||
}
|
||||
return pInfo->pUpdateRes;
|
||||
} else {
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
|
||||
}
|
||||
} else {
|
||||
pSDB->info.type = STREAM_NORMAL;
|
||||
getUpdateDataBlock(pInfo, true, pSDB, NULL);
|
||||
return pSDB;
|
||||
}
|
||||
setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes);
|
||||
if (pInfo->pUpdateRes->info.rows > 0) {
|
||||
prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
|
||||
return pInfo->pUpdateRes;
|
||||
}
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
|
||||
} else if (isStateWindow(pInfo)) {
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
|
||||
pInfo->updateResIndex = pInfo->pUpdateRes->info.rows;
|
||||
if (prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex)) {
|
||||
ASSERT(pInfo->pUpdateRes->info.rows == 0);
|
||||
// return empty data blcok
|
||||
return pInfo->pUpdateRes;
|
||||
}
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
|
||||
}
|
||||
|
||||
SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
|
||||
|
@ -1169,7 +1216,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
pOperator->status = OP_EXEC_DONE;
|
||||
} else if (pInfo->pUpdateInfo) {
|
||||
pInfo->tsArrayIndex = 0;
|
||||
getUpdateDataBlock(pInfo, true, pInfo->pRes, pInfo->pUpdateRes);
|
||||
checkUpdateData(pInfo, true, pInfo->pRes, true);
|
||||
setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes);
|
||||
if (pInfo->pUpdateRes->info.rows > 0) {
|
||||
if (pInfo->pUpdateRes->info.type == STREAM_CLEAR) {
|
||||
pInfo->updateResIndex = 0;
|
||||
|
@ -1180,9 +1228,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
|
||||
|
||||
} else if (pInfo->blockType == STREAM_INPUT__DATA_SCAN) {
|
||||
// check reader last status
|
||||
// if not match, reset status
|
||||
|
@ -1295,6 +1341,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
|
|||
pInfo->groupId = 0;
|
||||
pInfo->pPullDataRes = createPullDataBlock();
|
||||
pInfo->pStreamScanOp = pOperator;
|
||||
pInfo->deleteDataIndex = 0;
|
||||
pInfo->pDeleteDataRes = createPullDataBlock();
|
||||
|
||||
pOperator->name = "StreamBlockScanOperator";
|
||||
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
|
||||
|
@ -1327,6 +1375,7 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) {
|
|||
}
|
||||
|
||||
taosArrayDestroy(pInfo->scanCols);
|
||||
taosMemoryFreeClear(pInfo->pUser);
|
||||
}
|
||||
|
||||
static int32_t getSysTableDbNameColId(const char* pTable) {
|
||||
|
@ -1748,8 +1797,8 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) {
|
|||
getPerfDbMeta(&pSysDbTableMeta, &size);
|
||||
p->info.rows = buildDbTableInfoBlock(p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB);
|
||||
|
||||
relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
|
||||
pInfo->pRes->info.rows = p->info.rows;
|
||||
relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
|
||||
blockDataDestroy(p);
|
||||
|
||||
return pInfo->pRes->info.rows;
|
||||
|
|
|
@ -808,11 +808,31 @@ static void removeResult(SArray* pUpdated, TSKEY key) {
|
|||
static void removeResults(SArray* pWins, SArray* pUpdated) {
|
||||
int32_t size = taosArrayGetSize(pWins);
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
STimeWindow* pW = taosArrayGet(pWins, i);
|
||||
removeResult(pUpdated, pW->skey);
|
||||
SWinRes* pW = taosArrayGet(pWins, i);
|
||||
removeResult(pUpdated, pW->ts);
|
||||
}
|
||||
}
|
||||
|
||||
int64_t getWinReskey(void* data, int32_t index) {
|
||||
SArray* res = (SArray*)data;
|
||||
SWinRes* pos = taosArrayGet(res, index);
|
||||
return pos->ts;
|
||||
}
|
||||
|
||||
static void removeDeleteResults(SArray* pUpdated, SArray* pDelWins) {
|
||||
int32_t upSize = taosArrayGetSize(pUpdated);
|
||||
int32_t delSize = taosArrayGetSize(pDelWins);
|
||||
for (int32_t i = 0; i < upSize; i++) {
|
||||
SResKeyPos* pResKey = taosArrayGetP(pUpdated, i);
|
||||
int64_t key = *(int64_t*)pResKey->key;
|
||||
int32_t index = binarySearch(pDelWins, delSize, key, TSDB_ORDER_DESC, getWinReskey);
|
||||
if (index >= 0 && key == getWinReskey(pDelWins, index)) {
|
||||
taosArrayRemove(pDelWins, index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) {
|
||||
ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
|
||||
return pSup->maxTs != INT64_MIN && ts < pSup->maxTs - pSup->waterMark;
|
||||
|
@ -1264,6 +1284,38 @@ bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t
|
|||
return true;
|
||||
}
|
||||
|
||||
bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) {
|
||||
size_t bytes = sizeof(TSKEY);
|
||||
SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId);
|
||||
SResultRowPosition* p1 =
|
||||
(SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
|
||||
if (!p1) {
|
||||
// window has been closed
|
||||
return false;
|
||||
}
|
||||
SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId);
|
||||
// dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage);
|
||||
taosHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
|
||||
return true;
|
||||
}
|
||||
|
||||
void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pUpWins, SInterval* pInterval) {
|
||||
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
|
||||
TSKEY* tsStarts = (TSKEY*)pStartCol->pData;
|
||||
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
|
||||
uint64_t* groupIds = (uint64_t*)pGroupCol->pData;
|
||||
for (int32_t i = 0; i < pBlock->info.rows; i++) {
|
||||
SResultRowInfo dumyInfo;
|
||||
dumyInfo.cur.pageId = -1;
|
||||
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, pInterval->precision, NULL);
|
||||
doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]);
|
||||
if (pUpWins) {
|
||||
SWinRes winRes = {.ts = win.skey, .groupId = groupIds[i]};
|
||||
taosArrayPush(pUpWins, &winRes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t tsIndex,
|
||||
int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) {
|
||||
SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, tsIndex);
|
||||
|
@ -1279,13 +1331,11 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval*
|
|||
dumyInfo.cur.pageId = -1;
|
||||
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, NULL);
|
||||
step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
|
||||
uint64_t groupId = pBlock->info.groupId;
|
||||
if (pGpDatas) {
|
||||
groupId = pGpDatas[i];
|
||||
}
|
||||
bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), groupId, numOfOutput);
|
||||
uint64_t winGpId = pGpDatas ? pGpDatas[i] : pBlock->info.groupId;
|
||||
bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), winGpId, numOfOutput);
|
||||
if (pUpWins && res) {
|
||||
taosArrayPush(pUpWins, &win);
|
||||
SWinRes winRes = {.ts = win.skey, .groupId = winGpId};
|
||||
taosArrayPush(pUpWins, &winRes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1307,8 +1357,9 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
|
||||
SHashObj* pPullDataMap, SArray* closeWins) {
|
||||
static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
|
||||
SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins,
|
||||
SArray* pRecyPages, SDiskbasedBuf* pDiscBuf) {
|
||||
void* pIte = NULL;
|
||||
size_t keyLen = 0;
|
||||
while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
|
||||
|
@ -1342,6 +1393,11 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
|
|||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
ASSERT(pRecyPages != NULL);
|
||||
taosArrayPush(pRecyPages, &pPos->pageId);
|
||||
} else {
|
||||
SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId);
|
||||
// dBufSetBufPageRecycled(pDiscBuf, bufPage);
|
||||
}
|
||||
char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
|
||||
SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
|
||||
|
@ -1358,7 +1414,38 @@ static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) {
|
|||
SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
|
||||
ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);
|
||||
pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
|
||||
closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL);
|
||||
closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup,
|
||||
&pChInfo->interval, NULL, NULL, NULL, pChInfo->aggSup.pResultBuf);
|
||||
}
|
||||
}
|
||||
|
||||
static void freeAllPages(SArray* pageIds, SDiskbasedBuf* pDiskBuf) {
|
||||
int32_t size = taosArrayGetSize(pageIds);
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i);
|
||||
SFilePage* bufPage = getBufPage(pDiskBuf, pageId);
|
||||
// dBufSetBufPageRecycled(pDiskBuf, bufPage);
|
||||
}
|
||||
taosArrayClear(pageIds);
|
||||
}
|
||||
|
||||
static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlock) {
|
||||
blockDataCleanup(pBlock);
|
||||
int32_t size = taosArrayGetSize(pWins);
|
||||
if (*index == size) {
|
||||
*index = 0;
|
||||
taosArrayClear(pWins);
|
||||
return;
|
||||
}
|
||||
blockDataEnsureCapacity(pBlock, size - *index);
|
||||
SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
|
||||
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, DELETE_GROUPID_COLUMN_INDEX);
|
||||
for (int32_t i = *index; i < size; i++) {
|
||||
SWinRes* pWin = taosArrayGet(pWins, i);
|
||||
colDataAppend(pTsCol, pBlock->info.rows, (const char*)&pWin->ts, false);
|
||||
colDataAppend(pGroupCol, pBlock->info.rows, (const char*)&pWin->groupId, false);
|
||||
pBlock->info.rows++;
|
||||
(*index)++;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1374,27 +1461,37 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
|
|||
}
|
||||
|
||||
if (pOperator->status == OP_RES_TO_RETURN) {
|
||||
doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex ,pInfo->pDelRes);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
return pInfo->pDelRes;
|
||||
}
|
||||
|
||||
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
|
||||
if (pInfo->binfo.pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
|
||||
pOperator->status = OP_EXEC_DONE;
|
||||
freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
|
||||
}
|
||||
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
|
||||
}
|
||||
|
||||
SOperatorInfo* downstream = pOperator->pDownstream[0];
|
||||
|
||||
SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);
|
||||
SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos
|
||||
while (1) {
|
||||
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
|
||||
if (pBlock == NULL) {
|
||||
break;
|
||||
}
|
||||
printDataBlock(pBlock, "single interval recv");
|
||||
|
||||
if (pBlock->info.type == STREAM_CLEAR) {
|
||||
doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock,
|
||||
NULL);
|
||||
doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, 0,
|
||||
pOperator->exprSupp.numOfExprs, pBlock, NULL);
|
||||
qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
|
||||
continue;
|
||||
} if (pBlock->info.type == STREAM_DELETE_DATA) {
|
||||
doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval);
|
||||
continue;
|
||||
} else if (pBlock->info.type == STREAM_GET_ALL) {
|
||||
getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
|
||||
continue;
|
||||
|
@ -1416,14 +1513,19 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
|
|||
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
|
||||
hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated);
|
||||
}
|
||||
closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdated);
|
||||
pOperator->status = OP_RES_TO_RETURN;
|
||||
closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup,
|
||||
&pInfo->interval, NULL, pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
|
||||
|
||||
finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
|
||||
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
|
||||
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
|
||||
removeDeleteResults(pUpdated, pInfo->pDelWins);
|
||||
doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
return pInfo->pDelRes;
|
||||
}
|
||||
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
|
||||
|
||||
pOperator->status = OP_RES_TO_RETURN;
|
||||
printDataBlock(pInfo->binfo.pRes, "single interval");
|
||||
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
|
||||
}
|
||||
|
@ -1438,6 +1540,7 @@ void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) {
|
|||
SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
|
||||
cleanupBasicInfo(&pInfo->binfo);
|
||||
cleanupAggSup(&pInfo->aggSup);
|
||||
taosArrayDestroy(pInfo->pRecycledPages);
|
||||
}
|
||||
|
||||
void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
|
||||
|
@ -1448,12 +1551,13 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
|
|||
taosHashCleanup(pInfo->pPullDataMap);
|
||||
taosArrayDestroy(pInfo->pPullWins);
|
||||
blockDataDestroy(pInfo->pPullDataRes);
|
||||
taosArrayDestroy(pInfo->pRecycledPages);
|
||||
|
||||
if (pInfo->pChildren) {
|
||||
int32_t size = taosArrayGetSize(pInfo->pChildren);
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
|
||||
destroyIntervalOperatorInfo(pChildOp->info, numOfOutput);
|
||||
destroyStreamFinalIntervalOperatorInfo(pChildOp->info, numOfOutput);
|
||||
taosMemoryFreeClear(pChildOp->info);
|
||||
taosMemoryFreeClear(pChildOp);
|
||||
}
|
||||
|
@ -1520,6 +1624,28 @@ void increaseTs(SqlFunctionCtx* pCtx) {
|
|||
}
|
||||
}
|
||||
|
||||
SSDataBlock* createDeleteBlock() {
|
||||
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
|
||||
pBlock->info.hasVarCol = false;
|
||||
pBlock->info.groupId = 0;
|
||||
pBlock->info.rows = 0;
|
||||
pBlock->info.type = STREAM_DELETE_RESULT;
|
||||
pBlock->info.rowSize = sizeof(TSKEY) + sizeof(uint64_t);
|
||||
|
||||
pBlock->pDataBlock = taosArrayInit(2, sizeof(SColumnInfoData));
|
||||
SColumnInfoData infoData = {0};
|
||||
infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP;
|
||||
infoData.info.bytes = sizeof(TSKEY);
|
||||
// window start ts
|
||||
taosArrayPush(pBlock->pDataBlock, &infoData);
|
||||
|
||||
infoData.info.type = TSDB_DATA_TYPE_UBIGINT;
|
||||
infoData.info.bytes = sizeof(uint64_t);
|
||||
taosArrayPush(pBlock->pDataBlock, &infoData);
|
||||
|
||||
return pBlock;
|
||||
}
|
||||
|
||||
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
|
||||
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
|
||||
STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode,
|
||||
|
@ -1573,6 +1699,12 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
|
|||
goto _error;
|
||||
}
|
||||
}
|
||||
pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
|
||||
pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes));
|
||||
pInfo->delIndex = 0;
|
||||
// pInfo->pDelRes = createDeleteBlock(); todo(liuyao) for delete
|
||||
pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete
|
||||
pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete
|
||||
|
||||
initResultRowInfo(&pInfo->binfo.resultRowInfo);
|
||||
|
||||
|
@ -2219,28 +2351,44 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3
|
|||
}
|
||||
}
|
||||
|
||||
static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray,
|
||||
int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
|
||||
bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) {
|
||||
int32_t bytes = sizeof(TSKEY);
|
||||
SET_RES_WINDOW_KEY(pSup->keyBuf, &ts, bytes, groupId);
|
||||
SResultRowPosition* p1 =
|
||||
(SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
|
||||
return p1 != NULL;
|
||||
}
|
||||
|
||||
static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup,
|
||||
SArray* pWinArray, int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SArray* pUpdated) {
|
||||
int32_t size = taosArrayGetSize(pWinArray);
|
||||
if (!pInfo->pChildren) {
|
||||
return;
|
||||
}
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
STimeWindow* pParentWin = taosArrayGet(pWinArray, i);
|
||||
SWinRes* pWinRes = taosArrayGet(pWinArray, i);
|
||||
SResultRow* pCurResult = NULL;
|
||||
setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, pParentWin, true, &pCurResult, 0, pSup->pCtx, numOfOutput,
|
||||
STimeWindow ParentWin = {.skey = pWinRes->ts, .ekey = pWinRes->ts+1};
|
||||
setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &ParentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx, numOfOutput,
|
||||
pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
|
||||
int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
|
||||
bool find = true;
|
||||
for (int32_t j = 0; j < numOfChildren; j++) {
|
||||
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j);
|
||||
SIntervalAggOperatorInfo* pChInfo = pChildOp->info;
|
||||
SExprSupp* pChildSup = &pChildOp->exprSupp;
|
||||
|
||||
if (!hasIntervalWindow(&pChInfo->aggSup, pWinRes->ts, pWinRes->groupId)) {
|
||||
continue;
|
||||
}
|
||||
find = true;
|
||||
SResultRow* pChResult = NULL;
|
||||
setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, pParentWin, true, &pChResult, 0, pChildSup->pCtx,
|
||||
setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &ParentWin, true, &pChResult, pWinRes->groupId, pChildSup->pCtx,
|
||||
pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup, pTaskInfo);
|
||||
compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo);
|
||||
}
|
||||
if (find && pUpdated) {
|
||||
saveResultRow(pCurResult, pWinRes->groupId, pUpdated);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -2472,6 +2620,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
|
|||
if (!IS_FINAL_OP(pInfo)) {
|
||||
// semi interval operator clear disk buffer
|
||||
clearStreamIntervalOperator(pInfo);
|
||||
} else {
|
||||
freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
@ -2497,12 +2647,19 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
|
|||
printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
|
||||
return pInfo->pPullDataRes;
|
||||
}
|
||||
doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
|
||||
if (pInfo->pDelRes->info.rows != 0) {
|
||||
// process the rest of the data
|
||||
printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
|
||||
return pInfo->pDelRes;
|
||||
}
|
||||
}
|
||||
|
||||
while (1) {
|
||||
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
|
||||
if (pBlock == NULL) {
|
||||
clearSpecialDataBlock(pInfo->pUpdateRes);
|
||||
removeDeleteResults(pUpdated, pInfo->pDelWins);
|
||||
pOperator->status = OP_RES_TO_RETURN;
|
||||
qInfo("Stream Final Interval return data");
|
||||
break;
|
||||
|
@ -2514,7 +2671,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
|
|||
pBlock->info.type == STREAM_INVALID) {
|
||||
pInfo->binfo.pRes->info.type = pBlock->info.type;
|
||||
} else if (pBlock->info.type == STREAM_CLEAR) {
|
||||
SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
|
||||
SArray* pUpWins = taosArrayInit(8, sizeof(SWinRes));
|
||||
doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pInfo->primaryTsIndex, pOperator->exprSupp.numOfExprs,
|
||||
pBlock, pUpWins);
|
||||
if (IS_FINAL_OP(pInfo)) {
|
||||
|
@ -2525,8 +2682,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
|
|||
|
||||
doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildInfo->primaryTsIndex,
|
||||
pChildSup->numOfExprs, pBlock, NULL);
|
||||
rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs,
|
||||
pOperator->pTaskInfo);
|
||||
rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId,
|
||||
pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, NULL);
|
||||
taosArrayDestroy(pUpWins);
|
||||
continue;
|
||||
}
|
||||
|
@ -2535,11 +2692,25 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
|
|||
pInfo->returnUpdate = true;
|
||||
taosArrayDestroy(pUpWins);
|
||||
break;
|
||||
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
|
||||
doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval);
|
||||
if (IS_FINAL_OP(pInfo)) {
|
||||
int32_t childIndex = getChildIndex(pBlock);
|
||||
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
|
||||
SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
|
||||
SExprSupp* pChildSup = &pChildOp->exprSupp;
|
||||
doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval);
|
||||
rebuildIntervalWindow(pInfo, pSup, pInfo->pDelWins, pInfo->binfo.pRes->info.groupId,
|
||||
pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdated);
|
||||
continue;
|
||||
}
|
||||
removeResults(pInfo->pDelWins, pUpdated);
|
||||
break;
|
||||
} else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
|
||||
getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
|
||||
continue;
|
||||
} else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
|
||||
SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
|
||||
SArray* pUpWins = taosArrayInit(8, sizeof(SWinRes));
|
||||
doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
|
||||
removeResults(pUpWins, pUpdated);
|
||||
taosArrayDestroy(pUpWins);
|
||||
|
@ -2563,6 +2734,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
|
|||
if (!pChildOp) {
|
||||
longjmp(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
SStreamFinalIntervalOperatorInfo* pTmpInfo = pChildOp->info;
|
||||
pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
|
||||
taosArrayPush(pInfo->pChildren, &pChildOp);
|
||||
}
|
||||
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
|
||||
|
@ -2578,8 +2751,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
|
|||
|
||||
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
|
||||
if (IS_FINAL_OP(pInfo)) {
|
||||
closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap,
|
||||
pUpdated);
|
||||
closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup,
|
||||
&pInfo->interval, pInfo->pPullDataMap, pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
|
||||
closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
|
||||
}
|
||||
|
||||
|
@ -2607,6 +2780,13 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
|
|||
printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
|
||||
return pInfo->pPullDataRes;
|
||||
}
|
||||
|
||||
doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
|
||||
if (pInfo->pDelRes->info.rows != 0) {
|
||||
// process the rest of the data
|
||||
printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
|
||||
return pInfo->pDelRes;
|
||||
}
|
||||
// ASSERT(false);
|
||||
return NULL;
|
||||
}
|
||||
|
@ -2680,6 +2860,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
|
|||
for (int32_t i = 0; i < numOfChild; i++) {
|
||||
SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
|
||||
if (pChildOp) {
|
||||
SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
|
||||
pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
|
||||
taosArrayPush(pInfo->pChildren, &pChildOp);
|
||||
continue;
|
||||
}
|
||||
|
@ -2711,6 +2893,11 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
|
|||
pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
|
||||
pInfo->pPullDataRes = createPullDataBlock();
|
||||
pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
|
||||
// pInfo->pDelRes = createDeleteBlock(); // todo(liuyao) for delete
|
||||
pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete
|
||||
pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete
|
||||
pInfo->delIndex = 0;
|
||||
pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes));
|
||||
|
||||
pOperator->operatorType = pPhyNode->type;
|
||||
pOperator->blocking = true;
|
||||
|
@ -2851,9 +3038,9 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
|
|||
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
|
||||
pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
|
||||
pInfo->pDelIterator = NULL;
|
||||
pInfo->pDelRes = createOneDataBlock(pResBlock, false);
|
||||
pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
|
||||
blockDataEnsureCapacity(pInfo->pDelRes, 64);
|
||||
// pInfo->pDelRes = createDeleteBlock(); // todo(liuyao) for delete
|
||||
pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete
|
||||
pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete
|
||||
pInfo->pChildren = NULL;
|
||||
pInfo->isFinal = false;
|
||||
pInfo->pPhyNode = pPhyNode;
|
||||
|
@ -3205,6 +3392,11 @@ static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) {
|
|||
|
||||
void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
|
||||
blockDataCleanup(pBlock);
|
||||
int32_t size = taosHashGetSize(pStDeleted);
|
||||
if (size == 0) {
|
||||
return;
|
||||
}
|
||||
blockDataEnsureCapacity(pBlock, size);
|
||||
size_t keyLen = 0;
|
||||
while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
|
||||
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);
|
||||
|
@ -3979,9 +4171,9 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
|
|||
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
|
||||
pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
|
||||
pInfo->pDelIterator = NULL;
|
||||
pInfo->pDelRes = createOneDataBlock(pResBlock, false);
|
||||
pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
|
||||
blockDataEnsureCapacity(pInfo->pDelRes, 64);
|
||||
// pInfo->pDelRes = createDeleteBlock(); // todo(liuyao) for delete
|
||||
pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete
|
||||
pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete
|
||||
pInfo->pChildren = NULL;
|
||||
pInfo->ignoreExpiredData = pStateNode->window.igExpired;
|
||||
|
||||
|
|
|
@ -15,10 +15,10 @@
|
|||
|
||||
#include "builtins.h"
|
||||
#include "builtinsimpl.h"
|
||||
#include "cJSON.h"
|
||||
#include "querynodes.h"
|
||||
#include "scalar.h"
|
||||
#include "taoserror.h"
|
||||
#include "cJSON.h"
|
||||
|
||||
static int32_t buildFuncErrMsg(char* pErrBuf, int32_t len, int32_t errCode, const char* pFormat, ...) {
|
||||
va_list vArgList;
|
||||
|
@ -40,7 +40,7 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char
|
|||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName);
|
||||
}
|
||||
|
||||
#define TIME_UNIT_INVALID 1
|
||||
#define TIME_UNIT_INVALID 1
|
||||
#define TIME_UNIT_TOO_SMALL 2
|
||||
|
||||
static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) {
|
||||
|
@ -48,14 +48,19 @@ static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) {
|
|||
return TIME_UNIT_INVALID;
|
||||
}
|
||||
|
||||
if (TSDB_TIME_PRECISION_MILLI == dbPrec && 0 == strcasecmp(pVal->literal, "1u")) {
|
||||
if (TSDB_TIME_PRECISION_MILLI == dbPrec && (0 == strcasecmp(pVal->literal, "1u") ||
|
||||
0 == strcasecmp(pVal->literal, "1b"))) {
|
||||
return TIME_UNIT_TOO_SMALL;
|
||||
}
|
||||
|
||||
if (TSDB_TIME_PRECISION_MICRO == dbPrec && 0 == strcasecmp(pVal->literal, "1b")) {
|
||||
return TIME_UNIT_TOO_SMALL;
|
||||
}
|
||||
|
||||
if (pVal->literal[0] != '1' || (pVal->literal[1] != 'u' && pVal->literal[1] != 'a' &&
|
||||
pVal->literal[1] != 's' && pVal->literal[1] != 'm' &&
|
||||
pVal->literal[1] != 'h' && pVal->literal[1] != 'd' &&
|
||||
pVal->literal[1] != 'w')) {
|
||||
pVal->literal[1] != 'w' && pVal->literal[1] != 'b')) {
|
||||
return TIME_UNIT_INVALID;
|
||||
}
|
||||
|
||||
|
@ -600,7 +605,7 @@ static int32_t translateTopBot(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
|
|||
}
|
||||
|
||||
SValueNode* pValue = (SValueNode*)pParamNode1;
|
||||
if (pValue->node.resType.type != TSDB_DATA_TYPE_BIGINT) {
|
||||
if (!IS_INTEGER_TYPE(pValue->node.resType.type)) {
|
||||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
|
@ -696,13 +701,13 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len
|
|||
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
|
||||
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 1));
|
||||
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1));
|
||||
if (ret == TIME_UNIT_TOO_SMALL) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"ELAPSED function time unit parameter should be greater than db precision");
|
||||
} else if (ret == TIME_UNIT_INVALID) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"ELAPSED function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
|
||||
"ELAPSED function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -815,13 +820,13 @@ static int8_t validateHistogramBinType(char* binTypeStr) {
|
|||
}
|
||||
|
||||
static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* errMsg, int32_t msgLen) {
|
||||
const char *msg1 = "HISTOGRAM function requires four parameters";
|
||||
const char *msg3 = "HISTOGRAM function invalid format for binDesc parameter";
|
||||
const char *msg4 = "HISTOGRAM function binDesc parameter \"count\" should be in range [1, 1000]";
|
||||
const char *msg5 = "HISTOGRAM function bin/parameter should be in range [-DBL_MAX, DBL_MAX]";
|
||||
const char *msg6 = "HISTOGRAM function binDesc parameter \"width\" cannot be 0";
|
||||
const char *msg7 = "HISTOGRAM function binDesc parameter \"start\" cannot be 0 with \"log_bin\" type";
|
||||
const char *msg8 = "HISTOGRAM function binDesc parameter \"factor\" cannot be negative or equal to 0/1";
|
||||
const char* msg1 = "HISTOGRAM function requires four parameters";
|
||||
const char* msg3 = "HISTOGRAM function invalid format for binDesc parameter";
|
||||
const char* msg4 = "HISTOGRAM function binDesc parameter \"count\" should be in range [1, 1000]";
|
||||
const char* msg5 = "HISTOGRAM function bin/parameter should be in range [-DBL_MAX, DBL_MAX]";
|
||||
const char* msg6 = "HISTOGRAM function binDesc parameter \"width\" cannot be 0";
|
||||
const char* msg7 = "HISTOGRAM function binDesc parameter \"start\" cannot be 0 with \"log_bin\" type";
|
||||
const char* msg8 = "HISTOGRAM function binDesc parameter \"factor\" cannot be negative or equal to 0/1";
|
||||
|
||||
cJSON* binDesc = cJSON_Parse(binDescStr);
|
||||
int32_t numOfBins;
|
||||
|
@ -1004,8 +1009,8 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
|
|||
}
|
||||
|
||||
if (i == 3 && pValue->datum.i != 1 && pValue->datum.i != 0) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"HISTOGRAM function normalized parameter should be 0/1");
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"HISTOGRAM function normalized parameter should be 0/1");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1062,8 +1067,8 @@ static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32
|
|||
}
|
||||
|
||||
if (i == 3 && pValue->datum.i != 1 && pValue->datum.i != 0) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"HISTOGRAM function normalized parameter should be 0/1");
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"HISTOGRAM function normalized parameter should be 0/1");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1218,13 +1223,13 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32
|
|||
if (numOfParams == 4) {
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
|
||||
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 3));
|
||||
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 3));
|
||||
if (ret == TIME_UNIT_TOO_SMALL) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"STATEDURATION function time unit parameter should be greater than db precision");
|
||||
} else if (ret == TIME_UNIT_INVALID) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"STATEDURATION function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
|
||||
"STATEDURATION function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1432,10 +1437,6 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l
|
|||
|
||||
static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
|
||||
// first(col_list) will be rewritten as first(col)
|
||||
if (2 != LIST_LENGTH(pFunc->pParameterList)) { // input has two params c0,ts, is this a bug?
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
|
||||
uint8_t paraType = ((SExprNode*)pPara)->resType.type;
|
||||
int32_t paraBytes = ((SExprNode*)pPara)->resType.bytes;
|
||||
|
@ -1733,13 +1734,13 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
|
|||
// add database precision as param
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
|
||||
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 1));
|
||||
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1));
|
||||
if (ret == TIME_UNIT_TOO_SMALL) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"TIMETRUNCATE function time unit parameter should be greater than db precision");
|
||||
} else if (ret == TIME_UNIT_INVALID) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"TIMETRUNCATE function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
|
||||
"TIMETRUNCATE function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
|
||||
}
|
||||
|
||||
addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
|
||||
|
@ -1772,13 +1773,13 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
|
|||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
|
||||
if (3 == numOfParams) {
|
||||
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 2));
|
||||
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2));
|
||||
if (ret == TIME_UNIT_TOO_SMALL) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"TIMEDIFF function time unit parameter should be greater than db precision");
|
||||
} else if (ret == TIME_UNIT_INVALID) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"TIMEDIFF function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
|
||||
"TIMEDIFF function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -90,12 +90,14 @@ typedef struct SStddevRes {
|
|||
double result;
|
||||
int64_t count;
|
||||
union {
|
||||
double quadraticDSum;
|
||||
int64_t quadraticISum;
|
||||
double quadraticDSum;
|
||||
int64_t quadraticISum;
|
||||
uint64_t quadraticUSum;
|
||||
};
|
||||
union {
|
||||
double dsum;
|
||||
int64_t isum;
|
||||
double dsum;
|
||||
int64_t isum;
|
||||
uint64_t usum;
|
||||
};
|
||||
int16_t type;
|
||||
} SStddevRes;
|
||||
|
@ -1729,6 +1731,68 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
|
|||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_UTINYINT: {
|
||||
uint8_t* plist = (uint8_t*)pCol->pData;
|
||||
for (int32_t i = start; i < numOfRows + start; ++i) {
|
||||
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
numOfElem += 1;
|
||||
pStddevRes->count += 1;
|
||||
pStddevRes->usum += plist[i];
|
||||
pStddevRes->quadraticISum += plist[i] * plist[i];
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_USMALLINT: {
|
||||
uint16_t* plist = (uint16_t*)pCol->pData;
|
||||
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
|
||||
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
numOfElem += 1;
|
||||
pStddevRes->count += 1;
|
||||
pStddevRes->usum += plist[i];
|
||||
pStddevRes->quadraticISum += plist[i] * plist[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_UINT: {
|
||||
uint32_t* plist = (uint32_t*)pCol->pData;
|
||||
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
|
||||
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
numOfElem += 1;
|
||||
pStddevRes->count += 1;
|
||||
pStddevRes->usum += plist[i];
|
||||
pStddevRes->quadraticISum += plist[i] * plist[i];
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_UBIGINT: {
|
||||
uint64_t* plist = (uint64_t*)pCol->pData;
|
||||
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
|
||||
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
numOfElem += 1;
|
||||
pStddevRes->count += 1;
|
||||
pStddevRes->usum += plist[i];
|
||||
pStddevRes->quadraticISum += plist[i] * plist[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_FLOAT: {
|
||||
float* plist = (float*)pCol->pData;
|
||||
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
|
||||
|
@ -1771,9 +1835,12 @@ _stddev_over:
|
|||
|
||||
static void stddevTransferInfo(SStddevRes* pInput, SStddevRes* pOutput) {
|
||||
pOutput->type = pInput->type;
|
||||
if (IS_INTEGER_TYPE(pOutput->type)) {
|
||||
if (IS_SIGNED_NUMERIC_TYPE(pOutput->type)) {
|
||||
pOutput->quadraticISum += pInput->quadraticISum;
|
||||
pOutput->isum += pInput->isum;
|
||||
} else if (IS_UNSIGNED_NUMERIC_TYPE(pOutput->type)) {
|
||||
pOutput->quadraticUSum += pInput->quadraticUSum;
|
||||
pOutput->usum += pInput->usum;
|
||||
} else {
|
||||
pOutput->quadraticDSum += pInput->quadraticDSum;
|
||||
pOutput->dsum += pInput->dsum;
|
||||
|
@ -1848,6 +1915,22 @@ int32_t stddevInvertFunction(SqlFunctionCtx* pCtx) {
|
|||
LIST_STDDEV_SUB_N(pStddevRes->isum, int64_t);
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_UTINYINT: {
|
||||
LIST_STDDEV_SUB_N(pStddevRes->isum, uint8_t);
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_USMALLINT: {
|
||||
LIST_STDDEV_SUB_N(pStddevRes->isum, uint16_t);
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_UINT: {
|
||||
LIST_STDDEV_SUB_N(pStddevRes->isum, uint32_t);
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_UBIGINT: {
|
||||
LIST_STDDEV_SUB_N(pStddevRes->isum, uint64_t);
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_FLOAT: {
|
||||
LIST_STDDEV_SUB_N(pStddevRes->dsum, float);
|
||||
break;
|
||||
|
@ -1871,9 +1954,12 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
int32_t type = pStddevRes->type;
|
||||
double avg;
|
||||
|
||||
if (IS_INTEGER_TYPE(type)) {
|
||||
if (IS_SIGNED_NUMERIC_TYPE(type)) {
|
||||
avg = pStddevRes->isum / ((double)pStddevRes->count);
|
||||
pStddevRes->result = sqrt(fabs(pStddevRes->quadraticISum / ((double)pStddevRes->count) - avg * avg));
|
||||
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
|
||||
avg = pStddevRes->usum / ((double)pStddevRes->count);
|
||||
pStddevRes->result = sqrt(fabs(pStddevRes->quadraticUSum / ((double)pStddevRes->count) - avg * avg));
|
||||
} else {
|
||||
avg = pStddevRes->dsum / ((double)pStddevRes->count);
|
||||
pStddevRes->result = sqrt(fabs(pStddevRes->quadraticDSum / ((double)pStddevRes->count) - avg * avg));
|
||||
|
@ -1913,9 +1999,12 @@ int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
|
|||
SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
|
||||
SStddevRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
|
||||
|
||||
if (IS_INTEGER_TYPE(type)) {
|
||||
if (IS_SIGNED_NUMERIC_TYPE(type)) {
|
||||
pDBuf->isum += pSBuf->isum;
|
||||
pDBuf->quadraticISum += pSBuf->quadraticISum;
|
||||
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
|
||||
pDBuf->usum += pSBuf->usum;
|
||||
pDBuf->quadraticUSum += pSBuf->quadraticUSum;
|
||||
} else {
|
||||
pDBuf->dsum += pSBuf->dsum;
|
||||
pDBuf->quadraticDSum += pSBuf->quadraticDSum;
|
||||
|
|
|
@ -1565,6 +1565,10 @@ void constructUdfService(void *argsThread) {
|
|||
//TODO return value of uv_run
|
||||
uv_run(&udfc->uvLoop, UV_RUN_DEFAULT);
|
||||
uv_loop_close(&udfc->uvLoop);
|
||||
|
||||
uv_walk(&udfc->uvLoop, udfUdfdCloseWalkCb, NULL);
|
||||
uv_run(&udfc->uvLoop, UV_RUN_DEFAULT);
|
||||
uv_loop_close(&udfc->uvLoop);
|
||||
}
|
||||
|
||||
int32_t udfcOpen() {
|
||||
|
|
|
@ -65,9 +65,10 @@ void indexInit() {
|
|||
indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index");
|
||||
indexRefMgt = taosOpenRef(10, indexDestroy);
|
||||
}
|
||||
void indexCleanUp() {
|
||||
void indexCleanup() {
|
||||
// refacto later
|
||||
taosCleanUpScheduler(indexQhandle);
|
||||
taosCloseRef(indexRefMgt);
|
||||
}
|
||||
|
||||
typedef struct SIdxColInfo {
|
||||
|
|
|
@ -165,7 +165,7 @@ static int32_t valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
|
|||
memcpy(pDst->datum.p, pSrc->datum.p, len);
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_JSON:{
|
||||
case TSDB_DATA_TYPE_JSON: {
|
||||
int32_t len = getJsonValueLen(pSrc->datum.p);
|
||||
pDst->datum.p = taosMemoryCalloc(1, len);
|
||||
if (NULL == pDst->datum.p) {
|
||||
|
@ -368,6 +368,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
|
|||
static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) {
|
||||
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
|
||||
COPY_SCALAR_FIELD(joinType);
|
||||
CLONE_NODE_FIELD(pMergeCondition);
|
||||
CLONE_NODE_FIELD(pOnConditions);
|
||||
COPY_SCALAR_FIELD(isSingleTableJoin);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
@ -396,6 +397,7 @@ static int32_t logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModi
|
|||
COPY_SCALAR_FIELD(tableType);
|
||||
COPY_CHAR_ARRAY_FIELD(tableFName);
|
||||
COPY_OBJECT_FIELD(deleteTimeRange, sizeof(STimeWindow));
|
||||
CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
|
@ -19,8 +19,8 @@
|
|||
#include "query.h"
|
||||
#include "querynodes.h"
|
||||
#include "taoserror.h"
|
||||
#include "tjson.h"
|
||||
#include "tdatablock.h"
|
||||
#include "tjson.h"
|
||||
|
||||
static int32_t nodeToJson(const void* pObj, SJson* pJson);
|
||||
static int32_t jsonToNode(const SJson* pJson, void* pObj);
|
||||
|
@ -179,6 +179,8 @@ const char* nodesNodeName(ENodeType type) {
|
|||
return "ShowVnodeStmt";
|
||||
case QUERY_NODE_DELETE_STMT:
|
||||
return "DeleteStmt";
|
||||
case QUERY_NODE_INSERT_STMT:
|
||||
return "InsertStmt";
|
||||
case QUERY_NODE_LOGIC_PLAN_SCAN:
|
||||
return "LogicScan";
|
||||
case QUERY_NODE_LOGIC_PLAN_JOIN:
|
||||
|
@ -271,6 +273,8 @@ const char* nodesNodeName(ENodeType type) {
|
|||
return "PhysiDispatch";
|
||||
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
|
||||
return "PhysiInsert";
|
||||
case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT:
|
||||
return "PhysiQueryInsert";
|
||||
case QUERY_NODE_PHYSICAL_PLAN_DELETE:
|
||||
return "PhysiDelete";
|
||||
case QUERY_NODE_PHYSICAL_SUBPLAN:
|
||||
|
@ -1254,6 +1258,7 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) {
|
|||
|
||||
static const char* jkJoinLogicPlanJoinType = "JoinType";
|
||||
static const char* jkJoinLogicPlanOnConditions = "OnConditions";
|
||||
static const char* jkJoinLogicPlanMergeCondition = "MergeConditions";
|
||||
|
||||
static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj;
|
||||
|
@ -1262,6 +1267,9 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkJoinLogicPlanJoinType, pNode->joinType);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddObject(pJson, jkJoinLogicPlanMergeCondition, nodeToJson, pNode->pMergeCondition);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions);
|
||||
}
|
||||
|
@ -1617,6 +1625,7 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) {
|
|||
}
|
||||
|
||||
static const char* jkJoinPhysiPlanJoinType = "JoinType";
|
||||
static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition";
|
||||
static const char* jkJoinPhysiPlanOnConditions = "OnConditions";
|
||||
static const char* jkJoinPhysiPlanTargets = "Targets";
|
||||
|
||||
|
@ -1627,6 +1636,9 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanJoinType, pNode->joinType);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddObject(pJson, jkJoinPhysiPlanMergeCondition, nodeToJson, pNode->pMergeCondition);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pOnConditions);
|
||||
}
|
||||
|
@ -1648,6 +1660,9 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOnConditions);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = jsonToNodeObject(pJson, jkJoinPhysiPlanMergeCondition, &pNode->pMergeCondition);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets);
|
||||
}
|
||||
|
@ -2199,6 +2214,58 @@ static int32_t physiDispatchNodeToJson(const void* pObj, SJson* pJson) { return
|
|||
|
||||
static int32_t jsonToPhysiDispatchNode(const SJson* pJson, void* pObj) { return jsonToPhysicDataSinkNode(pJson, pObj); }
|
||||
|
||||
static const char* jkQueryInsertPhysiPlanTableId = "TableId";
|
||||
static const char* jkQueryInsertPhysiPlanTableType = "TableType";
|
||||
static const char* jkQueryInsertPhysiPlanTableFName = "TableFName";
|
||||
static const char* jkQueryInsertPhysiPlanVgId = "VgId";
|
||||
static const char* jkQueryInsertPhysiPlanEpSet = "EpSet";
|
||||
|
||||
static int32_t physiQueryInsertNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const SQueryInserterNode* pNode = (const SQueryInserterNode*)pObj;
|
||||
|
||||
int32_t code = physicDataSinkNodeToJson(pObj, pJson);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkQueryInsertPhysiPlanTableId, pNode->tableId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkQueryInsertPhysiPlanTableType, pNode->tableType);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddStringToObject(pJson, jkQueryInsertPhysiPlanTableFName, pNode->tableFName);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkQueryInsertPhysiPlanVgId, pNode->vgId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddObject(pJson, jkQueryInsertPhysiPlanEpSet, epSetToJson, &pNode->epSet);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t jsonToPhysiQueryInsertNode(const SJson* pJson, void* pObj) {
|
||||
SQueryInserterNode* pNode = (SQueryInserterNode*)pObj;
|
||||
|
||||
int32_t code = jsonToPhysicDataSinkNode(pJson, pObj);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetUBigIntValue(pJson, jkQueryInsertPhysiPlanTableId, &pNode->tableId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetTinyIntValue(pJson, jkQueryInsertPhysiPlanTableType, &pNode->tableType);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetStringValue(pJson, jkQueryInsertPhysiPlanTableFName, pNode->tableFName);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetIntValue(pJson, jkQueryInsertPhysiPlanVgId, &pNode->vgId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonToObject(pJson, jkQueryInsertPhysiPlanEpSet, jsonToEpSet, &pNode->epSet);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
static const char* jkDeletePhysiPlanTableId = "TableId";
|
||||
static const char* jkDeletePhysiPlanTableType = "TableType";
|
||||
static const char* jkDeletePhysiPlanTableFName = "TableFName";
|
||||
|
@ -2630,9 +2697,9 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) {
|
|||
case TSDB_DATA_TYPE_VARBINARY:
|
||||
code = tjsonAddStringToObject(pJson, jkValueDatum, varDataVal(pNode->datum.p));
|
||||
break;
|
||||
case TSDB_DATA_TYPE_JSON:{
|
||||
case TSDB_DATA_TYPE_JSON: {
|
||||
int32_t len = getJsonValueLen(pNode->datum.p);
|
||||
char* buf = taosMemoryCalloc( len * 2 + 1, sizeof(char));
|
||||
char* buf = taosMemoryCalloc(len * 2 + 1, sizeof(char));
|
||||
code = taosHexEncode(pNode->datum.p, buf, len);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
taosMemoryFree(buf);
|
||||
|
@ -2764,7 +2831,7 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) {
|
|||
}
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_JSON:{
|
||||
case TSDB_DATA_TYPE_JSON: {
|
||||
pNode->datum.p = taosMemoryCalloc(1, pNode->node.resType.bytes);
|
||||
if (NULL == pNode->datum.p) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
@ -4221,6 +4288,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
|
|||
return physiDispatchNodeToJson(pObj, pJson);
|
||||
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
|
||||
break;
|
||||
case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT:
|
||||
return physiQueryInsertNodeToJson(pObj, pJson);
|
||||
case QUERY_NODE_PHYSICAL_PLAN_DELETE:
|
||||
return physiDeleteNodeToJson(pObj, pJson);
|
||||
case QUERY_NODE_PHYSICAL_SUBPLAN:
|
||||
|
@ -4363,6 +4432,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
|
|||
return jsonToPhysiInterpFuncNode(pJson, pObj);
|
||||
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
|
||||
return jsonToPhysiDispatchNode(pJson, pObj);
|
||||
case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT:
|
||||
return jsonToPhysiQueryInsertNode(pJson, pObj);
|
||||
case QUERY_NODE_PHYSICAL_PLAN_DELETE:
|
||||
return jsonToPhysiDeleteNode(pJson, pObj);
|
||||
case QUERY_NODE_PHYSICAL_SUBPLAN:
|
||||
|
|
|
@ -470,6 +470,9 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
|
|||
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: {
|
||||
SJoinPhysiNode* pJoin = (SJoinPhysiNode*)pNode;
|
||||
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
|
||||
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
|
||||
res = walkPhysiPlan(pJoin->pMergeCondition, order, walker, pContext);
|
||||
}
|
||||
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
|
||||
res = walkPhysiPlan(pJoin->pOnConditions, order, walker, pContext);
|
||||
}
|
||||
|
|
|
@ -229,6 +229,8 @@ SNode* nodesMakeNode(ENodeType type) {
|
|||
return makeNode(type, sizeof(SKillStmt));
|
||||
case QUERY_NODE_DELETE_STMT:
|
||||
return makeNode(type, sizeof(SDeleteStmt));
|
||||
case QUERY_NODE_INSERT_STMT:
|
||||
return makeNode(type, sizeof(SInsertStmt));
|
||||
case QUERY_NODE_QUERY:
|
||||
return makeNode(type, sizeof(SQuery));
|
||||
case QUERY_NODE_LOGIC_PLAN_SCAN:
|
||||
|
@ -325,6 +327,8 @@ SNode* nodesMakeNode(ENodeType type) {
|
|||
return makeNode(type, sizeof(SDataDispatcherNode));
|
||||
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
|
||||
return makeNode(type, sizeof(SDataInserterNode));
|
||||
case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT:
|
||||
return makeNode(type, sizeof(SQueryInserterNode));
|
||||
case QUERY_NODE_PHYSICAL_PLAN_DELETE:
|
||||
return makeNode(type, sizeof(SDataDeleterNode));
|
||||
case QUERY_NODE_PHYSICAL_SUBPLAN:
|
||||
|
@ -690,6 +694,13 @@ void nodesDestroyNode(SNode* pNode) {
|
|||
nodesDestroyNode(pStmt->pTagCond);
|
||||
break;
|
||||
}
|
||||
case QUERY_NODE_INSERT_STMT: {
|
||||
SInsertStmt* pStmt = (SInsertStmt*)pNode;
|
||||
nodesDestroyNode(pStmt->pTable);
|
||||
nodesDestroyList(pStmt->pCols);
|
||||
nodesDestroyNode(pStmt->pQuery);
|
||||
break;
|
||||
}
|
||||
case QUERY_NODE_QUERY: {
|
||||
SQuery* pQuery = (SQuery*)pNode;
|
||||
nodesDestroyNode(pQuery->pRoot);
|
||||
|
@ -718,6 +729,7 @@ void nodesDestroyNode(SNode* pNode) {
|
|||
case QUERY_NODE_LOGIC_PLAN_JOIN: {
|
||||
SJoinLogicNode* pLogicNode = (SJoinLogicNode*)pNode;
|
||||
destroyLogicNode((SLogicNode*)pLogicNode);
|
||||
nodesDestroyNode(pLogicNode->pMergeCondition);
|
||||
nodesDestroyNode(pLogicNode->pOnConditions);
|
||||
break;
|
||||
}
|
||||
|
@ -828,6 +840,7 @@ void nodesDestroyNode(SNode* pNode) {
|
|||
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: {
|
||||
SJoinPhysiNode* pPhyNode = (SJoinPhysiNode*)pNode;
|
||||
destroyPhysiNode((SPhysiNode*)pPhyNode);
|
||||
nodesDestroyNode(pPhyNode->pMergeCondition);
|
||||
nodesDestroyNode(pPhyNode->pOnConditions);
|
||||
nodesDestroyList(pPhyNode->pTargets);
|
||||
break;
|
||||
|
@ -923,6 +936,11 @@ void nodesDestroyNode(SNode* pNode) {
|
|||
taosMemoryFreeClear(pSink->pData);
|
||||
break;
|
||||
}
|
||||
case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: {
|
||||
SQueryInserterNode* pSink = (SQueryInserterNode*)pNode;
|
||||
destroyDataSinkNode((SDataSinkNode*)pSink);
|
||||
break;
|
||||
}
|
||||
case QUERY_NODE_PHYSICAL_PLAN_DELETE: {
|
||||
SDataDeleterNode* pSink = (SDataDeleterNode*)pNode;
|
||||
destroyDataSinkNode((SDataSinkNode*)pSink);
|
||||
|
@ -1493,6 +1511,37 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char*
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols) {
|
||||
if (NULL == pCols) {
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
SCollectColumnsCxt cxt = {
|
||||
.errCode = TSDB_CODE_SUCCESS,
|
||||
.pTableAlias = pTableAlias,
|
||||
.collectType = type,
|
||||
.pCols = (NULL == *pCols ? nodesMakeList() : *pCols),
|
||||
.pColHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)};
|
||||
if (NULL == cxt.pCols || NULL == cxt.pColHash) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
*pCols = NULL;
|
||||
|
||||
nodesWalkExpr(node, collectColumns, &cxt);
|
||||
|
||||
taosHashCleanup(cxt.pColHash);
|
||||
if (TSDB_CODE_SUCCESS != cxt.errCode) {
|
||||
nodesDestroyList(cxt.pCols);
|
||||
return cxt.errCode;
|
||||
}
|
||||
if (LIST_LENGTH(cxt.pCols) > 0) {
|
||||
*pCols = cxt.pCols;
|
||||
} else {
|
||||
nodesDestroyList(cxt.pCols);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
typedef struct SCollectFuncsCxt {
|
||||
int32_t errCode;
|
||||
FFuncClassifier classifier;
|
||||
|
|
|
@ -210,6 +210,7 @@ SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName);
|
|||
SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName);
|
||||
SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName);
|
||||
SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere);
|
||||
SNode* createInsertStmt(SAstCreateContext* pCxt, SNode* pTable, SNodeList* pCols, SNode* pQuery);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
@ -259,7 +259,7 @@ multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C).
|
|||
|
||||
create_subtable_clause(A) ::=
|
||||
not_exists_opt(B) full_table_name(C) USING full_table_name(D)
|
||||
specific_tags_opt(E) TAGS NK_LP expression_list(F) NK_RP table_options(G). { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); }
|
||||
specific_cols_opt(E) TAGS NK_LP expression_list(F) NK_RP table_options(G). { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); }
|
||||
|
||||
%type multi_drop_clause { SNodeList* }
|
||||
%destructor multi_drop_clause { nodesDestroyList($$); }
|
||||
|
@ -268,10 +268,10 @@ multi_drop_clause(A) ::= multi_drop_clause(B) drop_table_clause(C).
|
|||
|
||||
drop_table_clause(A) ::= exists_opt(B) full_table_name(C). { A = createDropTableClause(pCxt, B, C); }
|
||||
|
||||
%type specific_tags_opt { SNodeList* }
|
||||
%destructor specific_tags_opt { nodesDestroyList($$); }
|
||||
specific_tags_opt(A) ::= . { A = NULL; }
|
||||
specific_tags_opt(A) ::= NK_LP col_name_list(B) NK_RP. { A = B; }
|
||||
%type specific_cols_opt { SNodeList* }
|
||||
%destructor specific_cols_opt { nodesDestroyList($$); }
|
||||
specific_cols_opt(A) ::= . { A = NULL; }
|
||||
specific_cols_opt(A) ::= NK_LP col_name_list(B) NK_RP. { A = B; }
|
||||
|
||||
full_table_name(A) ::= table_name(B). { A = createRealTableNode(pCxt, NULL, &B, NULL); }
|
||||
full_table_name(A) ::= db_name(B) NK_DOT table_name(C). { A = createRealTableNode(pCxt, &B, &C, NULL); }
|
||||
|
@ -515,6 +515,9 @@ cmd ::= DELETE FROM full_table_name(A) where_clause_opt(B).
|
|||
/************************************************ select **************************************************************/
|
||||
cmd ::= query_expression(A). { pCxt->pRootNode = A; }
|
||||
|
||||
/************************************************ insert **************************************************************/
|
||||
cmd ::= INSERT INTO full_table_name(A) specific_cols_opt(B) query_expression(C). { pCxt->pRootNode = createInsertStmt(pCxt, A, B, C); }
|
||||
|
||||
/************************************************ literal *************************************************************/
|
||||
literal(A) ::= NK_INTEGER(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); }
|
||||
literal(A) ::= NK_FLOAT(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B)); }
|
||||
|
@ -973,4 +976,4 @@ null_ordering_opt(A) ::= .
|
|||
null_ordering_opt(A) ::= NULLS FIRST. { A = NULL_ORDER_FIRST; }
|
||||
null_ordering_opt(A) ::= NULLS LAST. { A = NULL_ORDER_LAST; }
|
||||
|
||||
%fallback ID NK_BITNOT INSERT VALUES IMPORT NK_SEMI FILE.
|
||||
%fallback ID NK_BITNOT VALUES IMPORT NK_SEMI FILE.
|
||||
|
|
|
@ -385,6 +385,15 @@ SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType typ
|
|||
|
||||
SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight) {
|
||||
CHECK_PARSER_STATUS(pCxt);
|
||||
if (OP_TYPE_MINUS == type && QUERY_NODE_VALUE == nodeType(pLeft)) {
|
||||
SValueNode* pVal = (SValueNode*)pLeft;
|
||||
char* pNewLiteral = taosMemoryCalloc(1, strlen(pVal->literal) + 2);
|
||||
CHECK_OUT_OF_MEM(pNewLiteral);
|
||||
sprintf(pNewLiteral, "-%s", pVal->literal);
|
||||
taosMemoryFree(pVal->literal);
|
||||
pVal->literal = pNewLiteral;
|
||||
return pLeft;
|
||||
}
|
||||
SOperatorNode* op = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR);
|
||||
CHECK_OUT_OF_MEM(op);
|
||||
op->opType = type;
|
||||
|
@ -1653,3 +1662,13 @@ SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere) {
|
|||
}
|
||||
return (SNode*)pStmt;
|
||||
}
|
||||
|
||||
SNode* createInsertStmt(SAstCreateContext* pCxt, SNode* pTable, SNodeList* pCols, SNode* pQuery) {
|
||||
CHECK_PARSER_STATUS(pCxt);
|
||||
SInsertStmt* pStmt = (SInsertStmt*)nodesMakeNode(QUERY_NODE_INSERT_STMT);
|
||||
CHECK_OUT_OF_MEM(pStmt);
|
||||
pStmt->pTable = pTable;
|
||||
pStmt->pCols = pCols;
|
||||
pStmt->pQuery = pQuery;
|
||||
return (SNode*)pStmt;
|
||||
}
|
||||
|
|
|
@ -447,6 +447,14 @@ static int32_t collectMetaKeyFromDelete(SCollectMetaKeyCxt* pCxt, SDeleteStmt* p
|
|||
return collectMetaKeyFromRealTableImpl(pCxt, (SRealTableNode*)pStmt->pFromTable, AUTH_TYPE_WRITE);
|
||||
}
|
||||
|
||||
static int32_t collectMetaKeyFromInsert(SCollectMetaKeyCxt* pCxt, SInsertStmt* pStmt) {
|
||||
int32_t code = collectMetaKeyFromRealTableImpl(pCxt, (SRealTableNode*)pStmt->pTable, AUTH_TYPE_WRITE);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = collectMetaKeyFromQuery(pCxt, pStmt->pQuery);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t collectMetaKeyFromShowBlockDist(SCollectMetaKeyCxt* pCxt, SShowTableDistributedStmt* pStmt) {
|
||||
SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId};
|
||||
strcpy(name.dbname, pStmt->dbName);
|
||||
|
@ -554,6 +562,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
|
|||
return collectMetaKeyFromShowTransactions(pCxt, (SShowStmt*)pStmt);
|
||||
case QUERY_NODE_DELETE_STMT:
|
||||
return collectMetaKeyFromDelete(pCxt, (SDeleteStmt*)pStmt);
|
||||
case QUERY_NODE_INSERT_STMT:
|
||||
return collectMetaKeyFromInsert(pCxt, (SInsertStmt*)pStmt);
|
||||
case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT:
|
||||
return collectMetaKeyFromShowBlockDist(pCxt, (SShowTableDistributedStmt*)pStmt);
|
||||
case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
|
||||
|
|
|
@ -39,7 +39,7 @@ static int32_t checkAuth(SAuthCxt* pCxt, const char* pDbName, AUTH_TYPE type) {
|
|||
if (NULL != pCxt->pMetaCache) {
|
||||
code = getUserAuthFromCache(pCxt->pMetaCache, pParseCxt->pUser, dbFname, type, &pass);
|
||||
} else {
|
||||
SRequestConnInfo conn = {.pTrans = pParseCxt->pTransporter,
|
||||
SRequestConnInfo conn = {.pTrans = pParseCxt->pTransporter,
|
||||
.requestId = pParseCxt->requestId,
|
||||
.requestObjRefId = pParseCxt->requestRid,
|
||||
.mgmtEps = pParseCxt->mgmtEpSet};
|
||||
|
@ -88,6 +88,14 @@ static int32_t authDelete(SAuthCxt* pCxt, SDeleteStmt* pDelete) {
|
|||
return checkAuth(pCxt, ((SRealTableNode*)pDelete->pFromTable)->table.dbName, AUTH_TYPE_WRITE);
|
||||
}
|
||||
|
||||
static int32_t authInsert(SAuthCxt* pCxt, SInsertStmt* pInsert) {
|
||||
int32_t code = checkAuth(pCxt, ((SRealTableNode*)pInsert->pTable)->table.dbName, AUTH_TYPE_WRITE);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = authQuery(pCxt, pInsert->pQuery);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
|
||||
switch (nodeType(pStmt)) {
|
||||
case QUERY_NODE_SET_OPERATOR:
|
||||
|
@ -98,6 +106,8 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
|
|||
return authDropUser(pCxt, (SDropUserStmt*)pStmt);
|
||||
case QUERY_NODE_DELETE_STMT:
|
||||
return authDelete(pCxt, (SDeleteStmt*)pStmt);
|
||||
case QUERY_NODE_INSERT_STMT:
|
||||
return authInsert(pCxt, (SInsertStmt*)pStmt);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
|
@ -300,6 +300,14 @@ static int32_t calcConstDelete(SCalcConstContext* pCxt, SDeleteStmt* pDelete) {
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t calcConstInsert(SCalcConstContext* pCxt, SInsertStmt* pInsert) {
|
||||
int32_t code = calcConstFromTable(pCxt, pInsert->pTable);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = calcConstQuery(pCxt, pInsert->pQuery, false);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subquery) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
switch (nodeType(pStmt)) {
|
||||
|
@ -320,6 +328,9 @@ static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subque
|
|||
case QUERY_NODE_DELETE_STMT:
|
||||
code = calcConstDelete(pCxt, (SDeleteStmt*)pStmt);
|
||||
break;
|
||||
case QUERY_NODE_INSERT_STMT:
|
||||
code = calcConstInsert(pCxt, (SInsertStmt*)pStmt);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
|
@ -110,7 +110,7 @@ typedef struct SMemParam {
|
|||
static int32_t skipInsertInto(char** pSql, SMsgBuf* pMsg) {
|
||||
SToken sToken;
|
||||
NEXT_TOKEN(*pSql, sToken);
|
||||
if (TK_INSERT != sToken.type) {
|
||||
if (TK_INSERT != sToken.type && TK_IMPORT != sToken.type) {
|
||||
return buildSyntaxErrMsg(pMsg, "keyword INSERT is expected", sToken.z);
|
||||
}
|
||||
NEXT_TOKEN(*pSql, sToken);
|
||||
|
|
|
@ -558,11 +558,11 @@ static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColum
|
|||
pCol->node.resType = pExpr->resType;
|
||||
}
|
||||
|
||||
static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) {
|
||||
static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, bool igTags, SNodeList* pList) {
|
||||
if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) {
|
||||
const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta;
|
||||
int32_t nums =
|
||||
pMeta->tableInfo.numOfColumns + ((TSDB_SUPER_TABLE == pMeta->tableType) ? pMeta->tableInfo.numOfTags : 0);
|
||||
int32_t nums = pMeta->tableInfo.numOfColumns +
|
||||
(igTags ? 0 : ((TSDB_SUPER_TABLE == pMeta->tableType) ? pMeta->tableInfo.numOfTags : 0));
|
||||
for (int32_t i = 0; i < nums; ++i) {
|
||||
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
|
||||
if (NULL == pCol) {
|
||||
|
@ -878,6 +878,9 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal,
|
|||
}
|
||||
case TSDB_DATA_TYPE_VARCHAR:
|
||||
case TSDB_DATA_TYPE_VARBINARY: {
|
||||
if (strict && (pVal->node.resType.bytes > targetDt.bytes - VARSTR_HEADER_SIZE)) {
|
||||
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
|
||||
}
|
||||
pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1);
|
||||
if (NULL == pVal->datum.p) {
|
||||
return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY);
|
||||
|
@ -1333,6 +1336,9 @@ static int32_t rewriteSystemInfoFuncImpl(STranslateContext* pCxt, char* pLiteral
|
|||
pVal->isNull = true;
|
||||
} else {
|
||||
pVal->literal = pLiteral;
|
||||
if (IS_VAR_DATA_TYPE(pVal->node.resType.type)) {
|
||||
pVal->node.resType.bytes = strlen(pLiteral);
|
||||
}
|
||||
}
|
||||
if (DEAL_RES_ERROR != translateValue(pCxt, pVal)) {
|
||||
*pNode = (SNode*)pVal;
|
||||
|
@ -1928,7 +1934,7 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t createAllColumns(STranslateContext* pCxt, SNodeList** pCols) {
|
||||
static int32_t createAllColumns(STranslateContext* pCxt, bool igTags, SNodeList** pCols) {
|
||||
*pCols = nodesMakeList();
|
||||
if (NULL == *pCols) {
|
||||
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY);
|
||||
|
@ -1937,7 +1943,7 @@ static int32_t createAllColumns(STranslateContext* pCxt, SNodeList** pCols) {
|
|||
size_t nums = taosArrayGetSize(pTables);
|
||||
for (size_t i = 0; i < nums; ++i) {
|
||||
STableNode* pTable = taosArrayGetP(pTables, i);
|
||||
int32_t code = createColumnsByTable(pCxt, pTable, *pCols);
|
||||
int32_t code = createColumnsByTable(pCxt, pTable, igTags, *pCols);
|
||||
if (TSDB_CODE_SUCCESS != code) {
|
||||
return code;
|
||||
}
|
||||
|
@ -1974,7 +1980,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) {
|
|||
return (SNode*)pFunc;
|
||||
}
|
||||
|
||||
static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SNodeList** pOutput) {
|
||||
static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, bool igTags, SNodeList** pOutput) {
|
||||
STableNode* pTable = NULL;
|
||||
int32_t code = findTable(pCxt, pCol->tableAlias, &pTable);
|
||||
if (TSDB_CODE_SUCCESS == code && NULL == *pOutput) {
|
||||
|
@ -1984,7 +1990,7 @@ static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SN
|
|||
}
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = createColumnsByTable(pCxt, pTable, *pOutput);
|
||||
code = createColumnsByTable(pCxt, pTable, igTags, *pOutput);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
@ -2006,11 +2012,9 @@ static int32_t createMultiResFuncsParas(STranslateContext* pCxt, SNodeList* pSrc
|
|||
SNode* pPara = NULL;
|
||||
FOREACH(pPara, pSrcParas) {
|
||||
if (isStar(pPara)) {
|
||||
code = createAllColumns(pCxt, &pExprs);
|
||||
// The syntax definition ensures that * and other parameters do not appear at the same time
|
||||
break;
|
||||
code = createAllColumns(pCxt, true, &pExprs);
|
||||
} else if (isTableStar(pPara)) {
|
||||
code = createTableAllCols(pCxt, (SColumnNode*)pPara, &pExprs);
|
||||
code = createTableAllCols(pCxt, (SColumnNode*)pPara, true, &pExprs);
|
||||
} else {
|
||||
code = nodesListMakeStrictAppend(&pExprs, nodesCloneNode(pPara));
|
||||
}
|
||||
|
@ -2069,7 +2073,7 @@ static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
|
|||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
if (isStar(pNode)) {
|
||||
SNodeList* pCols = NULL;
|
||||
code = createAllColumns(pCxt, &pCols);
|
||||
code = createAllColumns(pCxt, false, &pCols);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
INSERT_LIST(pSelect->pProjectionList, pCols);
|
||||
ERASE_NODE(pSelect->pProjectionList);
|
||||
|
@ -2085,7 +2089,7 @@ static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
|
|||
}
|
||||
} else if (isTableStar(pNode)) {
|
||||
SNodeList* pCols = NULL;
|
||||
code = createTableAllCols(pCxt, (SColumnNode*)pNode, &pCols);
|
||||
code = createTableAllCols(pCxt, (SColumnNode*)pNode, false, &pCols);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
INSERT_LIST(pSelect->pProjectionList, pCols);
|
||||
ERASE_NODE(pSelect->pProjectionList);
|
||||
|
@ -2835,6 +2839,21 @@ static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) {
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t translateInsert(STranslateContext* pCxt, SInsertStmt* pInsert) {
|
||||
pCxt->pCurrStmt = (SNode*)pInsert;
|
||||
int32_t code = translateFrom(pCxt, pInsert->pTable);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = translateExprList(pCxt, pInsert->pCols);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = resetTranslateNamespace(pCxt);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = translateQuery(pCxt, pInsert->pQuery);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int64_t getUnitPerMinute(uint8_t precision) {
|
||||
switch (precision) {
|
||||
case TSDB_TIME_PRECISION_MILLI:
|
||||
|
@ -4604,6 +4623,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
|
|||
case QUERY_NODE_DELETE_STMT:
|
||||
code = translateDelete(pCxt, (SDeleteStmt*)pNode);
|
||||
break;
|
||||
case QUERY_NODE_INSERT_STMT:
|
||||
code = translateInsert(pCxt, (SInsertStmt*)pNode);
|
||||
break;
|
||||
case QUERY_NODE_CREATE_DATABASE_STMT:
|
||||
code = translateCreateDatabase(pCxt, (SCreateDatabaseStmt*)pNode);
|
||||
break;
|
||||
|
@ -5826,10 +5848,6 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
|
|||
return pCxt->errCode;
|
||||
}
|
||||
|
||||
if (IS_VAR_DATA_TYPE(pSchema->type) && strlen(pStmt->pVal->literal) > pSchema->bytes) {
|
||||
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pStmt->pVal->literal);
|
||||
}
|
||||
|
||||
pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type);
|
||||
if (targetDt.type == TSDB_DATA_TYPE_JSON) {
|
||||
pReq->isNull = 0;
|
||||
|
@ -6224,6 +6242,10 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
|
|||
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
|
||||
pQuery->msgType = TDMT_VND_DELETE;
|
||||
break;
|
||||
case QUERY_NODE_INSERT_STMT:
|
||||
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
|
||||
pQuery->msgType = TDMT_VND_SUBMIT;
|
||||
break;
|
||||
case QUERY_NODE_VNODE_MODIF_STMT:
|
||||
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
|
||||
pQuery->msgType = toMsgType(((SVnodeModifOpStmt*)pQuery->pRoot)->sqlNodeType);
|
||||
|
|
|
@ -236,7 +236,6 @@ int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char*
|
|||
|
||||
const char* prefix = "syntax error";
|
||||
if (sourceStr == NULL) {
|
||||
assert(additionalInfo != NULL);
|
||||
snprintf(pBuf->buf, pBuf->len, msgFormat1, additionalInfo);
|
||||
return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
|
||||
}
|
||||
|
@ -254,40 +253,25 @@ int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char*
|
|||
return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
|
||||
}
|
||||
|
||||
SSchema* getTableColumnSchema(const STableMeta* pTableMeta) {
|
||||
assert(pTableMeta != NULL);
|
||||
return (SSchema*)pTableMeta->schema;
|
||||
}
|
||||
SSchema* getTableColumnSchema(const STableMeta* pTableMeta) { return (SSchema*)pTableMeta->schema; }
|
||||
|
||||
static SSchema* getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) {
|
||||
assert(pTableMeta != NULL && pTableMeta->schema != NULL && colIndex >= 0 &&
|
||||
colIndex < (getNumOfColumns(pTableMeta) + getNumOfTags(pTableMeta)));
|
||||
|
||||
SSchema* pSchema = (SSchema*)pTableMeta->schema;
|
||||
return &pSchema[colIndex];
|
||||
}
|
||||
|
||||
SSchema* getTableTagSchema(const STableMeta* pTableMeta) {
|
||||
assert(pTableMeta != NULL &&
|
||||
(pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE));
|
||||
return getOneColumnSchema(pTableMeta, getTableInfo(pTableMeta).numOfColumns);
|
||||
}
|
||||
|
||||
int32_t getNumOfColumns(const STableMeta* pTableMeta) {
|
||||
assert(pTableMeta != NULL);
|
||||
// table created according to super table, use data from super table
|
||||
return getTableInfo(pTableMeta).numOfColumns;
|
||||
}
|
||||
|
||||
int32_t getNumOfTags(const STableMeta* pTableMeta) {
|
||||
assert(pTableMeta != NULL);
|
||||
return getTableInfo(pTableMeta).numOfTags;
|
||||
}
|
||||
int32_t getNumOfTags(const STableMeta* pTableMeta) { return getTableInfo(pTableMeta).numOfTags; }
|
||||
|
||||
STableComInfo getTableInfo(const STableMeta* pTableMeta) {
|
||||
assert(pTableMeta != NULL);
|
||||
return pTableMeta->tableInfo;
|
||||
}
|
||||
STableComInfo getTableInfo(const STableMeta* pTableMeta) { return pTableMeta->tableInfo; }
|
||||
|
||||
STableMeta* tableMetaDup(const STableMeta* pTableMeta) {
|
||||
size_t size = TABLE_META_SIZE(pTableMeta);
|
||||
|
|
|
@ -19,19 +19,28 @@
|
|||
#include "parInt.h"
|
||||
#include "parToken.h"
|
||||
|
||||
bool qIsInsertSql(const char* pStr, size_t length) {
|
||||
bool qIsInsertValuesSql(const char* pStr, size_t length) {
|
||||
if (NULL == pStr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const char* pSql = pStr;
|
||||
|
||||
int32_t index = 0;
|
||||
SToken t = tStrGetToken((char*)pStr, &index, false);
|
||||
if (TK_INSERT != t.type && TK_IMPORT != t.type) {
|
||||
return false;
|
||||
}
|
||||
|
||||
do {
|
||||
SToken t0 = tStrGetToken((char*)pStr, &index, false);
|
||||
if (t0.type != TK_NK_LP) {
|
||||
return t0.type == TK_INSERT || t0.type == TK_IMPORT;
|
||||
pStr += index;
|
||||
index = 0;
|
||||
t = tStrGetToken((char*)pStr, &index, false);
|
||||
if (TK_USING == t.type || TK_VALUES == t.type) {
|
||||
return true;
|
||||
}
|
||||
} while (1);
|
||||
} while (pStr - pSql < length);
|
||||
return false;
|
||||
}
|
||||
|
||||
static int32_t analyseSemantic(SParseContext* pCxt, SQuery* pQuery, SParseMetaCache* pMetaCache) {
|
||||
|
@ -148,7 +157,7 @@ static void rewriteExprAlias(SNode* pRoot) {
|
|||
|
||||
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
if (qIsInsertSql(pCxt->pSql, pCxt->sqlLen)) {
|
||||
if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) {
|
||||
code = parseInsertSql(pCxt, pQuery, NULL);
|
||||
} else {
|
||||
code = parseSqlIntoAst(pCxt, pQuery);
|
||||
|
@ -160,7 +169,7 @@ int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) {
|
|||
int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) {
|
||||
SParseMetaCache metaCache = {0};
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
if (qIsInsertSql(pCxt->pSql, pCxt->sqlLen)) {
|
||||
if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) {
|
||||
code = parseInsertSyntax(pCxt, pQuery, &metaCache);
|
||||
} else {
|
||||
code = parseSqlSyntax(pCxt, pQuery, &metaCache);
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -40,6 +40,12 @@ TEST_F(ParserExplainToSyncdbTest, grant) {
|
|||
run("GRANT READ, WRITE ON test.* TO wxy");
|
||||
}
|
||||
|
||||
TEST_F(ParserExplainToSyncdbTest, insert) {
|
||||
useDb("root", "test");
|
||||
|
||||
run("INSERT INTO t1 SELECT * FROM t1");
|
||||
}
|
||||
|
||||
// todo kill connection
|
||||
// todo kill query
|
||||
// todo kill stream
|
||||
|
|
|
@ -26,6 +26,7 @@ typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, void*, SLogicNode**);
|
|||
typedef int32_t (*FCreateSelectLogicNode)(SLogicPlanContext*, SSelectStmt*, SLogicNode**);
|
||||
typedef int32_t (*FCreateSetOpLogicNode)(SLogicPlanContext*, SSetOperator*, SLogicNode**);
|
||||
typedef int32_t (*FCreateDeleteLogicNode)(SLogicPlanContext*, SDeleteStmt*, SLogicNode**);
|
||||
typedef int32_t (*FCreateInsertLogicNode)(SLogicPlanContext*, SInsertStmt*, SLogicNode**);
|
||||
|
||||
static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable,
|
||||
SLogicNode** pLogicNode);
|
||||
|
@ -1262,6 +1263,47 @@ static int32_t createDeleteLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDele
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t creatInsertRootLogicNode(SLogicPlanContext* pCxt, SInsertStmt* pInsert, FCreateInsertLogicNode func,
|
||||
SLogicNode** pRoot) {
|
||||
return createRootLogicNode(pCxt, pInsert, pInsert->precision, (FCreateLogicNode)func, pRoot);
|
||||
}
|
||||
|
||||
static int32_t createVnodeModifLogicNodeByInsert(SLogicPlanContext* pCxt, SInsertStmt* pInsert,
|
||||
SLogicNode** pLogicNode) {
|
||||
SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY);
|
||||
if (NULL == pModify) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
SRealTableNode* pRealTable = (SRealTableNode*)pInsert->pTable;
|
||||
|
||||
pModify->modifyType = MODIFY_TABLE_TYPE_INSERT;
|
||||
pModify->tableId = pRealTable->pMeta->uid;
|
||||
pModify->tableType = pRealTable->pMeta->tableType;
|
||||
snprintf(pModify->tableFName, sizeof(pModify->tableFName), "%d.%s.%s", pCxt->pPlanCxt->acctId,
|
||||
pRealTable->table.dbName, pRealTable->table.tableName);
|
||||
TSWAP(pModify->pVgroupList, pRealTable->pVgroupList);
|
||||
|
||||
*pLogicNode = (SLogicNode*)pModify;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t createInsertLogicNode(SLogicPlanContext* pCxt, SInsertStmt* pInsert, SLogicNode** pLogicNode) {
|
||||
SLogicNode* pRoot = NULL;
|
||||
int32_t code = createQueryLogicNode(pCxt, pInsert->pQuery, &pRoot);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = creatInsertRootLogicNode(pCxt, pInsert, createVnodeModifLogicNodeByInsert, &pRoot);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
*pLogicNode = pRoot;
|
||||
} else {
|
||||
nodesDestroyNode((SNode*)pRoot);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode) {
|
||||
switch (nodeType(pStmt)) {
|
||||
case QUERY_NODE_SELECT_STMT:
|
||||
|
@ -1274,6 +1316,8 @@ static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogi
|
|||
return createSetOperatorLogicNode(pCxt, (SSetOperator*)pStmt, pLogicNode);
|
||||
case QUERY_NODE_DELETE_STMT:
|
||||
return createDeleteLogicNode(pCxt, (SDeleteStmt*)pStmt, pLogicNode);
|
||||
case QUERY_NODE_INSERT_STMT:
|
||||
return createInsertLogicNode(pCxt, (SInsertStmt*)pStmt, pLogicNode);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
|
@ -75,6 +75,11 @@ static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func)
|
|||
return NULL;
|
||||
}
|
||||
|
||||
static void optResetParent(SLogicNode* pNode) {
|
||||
SNode* pChild = NULL;
|
||||
FOREACH(pChild, pNode->pChildren) { ((SLogicNode*)pChild)->pParent = pNode; }
|
||||
}
|
||||
|
||||
EDealRes scanPathOptHaveNormalColImpl(SNode* pNode, void* pContext) {
|
||||
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
|
||||
// *((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType);
|
||||
|
@ -480,17 +485,12 @@ static int32_t pushDownCondOptPushCondToProject(SOptimizeContext* pCxt, SProject
|
|||
return pushDownCondOptAppendCond(&pProject->node.pConditions, pCond);
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptPushCondToJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) {
|
||||
return pushDownCondOptAppendCond(&pJoin->node.pConditions, pCond);
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) {
|
||||
switch (nodeType(pChild)) {
|
||||
case QUERY_NODE_LOGIC_PLAN_SCAN:
|
||||
return pushDownCondOptPushCondToScan(pCxt, (SScanLogicNode*)pChild, pCond);
|
||||
case QUERY_NODE_LOGIC_PLAN_PROJECT:
|
||||
return pushDownCondOptPushCondToProject(pCxt, (SProjectLogicNode*)pChild, pCond);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
planError("pushDownCondOptPushCondToChild failed, invalid logic plan node %s", nodesNodeName(nodeType(pChild)));
|
||||
return TSDB_CODE_PLAN_INTERNAL_ERROR;
|
||||
return pushDownCondOptAppendCond(&pChild->pConditions, pCond);
|
||||
}
|
||||
|
||||
static bool pushDownCondOptIsPriKey(SNode* pNode, SNodeList* pTableCols) {
|
||||
|
@ -554,13 +554,83 @@ static int32_t pushDownCondOptCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogic
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptPartJoinOnCondLogicCond(SJoinLogicNode* pJoin, SNode** ppMergeCond, SNode** ppOnCond) {
|
||||
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions);
|
||||
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
SNodeList* pOnConds = NULL;
|
||||
SNode* pCond = NULL;
|
||||
FOREACH(pCond, pLogicCond->pParameterList) {
|
||||
if (pushDownCondOptIsPriKeyEqualCond(pJoin, pCond)) {
|
||||
*ppMergeCond = nodesCloneNode(pCond);
|
||||
} else {
|
||||
code = nodesListMakeAppend(&pOnConds, nodesCloneNode(pCond));
|
||||
}
|
||||
}
|
||||
|
||||
SNode* pTempOnCond = NULL;
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = nodesMergeConds(&pTempOnCond, &pOnConds);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code && NULL != *ppMergeCond) {
|
||||
*ppOnCond = pTempOnCond;
|
||||
nodesDestroyNode(pJoin->pOnConditions);
|
||||
pJoin->pOnConditions = NULL;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else {
|
||||
nodesDestroyList(pOnConds);
|
||||
nodesDestroyNode(pTempOnCond);
|
||||
return TSDB_CODE_PLAN_INTERNAL_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptPartJoinOnCond(SJoinLogicNode* pJoin, SNode** ppMergeCond, SNode** ppOnCond) {
|
||||
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) &&
|
||||
LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) {
|
||||
return pushDownCondOptPartJoinOnCondLogicCond(pJoin, ppMergeCond, ppOnCond);
|
||||
}
|
||||
|
||||
if (pushDownCondOptIsPriKeyEqualCond(pJoin, pJoin->pOnConditions)) {
|
||||
*ppMergeCond = nodesCloneNode(pJoin->pOnConditions);
|
||||
*ppOnCond = NULL;
|
||||
nodesDestroyNode(pJoin->pOnConditions);
|
||||
pJoin->pOnConditions = NULL;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else {
|
||||
return TSDB_CODE_PLAN_INTERNAL_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
|
||||
int32_t code = pushDownCondOptCheckJoinOnCond(pCxt, pJoin);
|
||||
SNode* pJoinMergeCond = NULL;
|
||||
SNode* pJoinOnCond = NULL;
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = pushDownCondOptPartJoinOnCond(pJoin, &pJoinMergeCond, &pJoinOnCond);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
pJoin->pMergeCondition = pJoinMergeCond;
|
||||
pJoin->pOnConditions = pJoinOnCond;
|
||||
} else {
|
||||
nodesDestroyNode(pJoinMergeCond);
|
||||
nodesDestroyNode(pJoinOnCond);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
|
||||
if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (NULL == pJoin->node.pConditions) {
|
||||
return pushDownCondOptCheckJoinOnCond(pCxt, pJoin);
|
||||
int32_t code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE);
|
||||
pCxt->optimized = true;
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
SNode* pOnCond = NULL;
|
||||
|
@ -579,10 +649,13 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p
|
|||
pushDownCondOptPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE);
|
||||
pCxt->optimized = true;
|
||||
code = pushDownCondOptCheckJoinOnCond(pCxt, pJoin);
|
||||
} else {
|
||||
nodesDestroyNode(pOnCond);
|
||||
nodesDestroyNode(pLeftChildCond);
|
||||
|
@ -718,9 +791,7 @@ static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAg
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
// TODO: remove it after full implementation of pushing down to child
|
||||
if (1 != LIST_LENGTH(pAgg->node.pChildren) ||
|
||||
QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pAgg->node.pChildren, 0)) &&
|
||||
QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(nodesListGetNode(pAgg->node.pChildren, 0))) {
|
||||
if (1 != LIST_LENGTH(pAgg->node.pChildren)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
@ -747,6 +818,78 @@ static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAg
|
|||
return code;
|
||||
}
|
||||
|
||||
typedef struct SRewriteProjCondContext {
|
||||
SProjectLogicNode* pProj;
|
||||
int32_t errCode;
|
||||
} SRewriteProjCondContext;
|
||||
|
||||
static EDealRes rewriteProjectCondForPushDownImpl(SNode** ppNode, void* pContext) {
|
||||
SRewriteProjCondContext* pCxt = pContext;
|
||||
SProjectLogicNode* pProj = pCxt->pProj;
|
||||
if (QUERY_NODE_COLUMN == nodeType(*ppNode)) {
|
||||
SNode* pTarget = NULL;
|
||||
FOREACH(pTarget, pProj->node.pTargets) {
|
||||
if (nodesEqualNode(pTarget, *ppNode)) {
|
||||
SNode* pProjection = NULL;
|
||||
FOREACH(pProjection, pProj->pProjections) {
|
||||
if (0 == strcmp(((SExprNode*)pProjection)->aliasName, ((SColumnNode*)(*ppNode))->colName)) {
|
||||
SNode* pExpr = nodesCloneNode(pProjection);
|
||||
if (pExpr == NULL) {
|
||||
pCxt->errCode = terrno;
|
||||
return DEAL_RES_ERROR;
|
||||
}
|
||||
nodesDestroyNode(*ppNode);
|
||||
*ppNode = pExpr;
|
||||
} // end if expr alias name equal column name
|
||||
} // end for each project
|
||||
} // end if target node equals cond column node
|
||||
} // end for each targets
|
||||
return DEAL_RES_IGNORE_CHILD;
|
||||
}
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
||||
static int32_t rewriteProjectCondForPushDown(SOptimizeContext* pCxt, SProjectLogicNode* pProject,
|
||||
SNode** ppProjectCond) {
|
||||
SRewriteProjCondContext cxt = {.pProj = pProject, .errCode = TSDB_CODE_SUCCESS};
|
||||
SNode* pProjectCond = pProject->node.pConditions;
|
||||
nodesRewriteExpr(&pProjectCond, rewriteProjectCondForPushDownImpl, &cxt);
|
||||
*ppProjectCond = pProjectCond;
|
||||
pProject->node.pConditions = NULL;
|
||||
return cxt.errCode;
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptDealProject(SOptimizeContext* pCxt, SProjectLogicNode* pProject) {
|
||||
if (NULL == pProject->node.pConditions ||
|
||||
OPTIMIZE_FLAG_TEST_MASK(pProject->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
// TODO: remove it after full implementation of pushing down to child
|
||||
if (1 != LIST_LENGTH(pProject->node.pChildren)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (NULL != pProject->node.pLimit || NULL != pProject->node.pSlimit) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
SNode* pProjCond = NULL;
|
||||
code = rewriteProjectCondForPushDown(pCxt, pProject, &pProjCond);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProject->node.pChildren, 0);
|
||||
code = pushDownCondOptPushCondToChild(pCxt, pChild, &pProjCond);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
OPTIMIZE_FLAG_SET_MASK(pProject->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE);
|
||||
pCxt->optimized = true;
|
||||
} else {
|
||||
nodesDestroyNode(pProjCond);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
switch (nodeType(pLogicNode)) {
|
||||
|
@ -759,6 +902,9 @@ static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLog
|
|||
case QUERY_NODE_LOGIC_PLAN_AGG:
|
||||
code = pushDownCondOptDealAgg(pCxt, (SAggLogicNode*)pLogicNode);
|
||||
break;
|
||||
case QUERY_NODE_LOGIC_PLAN_PROJECT:
|
||||
code = pushDownCondOptDealProject(pCxt, (SProjectLogicNode*)pLogicNode);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
@ -1460,6 +1606,7 @@ static int32_t rewriteTailOptCreateSort(SIndefRowsFuncLogicNode* pIndef, SLogicN
|
|||
|
||||
pSort->groupSort = rewriteTailOptNeedGroupSort(pIndef);
|
||||
TSWAP(pSort->node.pChildren, pIndef->node.pChildren);
|
||||
optResetParent((SLogicNode*)pSort);
|
||||
pSort->node.precision = pIndef->node.precision;
|
||||
|
||||
SFunctionNode* pTail = NULL;
|
||||
|
@ -1667,6 +1814,7 @@ static int32_t rewriteUniqueOptCreateAgg(SIndefRowsFuncLogicNode* pIndef, SLogic
|
|||
}
|
||||
|
||||
TSWAP(pAgg->node.pChildren, pIndef->node.pChildren);
|
||||
optResetParent((SLogicNode*)pAgg);
|
||||
pAgg->node.precision = pIndef->node.precision;
|
||||
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
@ -1935,13 +2083,18 @@ static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimi
|
|||
static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) {
|
||||
char* pStr = NULL;
|
||||
nodesNodeToString((SNode*)pSubplan, false, &pStr, NULL);
|
||||
qDebugL("apply optimize %s rule: %s", pRuleName, pStr);
|
||||
if (NULL == pRuleName) {
|
||||
qDebugL("before optimize: %s", pStr);
|
||||
} else {
|
||||
qDebugL("apply optimize %s rule: %s", pRuleName, pStr);
|
||||
}
|
||||
taosMemoryFree(pStr);
|
||||
}
|
||||
|
||||
static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) {
|
||||
SOptimizeContext cxt = {.pPlanCxt = pCxt, .optimized = false};
|
||||
bool optimized = false;
|
||||
dumpLogicSubplan(NULL, pLogicSubplan);
|
||||
do {
|
||||
optimized = false;
|
||||
for (int32_t i = 0; i < optimizeRuleNum; ++i) {
|
||||
|
|
|
@ -612,10 +612,8 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
|
|||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
||||
pJoin->joinType = pJoinLogicNode->joinType;
|
||||
if (NULL != pJoinLogicNode->pOnConditions) {
|
||||
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pOnConditions,
|
||||
&pJoin->pOnConditions);
|
||||
}
|
||||
setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pMergeCondition,
|
||||
&pJoin->pMergeCondition);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = setListSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->node.pTargets,
|
||||
&pJoin->pTargets);
|
||||
|
@ -623,6 +621,21 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc);
|
||||
}
|
||||
|
||||
SNodeList* condCols = nodesMakeList();
|
||||
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) {
|
||||
code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &condCols);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = addDataBlockSlots(pCxt, condCols, pJoin->node.pOutputDataBlockDesc);
|
||||
nodesDestroyList(condCols);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) {
|
||||
code = setNodeSlotId(pCxt, ((SPhysiNode*)pJoin)->pOutputDataBlockDesc->dataBlockId, -1,
|
||||
pJoinLogicNode->pOnConditions, &pJoin->pOnConditions);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin);
|
||||
}
|
||||
|
@ -1483,12 +1496,60 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl
|
|||
return pSubplan;
|
||||
}
|
||||
|
||||
static int32_t buildInsertSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) {
|
||||
static int32_t buildInsertValuesSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) {
|
||||
pSubplan->msgType = pModify->msgType;
|
||||
pSubplan->execNode.epSet = pModify->pVgDataBlocks->vg.epSet;
|
||||
return createDataInserter(pCxt, pModify->pVgDataBlocks, &pSubplan->pDataSink);
|
||||
}
|
||||
|
||||
static int32_t createQueryInserter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan,
|
||||
SDataSinkNode** pSink) {
|
||||
SQueryInserterNode* pInserter = (SQueryInserterNode*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT);
|
||||
if (NULL == pInserter) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
pInserter->tableId = pModify->tableId;
|
||||
pInserter->tableType = pModify->tableType;
|
||||
strcpy(pInserter->tableFName, pModify->tableFName);
|
||||
pInserter->vgId = pModify->pVgroupList->vgroups[0].vgId;
|
||||
pInserter->epSet = pModify->pVgroupList->vgroups[0].epSet;
|
||||
vgroupInfoToNodeAddr(pModify->pVgroupList->vgroups, &pSubplan->execNode);
|
||||
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
||||
pInserter->sink.pInputDataBlockDesc =
|
||||
(SDataBlockDescNode*)nodesCloneNode((SNode*)pSubplan->pNode->pOutputDataBlockDesc);
|
||||
if (NULL == pInserter->sink.pInputDataBlockDesc) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
*pSink = (SDataSinkNode*)pInserter;
|
||||
} else {
|
||||
nodesDestroyNode((SNode*)pInserter);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t buildInsertSelectSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) {
|
||||
int32_t code =
|
||||
createPhysiNode(pCxt, (SLogicNode*)nodesListGetNode(pModify->node.pChildren, 0), pSubplan, &pSubplan->pNode);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = createQueryInserter(pCxt, pModify, pSubplan, &pSubplan->pDataSink);
|
||||
}
|
||||
pSubplan->msgType = TDMT_VND_SUBMIT;
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t buildInsertSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) {
|
||||
if (NULL == pModify->node.pChildren) {
|
||||
return buildInsertValuesSubplan(pCxt, pModify, pSubplan);
|
||||
}
|
||||
return buildInsertSelectSubplan(pCxt, pModify, pSubplan);
|
||||
}
|
||||
|
||||
static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, const SPhysiNode* pRoot,
|
||||
SDataSinkNode** pSink) {
|
||||
SDataDeleterNode* pDeleter = (SDataDeleterNode*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_DELETE);
|
||||
|
|
|
@ -82,29 +82,41 @@ static int32_t scaleOutByVgroups(SScaleOutContext* pCxt, SLogicSubplan* pSubplan
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t scaleOutForMerge(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) {
|
||||
return nodesListStrictAppend(pGroup, (SNode*)singleCloneSubLogicPlan(pCxt, pSubplan, level));
|
||||
}
|
||||
|
||||
static int32_t scaleOutForInsertValues(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level,
|
||||
SNodeList* pGroup) {
|
||||
SVnodeModifyLogicNode* pNode = (SVnodeModifyLogicNode*)pSubplan->pNode;
|
||||
size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks);
|
||||
for (int32_t i = 0; i < numOfVgroups; ++i) {
|
||||
SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level);
|
||||
if (NULL == pNewSubplan) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
((SVnodeModifyLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i);
|
||||
if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pGroup, (SNode*)pNewSubplan)) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t scaleOutForInsert(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) {
|
||||
SVnodeModifyLogicNode* pNode = (SVnodeModifyLogicNode*)pSubplan->pNode;
|
||||
if (NULL == pNode->node.pChildren) {
|
||||
return scaleOutForInsertValues(pCxt, pSubplan, level, pGroup);
|
||||
}
|
||||
return scaleOutForMerge(pCxt, pSubplan, level, pGroup);
|
||||
}
|
||||
|
||||
static int32_t scaleOutForModify(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) {
|
||||
SVnodeModifyLogicNode* pNode = (SVnodeModifyLogicNode*)pSubplan->pNode;
|
||||
if (MODIFY_TABLE_TYPE_DELETE == pNode->modifyType) {
|
||||
return scaleOutByVgroups(pCxt, pSubplan, level, pGroup);
|
||||
} else {
|
||||
size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks);
|
||||
for (int32_t i = 0; i < numOfVgroups; ++i) {
|
||||
SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level);
|
||||
if (NULL == pNewSubplan) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
((SVnodeModifyLogicNode*)pNewSubplan->pNode)->pVgDataBlocks =
|
||||
(SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i);
|
||||
if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pGroup, (SNode*)pNewSubplan)) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t scaleOutForMerge(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) {
|
||||
return nodesListStrictAppend(pGroup, (SNode*)singleCloneSubLogicPlan(pCxt, pSubplan, level));
|
||||
return scaleOutForInsert(pCxt, pSubplan, level, pGroup);
|
||||
}
|
||||
|
||||
static int32_t scaleOutForScan(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) {
|
||||
|
|
|
@ -20,6 +20,7 @@
|
|||
#define SPLIT_FLAG_MASK(n) (1 << n)
|
||||
|
||||
#define SPLIT_FLAG_STABLE_SPLIT SPLIT_FLAG_MASK(0)
|
||||
#define SPLIT_FLAG_INSERT_SPLIT SPLIT_FLAG_MASK(1)
|
||||
|
||||
#define SPLIT_FLAG_SET_MASK(val, mask) (val) |= (mask)
|
||||
#define SPLIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0)
|
||||
|
@ -1196,6 +1197,41 @@ static int32_t smaIndexSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
|
|||
return code;
|
||||
}
|
||||
|
||||
typedef struct SInsertSelectSplitInfo {
|
||||
SLogicNode* pQueryRoot;
|
||||
SLogicSubplan* pSubplan;
|
||||
} SInsertSelectSplitInfo;
|
||||
|
||||
static bool insSelSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode,
|
||||
SInsertSelectSplitInfo* pInfo) {
|
||||
if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY == nodeType(pNode) && 1 == LIST_LENGTH(pNode->pChildren) &&
|
||||
MODIFY_TABLE_TYPE_INSERT == ((SVnodeModifyLogicNode*)pNode)->modifyType) {
|
||||
pInfo->pQueryRoot = (SLogicNode*)nodesListGetNode(pNode->pChildren, 0);
|
||||
pInfo->pSubplan = pSubplan;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static int32_t insertSelectSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
|
||||
SInsertSelectSplitInfo info = {0};
|
||||
if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_INSERT_SPLIT, (FSplFindSplitNode)insSelSplFindSplitNode, &info)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t code = splCreateExchangeNodeForSubplan(pCxt, info.pSubplan, info.pQueryRoot, info.pSubplan->subplanType);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, (SNode*)splCreateScanSubplan(pCxt, info.pQueryRoot, 0));
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
info.pSubplan->subplanType = SUBPLAN_TYPE_MODIFY;
|
||||
SPLIT_FLAG_SET_MASK(info.pSubplan->splitFlag, SPLIT_FLAG_INSERT_SPLIT);
|
||||
}
|
||||
++(pCxt->groupId);
|
||||
pCxt->split = true;
|
||||
return code;
|
||||
}
|
||||
|
||||
typedef struct SQnodeSplitInfo {
|
||||
SLogicNode* pSplitNode;
|
||||
SLogicSubplan* pSubplan;
|
||||
|
@ -1203,7 +1239,8 @@ typedef struct SQnodeSplitInfo {
|
|||
|
||||
static bool qndSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode,
|
||||
SQnodeSplitInfo* pInfo) {
|
||||
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && NULL != pNode->pParent) {
|
||||
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && NULL != pNode->pParent &&
|
||||
((SScanLogicNode*)pNode)->scanSeq[0] < 1 && ((SScanLogicNode*)pNode)->scanSeq[1] < 1) {
|
||||
pInfo->pSplitNode = pNode;
|
||||
pInfo->pSubplan = pSubplan;
|
||||
return true;
|
||||
|
@ -1248,7 +1285,8 @@ static const SSplitRule splitRuleSet[] = {
|
|||
{.pName = "SingleTableJoinSplit", .splitFunc = singleTableJoinSplit},
|
||||
{.pName = "UnionAllSplit", .splitFunc = unionAllSplit},
|
||||
{.pName = "UnionDistinctSplit", .splitFunc = unionDistinctSplit},
|
||||
{.pName = "SmaIndexSplit", .splitFunc = smaIndexSplit}
|
||||
{.pName = "SmaIndexSplit", .splitFunc = smaIndexSplit},
|
||||
{.pName = "InsertSelectSplit", .splitFunc = insertSelectSplit}
|
||||
};
|
||||
// clang-format on
|
||||
|
||||
|
@ -1257,7 +1295,11 @@ static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule));
|
|||
static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) {
|
||||
char* pStr = NULL;
|
||||
nodesNodeToString((SNode*)pSubplan, false, &pStr, NULL);
|
||||
qDebugL("apply split %s rule: %s", pRuleName, pStr);
|
||||
if (NULL == pRuleName) {
|
||||
qDebugL("before split: %s", pStr);
|
||||
} else {
|
||||
qDebugL("apply split %s rule: %s", pRuleName, pStr);
|
||||
}
|
||||
taosMemoryFree(pStr);
|
||||
}
|
||||
|
||||
|
@ -1265,6 +1307,7 @@ static int32_t applySplitRule(SPlanContext* pCxt, SLogicSubplan* pSubplan) {
|
|||
SSplitContext cxt = {
|
||||
.pPlanCxt = pCxt, .queryId = pSubplan->id.queryId, .groupId = pSubplan->id.groupId + 1, .split = false};
|
||||
bool split = false;
|
||||
dumpLogicSubplan(NULL, pSubplan);
|
||||
do {
|
||||
split = false;
|
||||
for (int32_t i = 0; i < splitRuleNum; ++i) {
|
||||
|
@ -1292,8 +1335,16 @@ static void setVgroupsInfo(SLogicNode* pNode, SLogicSubplan* pSubplan) {
|
|||
FOREACH(pChild, pNode->pChildren) { setVgroupsInfo((SLogicNode*)pChild, pSubplan); }
|
||||
}
|
||||
|
||||
static bool needSplitSubplan(SLogicSubplan* pLogicSubplan) {
|
||||
if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY != nodeType(pLogicSubplan->pNode)) {
|
||||
return true;
|
||||
}
|
||||
SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)pLogicSubplan->pNode;
|
||||
return (MODIFY_TABLE_TYPE_INSERT == pModify->modifyType && NULL != pModify->node.pChildren);
|
||||
}
|
||||
|
||||
int32_t splitLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) {
|
||||
if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY == nodeType(pLogicSubplan->pNode)) {
|
||||
if (!needSplitSubplan(pLogicSubplan)) {
|
||||
setVgroupsInfo(pLogicSubplan->pNode, pLogicSubplan);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
|
|
@ -63,6 +63,10 @@ TEST_F(PlanBasicTest, uniqueFunc) {
|
|||
run("SELECT UNIQUE(c2 + 10), ts, c2 FROM t1 WHERE c1 > 10");
|
||||
|
||||
run("SELECT UNIQUE(c1) a FROM t1 ORDER BY a");
|
||||
|
||||
run("SELECT ts, UNIQUE(c1) FROM st1 PARTITION BY TBNAME");
|
||||
|
||||
run("SELECT TBNAME, UNIQUE(c1) FROM st1 PARTITION BY TBNAME");
|
||||
}
|
||||
|
||||
TEST_F(PlanBasicTest, tailFunc) {
|
||||
|
@ -81,6 +85,8 @@ TEST_F(PlanBasicTest, tailFunc) {
|
|||
run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1 LIMIT 5");
|
||||
|
||||
run("SELECT TAIL(c1, 2, 1) FROM st1s1 UNION ALL SELECT c1 FROM st1s2");
|
||||
|
||||
run("SELECT TAIL(c1, 1) FROM st2 WHERE jtag->'tag1' > 10");
|
||||
}
|
||||
|
||||
TEST_F(PlanBasicTest, interpFunc) {
|
||||
|
|
|
@ -81,3 +81,8 @@ TEST_F(PlanOptimizeTest, eliminateProjection) {
|
|||
run("SELECT c1 FROM st1s3");
|
||||
// run("select 1-abs(c1) from (select unique(c1) c1 from st1s3) order by 1 nulls first");
|
||||
}
|
||||
|
||||
TEST_F(PlanOptimizeTest, pushDownProjectCond) {
|
||||
useDb("root", "test");
|
||||
run("select 1-abs(c1) from (select unique(c1) c1 from st1s3) where 1-c1>5 order by 1 nulls first");
|
||||
}
|
|
@ -92,9 +92,8 @@ TEST_F(PlanOtherTest, delete) {
|
|||
run("DELETE FROM st1 WHERE ts > now - 2d and ts < now - 1d AND tag1 = 10");
|
||||
}
|
||||
|
||||
TEST_F(PlanOtherTest, queryPolicy) {
|
||||
TEST_F(PlanOtherTest, insert) {
|
||||
useDb("root", "test");
|
||||
|
||||
tsQueryPolicy = QUERY_POLICY_QNODE;
|
||||
run("SELECT COUNT(*) FROM st1");
|
||||
run("INSERT INTO t1 SELECT * FROM t1");
|
||||
}
|
||||
|
|
|
@ -78,6 +78,7 @@ static void parseArg(int argc, char* argv[]) {
|
|||
{"skipSql", required_argument, NULL, 's'},
|
||||
{"limitSql", required_argument, NULL, 'i'},
|
||||
{"log", required_argument, NULL, 'l'},
|
||||
{"queryPolicy", required_argument, NULL, 'q'},
|
||||
{0, 0, 0, 0}
|
||||
};
|
||||
// clang-format on
|
||||
|
@ -95,6 +96,9 @@ static void parseArg(int argc, char* argv[]) {
|
|||
case 'l':
|
||||
setLogLevel(optarg);
|
||||
break;
|
||||
case 'q':
|
||||
setQueryPolicy(optarg);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
|
@ -24,6 +24,7 @@
|
|||
#include "mockCatalogService.h"
|
||||
#include "parser.h"
|
||||
#include "planInt.h"
|
||||
#include "tglobal.h"
|
||||
|
||||
using namespace std;
|
||||
using namespace testing;
|
||||
|
@ -53,6 +54,7 @@ DumpModule g_dumpModule = DUMP_MODULE_NOTHING;
|
|||
int32_t g_skipSql = 0;
|
||||
int32_t g_limitSql = 0;
|
||||
int32_t g_logLevel = 131;
|
||||
int32_t g_queryPolicy = QUERY_POLICY_VNODE;
|
||||
|
||||
void setDumpModule(const char* pModule) {
|
||||
if (NULL == pModule) {
|
||||
|
@ -79,6 +81,7 @@ void setDumpModule(const char* pModule) {
|
|||
void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(pNum); }
|
||||
void setLimitSqlNum(const char* pNum) { g_limitSql = stoi(pNum); }
|
||||
void setLogLevel(const char* pLogLevel) { g_logLevel = stoi(pLogLevel); }
|
||||
void setQueryPolicy(const char* pQueryPolicy) { g_queryPolicy = stoi(pQueryPolicy); }
|
||||
|
||||
int32_t getLogLevel() { return g_logLevel; }
|
||||
|
||||
|
@ -105,7 +108,23 @@ class PlannerTestBaseImpl {
|
|||
}
|
||||
++sqlNum_;
|
||||
|
||||
switch (g_queryPolicy) {
|
||||
case QUERY_POLICY_VNODE:
|
||||
case QUERY_POLICY_HYBRID:
|
||||
case QUERY_POLICY_QNODE:
|
||||
runImpl(sql, g_queryPolicy);
|
||||
break;
|
||||
default:
|
||||
runImpl(sql, QUERY_POLICY_VNODE);
|
||||
runImpl(sql, QUERY_POLICY_HYBRID);
|
||||
runImpl(sql, QUERY_POLICY_QNODE);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void runImpl(const string& sql, int32_t queryPolicy) {
|
||||
reset();
|
||||
tsQueryPolicy = queryPolicy;
|
||||
try {
|
||||
SQuery* pQuery = nullptr;
|
||||
doParseSql(sql, &pQuery);
|
||||
|
|
|
@ -45,6 +45,7 @@ extern void setDumpModule(const char* pModule);
|
|||
extern void setSkipSqlNum(const char* pNum);
|
||||
extern void setLimitSqlNum(const char* pNum);
|
||||
extern void setLogLevel(const char* pLogLevel);
|
||||
extern void setQueryPolicy(const char* pQueryPolicy);
|
||||
extern int32_t getLogLevel();
|
||||
|
||||
#endif // PLAN_TEST_UTIL_H
|
||||
|
|
|
@ -171,17 +171,17 @@ char* jobTaskStatusStr(int32_t status) {
|
|||
switch (status) {
|
||||
case JOB_TASK_STATUS_NULL:
|
||||
return "NULL";
|
||||
case JOB_TASK_STATUS_NOT_START:
|
||||
return "NOT_START";
|
||||
case JOB_TASK_STATUS_EXECUTING:
|
||||
case JOB_TASK_STATUS_INIT:
|
||||
return "INIT";
|
||||
case JOB_TASK_STATUS_EXEC:
|
||||
return "EXECUTING";
|
||||
case JOB_TASK_STATUS_PARTIAL_SUCCEED:
|
||||
case JOB_TASK_STATUS_PART_SUCC:
|
||||
return "PARTIAL_SUCCEED";
|
||||
case JOB_TASK_STATUS_SUCCEED:
|
||||
case JOB_TASK_STATUS_SUCC:
|
||||
return "SUCCEED";
|
||||
case JOB_TASK_STATUS_FAILED:
|
||||
case JOB_TASK_STATUS_FAIL:
|
||||
return "FAILED";
|
||||
case JOB_TASK_STATUS_DROPPING:
|
||||
case JOB_TASK_STATUS_DROP:
|
||||
return "DROPPING";
|
||||
default:
|
||||
break;
|
||||
|
@ -200,7 +200,7 @@ SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* nam
|
|||
return s;
|
||||
}
|
||||
|
||||
void destroyQueryExecRes(SQueryExecRes* pRes) {
|
||||
void destroyQueryExecRes(SExecResult* pRes) {
|
||||
if (NULL == pRes || NULL == pRes->res) {
|
||||
return;
|
||||
}
|
||||
|
|
|
@ -226,8 +226,8 @@ typedef struct SQWorkerMgmt {
|
|||
#define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code))
|
||||
#define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code))
|
||||
#define QW_TASK_READY(status) \
|
||||
(status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || \
|
||||
status == JOB_TASK_STATUS_PARTIAL_SUCCEED)
|
||||
(status == JOB_TASK_STATUS_SUCC || status == JOB_TASK_STATUS_FAIL || status == JOB_TASK_STATUS_CANCELLED || \
|
||||
status == JOB_TASK_STATUS_PART_SUCC)
|
||||
#define QW_SET_QTID(id, qId, tId, eId) \
|
||||
do { \
|
||||
*(uint64_t *)(id) = (qId); \
|
||||
|
|
|
@ -19,7 +19,7 @@ int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus,
|
|||
int32_t code = 0;
|
||||
|
||||
if (oriStatus == newStatus) {
|
||||
if (newStatus == JOB_TASK_STATUS_EXECUTING || newStatus == JOB_TASK_STATUS_FAILED) {
|
||||
if (newStatus == JOB_TASK_STATUS_EXEC || newStatus == JOB_TASK_STATUS_FAIL) {
|
||||
*ignore = true;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
@ -29,47 +29,47 @@ int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus,
|
|||
|
||||
switch (oriStatus) {
|
||||
case JOB_TASK_STATUS_NULL:
|
||||
if (newStatus != JOB_TASK_STATUS_EXECUTING && newStatus != JOB_TASK_STATUS_FAILED &&
|
||||
newStatus != JOB_TASK_STATUS_NOT_START) {
|
||||
if (newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_FAIL &&
|
||||
newStatus != JOB_TASK_STATUS_INIT) {
|
||||
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
|
||||
}
|
||||
|
||||
break;
|
||||
case JOB_TASK_STATUS_NOT_START:
|
||||
if (newStatus != JOB_TASK_STATUS_DROPPING && newStatus != JOB_TASK_STATUS_EXECUTING
|
||||
&& newStatus != JOB_TASK_STATUS_FAILED) {
|
||||
case JOB_TASK_STATUS_INIT:
|
||||
if (newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC
|
||||
&& newStatus != JOB_TASK_STATUS_FAIL) {
|
||||
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
|
||||
}
|
||||
|
||||
break;
|
||||
case JOB_TASK_STATUS_EXECUTING:
|
||||
if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED && newStatus != JOB_TASK_STATUS_SUCCEED &&
|
||||
newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_DROPPING) {
|
||||
case JOB_TASK_STATUS_EXEC:
|
||||
if (newStatus != JOB_TASK_STATUS_PART_SUCC && newStatus != JOB_TASK_STATUS_SUCC &&
|
||||
newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_DROP) {
|
||||
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
|
||||
}
|
||||
|
||||
break;
|
||||
case JOB_TASK_STATUS_PARTIAL_SUCCEED:
|
||||
if (newStatus != JOB_TASK_STATUS_EXECUTING && newStatus != JOB_TASK_STATUS_SUCCEED &&
|
||||
newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_DROPPING) {
|
||||
case JOB_TASK_STATUS_PART_SUCC:
|
||||
if (newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_SUCC &&
|
||||
newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_DROP) {
|
||||
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
|
||||
}
|
||||
|
||||
break;
|
||||
case JOB_TASK_STATUS_SUCCEED:
|
||||
if (newStatus != JOB_TASK_STATUS_DROPPING && newStatus != JOB_TASK_STATUS_FAILED) {
|
||||
case JOB_TASK_STATUS_SUCC:
|
||||
if (newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_FAIL) {
|
||||
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
|
||||
}
|
||||
|
||||
break;
|
||||
case JOB_TASK_STATUS_FAILED:
|
||||
if (newStatus != JOB_TASK_STATUS_DROPPING) {
|
||||
case JOB_TASK_STATUS_FAIL:
|
||||
if (newStatus != JOB_TASK_STATUS_DROP) {
|
||||
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
|
||||
}
|
||||
break;
|
||||
|
||||
case JOB_TASK_STATUS_DROPPING:
|
||||
if (newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
|
||||
case JOB_TASK_STATUS_DROP:
|
||||
if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_PART_SUCC) {
|
||||
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
|
||||
}
|
||||
break;
|
||||
|
|
|
@ -206,7 +206,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
|
|||
|
||||
QW_TASK_DLOG_E("no data in sink and query end");
|
||||
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED);
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC);
|
||||
QW_ERR_RET(qwMallocFetchRsp(len, &rsp));
|
||||
|
||||
*rspMsg = rsp;
|
||||
|
@ -236,7 +236,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
|
|||
|
||||
if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) {
|
||||
QW_TASK_DLOG_E("task all data fetched, done");
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED);
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
@ -330,7 +330,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
|
|||
break;
|
||||
}
|
||||
|
||||
QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING));
|
||||
QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC));
|
||||
break;
|
||||
}
|
||||
case QW_PHASE_PRE_FETCH: {
|
||||
|
@ -447,7 +447,7 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp
|
|||
_return:
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code && QW_PHASE_POST_QUERY == phase) {
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PARTIAL_SUCCEED);
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC);
|
||||
}
|
||||
|
||||
if (rspConnection) {
|
||||
|
@ -467,7 +467,7 @@ _return:
|
|||
}
|
||||
|
||||
if (code) {
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAILED);
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL);
|
||||
}
|
||||
|
||||
QW_TASK_DLOG("end to handle event at phase %s, code:%x - %s", qwPhaseStr(phase), code, tstrerror(code));
|
||||
|
@ -499,7 +499,7 @@ int32_t qwPrerocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
|
|||
|
||||
ctx->ctrlConnInfo = qwMsg->connInfo;
|
||||
|
||||
QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_NOT_START));
|
||||
QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT));
|
||||
|
||||
_return:
|
||||
|
||||
|
@ -698,7 +698,7 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
|
|||
if (QW_IS_QUERY_RUNNING(ctx)) {
|
||||
atomic_store_8((int8_t *)&ctx->queryContinue, 1);
|
||||
} else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) {
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING);
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC);
|
||||
|
||||
atomic_store_8((int8_t *)&ctx->queryInQueue, 1);
|
||||
|
||||
|
@ -749,7 +749,7 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
|
|||
|
||||
if (QW_IS_QUERY_RUNNING(ctx)) {
|
||||
QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx));
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING);
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROP);
|
||||
} else if (ctx->phase > 0) {
|
||||
QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
|
||||
rsped = true;
|
||||
|
@ -770,7 +770,7 @@ _return:
|
|||
QW_UPDATE_RSP_CODE(ctx, code);
|
||||
}
|
||||
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAILED);
|
||||
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL);
|
||||
}
|
||||
|
||||
if (locked) {
|
||||
|
|
|
@ -1174,7 +1174,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
|
|||
int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 :
|
||||
(timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000);
|
||||
|
||||
timeUnit = timeUnit * 1000 / factor;
|
||||
int64_t unit = timeUnit * 1000 / factor;
|
||||
|
||||
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
|
||||
if (colDataIsNull_s(pInput[0].columnData, i)) {
|
||||
|
@ -1209,12 +1209,14 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
|
|||
NUM_TO_STRING(TSDB_DATA_TYPE_BIGINT, &timeVal, sizeof(buf), buf);
|
||||
int32_t tsDigits = (int32_t)strlen(buf);
|
||||
|
||||
switch (timeUnit) {
|
||||
case 0: { /* 1u */
|
||||
switch (unit) {
|
||||
case 0: { /* 1u or 1b */
|
||||
if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
|
||||
timeVal = timeVal / 1000 * 1000;
|
||||
//} else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
|
||||
// //timeVal = timeVal / 1000;
|
||||
if (timePrec == TSDB_TIME_PRECISION_NANO && timeUnit == 1) {
|
||||
timeVal = timeVal * 1;
|
||||
} else {
|
||||
timeVal = timeVal / 1000 * 1000;
|
||||
}
|
||||
} else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
|
||||
timeVal = timeVal * factor;
|
||||
} else {
|
||||
|
@ -1366,8 +1368,6 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
|
|||
int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 :
|
||||
(timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000);
|
||||
|
||||
timeUnit = timeUnit * 1000 / factor;
|
||||
|
||||
int32_t numOfRows = 0;
|
||||
for (int32_t i = 0; i < inputNum; ++i) {
|
||||
if (pInput[i].numOfRows > numOfRows) {
|
||||
|
@ -1447,9 +1447,14 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
|
|||
}
|
||||
}
|
||||
} else {
|
||||
switch(timeUnit) {
|
||||
case 0: { /* 1u */
|
||||
result = result / 1000;
|
||||
int64_t unit = timeUnit * 1000 / factor;
|
||||
switch(unit) {
|
||||
case 0: { /* 1u or 1b */
|
||||
if (timePrec == TSDB_TIME_PRECISION_NANO && timeUnit == 1) {
|
||||
result = result / 1;
|
||||
} else {
|
||||
result = result / 1000;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 1: { /* 1a */
|
||||
|
|
|
@ -52,6 +52,7 @@ typedef enum {
|
|||
SCH_OP_NULL = 0,
|
||||
SCH_OP_EXEC,
|
||||
SCH_OP_FETCH,
|
||||
SCH_OP_GET_STATUS,
|
||||
} SCH_OP_TYPE;
|
||||
|
||||
typedef struct SSchTrans {
|
||||
|
@ -97,13 +98,30 @@ typedef struct SSchStat {
|
|||
} SSchStat;
|
||||
|
||||
typedef struct SSchResInfo {
|
||||
SQueryResult* queryRes;
|
||||
SExecResult* execRes;
|
||||
void** fetchRes;
|
||||
schedulerExecFp execFp;
|
||||
schedulerFetchFp fetchFp;
|
||||
void* userParam;
|
||||
void* cbParam;
|
||||
} SSchResInfo;
|
||||
|
||||
typedef struct SSchOpEvent {
|
||||
SCH_OP_TYPE type;
|
||||
bool begin;
|
||||
SSchedulerReq *pReq;
|
||||
} SSchOpEvent;
|
||||
|
||||
typedef int32_t (*schStatusEnterFp)(void* pHandle, void* pParam);
|
||||
typedef int32_t (*schStatusLeaveFp)(void* pHandle, void* pParam);
|
||||
typedef int32_t (*schStatusEventFp)(void* pHandle, void* pParam, void* pEvent);
|
||||
|
||||
typedef struct SSchStatusFps {
|
||||
EJobTaskType status;
|
||||
schStatusEnterFp enterFp;
|
||||
schStatusLeaveFp leaveFp;
|
||||
schStatusEventFp eventFp;
|
||||
} SSchStatusFps;
|
||||
|
||||
typedef struct SSchedulerMgmt {
|
||||
uint64_t taskId; // sequential taksId
|
||||
uint64_t sId; // schedulerId
|
||||
|
@ -157,7 +175,7 @@ typedef struct SSchLevel {
|
|||
int32_t taskNum;
|
||||
int32_t taskLaunchedNum;
|
||||
int32_t taskDoneNum;
|
||||
SArray *subTasks; // Element is SQueryTask
|
||||
SArray *subTasks; // Element is SSchTask
|
||||
} SSchLevel;
|
||||
|
||||
typedef struct SSchTaskProfile {
|
||||
|
@ -195,12 +213,13 @@ typedef struct SSchTask {
|
|||
typedef struct SSchJobAttr {
|
||||
EExplainMode explainMode;
|
||||
bool queryJob;
|
||||
bool needFetch;
|
||||
bool needFlowCtrl;
|
||||
} SSchJobAttr;
|
||||
|
||||
typedef struct {
|
||||
int32_t op;
|
||||
bool sync;
|
||||
bool syncReq;
|
||||
} SSchOpStatus;
|
||||
|
||||
typedef struct SSchJob {
|
||||
|
@ -231,7 +250,7 @@ typedef struct SSchJob {
|
|||
SSchTask *fetchTask;
|
||||
int32_t errCode;
|
||||
SRWLatch resLock;
|
||||
SQueryExecRes execRes;
|
||||
SExecResult execRes;
|
||||
void *resData; //TODO free it or not
|
||||
int32_t resNumOfRows;
|
||||
SSchResInfo userRes;
|
||||
|
@ -292,19 +311,19 @@ extern SSchedulerMgmt schMgmt;
|
|||
#define SCH_GET_JOB_STATUS(job) atomic_load_8(&(job)->status)
|
||||
#define SCH_GET_JOB_STATUS_STR(job) jobTaskStatusStr(SCH_GET_JOB_STATUS(job))
|
||||
|
||||
#define SCH_JOB_IN_SYNC_OP(job) ((job)->opStatus.op && (job)->opStatus.sync)
|
||||
#define SCH_JOB_IN_ASYNC_EXEC_OP(job) (((job)->opStatus.op == SCH_OP_EXEC) && (!(job)->opStatus.sync))
|
||||
#define SCH_JOB_IN_ASYNC_FETCH_OP(job) (((job)->opStatus.op == SCH_OP_FETCH) && (!(job)->opStatus.sync))
|
||||
#define SCH_JOB_IN_SYNC_OP(job) ((job)->opStatus.op && (job)->opStatus.syncReq)
|
||||
#define SCH_JOB_IN_ASYNC_EXEC_OP(job) ((SCH_OP_EXEC == atomic_val_compare_exchange_32(&(job)->opStatus.op, SCH_OP_EXEC, SCH_OP_NULL)) && (!(job)->opStatus.syncReq))
|
||||
#define SCH_JOB_IN_ASYNC_FETCH_OP(job) ((SCH_OP_FETCH == atomic_val_compare_exchange_32(&(job)->opStatus.op, SCH_OP_FETCH, SCH_OP_NULL)) && (!(job)->opStatus.syncReq))
|
||||
|
||||
#define SCH_SET_JOB_NEED_FLOW_CTRL(_job) (_job)->attr.needFlowCtrl = true
|
||||
#define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl)
|
||||
#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_SRC_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level))
|
||||
|
||||
#define SCH_SET_JOB_TYPE(_job, type) (_job)->attr.queryJob = ((type) != SUBPLAN_TYPE_MODIFY)
|
||||
#define SCH_SET_JOB_TYPE(_job, type) do { if ((type) != SUBPLAN_TYPE_MODIFY) { (_job)->attr.queryJob = true; } } while (0)
|
||||
#define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob)
|
||||
#define SCH_JOB_NEED_FETCH(_job) SCH_IS_QUERY_JOB(_job)
|
||||
#define SCH_IS_WAIT_ALL_JOB(_job) (!SCH_IS_QUERY_JOB(_job))
|
||||
#define SCH_IS_NEED_DROP_JOB(_job) (SCH_IS_QUERY_JOB(_job))
|
||||
#define SCH_JOB_NEED_FETCH(_job) ((_job)->attr.needFetch)
|
||||
#define SCH_JOB_NEED_WAIT(_job) (!SCH_IS_QUERY_JOB(_job))
|
||||
#define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job))
|
||||
#define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode)
|
||||
#define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
|
||||
#define SCH_SUB_TASK_NETWORK_ERR(_code, _len) (SCH_NETWORK_ERR(_code) && ((_len) > 0))
|
||||
|
@ -329,9 +348,10 @@ extern SSchedulerMgmt schMgmt;
|
|||
#define SCH_TASK_WLOG(param, ...) \
|
||||
qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__)
|
||||
|
||||
#define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
|
||||
#define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
|
||||
#define SCH_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
|
||||
#define SCH_SET_ERRNO(_err) do { if (TSDB_CODE_SCH_IGNORE_ERROR != (_err)) { terrno = (_err); } } while (0)
|
||||
#define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { SCH_SET_ERRNO(_code); return _code; } } while (0)
|
||||
#define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { SCH_SET_ERRNO(_code); } return _code; } while (0)
|
||||
#define SCH_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { SCH_SET_ERRNO(code); goto _return; } } while (0)
|
||||
|
||||
#define SCH_LOCK(type, _lock) (SCH_READ == (type) ? taosRLockLatch(_lock) : taosWLockLatch(_lock))
|
||||
#define SCH_UNLOCK(type, _lock) (SCH_READ == (type) ? taosRUnLockLatch(_lock) : taosWUnLockLatch(_lock))
|
||||
|
@ -349,7 +369,7 @@ int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask);
|
|||
int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough);
|
||||
int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask);
|
||||
int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask);
|
||||
int32_t schFetchFromRemote(SSchJob *pJob);
|
||||
int32_t schLaunchFetchTask(SSchJob *pJob);
|
||||
int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode);
|
||||
int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray* taskAction);
|
||||
int32_t schCloneSMsgSendInfo(void *src, void **dst);
|
||||
|
@ -371,25 +391,45 @@ void schFreeRpcCtxVal(const void *arg);
|
|||
int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb);
|
||||
int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execId);
|
||||
int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync);
|
||||
int32_t schExecJobImpl(SSchedulerReq *pReq, SSchJob *pJob, bool sync);
|
||||
int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus);
|
||||
int32_t schCancelJob(SSchJob *pJob);
|
||||
int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode);
|
||||
uint64_t schGenTaskId(void);
|
||||
void schCloseJobRef(void);
|
||||
int32_t schExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes);
|
||||
int32_t schAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob);
|
||||
int32_t schFetchRows(SSchJob *pJob);
|
||||
int32_t schAsyncFetchRows(SSchJob *pJob);
|
||||
int32_t schJobFetchRows(SSchJob *pJob);
|
||||
int32_t schJobFetchRowsA(SSchJob *pJob);
|
||||
int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId);
|
||||
int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList);
|
||||
void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo);
|
||||
char* schGetOpStr(SCH_OP_TYPE type);
|
||||
int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync);
|
||||
int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob);
|
||||
int32_t schSetJobQueryRes(SSchJob* pJob, SQueryResult* pRes);
|
||||
int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq);
|
||||
int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq);
|
||||
int32_t schDumpJobExecRes(SSchJob* pJob, SExecResult* pRes);
|
||||
int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet* pEpSet);
|
||||
int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32_t rspCode);
|
||||
void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int32_t errCode);
|
||||
int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq);
|
||||
void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode);
|
||||
int32_t schProcessOnCbBegin(SSchJob** job, SSchTask** task, uint64_t qId, int64_t rId, uint64_t tId);
|
||||
void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask);
|
||||
bool schJobDone(SSchJob *pJob);
|
||||
int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask);
|
||||
int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask);
|
||||
int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param);
|
||||
int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq);
|
||||
int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int32_t errCode);
|
||||
int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask);
|
||||
void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode);
|
||||
int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry);
|
||||
int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode);
|
||||
int32_t schProcessOnJobPartialSuccess(SSchJob *pJob);
|
||||
void schFreeTask(SSchJob *pJob, SSchTask *pTask);
|
||||
void schDropTaskInHashList(SSchJob *pJob, SHashObj *list);
|
||||
int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level);
|
||||
int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask);
|
||||
int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel);
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
|
@ -14,16 +14,16 @@
|
|||
*/
|
||||
|
||||
#include "query.h"
|
||||
#include "schedulerInt.h"
|
||||
#include "schInt.h"
|
||||
|
||||
tsem_t schdRspSem;
|
||||
|
||||
void schdExecCallback(SQueryResult* pResult, void* param, int32_t code) {
|
||||
void schdExecCallback(SExecResult* pResult, void* param, int32_t code) {
|
||||
if (code) {
|
||||
pResult->code = code;
|
||||
}
|
||||
|
||||
*(SQueryResult*)param = *pResult;
|
||||
*(SExecResult*)param = *pResult;
|
||||
|
||||
taosMemoryFree(pResult);
|
||||
|
||||
|
|
|
@ -13,7 +13,7 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "schedulerInt.h"
|
||||
#include "schInt.h"
|
||||
#include "tmsg.h"
|
||||
#include "query.h"
|
||||
#include "catalog.h"
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -16,7 +16,7 @@
|
|||
#include "catalog.h"
|
||||
#include "command.h"
|
||||
#include "query.h"
|
||||
#include "schedulerInt.h"
|
||||
#include "schInt.h"
|
||||
#include "tmsg.h"
|
||||
#include "tref.h"
|
||||
#include "trpc.h"
|
||||
|
@ -37,7 +37,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy
|
|||
TMSG_INFO(msgType));
|
||||
}
|
||||
|
||||
if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
|
||||
if (taskStatus != JOB_TASK_STATUS_EXEC && taskStatus != JOB_TASK_STATUS_PART_SUCC) {
|
||||
SCH_TASK_DLOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus),
|
||||
TMSG_INFO(msgType));
|
||||
}
|
||||
|
@ -51,7 +51,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy
|
|||
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
|
||||
if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
|
||||
if (taskStatus != JOB_TASK_STATUS_EXEC && taskStatus != JOB_TASK_STATUS_PART_SUCC) {
|
||||
SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus),
|
||||
TMSG_INFO(msgType));
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
|
@ -76,7 +76,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy
|
|||
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
|
||||
if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
|
||||
if (taskStatus != JOB_TASK_STATUS_EXEC && taskStatus != JOB_TASK_STATUS_PART_SUCC) {
|
||||
SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus),
|
||||
TMSG_INFO(msgType));
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
|
@ -88,9 +88,21 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy
|
|||
}
|
||||
|
||||
// Note: no more task error processing, handled in function internal
|
||||
int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, char *msg, int32_t msgSize,
|
||||
int32_t rspCode) {
|
||||
int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) {
|
||||
int32_t code = 0;
|
||||
char *msg = pMsg->pData;
|
||||
int32_t msgSize = pMsg->len;
|
||||
int32_t msgType = pMsg->msgType;
|
||||
|
||||
bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode));
|
||||
SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, execId));
|
||||
|
||||
SCH_ERR_JRET(schValidateReceivedMsgType(pJob, pTask, msgType));
|
||||
|
||||
int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1);
|
||||
if (SCH_NEED_REDIRECT(reqType, rspCode, pMsg->len)) {
|
||||
SCH_RET(schHandleRedirect(pJob, pTask, (SDataBuf *)pMsg, rspCode));
|
||||
}
|
||||
|
||||
switch (msgType) {
|
||||
case TDMT_VND_CREATE_TABLE_RSP: {
|
||||
|
@ -308,7 +320,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SCH_ERR_JRET(schFetchFromRemote(pJob));
|
||||
SCH_ERR_JRET(schLaunchFetchTask(pJob));
|
||||
|
||||
taosMemoryFreeClear(msg);
|
||||
|
||||
|
@ -325,7 +337,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
|
|||
atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows));
|
||||
|
||||
if (rsp->completed) {
|
||||
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED);
|
||||
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC);
|
||||
}
|
||||
|
||||
SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed);
|
||||
|
@ -362,65 +374,24 @@ _return:
|
|||
|
||||
int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) {
|
||||
int32_t code = 0;
|
||||
int32_t msgType = pMsg->msgType;
|
||||
SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param;
|
||||
SSchTask *pTask = NULL;
|
||||
SSchJob *pJob = NULL;
|
||||
|
||||
SSchJob *pJob = schAcquireJob(pParam->refId);
|
||||
if (NULL == pJob) {
|
||||
qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:0x%" PRIx64,
|
||||
pParam->queryId, pParam->taskId, pParam->refId);
|
||||
SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED);
|
||||
}
|
||||
qDebug("begin to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, tstrerror(rspCode));
|
||||
|
||||
SCH_ERR_JRET(schGetTaskInJob(pJob, pParam->taskId, &pTask));
|
||||
|
||||
SCH_LOCK_TASK(pTask);
|
||||
SCH_ERR_RET(schProcessOnCbBegin(&pJob, &pTask, pParam->queryId, pParam->refId, pParam->taskId));
|
||||
|
||||
SCH_TASK_DLOG("rsp msg received, type:%s, handle:%p, code:%s", TMSG_INFO(msgType), pMsg->handle, tstrerror(rspCode));
|
||||
|
||||
if (pParam->execId != pTask->execId) {
|
||||
SCH_TASK_DLOG("execId %d mis-match current execId %d", pParam->execId, pTask->execId);
|
||||
goto _return;
|
||||
}
|
||||
|
||||
bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode));
|
||||
SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, pParam->execId));
|
||||
|
||||
int8_t status = 0;
|
||||
if (schJobNeedToStop(pJob, &status)) {
|
||||
SCH_TASK_ELOG("rsp will not be processed cause of job status %s, rspCode:0x%x", jobTaskStatusStr(status), rspCode);
|
||||
code = atomic_load_32(&pJob->errCode);
|
||||
goto _return;
|
||||
}
|
||||
|
||||
SCH_ERR_JRET(schValidateReceivedMsgType(pJob, pTask, msgType));
|
||||
|
||||
int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1);
|
||||
if (SCH_NEED_REDIRECT(reqType, rspCode, pMsg->len)) {
|
||||
code = schHandleRedirect(pJob, pTask, (SDataBuf *)pMsg, rspCode);
|
||||
goto _return;
|
||||
}
|
||||
|
||||
schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode);
|
||||
code = schHandleResponseMsg(pJob, pTask, pParam->execId, pMsg, rspCode);
|
||||
pMsg->pData = NULL;
|
||||
|
||||
_return:
|
||||
|
||||
if (pTask) {
|
||||
if (code) {
|
||||
schProcessOnTaskFailure(pJob, pTask, code);
|
||||
}
|
||||
|
||||
SCH_UNLOCK_TASK(pTask);
|
||||
}
|
||||
|
||||
if (pJob) {
|
||||
schReleaseJob(pParam->refId);
|
||||
}
|
||||
schProcessOnCbEnd(pJob, pTask, code);
|
||||
|
||||
taosMemoryFreeClear(pMsg->pData);
|
||||
taosMemoryFreeClear(param);
|
||||
|
||||
qDebug("end to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, tstrerror(rspCode));
|
||||
|
||||
SCH_RET(code);
|
||||
}
|
||||
|
||||
|
@ -451,6 +422,37 @@ int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) {
|
|||
}
|
||||
|
||||
|
||||
int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) {
|
||||
SSchedulerHbRsp rsp = {0};
|
||||
SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param;
|
||||
|
||||
if (code) {
|
||||
qError("hb rsp error:%s", tstrerror(code));
|
||||
SCH_ERR_JRET(code);
|
||||
}
|
||||
|
||||
if (tDeserializeSSchedulerHbRsp(pMsg->pData, pMsg->len, &rsp)) {
|
||||
qError("invalid hb rsp msg, size:%d", pMsg->len);
|
||||
SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
|
||||
}
|
||||
|
||||
SSchTrans trans = {0};
|
||||
trans.pTrans = pParam->pTrans;
|
||||
trans.pHandle = pMsg->handle;
|
||||
|
||||
SCH_ERR_JRET(schUpdateHbConnection(&rsp.epId, &trans));
|
||||
|
||||
SCH_ERR_JRET(schProcessOnTaskStatusRsp(&rsp.epId, rsp.taskStatus));
|
||||
|
||||
_return:
|
||||
|
||||
tFreeSSchedulerHbRsp(&rsp);
|
||||
taosMemoryFree(param);
|
||||
|
||||
SCH_RET(code);
|
||||
}
|
||||
|
||||
|
||||
int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bool isHb, SSchTrans *trans, void **pParam) {
|
||||
if (!isHb) {
|
||||
SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam));
|
||||
|
@ -692,36 +694,6 @@ _return:
|
|||
SCH_RET(code);
|
||||
}
|
||||
|
||||
int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) {
|
||||
SSchedulerHbRsp rsp = {0};
|
||||
SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param;
|
||||
|
||||
if (code) {
|
||||
qError("hb rsp error:%s", tstrerror(code));
|
||||
SCH_ERR_JRET(code);
|
||||
}
|
||||
|
||||
if (tDeserializeSSchedulerHbRsp(pMsg->pData, pMsg->len, &rsp)) {
|
||||
qError("invalid hb rsp msg, size:%d", pMsg->len);
|
||||
SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
|
||||
}
|
||||
|
||||
SSchTrans trans = {0};
|
||||
trans.pTrans = pParam->pTrans;
|
||||
trans.pHandle = pMsg->handle;
|
||||
|
||||
SCH_ERR_JRET(schUpdateHbConnection(&rsp.epId, &trans));
|
||||
|
||||
SCH_ERR_JRET(schProcessOnTaskStatusRsp(&rsp.epId, rsp.taskStatus));
|
||||
|
||||
_return:
|
||||
|
||||
tFreeSSchedulerHbRsp(&rsp);
|
||||
taosMemoryFree(param);
|
||||
|
||||
SCH_RET(code);
|
||||
}
|
||||
|
||||
int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb) {
|
||||
int32_t code = 0;
|
||||
int32_t msgType = TDMT_SCH_LINK_BROKEN;
|
||||
|
|
|
@ -0,0 +1,94 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "catalog.h"
|
||||
#include "command.h"
|
||||
#include "query.h"
|
||||
#include "schInt.h"
|
||||
#include "tmsg.h"
|
||||
#include "tref.h"
|
||||
#include "trpc.h"
|
||||
|
||||
int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) {
|
||||
int32_t code = 0;
|
||||
SCH_ERR_JRET(schUpdateJobStatus(pJob, status));
|
||||
|
||||
switch (status) {
|
||||
case JOB_TASK_STATUS_INIT:
|
||||
break;
|
||||
case JOB_TASK_STATUS_EXEC:
|
||||
SCH_ERR_JRET(schExecJob(pJob, (SSchedulerReq*)param));
|
||||
break;
|
||||
case JOB_TASK_STATUS_PART_SUCC:
|
||||
SCH_ERR_JRET(schProcessOnJobPartialSuccess(pJob));
|
||||
break;
|
||||
case JOB_TASK_STATUS_SUCC:
|
||||
break;
|
||||
case JOB_TASK_STATUS_FAIL:
|
||||
SCH_RET(schProcessOnJobFailure(pJob, (param ? *(int32_t*)param : 0)));
|
||||
break;
|
||||
case JOB_TASK_STATUS_DROP:
|
||||
SCH_ERR_JRET(schProcessOnJobDropped(pJob, *(int32_t*)param));
|
||||
|
||||
if (taosRemoveRef(schMgmt.jobRef, pJob->refId)) {
|
||||
SCH_JOB_ELOG("remove job from job list failed, refId:0x%" PRIx64, pJob->refId);
|
||||
} else {
|
||||
SCH_JOB_DLOG("job removed from jobRef list, refId:0x%" PRIx64, pJob->refId);
|
||||
}
|
||||
break;
|
||||
default: {
|
||||
SCH_JOB_ELOG("unknown job status %d", status);
|
||||
SCH_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
||||
_return:
|
||||
|
||||
SCH_RET(schProcessOnJobFailure(pJob, code));
|
||||
}
|
||||
|
||||
int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) {
|
||||
SSchJob *pJob = schAcquireJob(jobId);
|
||||
if (NULL == pJob) {
|
||||
qError("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId);
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
|
||||
*job = pJob;
|
||||
|
||||
SCH_RET(schProcessOnOpBegin(pJob, type, pReq));
|
||||
}
|
||||
|
||||
int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int32_t errCode) {
|
||||
int32_t code = errCode;
|
||||
|
||||
if (NULL == pJob) {
|
||||
SCH_RET(code);
|
||||
}
|
||||
|
||||
schProcessOnOpEnd(pJob, type, pReq, errCode);
|
||||
|
||||
if (TSDB_CODE_SCH_IGNORE_ERROR == errCode) {
|
||||
code = pJob->errCode;
|
||||
}
|
||||
|
||||
schReleaseJob(pJob->refId);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,830 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "catalog.h"
|
||||
#include "command.h"
|
||||
#include "query.h"
|
||||
#include "schInt.h"
|
||||
#include "tmsg.h"
|
||||
#include "tref.h"
|
||||
#include "trpc.h"
|
||||
|
||||
|
||||
|
||||
void schFreeTask(SSchJob *pJob, SSchTask *pTask) {
|
||||
schDeregisterTaskHb(pJob, pTask);
|
||||
|
||||
if (pTask->candidateAddrs) {
|
||||
taosArrayDestroy(pTask->candidateAddrs);
|
||||
}
|
||||
|
||||
taosMemoryFreeClear(pTask->msg);
|
||||
|
||||
if (pTask->children) {
|
||||
taosArrayDestroy(pTask->children);
|
||||
}
|
||||
|
||||
if (pTask->parents) {
|
||||
taosArrayDestroy(pTask->parents);
|
||||
}
|
||||
|
||||
if (pTask->execNodes) {
|
||||
taosHashCleanup(pTask->execNodes);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel) {
|
||||
pTask->plan = pPlan;
|
||||
pTask->level = pLevel;
|
||||
pTask->execId = -1;
|
||||
pTask->maxExecTimes = SCH_TASK_MAX_EXEC_TIMES;
|
||||
pTask->timeoutUsec = SCH_DEFAULT_TASK_TIMEOUT_USEC;
|
||||
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT);
|
||||
pTask->taskId = schGenTaskId();
|
||||
pTask->execNodes = taosHashInit(SCH_MAX_CANDIDATE_EP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
|
||||
if (NULL == pTask->execNodes) {
|
||||
SCH_TASK_ELOG("taosHashInit %d execNodes failed", SCH_MAX_CANDIDATE_EP_NUM);
|
||||
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) {
|
||||
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx);
|
||||
if (NULL == addr) {
|
||||
SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", pTask->candidateIdx,
|
||||
(int32_t)taosArrayGetSize(pTask->candidateAddrs));
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
|
||||
}
|
||||
|
||||
pTask->succeedAddr = *addr;
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execId) {
|
||||
SSchNodeInfo nodeInfo = {.addr = *addr, .handle = NULL};
|
||||
|
||||
if (taosHashPut(pTask->execNodes, &execId, sizeof(execId), &nodeInfo, sizeof(nodeInfo))) {
|
||||
SCH_TASK_ELOG("taosHashPut nodeInfo to execNodes failed, errno:%d", errno);
|
||||
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
SCH_TASK_DLOG("task execNode added, execId:%d", execId);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schDropTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int32_t execId) {
|
||||
if (NULL == pTask->execNodes) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (taosHashRemove(pTask->execNodes, &execId, sizeof(execId))) {
|
||||
SCH_TASK_ELOG("fail to remove execId %d from execNodeList", execId);
|
||||
} else {
|
||||
SCH_TASK_DLOG("execId %d removed from execNodeList", execId);
|
||||
}
|
||||
|
||||
if (execId != pTask->execId) { // ignore it
|
||||
SCH_TASK_DLOG("execId %d is not current execId %d", execId, pTask->execId);
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schUpdateTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int32_t execId) {
|
||||
if (taosHashGetSize(pTask->execNodes) <= 0) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SSchNodeInfo *nodeInfo = taosHashGet(pTask->execNodes, &execId, sizeof(execId));
|
||||
nodeInfo->handle = handle;
|
||||
|
||||
SCH_TASK_DLOG("handle updated to %p for execId %d", handle, execId);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId) {
|
||||
if (dropExecNode) {
|
||||
SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execId));
|
||||
}
|
||||
|
||||
SCH_SET_TASK_HANDLE(pTask, handle);
|
||||
|
||||
schUpdateTaskExecNode(pJob, pTask, handle, execId);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
// Note: no more task error processing, handled in function internal
|
||||
int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) {
|
||||
if (TSDB_CODE_SCH_IGNORE_ERROR == errCode) {
|
||||
return TSDB_CODE_SCH_IGNORE_ERROR;
|
||||
}
|
||||
|
||||
int8_t status = 0;
|
||||
if (schJobNeedToStop(pJob, &status)) {
|
||||
SCH_TASK_DLOG("no more task failure processing cause of job status %s", jobTaskStatusStr(status));
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR);
|
||||
}
|
||||
|
||||
if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXEC) {
|
||||
SCH_TASK_ELOG("task already not in EXEC status, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
|
||||
if (errCode == TSDB_CODE_SCH_TIMEOUT_ERROR) {
|
||||
SCH_LOG_TASK_WAIT_TS(pTask);
|
||||
} else {
|
||||
SCH_LOG_TASK_END_TS(pTask);
|
||||
}
|
||||
|
||||
bool needRetry = false;
|
||||
bool moved = false;
|
||||
int32_t taskDone = 0;
|
||||
|
||||
SCH_TASK_DLOG("taskOnFailure, code:%s", tstrerror(errCode));
|
||||
|
||||
SCH_ERR_RET(schTaskCheckSetRetry(pJob, pTask, errCode, &needRetry));
|
||||
|
||||
if (!needRetry) {
|
||||
SCH_TASK_ELOG("task failed and no more retry, code:%s", tstrerror(errCode));
|
||||
|
||||
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_FAIL);
|
||||
|
||||
if (SCH_JOB_NEED_WAIT(pJob)) {
|
||||
SCH_LOCK(SCH_WRITE, &pTask->level->lock);
|
||||
pTask->level->taskFailed++;
|
||||
taskDone = pTask->level->taskSucceed + pTask->level->taskFailed;
|
||||
SCH_UNLOCK(SCH_WRITE, &pTask->level->lock);
|
||||
|
||||
schUpdateJobErrCode(pJob, errCode);
|
||||
|
||||
if (taskDone < pTask->level->taskNum) {
|
||||
SCH_TASK_DLOG("need to wait other tasks, doneNum:%d, allNum:%d", taskDone, pTask->level->taskNum);
|
||||
SCH_RET(TSDB_CODE_SCH_IGNORE_ERROR);
|
||||
}
|
||||
|
||||
SCH_RET(atomic_load_32(&pJob->errCode));
|
||||
}
|
||||
} else {
|
||||
SCH_ERR_RET(schHandleTaskRetry(pJob, pTask));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SCH_RET(errCode);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Note: no more task error processing, handled in function internal
|
||||
int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
|
||||
bool moved = false;
|
||||
int32_t code = 0;
|
||||
|
||||
SCH_TASK_DLOG("taskOnSuccess, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
|
||||
|
||||
SCH_LOG_TASK_END_TS(pTask);
|
||||
|
||||
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PART_SUCC);
|
||||
|
||||
SCH_ERR_RET(schRecordTaskSucceedNode(pJob, pTask));
|
||||
|
||||
SCH_ERR_RET(schLaunchTasksInFlowCtrlList(pJob, pTask));
|
||||
|
||||
int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0;
|
||||
if (parentNum == 0) {
|
||||
int32_t taskDone = 0;
|
||||
if (SCH_JOB_NEED_WAIT(pJob)) {
|
||||
SCH_LOCK(SCH_WRITE, &pTask->level->lock);
|
||||
pTask->level->taskSucceed++;
|
||||
taskDone = pTask->level->taskSucceed + pTask->level->taskFailed;
|
||||
SCH_UNLOCK(SCH_WRITE, &pTask->level->lock);
|
||||
|
||||
if (taskDone < pTask->level->taskNum) {
|
||||
SCH_TASK_DLOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else if (taskDone > pTask->level->taskNum) {
|
||||
SCH_TASK_ELOG("taskDone number invalid, done:%d, total:%d", taskDone, pTask->level->taskNum);
|
||||
}
|
||||
|
||||
if (pTask->level->taskFailed > 0) {
|
||||
SCH_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, NULL));
|
||||
} else {
|
||||
SCH_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL));
|
||||
}
|
||||
} else {
|
||||
pJob->resNode = pTask->succeedAddr;
|
||||
}
|
||||
|
||||
pJob->fetchTask = pTask;
|
||||
|
||||
SCH_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL));
|
||||
}
|
||||
|
||||
/*
|
||||
if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CANDIDATE_EP_NUM) {
|
||||
strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn));
|
||||
job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port;
|
||||
|
||||
++job->dataSrcEps.numOfEps;
|
||||
}
|
||||
*/
|
||||
|
||||
for (int32_t i = 0; i < parentNum; ++i) {
|
||||
SSchTask *parent = *(SSchTask **)taosArrayGet(pTask->parents, i);
|
||||
int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1);
|
||||
|
||||
SCH_LOCK(SCH_WRITE, &parent->lock);
|
||||
SDownstreamSourceNode source = {.type = QUERY_NODE_DOWNSTREAM_SOURCE,
|
||||
.taskId = pTask->taskId,
|
||||
.schedId = schMgmt.sId,
|
||||
.execId = pTask->execId,
|
||||
.addr = pTask->succeedAddr};
|
||||
qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source);
|
||||
SCH_UNLOCK(SCH_WRITE, &parent->lock);
|
||||
|
||||
if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) {
|
||||
SCH_TASK_DLOG("all %d children task done, start to launch parent task 0x%" PRIx64, readyNum, parent->taskId);
|
||||
SCH_ERR_RET(schLaunchTask(pJob, parent));
|
||||
}
|
||||
}
|
||||
|
||||
SCH_ERR_RET(schLaunchJobLowerLevel(pJob, pTask));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) {
|
||||
if (SCH_IS_DATA_SRC_QRY_TASK(pTask)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (SCH_TASK_TIMEOUT(pTask) && JOB_TASK_STATUS_EXEC == pTask->status &&
|
||||
pJob->fetchTask != pTask && taosArrayGetSize(pTask->candidateAddrs) > 1) {
|
||||
SCH_TASK_DLOG("task execId %d will be rescheduled now", pTask->execId);
|
||||
schDropTaskOnExecNode(pJob, pTask);
|
||||
taosHashClear(pTask->execNodes);
|
||||
|
||||
SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, TSDB_CODE_SCH_TIMEOUT_ERROR));
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32_t rspCode) {
|
||||
int32_t code = 0;
|
||||
|
||||
if ((pTask->execId + 1) >= pTask->maxExecTimes) {
|
||||
SCH_TASK_DLOG("task no more retry since reach max try times, execId:%d", pTask->execId);
|
||||
schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, (void*)&rspCode);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SCH_TASK_DLOG("task will be redirected now, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
|
||||
|
||||
schDropTaskOnExecNode(pJob, pTask);
|
||||
taosHashClear(pTask->execNodes);
|
||||
SCH_ERR_JRET(schRemoveTaskFromExecList(pJob, pTask));
|
||||
schDeregisterTaskHb(pJob, pTask);
|
||||
atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1);
|
||||
taosMemoryFreeClear(pTask->msg);
|
||||
pTask->msgLen = 0;
|
||||
pTask->lastMsgType = 0;
|
||||
memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr));
|
||||
|
||||
if (SCH_IS_DATA_SRC_QRY_TASK(pTask)) {
|
||||
if (pData) {
|
||||
SCH_ERR_JRET(schUpdateTaskCandidateAddr(pJob, pTask, pData->pEpSet));
|
||||
}
|
||||
|
||||
if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) {
|
||||
if (JOB_TASK_STATUS_EXEC == SCH_GET_TASK_STATUS(pTask)) {
|
||||
SCH_ERR_JRET(schLaunchTasksInFlowCtrlList(pJob, pTask));
|
||||
}
|
||||
}
|
||||
|
||||
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT);
|
||||
|
||||
SCH_ERR_JRET(schLaunchTask(pJob, pTask));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
// merge plan
|
||||
|
||||
pTask->childReady = 0;
|
||||
|
||||
qClearSubplanExecutionNode(pTask->plan);
|
||||
|
||||
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT);
|
||||
|
||||
int32_t childrenNum = taosArrayGetSize(pTask->children);
|
||||
for (int32_t i = 0; i < childrenNum; ++i) {
|
||||
SSchTask* pChild = taosArrayGetP(pTask->children, i);
|
||||
SCH_LOCK_TASK(pChild);
|
||||
schDoTaskRedirect(pJob, pChild, NULL, rspCode);
|
||||
SCH_UNLOCK_TASK(pChild);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
||||
_return:
|
||||
|
||||
SCH_RET(schProcessOnTaskFailure(pJob, pTask, code));
|
||||
}
|
||||
|
||||
int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32_t rspCode) {
|
||||
int32_t code = 0;
|
||||
|
||||
if (SCH_IS_DATA_SRC_QRY_TASK(pTask)) {
|
||||
if (NULL == pData->pEpSet) {
|
||||
SCH_TASK_ELOG("no epset updated while got error %s", tstrerror(rspCode));
|
||||
SCH_ERR_JRET(rspCode);
|
||||
}
|
||||
}
|
||||
|
||||
SCH_RET(schDoTaskRedirect(pJob, pTask, pData, rspCode));
|
||||
|
||||
_return:
|
||||
|
||||
SCH_RET(schProcessOnTaskFailure(pJob, pTask, code));
|
||||
}
|
||||
|
||||
int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) {
|
||||
int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES);
|
||||
if (0 != code) {
|
||||
if (HASH_NODE_EXIST(code)) {
|
||||
SCH_TASK_ELOG("task already in execTask list, code:%x", code);
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
|
||||
}
|
||||
|
||||
SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno);
|
||||
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
SCH_TASK_DLOG("task added to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
/*
|
||||
int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
|
||||
if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) {
|
||||
SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
|
||||
} else {
|
||||
SCH_TASK_DLOG("task removed from execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks));
|
||||
}
|
||||
|
||||
int32_t code = taosHashPut(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES);
|
||||
if (0 != code) {
|
||||
if (HASH_NODE_EXIST(code)) {
|
||||
*moved = true;
|
||||
SCH_TASK_ELOG("task already in succTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
|
||||
SCH_TASK_ELOG("taosHashPut task to succTask list failed, errno:%d", errno);
|
||||
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
*moved = true;
|
||||
|
||||
SCH_TASK_DLOG("task moved to succTask list, numOfTasks:%d", taosHashGetSize(pJob->succTasks));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
|
||||
*moved = false;
|
||||
|
||||
if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) {
|
||||
SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
|
||||
}
|
||||
|
||||
int32_t code = taosHashPut(pJob->failTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES);
|
||||
if (0 != code) {
|
||||
if (HASH_NODE_EXIST(code)) {
|
||||
*moved = true;
|
||||
|
||||
SCH_TASK_WLOG("task already in failTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
|
||||
SCH_TASK_ELOG("taosHashPut task to failTask list failed, errno:%d", errno);
|
||||
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
*moved = true;
|
||||
|
||||
SCH_TASK_DLOG("task moved to failTask list, numOfTasks:%d", taosHashGetSize(pJob->failTasks));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
|
||||
if (0 != taosHashRemove(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId))) {
|
||||
SCH_TASK_WLOG("remove task from succTask list failed, may not exist, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
|
||||
}
|
||||
|
||||
int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES);
|
||||
if (0 != code) {
|
||||
if (HASH_NODE_EXIST(code)) {
|
||||
*moved = true;
|
||||
|
||||
SCH_TASK_ELOG("task already in execTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
|
||||
SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno);
|
||||
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
*moved = true;
|
||||
|
||||
SCH_TASK_DLOG("task moved to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
*/
|
||||
|
||||
int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) {
|
||||
if (TSDB_CODE_SCH_TIMEOUT_ERROR == errCode) {
|
||||
pTask->maxExecTimes++;
|
||||
if (pTask->timeoutUsec < SCH_MAX_TASK_TIMEOUT_USEC) {
|
||||
pTask->timeoutUsec *= 2;
|
||||
if (pTask->timeoutUsec > SCH_MAX_TASK_TIMEOUT_USEC) {
|
||||
pTask->timeoutUsec = SCH_MAX_TASK_TIMEOUT_USEC;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((pTask->execId + 1) >= pTask->maxExecTimes) {
|
||||
*needRetry = false;
|
||||
SCH_TASK_DLOG("task no more retry since reach max try times, execId:%d", pTask->execId);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (!SCH_NEED_RETRY(pTask->lastMsgType, errCode)) {
|
||||
*needRetry = false;
|
||||
SCH_TASK_DLOG("task no more retry cause of errCode, errCode:%x - %s", errCode, tstrerror(errCode));
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (SCH_IS_DATA_SRC_TASK(pTask)) {
|
||||
if ((pTask->execId + 1) >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) {
|
||||
*needRetry = false;
|
||||
SCH_TASK_DLOG("task no more retry since all ep tried, execId:%d, epNum:%d", pTask->execId,
|
||||
SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode));
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
} else {
|
||||
int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs);
|
||||
|
||||
if ((pTask->candidateIdx + 1) >= candidateNum && (TSDB_CODE_SCH_TIMEOUT_ERROR != errCode)) {
|
||||
*needRetry = false;
|
||||
SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d",
|
||||
pTask->candidateIdx, candidateNum);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
*needRetry = true;
|
||||
SCH_TASK_DLOG("task need the %dth retry, errCode:%x - %s", pTask->execId + 1, errCode, tstrerror(errCode));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) {
|
||||
atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1);
|
||||
|
||||
SCH_ERR_RET(schRemoveTaskFromExecList(pJob, pTask));
|
||||
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT);
|
||||
|
||||
if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) {
|
||||
SCH_ERR_RET(schLaunchTasksInFlowCtrlList(pJob, pTask));
|
||||
}
|
||||
|
||||
schDeregisterTaskHb(pJob, pTask);
|
||||
|
||||
if (SCH_IS_DATA_SRC_TASK(pTask)) {
|
||||
SCH_SWITCH_EPSET(&pTask->plan->execNode);
|
||||
} else {
|
||||
int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs);
|
||||
if (++pTask->candidateIdx >= candidateNum) {
|
||||
pTask->candidateIdx = 0;
|
||||
}
|
||||
}
|
||||
|
||||
SCH_ERR_RET(schLaunchTask(pJob, pTask));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) {
|
||||
int32_t addNum = 0;
|
||||
int32_t nodeNum = 0;
|
||||
|
||||
if (pJob->nodeList) {
|
||||
nodeNum = taosArrayGetSize(pJob->nodeList);
|
||||
|
||||
for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) {
|
||||
SQueryNodeLoad *nload = taosArrayGet(pJob->nodeList, i);
|
||||
SQueryNodeAddr *naddr = &nload->addr;
|
||||
|
||||
if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) {
|
||||
SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno);
|
||||
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
SCH_TASK_DLOG("set %dth candidate addr, id %d, fqdn:%s, port:%d", i, naddr->nodeId, SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port);
|
||||
|
||||
++addNum;
|
||||
}
|
||||
}
|
||||
|
||||
if (addNum <= 0) {
|
||||
SCH_TASK_ELOG("no available execNode as candidates, nodeNum:%d", nodeNum);
|
||||
SCH_ERR_RET(TSDB_CODE_TSC_NO_EXEC_NODE);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) {
|
||||
if (NULL != pTask->candidateAddrs) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
pTask->candidateIdx = 0;
|
||||
pTask->candidateAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr));
|
||||
if (NULL == pTask->candidateAddrs) {
|
||||
SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCH_MAX_CANDIDATE_EP_NUM);
|
||||
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
if (pTask->plan->execNode.epSet.numOfEps > 0) {
|
||||
if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) {
|
||||
SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", errno);
|
||||
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
SCH_TASK_DLOG("use execNode in plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (SCH_IS_DATA_SRC_QRY_TASK(pTask)) {
|
||||
SCH_TASK_ELOG("no execNode specifed for data src task, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps);
|
||||
SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
|
||||
}
|
||||
|
||||
SCH_ERR_RET(schSetAddrsFromNodeList(pJob, pTask));
|
||||
|
||||
/*
|
||||
for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) {
|
||||
strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i]));
|
||||
epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i];
|
||||
|
||||
++epSet->numOfEps;
|
||||
}
|
||||
*/
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet* pEpSet) {
|
||||
if (NULL == pTask->candidateAddrs || 1 != taosArrayGetSize(pTask->candidateAddrs)) {
|
||||
SCH_TASK_ELOG("not able to update cndidate addr, addr num %d", (int32_t)(pTask->candidateAddrs ? taosArrayGetSize(pTask->candidateAddrs): 0));
|
||||
SCH_ERR_RET(TSDB_CODE_APP_ERROR);
|
||||
}
|
||||
|
||||
SQueryNodeAddr* pAddr = taosArrayGet(pTask->candidateAddrs, 0);
|
||||
|
||||
SEp* pOld = &pAddr->epSet.eps[pAddr->epSet.inUse];
|
||||
SEp* pNew = &pEpSet->eps[pEpSet->inUse];
|
||||
|
||||
SCH_TASK_DLOG("update task ep from %s:%d to %s:%d", pOld->fqdn, pOld->port, pNew->fqdn, pNew->port);
|
||||
|
||||
memcpy(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask) {
|
||||
int32_t code = taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId));
|
||||
if (code) {
|
||||
SCH_TASK_ELOG("task failed to rm from execTask list, code:%x", code);
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) {
|
||||
if (NULL == pTask->execNodes) {
|
||||
SCH_TASK_DLOG("no exec address, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
|
||||
return;
|
||||
}
|
||||
|
||||
int32_t size = (int32_t)taosHashGetSize(pTask->execNodes);
|
||||
|
||||
if (size <= 0) {
|
||||
SCH_TASK_DLOG("task has no execNodes, no need to drop it, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
|
||||
return;
|
||||
}
|
||||
|
||||
SSchNodeInfo *nodeInfo = taosHashIterate(pTask->execNodes, NULL);
|
||||
while (nodeInfo) {
|
||||
SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle);
|
||||
|
||||
schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK);
|
||||
|
||||
nodeInfo = taosHashIterate(pTask->execNodes, nodeInfo);
|
||||
}
|
||||
|
||||
SCH_TASK_DLOG("task has been dropped on %d exec nodes", size);
|
||||
}
|
||||
|
||||
|
||||
|
||||
int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList) {
|
||||
int32_t taskNum = (int32_t)taosArrayGetSize(pStatusList);
|
||||
SSchTask *pTask = NULL;
|
||||
SSchJob *pJob = NULL;
|
||||
|
||||
qDebug("%d task status in hb rsp from nodeId:%d, fqdn:%s, port:%d", taskNum, pEpId->nodeId, pEpId->ep.fqdn, pEpId->ep.port);
|
||||
|
||||
for (int32_t i = 0; i < taskNum; ++i) {
|
||||
STaskStatus *pStatus = taosArrayGet(pStatusList, i);
|
||||
int32_t code = 0;
|
||||
|
||||
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s",
|
||||
pStatus->queryId, pStatus->taskId, pStatus->execId, jobTaskStatusStr(pStatus->status));
|
||||
|
||||
if (schProcessOnCbBegin(&pJob, &pTask, pStatus->queryId, pStatus->refId, pStatus->taskId)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (pStatus->execId != pTask->execId) {
|
||||
//TODO
|
||||
SCH_TASK_DLOG("execId %d mis-match current execId %d", pStatus->execId, pTask->execId);
|
||||
schProcessOnCbEnd(pJob, pTask, 0);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (pStatus->status == JOB_TASK_STATUS_FAIL) {
|
||||
// RECORD AND HANDLE ERROR!!!!
|
||||
schProcessOnCbEnd(pJob, pTask, 0);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (pStatus->status == JOB_TASK_STATUS_INIT) {
|
||||
code = schRescheduleTask(pJob, pTask);
|
||||
}
|
||||
|
||||
schProcessOnCbEnd(pJob, pTask, code);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) {
|
||||
int8_t status = 0;
|
||||
int32_t code = 0;
|
||||
|
||||
atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1);
|
||||
pTask->execId++;
|
||||
|
||||
SCH_TASK_DLOG("start to launch task's %dth exec", pTask->execId);
|
||||
|
||||
SCH_LOG_TASK_START_TS(pTask);
|
||||
|
||||
if (schJobNeedToStop(pJob, &status)) {
|
||||
SCH_TASK_DLOG("no need to launch task cause of job status %s", jobTaskStatusStr(status));
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR);
|
||||
}
|
||||
|
||||
// NOTE: race condition: the task should be put into the hash table before send msg to server
|
||||
if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXEC) {
|
||||
SCH_ERR_RET(schPushTaskToExecList(pJob, pTask));
|
||||
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXEC);
|
||||
}
|
||||
|
||||
SSubplan *plan = pTask->plan;
|
||||
|
||||
if (NULL == pTask->msg) { // TODO add more detailed reason for failure
|
||||
code = qSubPlanToString(plan, &pTask->msg, &pTask->msgLen);
|
||||
if (TSDB_CODE_SUCCESS != code) {
|
||||
SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg,
|
||||
pTask->msgLen);
|
||||
SCH_ERR_RET(code);
|
||||
} else {
|
||||
SCH_TASK_DLOGL("physical plan len:%d, %s", pTask->msgLen, pTask->msg);
|
||||
}
|
||||
}
|
||||
|
||||
SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask));
|
||||
|
||||
if (SCH_IS_QUERY_JOB(pJob)) {
|
||||
SCH_ERR_RET(schEnsureHbConnection(pJob, pTask));
|
||||
}
|
||||
|
||||
SCH_ERR_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
// Note: no more error processing, handled in function internal
|
||||
int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) {
|
||||
bool enough = false;
|
||||
int32_t code = 0;
|
||||
|
||||
SCH_SET_TASK_HANDLE(pTask, NULL);
|
||||
|
||||
if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) {
|
||||
SCH_ERR_JRET(schCheckIncTaskFlowQuota(pJob, pTask, &enough));
|
||||
|
||||
if (enough) {
|
||||
SCH_ERR_JRET(schLaunchTaskImpl(pJob, pTask));
|
||||
}
|
||||
} else {
|
||||
SCH_ERR_JRET(schLaunchTaskImpl(pJob, pTask));
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
||||
_return:
|
||||
|
||||
SCH_RET(schProcessOnTaskFailure(pJob, pTask, code));
|
||||
}
|
||||
|
||||
int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level) {
|
||||
SCH_ERR_RET(schChkJobNeedFlowCtrl(pJob, level));
|
||||
|
||||
for (int32_t i = 0; i < level->taskNum; ++i) {
|
||||
SSchTask *pTask = taosArrayGet(level->subTasks, i);
|
||||
|
||||
SCH_ERR_RET(schLaunchTask(pJob, pTask));
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) {
|
||||
if (!SCH_JOB_NEED_DROP(pJob)) {
|
||||
return;
|
||||
}
|
||||
|
||||
void *pIter = taosHashIterate(list, NULL);
|
||||
while (pIter) {
|
||||
SSchTask *pTask = *(SSchTask **)pIter;
|
||||
|
||||
schDropTaskOnExecNode(pJob, pTask);
|
||||
|
||||
pIter = taosHashIterate(list, pIter);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Note: no more error processing, handled in function internal
|
||||
int32_t schLaunchFetchTask(SSchJob *pJob) {
|
||||
int32_t code = 0;
|
||||
|
||||
void *resData = atomic_load_ptr(&pJob->resData);
|
||||
if (resData) {
|
||||
SCH_JOB_DLOG("res already fetched, res:%p", resData);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_SCH_FETCH));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
||||
_return:
|
||||
|
||||
SCH_RET(schProcessOnTaskFailure(pJob, pJob->fetchTask, code));
|
||||
}
|
||||
|
||||
|
|
@ -16,11 +16,25 @@
|
|||
#include "catalog.h"
|
||||
#include "command.h"
|
||||
#include "query.h"
|
||||
#include "schedulerInt.h"
|
||||
#include "schInt.h"
|
||||
#include "tmsg.h"
|
||||
#include "tref.h"
|
||||
#include "trpc.h"
|
||||
|
||||
FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) {
|
||||
qDebug("sch acquire jobId:0x%"PRIx64, refId);
|
||||
return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId);
|
||||
}
|
||||
|
||||
FORCE_INLINE int32_t schReleaseJob(int64_t refId) {
|
||||
if (0 == refId) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
qDebug("sch release jobId:0x%"PRIx64, refId);
|
||||
return taosReleaseRef(schMgmt.jobRef, refId);
|
||||
}
|
||||
|
||||
char* schGetOpStr(SCH_OP_TYPE type) {
|
||||
switch (type) {
|
||||
case SCH_OP_NULL:
|
||||
|
@ -29,6 +43,8 @@ char* schGetOpStr(SCH_OP_TYPE type) {
|
|||
return "EXEC";
|
||||
case SCH_OP_FETCH:
|
||||
return "FETCH";
|
||||
case SCH_OP_GET_STATUS:
|
||||
return "GET STATUS";
|
||||
default:
|
||||
return "UNKNOWN";
|
||||
}
|
||||
|
@ -283,3 +299,20 @@ void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo) {
|
|||
taosMemoryFree(msgSendInfo);
|
||||
}
|
||||
|
||||
int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) {
|
||||
int32_t s = taosHashGetSize(pTaskList);
|
||||
if (s <= 0) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SSchTask **task = taosHashGet(pTaskList, &taskId, sizeof(taskId));
|
||||
if (NULL == task || NULL == (*task)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
*pTask = *task;
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -16,7 +16,7 @@
|
|||
#include "catalog.h"
|
||||
#include "command.h"
|
||||
#include "query.h"
|
||||
#include "schedulerInt.h"
|
||||
#include "schInt.h"
|
||||
#include "tmsg.h"
|
||||
#include "tref.h"
|
||||
#include "trpc.h"
|
||||
|
@ -67,121 +67,45 @@ int32_t schedulerInit(SSchedulerCfg *cfg) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJobId, SQueryResult *pRes) {
|
||||
qDebug("scheduler sync exec job start");
|
||||
int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJobId) {
|
||||
qDebug("scheduler %s exec job start", pReq->syncReq ? "SYNC" : "ASYNC");
|
||||
|
||||
int32_t code = 0;
|
||||
SSchJob *pJob = NULL;
|
||||
SCH_ERR_JRET(schInitJob(pReq, &pJob));
|
||||
|
||||
*pJobId = pJob->refId;
|
||||
|
||||
SCH_ERR_JRET(schExecJobImpl(pReq, pJob, true));
|
||||
SCH_ERR_JRET(schInitJob(pJobId, pReq));
|
||||
|
||||
SCH_ERR_JRET(schHandleOpBeginEvent(*pJobId, &pJob, SCH_OP_EXEC, pReq));
|
||||
|
||||
SCH_ERR_JRET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_INIT, pReq));
|
||||
|
||||
SCH_ERR_JRET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_EXEC, pReq));
|
||||
|
||||
_return:
|
||||
|
||||
if (code && NULL == pJob) {
|
||||
qDestroyQueryPlan(pReq->pDag);
|
||||
}
|
||||
|
||||
if (pJob) {
|
||||
schSetJobQueryRes(pJob, pRes);
|
||||
schReleaseJob(pJob->refId);
|
||||
}
|
||||
|
||||
return code;
|
||||
SCH_RET(schHandleOpEndEvent(pJob, SCH_OP_EXEC, pReq, code));
|
||||
}
|
||||
|
||||
int32_t schedulerAsyncExecJob(SSchedulerReq *pReq, int64_t *pJobId) {
|
||||
qDebug("scheduler async exec job start");
|
||||
int32_t schedulerFetchRows(int64_t jobId, SSchedulerReq *pReq) {
|
||||
qDebug("scheduler %s fetch rows start", pReq->syncReq ? "SYNC" : "ASYNC");
|
||||
|
||||
int32_t code = 0;
|
||||
int32_t code = 0;
|
||||
SSchJob *pJob = NULL;
|
||||
SCH_ERR_JRET(schInitJob(pReq, &pJob));
|
||||
|
||||
*pJobId = pJob->refId;
|
||||
|
||||
SCH_ERR_JRET(schExecJobImpl(pReq, pJob, false));
|
||||
SCH_ERR_JRET(schHandleOpBeginEvent(jobId, &pJob, SCH_OP_FETCH, pReq));
|
||||
|
||||
SCH_ERR_JRET(schJobFetchRows(pJob));
|
||||
|
||||
_return:
|
||||
|
||||
if (code && NULL == pJob) {
|
||||
qDestroyQueryPlan(pReq->pDag);
|
||||
}
|
||||
|
||||
if (pJob) {
|
||||
schReleaseJob(pJob->refId);
|
||||
}
|
||||
|
||||
return code;
|
||||
SCH_RET(schHandleOpEndEvent(pJob, SCH_OP_FETCH, pReq, code));
|
||||
}
|
||||
|
||||
int32_t schedulerFetchRows(int64_t job, void **pData) {
|
||||
qDebug("scheduler sync fetch rows start");
|
||||
|
||||
if (NULL == pData) {
|
||||
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
|
||||
}
|
||||
|
||||
int32_t schedulerGetTasksStatus(int64_t jobId, SArray *pSub) {
|
||||
int32_t code = 0;
|
||||
SSchJob *pJob = schAcquireJob(job);
|
||||
if (NULL == pJob) {
|
||||
qError("acquire job from jobRef list failed, may be dropped, jobId:0x%" PRIx64, job);
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
SSchJob *pJob = NULL;
|
||||
|
||||
SCH_ERR_RET(schBeginOperation(pJob, SCH_OP_FETCH, true));
|
||||
|
||||
pJob->userRes.fetchRes = pData;
|
||||
code = schFetchRows(pJob);
|
||||
|
||||
schReleaseJob(job);
|
||||
|
||||
SCH_RET(code);
|
||||
}
|
||||
|
||||
void schedulerAsyncFetchRows(int64_t job, schedulerFetchFp fp, void* param) {
|
||||
qDebug("scheduler async fetch rows start");
|
||||
|
||||
int32_t code = 0;
|
||||
if (NULL == fp || NULL == param) {
|
||||
SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
|
||||
}
|
||||
|
||||
SSchJob *pJob = schAcquireJob(job);
|
||||
if (NULL == pJob) {
|
||||
qError("acquire sch job from job list failed, may be dropped, jobId:0x%" PRIx64, job);
|
||||
SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
|
||||
SCH_ERR_JRET(schBeginOperation(pJob, SCH_OP_FETCH, false));
|
||||
|
||||
pJob->userRes.fetchFp = fp;
|
||||
pJob->userRes.userParam = param;
|
||||
|
||||
SCH_ERR_JRET(schAsyncFetchRows(pJob));
|
||||
|
||||
_return:
|
||||
|
||||
if (code) {
|
||||
fp(NULL, param, code);
|
||||
}
|
||||
|
||||
schReleaseJob(job);
|
||||
}
|
||||
|
||||
int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub) {
|
||||
int32_t code = 0;
|
||||
SSchJob *pJob = schAcquireJob(job);
|
||||
if (NULL == pJob) {
|
||||
qDebug("acquire job from jobRef list failed, may not started or dropped, refId:0x%" PRIx64, job);
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
|
||||
if (pJob->status < JOB_TASK_STATUS_NOT_START || pJob->levelNum <= 0 || NULL == pJob->levels) {
|
||||
qDebug("job not initialized or not executable job, refId:0x%" PRIx64, job);
|
||||
SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
SCH_ERR_JRET(schHandleOpBeginEvent(jobId, &pJob, SCH_OP_GET_STATUS, NULL));
|
||||
|
||||
for (int32_t i = pJob->levelNum - 1; i >= 0; --i) {
|
||||
SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
|
||||
|
@ -198,23 +122,7 @@ int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub) {
|
|||
|
||||
_return:
|
||||
|
||||
schReleaseJob(job);
|
||||
|
||||
SCH_RET(code);
|
||||
}
|
||||
|
||||
int32_t scheduleCancelJob(int64_t job) {
|
||||
SSchJob *pJob = schAcquireJob(job);
|
||||
if (NULL == pJob) {
|
||||
qError("acquire job from jobRef list failed, may be dropped, jobId:0x%" PRIx64, job);
|
||||
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
|
||||
}
|
||||
|
||||
int32_t code = schCancelJob(pJob);
|
||||
|
||||
schReleaseJob(job);
|
||||
|
||||
SCH_RET(code);
|
||||
SCH_RET(schHandleOpEndEvent(pJob, SCH_OP_GET_STATUS, NULL, code));
|
||||
}
|
||||
|
||||
void schedulerStopQueryHb(void *pTrans) {
|
||||
|
@ -225,33 +133,23 @@ void schedulerStopQueryHb(void *pTrans) {
|
|||
schCleanClusterHb(pTrans);
|
||||
}
|
||||
|
||||
void schedulerFreeJob(int64_t* job, int32_t errCode) {
|
||||
if (0 == *job) {
|
||||
void schedulerFreeJob(int64_t* jobId, int32_t errCode) {
|
||||
if (0 == *jobId) {
|
||||
return;
|
||||
}
|
||||
|
||||
SSchJob *pJob = schAcquireJob(*job);
|
||||
|
||||
SSchJob *pJob = schAcquireJob(*jobId);
|
||||
if (NULL == pJob) {
|
||||
qError("acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *job);
|
||||
*job = 0;
|
||||
qError("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId);
|
||||
return;
|
||||
}
|
||||
|
||||
int32_t code = schProcessOnJobDropped(pJob, errCode);
|
||||
if (TSDB_CODE_SCH_JOB_IS_DROPPING == code) {
|
||||
SCH_JOB_DLOG("sch job is already dropping, refId:0x%" PRIx64, *job);
|
||||
*job = 0;
|
||||
if (schJobDone(pJob)) {
|
||||
return;
|
||||
}
|
||||
|
||||
SCH_JOB_DLOG("start to remove job from jobRef list, refId:0x%" PRIx64, *job);
|
||||
|
||||
if (taosRemoveRef(schMgmt.jobRef, *job)) {
|
||||
SCH_JOB_ELOG("remove job from job list failed, refId:0x%" PRIx64, *job);
|
||||
}
|
||||
|
||||
schReleaseJob(*job);
|
||||
*job = 0;
|
||||
schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, (void*)&errCode);
|
||||
*jobId = 0;
|
||||
}
|
||||
|
||||
void schedulerDestroy(void) {
|
||||
|
|
|
@ -50,7 +50,7 @@
|
|||
#pragma GCC diagnostic ignored "-Wreturn-type"
|
||||
#pragma GCC diagnostic ignored "-Wformat"
|
||||
|
||||
#include "schedulerInt.h"
|
||||
#include "schInt.h"
|
||||
#include "stub.h"
|
||||
#include "tref.h"
|
||||
|
||||
|
@ -87,7 +87,7 @@ void schtInitLogFile() {
|
|||
|
||||
}
|
||||
|
||||
void schtQueryCb(SQueryResult* pResult, void* param, int32_t code) {
|
||||
void schtQueryCb(SExecResult* pResult, void* param, int32_t code) {
|
||||
assert(TSDB_CODE_SUCCESS == code);
|
||||
*(int32_t*)param = 1;
|
||||
}
|
||||
|
@ -507,14 +507,15 @@ void* schtRunJobThread(void *aa) {
|
|||
SRequestConnInfo conn = {0};
|
||||
conn.pTrans = mockPointer;
|
||||
SSchedulerReq req = {0};
|
||||
req.syncReq = false;
|
||||
req.pConn = &conn;
|
||||
req.pNodeList = qnodeList;
|
||||
req.pDag = &dag;
|
||||
req.sql = "select * from tb";
|
||||
req.execFp = schtQueryCb;
|
||||
req.execParam = &queryDone;
|
||||
req.cbParam = &queryDone;
|
||||
|
||||
code = schedulerAsyncExecJob(&req, &queryJobRefId);
|
||||
code = schedulerExecJob(&req, &queryJobRefId);
|
||||
assert(code == 0);
|
||||
|
||||
pJob = schAcquireJob(queryJobRefId);
|
||||
|
@ -584,7 +585,10 @@ void* schtRunJobThread(void *aa) {
|
|||
atomic_store_32(&schtStartFetch, 1);
|
||||
|
||||
void *data = NULL;
|
||||
code = schedulerFetchRows(queryJobRefId, &data);
|
||||
req.syncReq = true;
|
||||
req.pFetchRes = &data;
|
||||
|
||||
code = schedulerFetchRows(queryJobRefId, &req);
|
||||
assert(code == 0 || code);
|
||||
|
||||
if (0 == code) {
|
||||
|
@ -594,7 +598,7 @@ void* schtRunJobThread(void *aa) {
|
|||
}
|
||||
|
||||
data = NULL;
|
||||
code = schedulerFetchRows(queryJobRefId, &data);
|
||||
code = schedulerFetchRows(queryJobRefId, &req);
|
||||
assert(code == 0 || code);
|
||||
|
||||
schtFreeQueryJob(0);
|
||||
|
@ -658,15 +662,15 @@ TEST(queryTest, normalCase) {
|
|||
|
||||
SRequestConnInfo conn = {0};
|
||||
conn.pTrans = mockPointer;
|
||||
SSchedulerReq req = {0};
|
||||
SSchedulerReq req = {0};
|
||||
req.pConn = &conn;
|
||||
req.pNodeList = qnodeList;
|
||||
req.pDag = &dag;
|
||||
req.sql = "select * from tb";
|
||||
req.execFp = schtQueryCb;
|
||||
req.execParam = &queryDone;
|
||||
req.cbParam = &queryDone;
|
||||
|
||||
code = schedulerAsyncExecJob(&req, &job);
|
||||
code = schedulerExecJob(&req, &job);
|
||||
ASSERT_EQ(code, 0);
|
||||
|
||||
|
||||
|
@ -709,7 +713,10 @@ TEST(queryTest, normalCase) {
|
|||
taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job);
|
||||
|
||||
void *data = NULL;
|
||||
code = schedulerFetchRows(job, &data);
|
||||
req.syncReq = true;
|
||||
req.pFetchRes = &data;
|
||||
|
||||
code = schedulerFetchRows(job, &req);
|
||||
ASSERT_EQ(code, 0);
|
||||
|
||||
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data;
|
||||
|
@ -718,7 +725,7 @@ TEST(queryTest, normalCase) {
|
|||
taosMemoryFreeClear(data);
|
||||
|
||||
data = NULL;
|
||||
code = schedulerFetchRows(job, &data);
|
||||
code = schedulerFetchRows(job, &req);
|
||||
ASSERT_EQ(code, 0);
|
||||
ASSERT_TRUE(data == NULL);
|
||||
|
||||
|
@ -768,8 +775,8 @@ TEST(queryTest, readyFirstCase) {
|
|||
req.pDag = &dag;
|
||||
req.sql = "select * from tb";
|
||||
req.execFp = schtQueryCb;
|
||||
req.execParam = &queryDone;
|
||||
code = schedulerAsyncExecJob(&req, &job);
|
||||
req.cbParam = &queryDone;
|
||||
code = schedulerExecJob(&req, &job);
|
||||
ASSERT_EQ(code, 0);
|
||||
|
||||
|
||||
|
@ -813,7 +820,9 @@ TEST(queryTest, readyFirstCase) {
|
|||
taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job);
|
||||
|
||||
void *data = NULL;
|
||||
code = schedulerFetchRows(job, &data);
|
||||
req.syncReq = true;
|
||||
req.pFetchRes = &data;
|
||||
code = schedulerFetchRows(job, &req);
|
||||
ASSERT_EQ(code, 0);
|
||||
|
||||
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data;
|
||||
|
@ -822,7 +831,7 @@ TEST(queryTest, readyFirstCase) {
|
|||
taosMemoryFreeClear(data);
|
||||
|
||||
data = NULL;
|
||||
code = schedulerFetchRows(job, &data);
|
||||
code = schedulerFetchRows(job, &req);
|
||||
ASSERT_EQ(code, 0);
|
||||
ASSERT_TRUE(data == NULL);
|
||||
|
||||
|
@ -875,9 +884,9 @@ TEST(queryTest, flowCtrlCase) {
|
|||
req.pDag = &dag;
|
||||
req.sql = "select * from tb";
|
||||
req.execFp = schtQueryCb;
|
||||
req.execParam = &queryDone;
|
||||
req.cbParam = &queryDone;
|
||||
|
||||
code = schedulerAsyncExecJob(&req, &job);
|
||||
code = schedulerExecJob(&req, &job);
|
||||
ASSERT_EQ(code, 0);
|
||||
|
||||
|
||||
|
@ -925,7 +934,9 @@ TEST(queryTest, flowCtrlCase) {
|
|||
taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job);
|
||||
|
||||
void *data = NULL;
|
||||
code = schedulerFetchRows(job, &data);
|
||||
req.syncReq = true;
|
||||
req.pFetchRes = &data;
|
||||
code = schedulerFetchRows(job, &req);
|
||||
ASSERT_EQ(code, 0);
|
||||
|
||||
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data;
|
||||
|
@ -934,7 +945,7 @@ TEST(queryTest, flowCtrlCase) {
|
|||
taosMemoryFreeClear(data);
|
||||
|
||||
data = NULL;
|
||||
code = schedulerFetchRows(job, &data);
|
||||
code = schedulerFetchRows(job, &req);
|
||||
ASSERT_EQ(code, 0);
|
||||
ASSERT_TRUE(data == NULL);
|
||||
|
||||
|
@ -978,7 +989,7 @@ TEST(insertTest, normalCase) {
|
|||
TdThread thread1;
|
||||
taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId);
|
||||
|
||||
SQueryResult res = {0};
|
||||
SExecResult res = {0};
|
||||
|
||||
SRequestConnInfo conn = {0};
|
||||
conn.pTrans = mockPointer;
|
||||
|
@ -988,9 +999,9 @@ TEST(insertTest, normalCase) {
|
|||
req.pDag = &dag;
|
||||
req.sql = "insert into tb values(now,1)";
|
||||
req.execFp = schtQueryCb;
|
||||
req.execParam = NULL;
|
||||
req.cbParam = NULL;
|
||||
|
||||
code = schedulerExecJob(&req, &insertJobRefId, &res);
|
||||
code = schedulerExecJob(&req, &insertJobRefId);
|
||||
ASSERT_EQ(code, 0);
|
||||
ASSERT_EQ(res.numOfRows, 20);
|
||||
|
||||
|
|
|
@ -36,7 +36,7 @@ typedef struct SRaftCfg {
|
|||
TdFilePtr pFile;
|
||||
char path[TSDB_FILENAME_LEN * 2];
|
||||
int8_t isStandBy;
|
||||
int8_t snapshotEnable;
|
||||
int8_t snapshotStrategy;
|
||||
SyncIndex lastConfigIndex;
|
||||
|
||||
SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT];
|
||||
|
@ -49,20 +49,20 @@ int32_t raftCfgClose(SRaftCfg *pRaftCfg);
|
|||
int32_t raftCfgPersist(SRaftCfg *pRaftCfg);
|
||||
int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex);
|
||||
|
||||
cJSON * syncCfg2Json(SSyncCfg *pSyncCfg);
|
||||
char * syncCfg2Str(SSyncCfg *pSyncCfg);
|
||||
char * syncCfg2SimpleStr(SSyncCfg *pSyncCfg);
|
||||
cJSON *syncCfg2Json(SSyncCfg *pSyncCfg);
|
||||
char *syncCfg2Str(SSyncCfg *pSyncCfg);
|
||||
char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg);
|
||||
int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg);
|
||||
int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg);
|
||||
|
||||
cJSON * raftCfg2Json(SRaftCfg *pRaftCfg);
|
||||
char * raftCfg2Str(SRaftCfg *pRaftCfg);
|
||||
cJSON *raftCfg2Json(SRaftCfg *pRaftCfg);
|
||||
char *raftCfg2Str(SRaftCfg *pRaftCfg);
|
||||
int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg);
|
||||
int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg);
|
||||
|
||||
typedef struct SRaftCfgMeta {
|
||||
int8_t isStandBy;
|
||||
int8_t snapshotEnable;
|
||||
int8_t snapshotStrategy;
|
||||
SyncIndex lastConfigIndex;
|
||||
} SRaftCfgMeta;
|
||||
|
||||
|
|
|
@ -83,9 +83,9 @@ typedef struct SSyncSnapshotReceiver {
|
|||
|
||||
SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId);
|
||||
void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver);
|
||||
int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg);
|
||||
int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver);
|
||||
bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver);
|
||||
int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg);
|
||||
int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver);
|
||||
bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver);
|
||||
|
||||
cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver);
|
||||
char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver);
|
||||
|
|
|
@ -96,12 +96,20 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) {
|
|||
return ret;
|
||||
}
|
||||
|
||||
if (pSyncNode->pRaftCfg->snapshotEnable) {
|
||||
ret = syncNodeRequestVotePeersSnapshot(pSyncNode);
|
||||
} else {
|
||||
ret = syncNodeRequestVotePeers(pSyncNode);
|
||||
}
|
||||
switch (pSyncNode->pRaftCfg->snapshotStrategy) {
|
||||
case SYNC_STRATEGY_NO_SNAPSHOT:
|
||||
ret = syncNodeRequestVotePeers(pSyncNode);
|
||||
break;
|
||||
|
||||
case SYNC_STRATEGY_STANDARD_SNAPSHOT:
|
||||
case SYNC_STRATEGY_WAL_FIRST:
|
||||
ret = syncNodeRequestVotePeersSnapshot(pSyncNode);
|
||||
break;
|
||||
|
||||
default:
|
||||
ret = syncNodeRequestVotePeers(pSyncNode);
|
||||
break;
|
||||
}
|
||||
ASSERT(ret == 0);
|
||||
syncNodeResetElectTimer(pSyncNode);
|
||||
|
||||
|
|
|
@ -158,13 +158,13 @@ int32_t syncSetStandby(int64_t rid) {
|
|||
}
|
||||
|
||||
if (pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER) {
|
||||
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
|
||||
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
|
||||
terrno = TSDB_CODE_SYN_IS_LEADER;
|
||||
} else {
|
||||
terrno = TSDB_CODE_SYN_STANDBY_NOT_READY;
|
||||
}
|
||||
sError("failed to set standby since it is not follower, state:%s rid:%" PRId64, syncStr(pSyncNode->state), rid);
|
||||
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
@ -620,6 +620,7 @@ int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) {
|
|||
|
||||
SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid);
|
||||
if (pSyncNode == NULL) {
|
||||
taosReleaseRef(tsNodeRefId, rid);
|
||||
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
|
||||
return -1;
|
||||
}
|
||||
|
@ -671,12 +672,12 @@ int32_t syncNodeProposeBatch(SSyncNode* pSyncNode, SRpcMsg* pMsgArr, bool* pIsWe
|
|||
}
|
||||
|
||||
if (arrSize > SYNC_MAX_BATCH_SIZE) {
|
||||
syncNodeErrorLog(pSyncNode, "sync propose match batch error");
|
||||
syncNodeErrorLog(pSyncNode, "sync propose batch error");
|
||||
terrno = TSDB_CODE_SYN_BATCH_ERROR;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
|
||||
if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) {
|
||||
syncNodeErrorLog(pSyncNode, "sync propose not leader");
|
||||
terrno = TSDB_CODE_SYN_NOT_LEADER;
|
||||
return -1;
|
||||
|
@ -706,13 +707,36 @@ int32_t syncNodeProposeBatch(SSyncNode* pSyncNode, SRpcMsg* pMsgArr, bool* pIsWe
|
|||
syncClientRequestBatch2RpcMsg(pSyncMsg, &rpcMsg);
|
||||
taosMemoryFree(pSyncMsg); // only free msg body, do not free rpc msg content
|
||||
|
||||
if (pSyncNode->FpEqMsg != NULL && (*pSyncNode->FpEqMsg)(pSyncNode->msgcb, &rpcMsg) == 0) {
|
||||
// enqueue msg ok
|
||||
if (pSyncNode->replicaNum == 1 && pSyncNode->vgId != 1) {
|
||||
int32_t code = syncNodeOnClientRequestBatchCb(pSyncNode, pSyncMsg);
|
||||
if (code == 0) {
|
||||
// update rpc msg applyIndex
|
||||
SRpcMsg* msgArr = syncClientRequestBatchRpcMsgArr(pSyncMsg);
|
||||
ASSERT(arrSize == pSyncMsg->dataCount);
|
||||
for (int i = 0; i < arrSize; ++i) {
|
||||
pMsgArr[i].info.conn.applyIndex = msgArr[i].info.conn.applyIndex;
|
||||
syncRespMgrDel(pSyncNode->pSyncRespMgr, raftArr[i].seqNum);
|
||||
}
|
||||
|
||||
rpcFreeCont(rpcMsg.pCont);
|
||||
terrno = 0;
|
||||
return 1;
|
||||
|
||||
} else {
|
||||
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
|
||||
return -1;
|
||||
}
|
||||
|
||||
} else {
|
||||
sError("enqueue msg error, FpEqMsg is NULL");
|
||||
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
|
||||
return -1;
|
||||
if (pSyncNode->FpEqMsg != NULL && (*pSyncNode->FpEqMsg)(pSyncNode->msgcb, &rpcMsg) == 0) {
|
||||
// enqueue msg ok
|
||||
return 0;
|
||||
|
||||
} else {
|
||||
sError("vgId:%d, enqueue msg error, FpEqMsg is NULL", pSyncNode->vgId);
|
||||
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
@ -729,7 +753,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) {
|
|||
if (pSyncNode->changing && pMsg->msgType != TDMT_SYNC_CONFIG_CHANGE_FINISH) {
|
||||
ret = -1;
|
||||
terrno = TSDB_CODE_SYN_PROPOSE_NOT_READY;
|
||||
sError("sync propose not ready, type:%s,%d", TMSG_INFO(pMsg->msgType), pMsg->msgType);
|
||||
sError("vgId:%d, sync propose not ready, type:%s,%d", pSyncNode->vgId, TMSG_INFO(pMsg->msgType), pMsg->msgType);
|
||||
goto _END;
|
||||
}
|
||||
|
||||
|
@ -738,7 +762,8 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) {
|
|||
if (!syncNodeCanChange(pSyncNode)) {
|
||||
ret = -1;
|
||||
terrno = TSDB_CODE_SYN_RECONFIG_NOT_READY;
|
||||
sError("sync reconfig not ready, type:%s,%d", TMSG_INFO(pMsg->msgType), pMsg->msgType);
|
||||
sError("vgId:%d, sync reconfig not ready, type:%s,%d", pSyncNode->vgId, TMSG_INFO(pMsg->msgType),
|
||||
pMsg->msgType);
|
||||
goto _END;
|
||||
}
|
||||
|
||||
|
@ -779,7 +804,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) {
|
|||
} else {
|
||||
ret = -1;
|
||||
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
|
||||
sError("enqueue msg error, FpEqMsg is NULL");
|
||||
sError("vgId:%d, enqueue msg error, FpEqMsg is NULL", pSyncNode->vgId);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -789,7 +814,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) {
|
|||
} else {
|
||||
ret = -1;
|
||||
terrno = TSDB_CODE_SYN_NOT_LEADER;
|
||||
sError("sync propose not leader, %s", syncUtilState2String(pSyncNode->state));
|
||||
sError("vgId:%d, sync propose not leader, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state));
|
||||
goto _END;
|
||||
}
|
||||
|
||||
|
@ -819,7 +844,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
|
|||
// create a new raft config file
|
||||
SRaftCfgMeta meta;
|
||||
meta.isStandBy = pSyncInfo->isStandBy;
|
||||
meta.snapshotEnable = pSyncInfo->snapshotStrategy;
|
||||
meta.snapshotStrategy = pSyncInfo->snapshotStrategy;
|
||||
meta.lastConfigIndex = SYNC_INDEX_INVALID;
|
||||
ret = raftCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), meta, pSyncNode->configPath);
|
||||
ASSERT(ret == 0);
|
||||
|
@ -968,7 +993,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
|
|||
pSyncNode->FpOnSnapshotSend = syncNodeOnSnapshotSendCb;
|
||||
pSyncNode->FpOnSnapshotRsp = syncNodeOnSnapshotRspCb;
|
||||
|
||||
if (pSyncNode->pRaftCfg->snapshotEnable) {
|
||||
if (pSyncNode->pRaftCfg->snapshotStrategy) {
|
||||
sInfo("sync node use snapshot");
|
||||
pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteSnapshotCb;
|
||||
pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplySnapshotCb;
|
||||
|
@ -1106,7 +1131,7 @@ void syncNodeClose(SSyncNode* pSyncNode) {
|
|||
// option
|
||||
// bool syncNodeSnapshotEnable(SSyncNode* pSyncNode) { return pSyncNode->pRaftCfg->snapshotEnable; }
|
||||
|
||||
ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->pRaftCfg->snapshotEnable; }
|
||||
ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->pRaftCfg->snapshotStrategy; }
|
||||
|
||||
// ping --------------
|
||||
int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) {
|
||||
|
@ -2488,6 +2513,9 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p
|
|||
ASSERT(0);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// update rpc msg conn apply.index
|
||||
msgArr[i].info.conn.applyIndex = pEntry->index;
|
||||
}
|
||||
|
||||
// fsync once
|
||||
|
@ -2495,6 +2523,15 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p
|
|||
SWal* pWal = pData->pWal;
|
||||
walFsync(pWal, true);
|
||||
|
||||
if (ths->replicaNum > 1) {
|
||||
// if multi replica, start replicate right now
|
||||
syncNodeReplicate(ths);
|
||||
|
||||
} else if (ths->replicaNum == 1) {
|
||||
// one replica
|
||||
syncMaybeAdvanceCommitIndex(ths);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
@ -101,7 +101,7 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) {
|
|||
|
||||
char *syncCfg2Str(SSyncCfg *pSyncCfg) {
|
||||
cJSON *pJson = syncCfg2Json(pSyncCfg);
|
||||
char * serialized = cJSON_Print(pJson);
|
||||
char *serialized = cJSON_Print(pJson);
|
||||
cJSON_Delete(pJson);
|
||||
return serialized;
|
||||
}
|
||||
|
@ -109,7 +109,7 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) {
|
|||
char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) {
|
||||
if (pSyncCfg != NULL) {
|
||||
int32_t len = 512;
|
||||
char * s = taosMemoryMalloc(len);
|
||||
char *s = taosMemoryMalloc(len);
|
||||
memset(s, 0, len);
|
||||
|
||||
snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex);
|
||||
|
@ -182,7 +182,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
|
|||
cJSON *pRoot = cJSON_CreateObject();
|
||||
cJSON_AddItemToObject(pRoot, "SSyncCfg", syncCfg2Json(&(pRaftCfg->cfg)));
|
||||
cJSON_AddNumberToObject(pRoot, "isStandBy", pRaftCfg->isStandBy);
|
||||
cJSON_AddNumberToObject(pRoot, "snapshotEnable", pRaftCfg->snapshotEnable);
|
||||
cJSON_AddNumberToObject(pRoot, "snapshotStrategy", pRaftCfg->snapshotStrategy);
|
||||
|
||||
char buf64[128];
|
||||
snprintf(buf64, sizeof(buf64), "%ld", pRaftCfg->lastConfigIndex);
|
||||
|
@ -205,7 +205,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
|
|||
|
||||
char *raftCfg2Str(SRaftCfg *pRaftCfg) {
|
||||
cJSON *pJson = raftCfg2Json(pRaftCfg);
|
||||
char * serialized = cJSON_Print(pJson);
|
||||
char *serialized = cJSON_Print(pJson);
|
||||
cJSON_Delete(pJson);
|
||||
return serialized;
|
||||
}
|
||||
|
@ -228,7 +228,7 @@ int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) {
|
|||
SRaftCfg raftCfg;
|
||||
raftCfg.cfg = *pCfg;
|
||||
raftCfg.isStandBy = meta.isStandBy;
|
||||
raftCfg.snapshotEnable = meta.snapshotEnable;
|
||||
raftCfg.snapshotStrategy = meta.snapshotStrategy;
|
||||
raftCfg.lastConfigIndex = meta.lastConfigIndex;
|
||||
raftCfg.configIndexCount = 1;
|
||||
memset(raftCfg.configIndexArr, 0, sizeof(raftCfg.configIndexArr));
|
||||
|
@ -257,8 +257,8 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) {
|
|||
cJSON *pJsonIsStandBy = cJSON_GetObjectItem(pJson, "isStandBy");
|
||||
pRaftCfg->isStandBy = cJSON_GetNumberValue(pJsonIsStandBy);
|
||||
|
||||
cJSON *pJsonSnapshotEnable = cJSON_GetObjectItem(pJson, "snapshotEnable");
|
||||
pRaftCfg->snapshotEnable = cJSON_GetNumberValue(pJsonSnapshotEnable);
|
||||
cJSON *pJsonSnapshotStrategy = cJSON_GetObjectItem(pJson, "snapshotStrategy");
|
||||
pRaftCfg->snapshotStrategy = cJSON_GetNumberValue(pJsonSnapshotStrategy);
|
||||
|
||||
cJSON *pJsonLastConfigIndex = cJSON_GetObjectItem(pJson, "lastConfigIndex");
|
||||
pRaftCfg->lastConfigIndex = atoll(cJSON_GetStringValue(pJsonLastConfigIndex));
|
||||
|
@ -280,7 +280,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) {
|
|||
(pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring);
|
||||
}
|
||||
|
||||
cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
|
||||
cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
|
||||
int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg));
|
||||
ASSERT(code == 0);
|
||||
|
||||
|
|
|
@ -132,10 +132,6 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) {
|
|||
SyncIndex preLogIndex = syncNodeGetPreIndex(pSyncNode, nextIndex);
|
||||
SyncTerm preLogTerm = syncNodeGetPreTerm(pSyncNode, nextIndex);
|
||||
if (preLogTerm == SYNC_TERM_INVALID) {
|
||||
SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(pSyncNode, pDestId);
|
||||
ASSERT(pSender != NULL);
|
||||
ASSERT(!snapshotSenderIsStart(pSender));
|
||||
|
||||
SyncIndex newNextIndex = syncNodeGetLastIndex(pSyncNode) + 1;
|
||||
syncIndexMgrSetIndex(pSyncNode->pNextIndex, pDestId, newNextIndex);
|
||||
syncIndexMgrSetIndex(pSyncNode->pMatchIndex, pDestId, SYNC_INDEX_INVALID);
|
||||
|
@ -145,26 +141,32 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
// entry pointer array
|
||||
SSyncRaftEntry* entryPArr[SYNC_MAX_BATCH_SIZE];
|
||||
memset(entryPArr, 0, sizeof(entryPArr));
|
||||
|
||||
// get entry batch
|
||||
int32_t getCount = 0;
|
||||
SyncIndex getEntryIndex = nextIndex;
|
||||
for (int32_t i = 0; i < pSyncNode->batchSize; ++i) {
|
||||
SSyncRaftEntry* pEntry;
|
||||
SSyncRaftEntry* pEntry = NULL;
|
||||
int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, getEntryIndex, &pEntry);
|
||||
if (code == 0) {
|
||||
ASSERT(pEntry != NULL);
|
||||
entryPArr[i] = pEntry;
|
||||
getCount++;
|
||||
getEntryIndex++;
|
||||
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// build msg
|
||||
SyncAppendEntriesBatch* pMsg = syncAppendEntriesBatchBuild(entryPArr, getCount, pSyncNode->vgId);
|
||||
ASSERT(pMsg != NULL);
|
||||
|
||||
// free entries
|
||||
for (int32_t i = 0; i < pSyncNode->batchSize; ++i) {
|
||||
SSyncRaftEntry* pEntry = entryPArr[i];
|
||||
if (pEntry != NULL) {
|
||||
|
@ -197,12 +199,6 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) {
|
|||
syncIndexMgrLog2("begin append entries peers pNextIndex:", pSyncNode->pNextIndex);
|
||||
syncIndexMgrLog2("begin append entries peers pMatchIndex:", pSyncNode->pMatchIndex);
|
||||
logStoreSimpleLog2("begin append entries peers LogStore:", pSyncNode->pLogStore);
|
||||
if (gRaftDetailLog) {
|
||||
SSnapshot snapshot;
|
||||
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
|
||||
sTrace("begin append entries peers, snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu",
|
||||
snapshot.lastApplyIndex, snapshot.lastApplyTerm);
|
||||
}
|
||||
|
||||
int32_t ret = 0;
|
||||
for (int i = 0; i < pSyncNode->peersNum; ++i) {
|
||||
|
@ -224,9 +220,6 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
// batch optimized
|
||||
// SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex);
|
||||
|
||||
// prepare entry
|
||||
SyncAppendEntries* pMsg = NULL;
|
||||
|
||||
|
@ -283,11 +276,24 @@ int32_t syncNodeReplicate(SSyncNode* pSyncNode) {
|
|||
// start replicate
|
||||
int32_t ret = 0;
|
||||
|
||||
if (pSyncNode->pRaftCfg->snapshotEnable) {
|
||||
ret = syncNodeAppendEntriesPeersSnapshot(pSyncNode);
|
||||
} else {
|
||||
ret = syncNodeAppendEntriesPeers(pSyncNode);
|
||||
switch (pSyncNode->pRaftCfg->snapshotStrategy) {
|
||||
case SYNC_STRATEGY_NO_SNAPSHOT:
|
||||
ret = syncNodeAppendEntriesPeers(pSyncNode);
|
||||
break;
|
||||
|
||||
case SYNC_STRATEGY_STANDARD_SNAPSHOT:
|
||||
ret = syncNodeAppendEntriesPeersSnapshot(pSyncNode);
|
||||
break;
|
||||
|
||||
case SYNC_STRATEGY_WAL_FIRST:
|
||||
ret = syncNodeAppendEntriesPeersSnapshot2(pSyncNode);
|
||||
break;
|
||||
|
||||
default:
|
||||
ret = syncNodeAppendEntriesPeers(pSyncNode);
|
||||
break;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
|
@ -14,6 +14,7 @@
|
|||
*/
|
||||
|
||||
#include "syncRespMgr.h"
|
||||
#include "syncRaftEntry.h"
|
||||
#include "syncRaftStore.h"
|
||||
|
||||
SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) {
|
||||
|
@ -116,4 +117,59 @@ void syncRespClean(SSyncRespMgr *pObj) {
|
|||
taosThreadMutexUnlock(&(pObj->mutex));
|
||||
}
|
||||
|
||||
void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) {}
|
||||
void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) {
|
||||
SRespStub *pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, NULL);
|
||||
int cnt = 0;
|
||||
SSyncNode *pSyncNode = pObj->data;
|
||||
|
||||
SArray *delIndexArray = taosArrayInit(0, sizeof(SyncIndex));
|
||||
ASSERT(delIndexArray != NULL);
|
||||
|
||||
while (pStub) {
|
||||
size_t len;
|
||||
void *key = taosHashGetKey(pStub, &len);
|
||||
SyncIndex *pIndex = (SyncIndex *)key;
|
||||
|
||||
int64_t nowMS = taosGetTimestampMs();
|
||||
if (nowMS - pStub->createTime > ttl) {
|
||||
taosArrayPush(delIndexArray, pIndex);
|
||||
cnt++;
|
||||
|
||||
SSyncRaftEntry *pEntry = NULL;
|
||||
int32_t code = 0;
|
||||
if (pSyncNode->pLogStore != NULL) {
|
||||
code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, *pIndex, &pEntry);
|
||||
if (code == 0 && pEntry != NULL) {
|
||||
SFsmCbMeta cbMeta = {0};
|
||||
cbMeta.index = pEntry->index;
|
||||
cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(pSyncNode, cbMeta.index);
|
||||
cbMeta.isWeak = pEntry->isWeak;
|
||||
cbMeta.code = TSDB_CODE_SYN_TIMEOUT;
|
||||
cbMeta.state = pSyncNode->state;
|
||||
cbMeta.seqNum = pEntry->seqNum;
|
||||
cbMeta.term = pEntry->term;
|
||||
cbMeta.currentTerm = pSyncNode->pRaftStore->currentTerm;
|
||||
cbMeta.flag = 0;
|
||||
|
||||
SRpcMsg rpcMsg = pStub->rpcMsg;
|
||||
rpcMsg.pCont = rpcMallocCont(pEntry->dataLen);
|
||||
memcpy(rpcMsg.pCont, pEntry->data, pEntry->dataLen);
|
||||
pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, cbMeta);
|
||||
|
||||
syncEntryDestory(pEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, pStub);
|
||||
}
|
||||
|
||||
int32_t arraySize = taosArrayGetSize(delIndexArray);
|
||||
sDebug("vgId:%d, resp clean by ttl, cnt:%d, array-size:%d", pSyncNode->vgId, cnt, arraySize);
|
||||
|
||||
for (int32_t i = 0; i < arraySize; ++i) {
|
||||
SyncIndex *pIndex = taosArrayGet(delIndexArray, i);
|
||||
taosHashRemove(pObj->pRespHash, pIndex, sizeof(SyncIndex));
|
||||
}
|
||||
taosArrayDestroy(delIndexArray);
|
||||
}
|
||||
|
|
|
@ -22,9 +22,11 @@
|
|||
#include "wal.h"
|
||||
|
||||
//----------------------------------
|
||||
static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm,
|
||||
SyncSnapshotSend *pBeginMsg);
|
||||
static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg);
|
||||
static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg);
|
||||
static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg);
|
||||
static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver);
|
||||
static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg);
|
||||
static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg);
|
||||
|
||||
//----------------------------------
|
||||
SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) {
|
||||
|
@ -68,7 +70,9 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {
|
|||
// close reader
|
||||
if (pSender->pReader != NULL) {
|
||||
int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader);
|
||||
ASSERT(ret == 0);
|
||||
if (ret != 0) {
|
||||
syncNodeErrorLog(pSender->pSyncNode, "stop reader error");
|
||||
}
|
||||
pSender->pReader = NULL;
|
||||
}
|
||||
|
||||
|
@ -79,7 +83,12 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {
|
|||
|
||||
bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; }
|
||||
|
||||
// begin send snapshot by snapshot, pReader
|
||||
// begin send snapshot by param, snapshot, pReader
|
||||
//
|
||||
// action:
|
||||
// 1. assert reader not start
|
||||
// 2. update state
|
||||
// 3. send first snapshot block
|
||||
int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapshotParam, SSnapshot snapshot,
|
||||
void *pReader) {
|
||||
ASSERT(!snapshotSenderIsStart(pSender));
|
||||
|
@ -98,7 +107,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapsho
|
|||
|
||||
// update term
|
||||
pSender->term = pSender->pSyncNode->pRaftStore->currentTerm;
|
||||
++(pSender->privateTerm);
|
||||
++(pSender->privateTerm); // increase private term
|
||||
|
||||
// update state
|
||||
pSender->finish = false;
|
||||
|
@ -114,9 +123,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapsho
|
|||
|
||||
code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore,
|
||||
pSender->snapshot.lastConfigIndex, &pEntry);
|
||||
if (code == 0) {
|
||||
ASSERT(pEntry != NULL);
|
||||
|
||||
if (code == 0 && pEntry != NULL) {
|
||||
SRpcMsg rpcMsg;
|
||||
syncEntry2OriginalRpc(pEntry, &rpcMsg);
|
||||
|
||||
|
@ -207,6 +214,8 @@ int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) {
|
|||
pSender->start = false;
|
||||
pSender->finish = finish;
|
||||
|
||||
// do not update term, maybe print
|
||||
|
||||
// event log
|
||||
do {
|
||||
char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender stop");
|
||||
|
@ -243,6 +252,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) {
|
|||
pMsg->srcId = pSender->pSyncNode->myRaftId;
|
||||
pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex];
|
||||
pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm;
|
||||
pMsg->beginIndex = pSender->snapshotParam.start;
|
||||
pMsg->lastIndex = pSender->snapshot.lastApplyIndex;
|
||||
pMsg->lastTerm = pSender->snapshot.lastApplyTerm;
|
||||
pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex;
|
||||
|
@ -281,11 +291,13 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
|
|||
pMsg->srcId = pSender->pSyncNode->myRaftId;
|
||||
pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex];
|
||||
pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm;
|
||||
pMsg->beginIndex = pSender->snapshotParam.start;
|
||||
pMsg->lastIndex = pSender->snapshot.lastApplyIndex;
|
||||
pMsg->lastTerm = pSender->snapshot.lastApplyTerm;
|
||||
pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex;
|
||||
pMsg->lastConfig = pSender->lastConfig;
|
||||
pMsg->seq = pSender->seq;
|
||||
pMsg->privateTerm = pSender->privateTerm;
|
||||
memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen);
|
||||
|
||||
// send msg
|
||||
|
@ -305,6 +317,12 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {
|
||||
ASSERT(pMsg->ack == pSender->seq);
|
||||
pSender->ack = pMsg->ack;
|
||||
++(pSender->seq);
|
||||
}
|
||||
|
||||
cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) {
|
||||
char u64buf[128];
|
||||
cJSON *pRoot = cJSON_CreateObject();
|
||||
|
@ -371,10 +389,11 @@ char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) {
|
|||
syncUtilU642Addr(destId.addr, host, sizeof(host), &port);
|
||||
|
||||
snprintf(s, len,
|
||||
"%s {%p laindex:%ld laterm:%lu lcindex:%ld seq:%d ack:%d finish:%d pterm:%lu replica-index:%d %s:%d}", event,
|
||||
pSender, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm,
|
||||
pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, pSender->finish, pSender->privateTerm,
|
||||
pSender->replicaIndex, host, port);
|
||||
"%s {%p s-param:%ld e-param:%ld laindex:%ld laterm:%lu lcindex:%ld seq:%d ack:%d finish:%d pterm:%lu "
|
||||
"replica-index:%d %s:%d}",
|
||||
event, pSender, pSender->snapshotParam.start, pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex,
|
||||
pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack,
|
||||
pSender->finish, pSender->privateTerm, pSender->replicaIndex, host, port);
|
||||
|
||||
return s;
|
||||
}
|
||||
|
@ -429,11 +448,10 @@ bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceive
|
|||
// static do start by privateTerm, pBeginMsg
|
||||
// receive first snapshot data
|
||||
// write first block data
|
||||
static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm,
|
||||
SyncSnapshotSend *pBeginMsg) {
|
||||
static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) {
|
||||
// update state
|
||||
pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm;
|
||||
pReceiver->privateTerm = privateTerm;
|
||||
pReceiver->privateTerm = pBeginMsg->privateTerm;
|
||||
pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN;
|
||||
pReceiver->fromId = pBeginMsg->srcId;
|
||||
pReceiver->start = true;
|
||||
|
@ -445,7 +463,7 @@ static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm p
|
|||
pReceiver->snapshotParam.start = pBeginMsg->beginIndex;
|
||||
pReceiver->snapshotParam.end = pBeginMsg->lastIndex;
|
||||
|
||||
// write data
|
||||
// start writer
|
||||
ASSERT(pReceiver->pWriter == NULL);
|
||||
int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm,
|
||||
&(pReceiver->snapshotParam), &(pReceiver->pWriter));
|
||||
|
@ -481,10 +499,10 @@ static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) {
|
|||
|
||||
// if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver
|
||||
// if already start, force close, start again
|
||||
int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) {
|
||||
int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) {
|
||||
if (!snapshotReceiverIsStart(pReceiver)) {
|
||||
// first start
|
||||
snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg);
|
||||
snapshotReceiverDoStart(pReceiver, pBeginMsg);
|
||||
|
||||
} else {
|
||||
// already start
|
||||
|
@ -494,12 +512,14 @@ int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm private
|
|||
snapshotReceiverForceStop(pReceiver);
|
||||
|
||||
// start again
|
||||
snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg);
|
||||
snapshotReceiverDoStart(pReceiver, pBeginMsg);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// just set start = false
|
||||
// FpSnapshotStopWrite should not be called, assert writer == NULL
|
||||
int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
|
||||
if (pReceiver->pWriter != NULL) {
|
||||
int32_t ret =
|
||||
|
@ -522,6 +542,7 @@ int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
// when recv last snapshot block, apply data into snapshot
|
||||
static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) {
|
||||
ASSERT(pMsg->seq == SYNC_SNAPSHOT_SEQ_END);
|
||||
|
||||
|
@ -550,7 +571,7 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
|
|||
pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex;
|
||||
}
|
||||
|
||||
// stop writer
|
||||
// stop writer, apply data
|
||||
code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true);
|
||||
if (code != 0) {
|
||||
syncNodeErrorLog(pReceiver->pSyncNode, "snapshot stop writer true error");
|
||||
|
@ -579,15 +600,20 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
|
|||
return 0;
|
||||
}
|
||||
|
||||
// apply data block
|
||||
// update progress
|
||||
static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) {
|
||||
ASSERT(pMsg->seq == pReceiver->ack + 1);
|
||||
|
||||
if (pReceiver->pWriter != NULL) {
|
||||
if (pMsg->dataLen > 0) {
|
||||
// apply data block
|
||||
int32_t code = pReceiver->pSyncNode->pFsm->FpSnapshotDoWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter,
|
||||
pMsg->data, pMsg->dataLen);
|
||||
ASSERT(code == 0);
|
||||
}
|
||||
|
||||
// update progress
|
||||
pReceiver->ack = pMsg->seq;
|
||||
|
||||
// event log
|
||||
|
@ -665,14 +691,23 @@ char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event)
|
|||
uint16_t port;
|
||||
syncUtilU642Addr(fromId.addr, host, sizeof(host), &port);
|
||||
|
||||
snprintf(s, len, "%s {%p start:%d ack:%d term:%lu pterm:%lu from:%s:%d laindex:%ld laterm:%lu lcindex:%ld}", event,
|
||||
pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term, pReceiver->privateTerm, host, port,
|
||||
pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm, pReceiver->snapshot.lastConfigIndex);
|
||||
snprintf(s, len,
|
||||
"%s {%p start:%d ack:%d term:%lu pterm:%lu from:%s:%d s-param:%ld e-param:%ld laindex:%ld laterm:%lu "
|
||||
"lcindex:%ld}",
|
||||
event, pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term, pReceiver->privateTerm, host, port,
|
||||
pReceiver->snapshotParam.start, pReceiver->snapshotParam.end, pReceiver->snapshot.lastApplyIndex,
|
||||
pReceiver->snapshot.lastApplyTerm, pReceiver->snapshot.lastConfigIndex);
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
// receiver do something
|
||||
// receiver on message
|
||||
//
|
||||
// condition 1, recv SYNC_SNAPSHOT_SEQ_BEGIN, start receiver, update privateTerm
|
||||
// condition 2, recv SYNC_SNAPSHOT_SEQ_END, finish receiver(apply snapshot data, update commit index, maybe reconfig)
|
||||
// condition 3, recv SYNC_SNAPSHOT_SEQ_FORCE_CLOSE, force close
|
||||
// condition 4, got data, update ack
|
||||
//
|
||||
int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
||||
// get receiver
|
||||
SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
|
||||
|
@ -683,11 +718,13 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
|||
if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) {
|
||||
if (pMsg->term == pSyncNode->pRaftStore->currentTerm) {
|
||||
if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) {
|
||||
// condition 1
|
||||
// begin, no data
|
||||
snapshotReceiverStart(pReceiver, pMsg->privateTerm, pMsg);
|
||||
snapshotReceiverStart(pReceiver, pMsg);
|
||||
needRsp = true;
|
||||
|
||||
} else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) {
|
||||
// condition 2
|
||||
// end, finish FSM
|
||||
code = snapshotReceiverFinish(pReceiver, pMsg);
|
||||
if (code == 0) {
|
||||
|
@ -697,7 +734,6 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
|||
|
||||
// maybe update lastconfig
|
||||
if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) {
|
||||
// int32_t oldReplicaNum = pSyncNode->replicaNum;
|
||||
SSyncCfg oldSyncCfg = pSyncNode->pRaftCfg->cfg;
|
||||
|
||||
// update new config myIndex
|
||||
|
@ -709,11 +745,13 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
|||
}
|
||||
|
||||
} else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) {
|
||||
// condition 3
|
||||
// force close
|
||||
snapshotReceiverForceStop(pReceiver);
|
||||
needRsp = false;
|
||||
|
||||
} else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) {
|
||||
// condition 4
|
||||
// transfering
|
||||
if (pMsg->seq == pReceiver->ack + 1) {
|
||||
snapshotReceiverGotData(pReceiver, pMsg);
|
||||
|
@ -752,6 +790,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
|||
syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg);
|
||||
syncSnapshotRspDestroy(pRspMsg);
|
||||
}
|
||||
|
||||
} else {
|
||||
// error log
|
||||
do {
|
||||
|
@ -759,6 +798,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
|||
syncNodeErrorLog(pSyncNode, eventLog);
|
||||
taosMemoryFree(eventLog);
|
||||
} while (0);
|
||||
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
// error log
|
||||
|
@ -767,19 +808,19 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
|||
syncNodeErrorLog(pSyncNode, eventLog);
|
||||
taosMemoryFree(eventLog);
|
||||
} while (0);
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {
|
||||
ASSERT(pMsg->ack == pSender->seq);
|
||||
pSender->ack = pMsg->ack;
|
||||
++(pSender->seq);
|
||||
}
|
||||
|
||||
// sender receives ack, set seq = ack + 1, send msg from seq
|
||||
// if ack == SYNC_SNAPSHOT_SEQ_END, stop sender
|
||||
// sender on message
|
||||
//
|
||||
// condition 1 sender receives SYNC_SNAPSHOT_SEQ_END, close sender
|
||||
// condition 2 sender receives ack, set seq = ack + 1, send msg from seq
|
||||
// condition 3 sender receives error msg, just print error log
|
||||
//
|
||||
int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) {
|
||||
// if already drop replica, do not process
|
||||
if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
|
||||
|
@ -794,12 +835,14 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) {
|
|||
// state, term, seq/ack
|
||||
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
|
||||
if (pMsg->term == pSyncNode->pRaftStore->currentTerm) {
|
||||
// receiver ack is finish, close sender
|
||||
// condition 1
|
||||
// receive ack is finish, close sender
|
||||
if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) {
|
||||
snapshotSenderStop(pSender, true);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// condition 2
|
||||
// send next msg
|
||||
if (pMsg->ack == pSender->seq) {
|
||||
// update sender ack
|
||||
|
@ -807,6 +850,7 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) {
|
|||
snapshotSend(pSender);
|
||||
|
||||
} else if (pMsg->ack == pSender->seq - 1) {
|
||||
// maybe resend
|
||||
snapshotReSend(pSender);
|
||||
|
||||
} else {
|
||||
|
|
|
@ -8,7 +8,13 @@ void print(SHashObj *pNextIndex) {
|
|||
printf("----------------\n");
|
||||
uint64_t *p = (uint64_t *)taosHashIterate(pNextIndex, NULL);
|
||||
while (p) {
|
||||
printf("%lu \n", *p);
|
||||
|
||||
size_t len;
|
||||
void* key = taosHashGetKey(p, &len);
|
||||
|
||||
SRaftId *pRaftId = (SRaftId*)key;
|
||||
|
||||
printf("key:<%lu, %d>, value:%lu \n", pRaftId->addr, pRaftId->vgId, *p);
|
||||
p = (uint64_t *)taosHashIterate(pNextIndex, p);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -83,7 +83,7 @@ void test3() {
|
|||
} else {
|
||||
SRaftCfgMeta meta;
|
||||
meta.isStandBy = 7;
|
||||
meta.snapshotEnable = 9;
|
||||
meta.snapshotStrategy = 9;
|
||||
meta.lastConfigIndex = 789;
|
||||
raftCfgCreateFile(pCfg, meta, s);
|
||||
printf("%s create json file: %s \n", (char*)__FUNCTION__, s);
|
||||
|
@ -108,7 +108,7 @@ void test5() {
|
|||
|
||||
pCfg->cfg.myIndex = taosGetTimestampSec();
|
||||
pCfg->isStandBy += 2;
|
||||
pCfg->snapshotEnable += 3;
|
||||
pCfg->snapshotStrategy += 3;
|
||||
pCfg->lastConfigIndex += 1000;
|
||||
|
||||
pCfg->configIndexCount = 5;
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue