diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 88fa0e728f..3d9228f05c 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -77,7 +77,9 @@ typedef struct SDataBlockInfo { int16_t numOfCols; int16_t hasVarCol; int32_t capacity; - EStreamType type; + // TODO: optimize and remove following + int32_t childId; // used for stream, do not serialize + EStreamType type; // used for stream, do not serialize } SDataBlockInfo; typedef struct SSDataBlock { @@ -105,14 +107,14 @@ typedef struct SColumnInfoData { } SColumnInfoData; typedef struct SQueryTableDataCond { - //STimeWindow twindow; + // STimeWindow twindow; int32_t order; // desc|asc order to iterate the data block int32_t numOfCols; - SColumnInfo *colList; + SColumnInfo* colList; bool loadExternalRows; // load external rows or not int32_t type; // data block load type: int32_t numOfTWindows; - STimeWindow *twindows; + STimeWindow* twindows; } SQueryTableDataCond; void* blockDataDestroy(SSDataBlock* pBlock); @@ -202,17 +204,17 @@ typedef struct SExprInfo { } SExprInfo; typedef struct { - const char* key; - int32_t keyLen; - uint8_t type; - union{ + const char* key; + int32_t keyLen; + uint8_t type; + union { const char* value; int64_t i; uint64_t u; double d; float f; }; - int32_t length; + int32_t length; } SSmlKv; #define QUERY_ASC_FORWARD_STEP 1 @@ -220,7 +222,7 @@ typedef struct { #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) -#define SORT_QSORT_T 0x1 +#define SORT_QSORT_T 0x1 #define SORT_SPILLED_MERGE_SORT_T 0x2 typedef struct SSortExecInfo { int32_t sortMethod; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5019dcc51c..23945cd500 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -177,8 +177,8 @@ typedef struct { typedef struct SField { char name[TSDB_COL_NAME_LEN]; uint8_t type; - int32_t bytes; int8_t flags; + int32_t bytes; } SField; typedef struct SRetention { @@ -1123,13 +1123,13 @@ typedef struct { SSchema* pSchemas; } STableMetaRsp; -typedef struct { +typedef struct { STableMetaRsp* pMeta; } SMAlterStbRsp; -int32_t tEncodeSMAlterStbRsp(SEncoder *pEncoder, const SMAlterStbRsp *pRsp); -int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp); -void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp); +int32_t tEncodeSMAlterStbRsp(SEncoder* pEncoder, const SMAlterStbRsp* pRsp); +int32_t tDecodeSMAlterStbRsp(SDecoder* pDecoder, SMAlterStbRsp* pRsp); +void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp); int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); @@ -2304,23 +2304,23 @@ typedef struct { } SVgEpSet; typedef struct { - int8_t version; // for compatibility(default 0) - int8_t intervalUnit; // MACRO: TIME_UNIT_XXX - int8_t slidingUnit; // MACRO: TIME_UNIT_XXX - int8_t timezoneInt; // sma data expired if timezone changes. - int32_t dstVgId; - char indexName[TSDB_INDEX_NAME_LEN]; - int32_t exprLen; - int32_t tagsFilterLen; - int32_t numOfVgroups; - int64_t indexUid; - tb_uid_t tableUid; // super/child/common table uid - int64_t interval; - int64_t offset; // use unit by precision of DB - int64_t sliding; - char* expr; // sma expression - char* tagsFilter; - SVgEpSet vgEpSet[]; + int8_t version; // for compatibility(default 0) + int8_t intervalUnit; // MACRO: TIME_UNIT_XXX + int8_t slidingUnit; // MACRO: TIME_UNIT_XXX + int8_t timezoneInt; // sma data expired if timezone changes. + int32_t dstVgId; + char indexName[TSDB_INDEX_NAME_LEN]; + int32_t exprLen; + int32_t tagsFilterLen; + int32_t numOfVgroups; + int64_t indexUid; + tb_uid_t tableUid; // super/child/common table uid + int64_t interval; + int64_t offset; // use unit by precision of DB + int64_t sliding; + char* expr; // sma expression + char* tagsFilter; + SVgEpSet* pVgEpSet; } STSma; // Time-range-wise SMA typedef STSma SVCreateTSmaReq; @@ -2406,7 +2406,7 @@ static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) { } typedef struct { - int64_t indexUid; + int64_t indexUid; STimeWindow queryWindow; } SVGetTsmaExpWndsReq; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 29fabb5418..e3ed8e1d67 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -195,6 +195,7 @@ typedef enum ENodeType { QUERY_NODE_LOGIC_PLAN_FILL, QUERY_NODE_LOGIC_PLAN_SORT, QUERY_NODE_LOGIC_PLAN_PARTITION, + QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC, QUERY_NODE_LOGIC_SUBPLAN, QUERY_NODE_LOGIC_PLAN, @@ -211,7 +212,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_MERGE, QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, - QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, @@ -222,6 +223,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW, QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW, QUERY_NODE_PHYSICAL_PLAN_PARTITION, + QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, QUERY_NODE_PHYSICAL_PLAN_DISPATCH, QUERY_NODE_PHYSICAL_PLAN_INSERT, QUERY_NODE_PHYSICAL_PLAN_DELETE, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 38c3f150ef..00380724b3 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -42,6 +42,7 @@ typedef struct SScanLogicNode { SNodeList* pScanPseudoCols; int8_t tableType; uint64_t tableId; + uint64_t stableId; SVgroupsInfo* pVgroupList; EScanType scanType; uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count @@ -86,6 +87,11 @@ typedef struct SProjectLogicNode { int64_t soffset; } SProjectLogicNode; +typedef struct SIndefRowsFuncLogicNode { + SLogicNode node; + SNodeList* pVectorFuncs; +} SIndefRowsFuncLogicNode; + typedef enum EModifyTableType { MODIFY_TABLE_TYPE_INSERT = 1, MODIFY_TABLE_TYPE_DELETE } EModifyTableType; typedef struct SVnodeModifyLogicNode { @@ -94,7 +100,7 @@ typedef struct SVnodeModifyLogicNode { int32_t msgType; SArray* pDataBlocks; SVgDataBlocks* pVgDataBlocks; - SNode* pModifyRows; // SColumnNode + SNode* pAffectedRows; // SColumnNode uint64_t tableId; int8_t tableType; // table type char tableFName[TSDB_TABLE_FNAME_LEN]; @@ -109,6 +115,7 @@ typedef struct SExchangeLogicNode { typedef struct SMergeLogicNode { SLogicNode node; SNodeList* pMergeKeys; + SNodeList* pInputs; int32_t numOfChannels; int32_t srcGroupId; } SMergeLogicNode; @@ -117,7 +124,7 @@ typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW typedef enum EIntervalAlgorithm { INTERVAL_ALGO_HASH = 1, - INTERVAL_ALGO_SORT_MERGE, + INTERVAL_ALGO_MERGE, INTERVAL_ALGO_STREAM_FINAL, INTERVAL_ALGO_STREAM_SEMI, INTERVAL_ALGO_STREAM_SINGLE, @@ -220,6 +227,7 @@ typedef struct SScanPhysiNode { SNodeList* pScanCols; SNodeList* pScanPseudoCols; uint64_t uid; // unique id of the table + uint64_t suid; int8_t tableType; SName tableName; } SScanPhysiNode; @@ -264,6 +272,12 @@ typedef struct SProjectPhysiNode { int64_t soffset; } SProjectPhysiNode; +typedef struct SIndefRowsFuncPhysiNode { + SPhysiNode node; + SNodeList* pExprs; + SNodeList* pVectorFuncs; +} SIndefRowsFuncPhysiNode; + typedef struct SJoinPhysiNode { SPhysiNode node; EJoinType joinType; @@ -296,6 +310,7 @@ typedef struct SExchangePhysiNode { typedef struct SMergePhysiNode { SPhysiNode node; SNodeList* pMergeKeys; + SNodeList* pTargets; int32_t numOfChannels; int32_t srcGroupId; } SMergePhysiNode; @@ -319,7 +334,7 @@ typedef struct SIntervalPhysiNode { int8_t slidingUnit; } SIntervalPhysiNode; -typedef SIntervalPhysiNode SSortMergeIntervalPhysiNode; +typedef SIntervalPhysiNode SMergeIntervalPhysiNode; typedef SIntervalPhysiNode SStreamIntervalPhysiNode; typedef SIntervalPhysiNode SStreamFinalIntervalPhysiNode; typedef SIntervalPhysiNode SStreamSemiIntervalPhysiNode; @@ -388,6 +403,7 @@ typedef struct SDataDeleterNode { int8_t tableType; // table type char tableFName[TSDB_TABLE_FNAME_LEN]; STimeWindow deleteTimeRange; + SNode* pAffectedRows; } SDataDeleterNode; typedef struct SSubplan { diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index a17be44846..a717a6b333 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -224,7 +224,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_SCHEDULER_RETRY_ERROR(_code) \ ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) -#define REQUEST_MAX_TRY_TIMES 1 +#define REQUEST_TOTAL_EXEC_TIMES 2 #define qFatal(...) \ do { \ diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index f7ad7b4ed8..bb0b6dc0a0 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -24,8 +24,8 @@ extern "C" { #endif -#ifndef _TSTREAM_H_ -#define _TSTREAM_H_ +#ifndef _STREAM_H_ +#define _STREAM_H_ typedef struct SStreamTask SStreamTask; @@ -39,6 +39,7 @@ enum { TASK_INPUT_STATUS__NORMAL = 1, TASK_INPUT_STATUS__BLOCKED, TASK_INPUT_STATUS__RECOVER, + TASK_INPUT_STATUS__PROCESSING, TASK_INPUT_STATUS__STOP, TASK_INPUT_STATUS__FAILED, }; @@ -60,6 +61,10 @@ enum { STREAM_INPUT__CHECKPOINT, }; +typedef struct { + int8_t type; +} SStreamQueueItem; + typedef struct { int8_t type; int64_t ver; @@ -80,55 +85,51 @@ typedef struct { int8_t type; } SStreamCheckpoint; +enum { + STREAM_QUEUE__SUCESS = 1, + STREAM_QUEUE__FAILED, + STREAM_QUEUE__PROCESSING, +}; + typedef struct { STaosQueue* queue; STaosQall* qall; void* qItem; - int8_t failed; -} SStreamQ; + int8_t status; +} SStreamQueue; -static FORCE_INLINE void* streamQCurItem(SStreamQ* queue) { - // - return queue->qItem; +SStreamQueue* streamQueueOpen(); +void streamQueueClose(SStreamQueue* queue); + +static FORCE_INLINE void streamQueueProcessSuccess(SStreamQueue* queue) { + ASSERT(atomic_load_8(&queue->status) == STREAM_QUEUE__PROCESSING); + queue->qItem = NULL; + atomic_store_8(&queue->status, STREAM_QUEUE__SUCESS); } -static FORCE_INLINE void* streamQNextItem(SStreamQ* queue) { - int8_t failed = atomic_load_8(&queue->failed); - if (failed) { +static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) { + ASSERT(atomic_load_8(&queue->status) == STREAM_QUEUE__PROCESSING); + atomic_store_8(&queue->status, STREAM_QUEUE__FAILED); +} + +static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; } + +static FORCE_INLINE void* streamQueueNextItem(SStreamQueue* queue) { + int8_t dequeueFlag = atomic_exchange_8(&queue->status, STREAM_QUEUE__PROCESSING); + if (dequeueFlag == STREAM_QUEUE__FAILED) { ASSERT(queue->qItem != NULL); - return streamQCurItem(queue); + return streamQueueCurItem(queue); } else { taosGetQitem(queue->qall, &queue->qItem); if (queue->qItem == NULL) { taosReadAllQitems(queue->queue, queue->qall); taosGetQitem(queue->qall, &queue->qItem); } - return streamQCurItem(queue); + return streamQueueCurItem(queue); } } -static FORCE_INLINE void streamQSetFail(SStreamQ* queue) { atomic_store_8(&queue->failed, 1); } - -static FORCE_INLINE void streamQSetSuccess(SStreamQ* queue) { atomic_store_8(&queue->failed, 0); } - -static FORCE_INLINE SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) { - SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); - if (pDataSubmit == NULL) return NULL; - pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t)); - if (pDataSubmit->dataRef == NULL) goto FAIL; - pDataSubmit->data = pReq; - *pDataSubmit->dataRef = 1; - pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT; - return pDataSubmit; -FAIL: - taosFreeQitem(pDataSubmit); - return NULL; -} - -static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) { - // - atomic_add_fetch_32(pDataSubmit->dataRef, 1); -} +SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq); static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) { int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); @@ -141,9 +142,31 @@ static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit); +#if 0 int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput); void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput); +static FORCE_INLINE int32_t streamEnqueue1(SStreamQueue* queue, SStreamQueueItem* pItem) { + int8_t inputStatus = atomic_load_8(&queue->enqueueStatus); + if (inputStatus == TASK_INPUT_STATUS__NORMAL) { + if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem); + if (pSubmitClone == NULL) { + atomic_store_8(&queue->enqueueStatus, TASK_INPUT_STATUS__FAILED); + return -1; + } + taosWriteQitem(queue->queue, pSubmitClone); + } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) { + taosWriteQitem(queue->queue, pItem); + } else if (pItem->type == STREAM_INPUT__CHECKPOINT) { + taosWriteQitem(queue->queue, pItem); + } + return 0; + } + return 0; +} +#endif + typedef struct { int8_t parallelizable; char* qmsg; @@ -236,13 +259,17 @@ struct SStreamTask { int8_t dispatchType; int16_t dispatchMsgType; + // node info + int32_t childId; int32_t nodeId; SEpSet epSet; // exec STaskExec exec; - // local sink + // TODO: merge sink and dispatch + + // local sink union { STaskSinkTb tbSink; STaskSinkSma smaSink; @@ -258,25 +285,61 @@ struct SStreamTask { int8_t inputStatus; int8_t outputStatus; - +#if 0 STaosQueue* inputQ; STaosQall* inputQAll; STaosQueue* outputQ; STaosQall* outputQAll; +#endif + + SStreamQueue* inputQueue; + SStreamQueue* outputQueue; // application storage void* ahandle; }; -SStreamTask* tNewSStreamTask(int64_t streamId); +SStreamTask* tNewSStreamTask(int64_t streamId, int32_t childId); int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask); void tFreeSStreamTask(SStreamTask* pTask); -typedef struct { - // SMsgHead head; - SStreamTask* task; -} SStreamTaskDeployReq; +static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { + while (1) { + int8_t inputStatus = + atomic_val_compare_exchange_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL, TASK_INPUT_STATUS__PROCESSING); + if (inputStatus == TASK_INPUT_STATUS__NORMAL) { + break; + } + ASSERT(0); + } + + if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem); + if (pSubmitClone == NULL) { + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); + return -1; + } + taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); + } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) { + taosWriteQitem(pTask->inputQueue->queue, pItem); + } else if (pItem->type == STREAM_INPUT__CHECKPOINT) { + taosWriteQitem(pTask->inputQueue->queue, pItem); + } + + // TODO: back pressure + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL); + return 0; +} + +static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) { + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); +} + +static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) { + taosWriteQitem(pTask->outputQueue->queue, pBlock); + return 0; +} typedef struct { int32_t reserved; @@ -289,6 +352,11 @@ typedef struct { SArray* data; // SArray } SStreamTaskExecReq; +typedef struct { + // SMsgHead head; + SStreamTask* task; +} SStreamTaskDeployReq; + int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq); void* tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq); void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq); @@ -297,6 +365,12 @@ typedef struct { int32_t reserved; } SStreamTaskExecRsp; +typedef struct { + SMsgHead head; + int64_t streamId; + int32_t taskId; +} SStreamTaskRunReq; + typedef struct { // SMsgHead head; int64_t streamId; @@ -304,21 +378,18 @@ typedef struct { SArray* res; // SArray } SStreamSinkReq; -typedef struct { - SMsgHead head; - int64_t streamId; - int32_t taskId; -} SStreamTaskRunReq; - typedef struct { int64_t streamId; int32_t taskId; int32_t sourceTaskId; int32_t sourceVg; + int32_t sourceChildId; #if 0 int64_t sourceVer; #endif - SArray* data; // SArray + int32_t blockNum; + SArray* dataLen; // SArray + SArray* data; // SArray } SStreamDispatchReq; typedef struct { @@ -340,6 +411,8 @@ typedef struct { int8_t inputStatus; } SStreamTaskRecoverRsp; +int32_t streamTriggerByWrite(SStreamTask* pTask, int32_t vgId, SMsgCb* pMsgCb); + int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input); int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input); int32_t streamDequeueOutput(SStreamTask* pTask, void** output); @@ -356,4 +429,4 @@ int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) } #endif -#endif /* ifndef _TSTREAM_H_ */ +#endif /* ifndef _STREAM_H_ */ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index cbef0b46a7..842c8a44e1 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -218,6 +218,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0390) #define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0391) #define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0392) +#define TSDB_CODE_MND_VGROUP_UN_CHANGED TAOS_DEF_ERROR_CODE(0, 0x0393) +#define TSDB_CODE_MND_HAS_OFFLINE_DNODE TAOS_DEF_ERROR_CODE(0, 0x0394) // mnode-stable #define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0) @@ -235,6 +237,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x03AC) #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AD) #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE) +#define TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03AF) #define TSDB_CODE_MND_SINGLE_STB_MODE_DB TAOS_DEF_ERROR_CODE(0, 0x03B0) // mnode-infoSchema diff --git a/include/util/tcompare.h b/include/util/tcompare.h index ed07ae1c9c..cc9e8ae464 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -105,8 +105,6 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight); int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight); int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight); -int32_t compareJsonContainsKey(const void *pLeft, const void *pRight); - __compar_fn_t getComparFunc(int32_t type, int32_t optr); __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); int32_t doCompare(const char *a, const char *b, int32_t type, size_t size); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index eaf18884c6..0630704d93 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -212,6 +212,9 @@ typedef struct SRequestObj { SArray* tableList; SQueryExecMetric metric; SRequestSendRecvBody body; + + uint32_t prevCode; //previous error code: todo refactor, add update flag for catalog + uint32_t retry; } SRequestObj; typedef struct SSyncQueryParam { @@ -263,8 +266,8 @@ extern SAppInfo appInfo; extern int32_t clientReqRefPool; extern int32_t clientConnRefPool; -extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); -int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); +__async_send_cb_fn_t getMsgRspHandle(int32_t msgType); + SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); void* createTscObj(const char* user, const char* auth, const char* db, int32_t connType, SAppInstInfo* pAppInfo); @@ -274,7 +277,7 @@ int32_t releaseTscObj(int64_t rid); uint64_t generateRequestId(); -void* createRequest(STscObj* pObj, void* param, int32_t type); +void* createRequest(STscObj* pObj, int32_t type); void destroyRequest(SRequestObj* pRequest); SRequestObj* acquireRequest(int64_t rid); int32_t releaseRequest(int64_t rid); @@ -290,8 +293,6 @@ void* openTransporter(const char* user, const char* auth, int32_t numOfThreads); bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); -void initMsgHandleFp(); - TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, uint16_t port, int connType); @@ -325,6 +326,9 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery); int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList); +void doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta); +int32_t removeMeta(STscObj* pTscObj, SArray* tbList);// todo move to clientImpl.c and become a static function +int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);// todo move to xxx #ifdef __cplusplus } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index be0a41ba40..75e1884360 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -177,7 +177,7 @@ STscObj *acquireTscObj(int64_t rid) { return (STscObj *)taosAcquireRef(clientCon int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, rid); } -void *createRequest(STscObj *pObj, void *param, int32_t type) { +void *createRequest(STscObj *pObj, int32_t type) { assert(pObj != NULL); SRequestObj *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj)); @@ -191,8 +191,6 @@ void *createRequest(STscObj *pObj, void *param, int32_t type) { pRequest->requestId = generateRequestId(); pRequest->metric.start = taosGetTimestampUs(); - pRequest->body.param = param; - pRequest->type = type; pRequest->pTscObj = pObj; pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); @@ -280,7 +278,6 @@ void taos_init_imp(void) { return; } - initMsgHandleFp(); initQueryModuleMsgHandle(); rpcInit(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ca29261755..62026dd577 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -133,7 +133,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, } int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest) { - *pRequest = createRequest(pTscObj, NULL, TSDB_SQL_SELECT); + *pRequest = createRequest(pTscObj, TSDB_SQL_SELECT); if (*pRequest == NULL) { tscError("failed to malloc sqlObj"); return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -207,6 +207,7 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, false); } + return code; } @@ -235,6 +236,31 @@ static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { return pRequest->pTscObj->pAppInfo; } +void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { + SRetrieveTableRsp* pRsp = NULL; + + int32_t code = qExecCommand(pQuery->pRoot, &pRsp); + if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, false); + } + + SReqResultInfo* pResultInfo = &pRequest->body.resInfo; + + if (pRequest->code != TSDB_CODE_SUCCESS) { + pResultInfo->numOfRows = 0; + pRequest->code = code; + tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); + } else { + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, + pRequest->requestId); + } + + pRequest->body.queryFp(pRequest->body.param, pRequest, 0); +// pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); +} + int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { // drop table if exists not_exists_table if (NULL == pQuery->pCmdMsg) { @@ -509,19 +535,20 @@ int32_t handleAlterTbExecRes(void* res, SCatalog* pCatalog) { return catalogUpdateTableMeta(pCatalog, (STableMetaRsp*)res); } -int32_t handleExecRes(SRequestObj* pRequest) { +int32_t handleQueryExecRsp(SRequestObj* pRequest) { if (NULL == pRequest->body.resInfo.execRes.res) { return TSDB_CODE_SUCCESS; } - int32_t code = 0; SCatalog* pCatalog = NULL; - code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + SAppInstInfo* pAppInfo = getAppInfo(pRequest); + + int32_t code = catalogGetHandle(pAppInfo->clusterId, &pCatalog); if (code) { return code; } - SEpSet epset = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + SEpSet epset = getEpSet_s(&pAppInfo->mgmtEp); SQueryExecRes* pRes = &pRequest->body.resInfo.execRes; switch (pRes->msgType) { @@ -539,8 +566,9 @@ int32_t handleExecRes(SRequestObj* pRequest) { break; } default: - tscError("invalid exec result for request type %d", pRequest->type); - return TSDB_CODE_APP_ERROR; + tscError("0x%"PRIx64", invalid exec result for request type %d, reqId:0x%"PRIx64, pRequest->self, + pRequest->type, pRequest->requestId); + code = TSDB_CODE_APP_ERROR; } return code; @@ -548,6 +576,25 @@ int32_t handleExecRes(SRequestObj* pRequest) { void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { SRequestObj* pRequest = (SRequestObj*) param; + pRequest->code = code; + + STscObj* pTscObj = pRequest->pTscObj; + if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) { + tscDebug("0x%"PRIx64" client retry to handle the error, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); + pRequest->prevCode = code; + doAsyncQuery(pRequest, true); + return; + } + + if (code == TSDB_CODE_SUCCESS) { + code = handleQueryExecRsp(pRequest); + ASSERT(pRequest->code == TSDB_CODE_SUCCESS); + pRequest->code = code; + } + + if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) { + removeMeta(pTscObj, pRequest->tableList); + } // return to client pRequest->body.queryFp(pRequest->body.param, pRequest, code); @@ -583,7 +630,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue qDestroyQuery(pQuery); } - handleExecRes(pRequest); + handleQueryExecRsp(pRequest); if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; @@ -617,13 +664,12 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { } void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) { - void* pRes = NULL; - int32_t code = 0; + switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: - code = execLocalCmd(pRequest, pQuery); - break; + asyncExecLocalCmd(pRequest, pQuery); + return; case QUERY_EXEC_MODE_RPC: code = asyncExecDdlQuery(pRequest, pQuery); break; @@ -649,11 +695,9 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) { if (TSDB_CODE_SUCCESS == code) { schedulerAsyncExecJob(pAppInfo->pTransporter, pNodeList, pRequest->body.pDag, &pRequest->body.queryJob, pRequest->sqlstr, pRequest->metric.start, schedulerExecCb, pRequest); - // if (NULL != pRes) { - // code = validateSversion(pRequest, pRes); - // } } + //todo not to be released here taosArrayDestroy(pNodeList); break; } @@ -671,12 +715,6 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) { if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } - - // if (res) { - // *res = pRes; - // } else { -// freeRequestRes(pRequest, pRes); -// pRes = NULL; } int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { @@ -750,7 +788,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { pRequest->code = code; break; } - } while (retryNum++ < REQUEST_MAX_TRY_TIMES); + } while (retryNum++ < REQUEST_TOTAL_EXEC_TIMES); if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) { removeMeta(pTscObj, pRequest->tableList); @@ -808,7 +846,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t return pTscObj; } - SRequestObj* pRequest = createRequest(pTscObj, param, TDMT_MND_CONNECT); + SRequestObj* pRequest = createRequest(pTscObj, TDMT_MND_CONNECT); if (pRequest == NULL) { destroyTscObj(pTscObj); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -850,7 +888,7 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestId = pRequest->requestId; - pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)]; + pMsgSendInfo->fp = getMsgRspHandle(pMsgSendInfo->msgType); pMsgSendInfo->param = pRequest; SConnectReq connectReq = {0}; @@ -1429,7 +1467,7 @@ TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* de } code = statusRsp.statusCode; - if (details != NULL && statusRsp.details != NULL) { + if (details != NULL) { tstrncpy(details, statusRsp.details, maxlen); } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index bddc32ea20..d192b65bf0 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -30,6 +30,7 @@ #define TSC_VAR_RELEASED 0 static int32_t sentinel = TSC_VAR_NOT_RELEASE; +static int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt); int taos_options(TSDB_OPTION option, const void *arg, ...) { static int32_t lock = 0; @@ -192,7 +193,7 @@ TAOS_RES *taos_query(TAOS *taos, const char *sql) { STscObj* pTscObj = (STscObj*)taos; #if SYNC_ON_TOP_OF_ASYNC - SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(struct SSyncQueryParam)); + SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); tsem_init(¶m->sem, 0, 0); taos_query_a(pTscObj, sql, syncQueryFn, param); @@ -608,36 +609,38 @@ void retrieveMetaCallback(SMetaData* pResultMeta, void* param, int32_t code) { SQuery* pQuery = pWrapper->pQuery; SRequestObj* pRequest = pWrapper->pRequest; - if (code != TSDB_CODE_SUCCESS) { - goto _error; + if (code == TSDB_CODE_SUCCESS) { + code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery); } - code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery); - if (code != TSDB_CODE_SUCCESS) { - goto _error; + if (code == TSDB_CODE_SUCCESS) { + if (pQuery->haveResultSet) { + setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols); + setResPrecision(&pRequest->body.resInfo, pQuery->precision); + } + + TSWAP(pRequest->dbList, (pQuery)->pDbList); + TSWAP(pRequest->tableList, (pQuery)->pTableList); + + taosMemoryFree(pWrapper); + launchAsyncQuery(pRequest, pQuery); + } else { + if (NEED_CLIENT_HANDLE_ERROR(code)) { + tscDebug("0x%"PRIx64" client retry to handle the error, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); + pRequest->prevCode = code; + doAsyncQuery(pRequest, true); + return; + } + + // return to app directly + taosMemoryFree(pWrapper); + tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); + pRequest->code = code; + pRequest->body.queryFp(pRequest->body.param, pRequest, code); } - - if (pQuery->haveResultSet) { - setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, (pQuery)->numOfResCols); - setResPrecision(&pRequest->body.resInfo, (pQuery)->precision); - } - - TSWAP(pRequest->dbList, (pQuery)->pDbList); - TSWAP(pRequest->tableList, (pQuery)->pTableList); - - taosMemoryFree(pWrapper); - launchAsyncQuery(pRequest, pQuery); - return; - - _error: - taosMemoryFree(pWrapper); - tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, reqId:%" PRIx64, pRequest->self, tstrerror(code), - pRequest->requestId); - pRequest->code = code; - pRequest->body.queryFp(pRequest->body.param, pRequest, code); } -// todo add retry before return user's callback void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { ASSERT(fp != NULL); @@ -657,24 +660,27 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param } SRequestObj *pRequest = NULL; - int32_t retryNum = 0; - int32_t code = 0; - - // while (retryNum++ < REQUEST_MAX_TRY_TIMES) { - code = buildRequest(taos, sql, sqlLen, &pRequest); + int32_t code = buildRequest(taos, sql, sqlLen, &pRequest); if (code != TSDB_CODE_SUCCESS) { terrno = code; - fp(param, NULL, code); + fp(param, NULL, terrno); return; } pRequest->body.queryFp = fp; pRequest->body.param = param; + doAsyncQuery(pRequest, false); +} - STscObj *pTscObj = pRequest->pTscObj; +int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt) { + const STscObj *pTscObj = pRequest->pTscObj; - SParseContext* pCxt = taosMemoryCalloc(1, sizeof(SParseContext)); - *pCxt = (SParseContext){.requestId = pRequest->requestId, + *pCxt = taosMemoryCalloc(1, sizeof(SParseContext)); + if (*pCxt == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + **pCxt = (SParseContext){.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = pRequest->pDb, .topicQuery = false, @@ -687,6 +693,22 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param .pUser = pTscObj->user, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .async = true,}; + return TSDB_CODE_SUCCESS; +} + +void doAsyncQuery(SRequestObj* pRequest, bool updateMetaForce) { + SParseContext* pCxt = NULL; + STscObj *pTscObj = pRequest->pTscObj; + + if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) { + pRequest->code = pRequest->prevCode; + goto _error; + } + + int32_t code = createParseContext(pRequest, &pCxt); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } pCxt->mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCxt->pCatalog); @@ -694,39 +716,36 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param goto _error; } - SQuery * pQuery = NULL; - SCatalogReq catalogReq = {0}; + SQuery *pQuery = NULL; + + SCatalogReq catalogReq = {.forceUpdate = updateMetaForce}; code = qParseSqlSyntax(pCxt, &pQuery, &catalogReq); if (code != TSDB_CODE_SUCCESS) { goto _error; } SqlParseWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SqlParseWrapper)); + if (pWrapper == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + pWrapper->pCtx = pCxt; pWrapper->pQuery = pQuery; pWrapper->pRequest = pRequest; pWrapper->catalogReq = catalogReq; code = catalogAsyncGetAllMeta(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, pRequest->requestId, - &catalogReq, retrieveMetaCallback, pWrapper, &pRequest->body.queryJob); - - if (code != TSDB_CODE_SUCCESS) { - goto _error; + &catalogReq, retrieveMetaCallback, pWrapper, &pRequest->body.queryJob); + if (code == TSDB_CODE_SUCCESS) { + return; } - return; - - // todo handle the retry process - - // if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) { - // TSWAP(pRequest->dbList, (pQuery)->pDbList); - // TSWAP(pRequest->tableList, (pQuery)->pTableList); - // } - _error: + tscError("0x%"PRIx64" error happens, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); terrno = code; pRequest->code = code; - fp(param, pRequest, code); + pRequest->body.queryFp(pRequest->body.param, pRequest, code); } static void fetchCallback(void* pResult, void* param, int32_t code) { @@ -751,14 +770,15 @@ static void fetchCallback(void* pResult, void* param, int32_t code) { pRequest->code = setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResultInfo->pData, true, false); if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; - pRequest->code = code; - pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); + tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); + } else { + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, + pRequest->requestId); } - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, - pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); - pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 2d5199f181..14e2798d3b 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -21,8 +21,6 @@ #include "tdef.h" #include "tname.h" -int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); - static void setErrno(SRequestObj* pRequest, int32_t code) { pRequest->code = code; terrno = code; @@ -107,10 +105,7 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) { assert(pRequest != NULL); pMsgSendInfo->msgInfo = pRequest->body.requestMsg; - - pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL) - ? genericRspCallback - : handleRequestRspFp[TMSG_INDEX(pRequest->type)]; + pMsgSendInfo->fp = getMsgRspHandle(pRequest->type); return pMsgSendInfo; } @@ -209,7 +204,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } -int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t processCreateSTableRsp(void* param, const SDataBuf* pMsg, int32_t code) { assert(pMsg != NULL && param != NULL); SRequestObj* pRequest = param; @@ -219,6 +214,7 @@ int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) { } if (pRequest->body.queryFp != NULL) { + removeMeta(pRequest->pTscObj, pRequest->tableList); pRequest->body.queryFp(pRequest->body.param, pRequest, code); } else { tsem_post(&pRequest->body.rspSem); @@ -251,30 +247,54 @@ int32_t processAlterStbRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); - tsem_post(&pRequest->body.rspSem); - return code; + } else { + SMAlterStbRsp alterRsp = {0}; + SDecoder coder = {0}; + tDecoderInit(&coder, pMsg->pData, pMsg->len); + tDecodeSMAlterStbRsp(&coder, &alterRsp); + tDecoderClear(&coder); + + pRequest->body.resInfo.execRes.msgType = TDMT_MND_ALTER_STB; + pRequest->body.resInfo.execRes.res = alterRsp.pMeta; } - SMAlterStbRsp alterRsp = {0}; - SDecoder coder = {0}; - tDecoderInit(&coder, pMsg->pData, pMsg->len); - tDecodeSMAlterStbRsp(&coder, &alterRsp); - tDecoderClear(&coder); + if (pRequest->body.queryFp != NULL) { + SQueryExecRes* pRes = &pRequest->body.resInfo.execRes; - pRequest->body.resInfo.execRes.msgType = TDMT_MND_ALTER_STB; - pRequest->body.resInfo.execRes.res = alterRsp.pMeta; + if (code == TSDB_CODE_SUCCESS) { + SCatalog* pCatalog = NULL; + int32_t ret = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (pRes->res != NULL) { + ret = handleAlterTbExecRes(pRes->res, pCatalog); + } - tsem_post(&pRequest->body.rspSem); + if (ret != TSDB_CODE_SUCCESS) { + code = ret; + } + } + + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + } else { + tsem_post(&pRequest->body.rspSem); + } return code; } - -// todo refactor: this arraylist is too large -void initMsgHandleFp() { - handleRequestRspFp[TMSG_INDEX(TDMT_MND_CONNECT)] = processConnectRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = processCreateDbRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = processCreateTableRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = processAlterStbRsp; +__async_send_cb_fn_t getMsgRspHandle(int32_t msgType) { + switch (msgType) { + case TDMT_MND_CONNECT: + return processConnectRsp; + case TDMT_MND_CREATE_DB: + return processCreateDbRsp; + case TDMT_MND_USE_DB: + return processUseDbRsp; + case TDMT_MND_CREATE_STB: + return processCreateSTableRsp; + case TDMT_MND_DROP_DB: + return processDropDbRsp; + case TDMT_MND_ALTER_STB: + return processAlterStbRsp; + default: + return genericRspCallback; + } } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 5643af746d..4cef67b34d 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2364,7 +2364,7 @@ static int32_t isSchemalessDb(SSmlHandle* info){ */ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { - SRequestObj* request = (SRequestObj*)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT); + SRequestObj* request = (SRequestObj*)createRequest((STscObj *)taos, TSDB_SQL_INSERT); if(!request){ uError("SML:taos_schemaless_insert error request is null"); return NULL; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index a0f33627c4..f16baba0e7 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -778,7 +778,35 @@ TEST(testCase, async_api_test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - taos_query_a(pConn, "drop table test.tm0", queryCallback, pConn); + taos_query(pConn, "use test"); + + TAOS_RES* pRes = taos_query(pConn, "select * from t1"); + + taos_query(pConn, "alter table t1 add column b int"); + pRes = taos_query(pConn, "insert into t1 values(now, 1, 2)"); + if (taos_errno(pRes) != 0) { + printf("failed, reason:%s\n", taos_errstr(pRes)); + } + +// int32_t n = 0; +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t* length = taos_fetch_lengths(pRes); +// for(int32_t i = 0; i < numOfFields; ++i) { +// printf("(%d):%d " , i, length[i]); +// } +// printf("\n"); +// +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// memset(str, 0, sizeof(str)); +// } + + taos_query_a(pConn, "alter table test.m1 comment 'abcde' ", queryCallback, pConn); getchar(); taos_close(pConn); } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 1358d45f30..3bfd26bb43 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -486,7 +486,7 @@ TEST(testCase, smlProcess_influx_Test) { pRes = taos_query(taos, "use inflx_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -607,7 +607,7 @@ TEST(testCase, smlParseLine_error_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -656,7 +656,7 @@ TEST(testCase, smlProcess_telnet_Test) { pRes = taos_query(taos, "use telnet_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -710,7 +710,7 @@ TEST(testCase, smlProcess_json1_Test) { pRes = taos_query(taos, "use json_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -779,7 +779,7 @@ TEST(testCase, smlProcess_json2_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -823,7 +823,7 @@ TEST(testCase, smlProcess_json3_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -895,7 +895,7 @@ TEST(testCase, smlProcess_json4_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -957,7 +957,7 @@ TEST(testCase, smlParseTelnetLine_error_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -1006,7 +1006,7 @@ TEST(testCase, smlParseTelnetLine_diff_type_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -1033,7 +1033,7 @@ TEST(testCase, smlParseTelnetLine_json_error_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -1101,7 +1101,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -1146,7 +1146,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -1191,7 +1191,7 @@ TEST(testCase, sml_TD15662_Test) { pRes = taos_query(taos, "use db_15662"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); @@ -1218,7 +1218,7 @@ TEST(testCase, sml_TD15735_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -1244,7 +1244,7 @@ TEST(testCase, sml_TD15742_Test) { pRes = taos_query(taos, "use TD15742"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c615af705a..d9293433ea 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1590,7 +1590,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks for (int32_t k = 0; k < colNum; ++k) { // iterate by column SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); STColumn* pCol = &pTSchema->columns[k]; - ASSERT(pCol->type == pColInfoData->info.type); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: @@ -1614,20 +1613,39 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_MEDIUMBLOB: uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); TASSERT(0); break; default: if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pColInfoData->info.type, TD_VTYPE_NORM, var, true, offset, k); + char tv[8] = {0}; + if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, k); } else { uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); TASSERT(0); } break; } - offset += TYPE_BYTES[pColInfoData->info.type]; + offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation } dataLen += TD_ROW_LEN(rb.pBuf); #ifdef TD_DEBUG_PRINT_ROW diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 68c0eed1fb..773e430b83 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -694,7 +694,6 @@ void tFreeSMAltertbReq(SMAlterStbReq *pReq) { pReq->pFields = NULL; } - int32_t tSerializeSEpSet(void *buf, int32_t bufLen, const SEpSet *pEpset) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3676,12 +3675,12 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1; } for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { - if (tEncodeI32(pCoder, pSma->vgEpSet[v].vgId) < 0) return -1; - if (tEncodeI8(pCoder, pSma->vgEpSet[v].epSet.inUse) < 0) return -1; - int8_t numOfEps = pSma->vgEpSet[v].epSet.numOfEps; + if (tEncodeI32(pCoder, pSma->pVgEpSet[v].vgId) < 0) return -1; + if (tEncodeI8(pCoder, pSma->pVgEpSet[v].epSet.inUse) < 0) return -1; + int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps; if (tEncodeI8(pCoder, numOfEps) < 0) return -1; for (int32_t n = 0; n < numOfEps; ++n) { - const SEp *pEp = &pSma->vgEpSet[v].epSet.eps[n]; + const SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n]; if (tEncodeCStr(pCoder, pEp->fqdn) < 0) return -1; if (tEncodeU16(pCoder, pEp->port) < 0) return -1; } @@ -3714,15 +3713,25 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { } else { pSma->tagsFilter = NULL; } - for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { - if (tDecodeI32(pCoder, &pSma->vgEpSet[v].vgId) < 0) return -1; - if (tDecodeI8(pCoder, &pSma->vgEpSet[v].epSet.inUse) < 0) return -1; - if (tDecodeI8(pCoder, &pSma->vgEpSet[v].epSet.numOfEps) < 0) return -1; - int8_t numOfEps = pSma->vgEpSet[v].epSet.numOfEps; - for (int32_t n = 0; n < numOfEps; ++n) { - SEp *pEp = &pSma->vgEpSet[v].epSet.eps[n]; - if (tDecodeCStrTo(pCoder, pEp->fqdn) < 0) return -1; - if (tDecodeU16(pCoder, &pEp->port) < 0) return -1; + if (pSma->numOfVgroups > 0) { + pSma->pVgEpSet = (SVgEpSet *)tDecoderMalloc(pCoder, pSma->numOfVgroups * sizeof(SVgEpSet)); + if (!pSma->pVgEpSet) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + memset(pSma->pVgEpSet, 0, pSma->numOfVgroups * sizeof(SVgEpSet)); + + for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { + if (tDecodeI32(pCoder, &pSma->pVgEpSet[v].vgId) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.inUse) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.numOfEps) < 0) return -1; + int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps; + for (int32_t n = 0; n < numOfEps; ++n) { + SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n]; + if (tDecodeCStrTo(pCoder, pEp->fqdn) < 0) return -1; + if (tDecodeU16(pCoder, &pEp->port) < 0) return -1; + } } } @@ -3767,7 +3776,7 @@ int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { return 0; } -int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder* pCoder, const SVGetTsmaExpWndsReq* pReq) { +int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder *pCoder, const SVGetTsmaExpWndsReq *pReq) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1; @@ -3775,10 +3784,10 @@ int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder* pCoder, const SVGetTsmaExpWndsReq* if (tEncodeI64(pCoder, pReq->queryWindow.ekey) < 0) return -1; tEndEncode(pCoder); - return 0; + return 0; } -int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder* pCoder, SVGetTsmaExpWndsReq* pReq) { +int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder *pCoder, SVGetTsmaExpWndsReq *pReq) { if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; @@ -3789,7 +3798,7 @@ int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder* pCoder, SVGetTsmaExpWndsReq* pReq) return 0; } -int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder* pCoder, const SVGetTsmaExpWndsRsp* pReq) { +int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder *pCoder, const SVGetTsmaExpWndsRsp *pReq) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1; @@ -3877,7 +3886,7 @@ int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { return 0; } -int32_t tEncodeSVDeleteRsp(SEncoder* pCoder, const SVDeleteRsp* pReq) { +int32_t tEncodeSVDeleteRsp(SEncoder *pCoder, const SVDeleteRsp *pReq) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeI64(pCoder, pReq->affectedRows) < 0) return -1; @@ -3886,7 +3895,7 @@ int32_t tEncodeSVDeleteRsp(SEncoder* pCoder, const SVDeleteRsp* pReq) { return 0; } -int32_t tDecodeSVDeleteRsp(SDecoder* pCoder, SVDeleteRsp* pReq) { +int32_t tDecodeSVDeleteRsp(SDecoder *pCoder, SVDeleteRsp *pReq) { if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI64(pCoder, &pReq->affectedRows) < 0) return -1; @@ -4527,7 +4536,7 @@ int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) { } int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) { - int32_t meta = 0; + int32_t meta = 0; SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -4568,7 +4577,7 @@ int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp) { } int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) { - int32_t meta = 0; + int32_t meta = 0; SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -4584,7 +4593,7 @@ int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp return 0; } -void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp) { +void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) { if (NULL == pRsp) { return; } @@ -4594,6 +4603,3 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp) { taosMemoryFree(pRsp->pMeta); } } - - - diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 83a36f4b0d..8ea172ef94 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -298,28 +298,30 @@ typedef struct { } SVgObj; typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; - char stb[TSDB_TABLE_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createdTime; - int64_t uid; - int64_t stbUid; - int64_t dbUid; - int8_t intervalUnit; - int8_t slidingUnit; - int8_t timezone; - int32_t dstVgId; // for stream - int64_t interval; - int64_t offset; - int64_t sliding; - int32_t exprLen; // strlen + 1 - int32_t tagsFilterLen; - int32_t sqlLen; - int32_t astLen; - char* expr; - char* tagsFilter; - char* sql; - char* ast; + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createdTime; + int64_t uid; + int64_t stbUid; + int64_t dbUid; + int8_t intervalUnit; + int8_t slidingUnit; + int8_t timezone; + int32_t dstVgId; // for stream + int64_t interval; + int64_t offset; + int64_t sliding; + int32_t exprLen; // strlen + 1 + int32_t tagsFilterLen; + int32_t sqlLen; + int32_t astLen; + int32_t numOfVgroups; + char* expr; + char* tagsFilter; + char* sql; + char* ast; + SVgEpSet* pVgEpSet; } SSmaObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h index c76186c0a2..cf1e7422be 100644 --- a/source/dnode/mnode/impl/inc/mndDnode.h +++ b/source/dnode/mnode/impl/inc/mndDnode.h @@ -28,7 +28,7 @@ SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId); void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode); SEpSet mndGetDnodeEpset(SDnodeObj *pDnode); int32_t mndGetDnodeSize(SMnode *pMnode); -bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs); +bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndMnode.h b/source/dnode/mnode/impl/inc/mndMnode.h index fd62b3ce75..a433af9947 100644 --- a/source/dnode/mnode/impl/inc/mndMnode.h +++ b/source/dnode/mnode/impl/inc/mndMnode.h @@ -28,6 +28,7 @@ SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId); void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj); bool mndIsMnode(SMnode *pMnode, int32_t dnodeId); void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet); +int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index 4aa18ea591..4becad6da2 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -37,7 +37,7 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); -bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *colIds); +int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 2119320de5..89f93d30b5 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -30,15 +30,21 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); -int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup); -int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); -SArray *mndBuildDnodesArray(SMnode *pMnode); -int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray); -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pVgId); +SArray *mndBuildDnodesArray(SMnode *, int32_t exceptDnodeId); +int32_t mndAllocSmaVgroup(SMnode *, SDbObj *pDb, SVgObj *pVgroup); +int32_t mndAllocVgroup(SMnode *, SDbObj *pDb, SVgObj **ppVgroups); +int32_t mndAddVnodeToVgroup(SMnode *, SVgObj *pVgroup, SArray *pArray); +int32_t mndRemoveVnodeFromVgroup(SMnode *, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid); +int32_t mndAddCreateVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool standby); +int32_t mndAddAlterVnodeConfirmAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup); +int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType); +int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo); +int32_t mndSetMoveVgroupInfoToTrans(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vn, SArray *pArray); +int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId); -void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen, bool standby); -void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *cntlen, bool standby); +void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildAlterVnodeReq(SMnode *, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 38a166900d..ed07e15c63 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -299,7 +299,7 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } @@ -409,7 +409,7 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 1ce1a2590a..61d6e7be6c 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -263,111 +263,6 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, - bool standby) { - STransAction action = {0}; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen, standby); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_CREATE_VNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - - return 0; -} - -static int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { - STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgroup); - - int32_t contLen = sizeof(SMsgHead); - SMsgHead *pHead = taosMemoryMalloc(contLen); - if (pHead == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pHead->contLen = htonl(contLen); - pHead->vgId = htonl(pVgroup->vgId); - - action.pCont = pHead; - action.contLen = contLen; - action.msgType = TDMT_VND_ALTER_CONFIRM; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pHead); - return -1; - } - - return 0; -} - -static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType) { - STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgroup); - - int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeReq(pMnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = msgType; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - - return 0; -} - -static int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, - bool isRedo) { - STransAction action = {0}; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - - if (isRedo) { - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } else { - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - return 0; -} - static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { char *pos = strstr(dbName, TS_PATH_DELIMITER); if (pos == NULL) { @@ -795,7 +690,7 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; - SArray *pArray = mndBuildDnodesArray(pMnode); + SArray *pArray = mndBuildDnodesArray(pMnode, 0); while (1) { SVgObj *pVgroup = NULL; @@ -1497,20 +1392,26 @@ char *buildRetension(SArray *pRetension) { int64_t v1 = getValOfDiffPrecision(p->freqUnit, p->freq); int64_t v2 = getValOfDiffPrecision(p->keepUnit, p->keep); - len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c,", v1, p->freqUnit, v2, p->keepUnit); - - p = taosArrayGet(pRetension, 1); - - v1 = getValOfDiffPrecision(p->freqUnit, p->freq); - v2 = getValOfDiffPrecision(p->keepUnit, p->keep); - len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c,", v1, p->freqUnit, v2, p->keepUnit); - - p = taosArrayGet(pRetension, 2); - - v1 = getValOfDiffPrecision(p->freqUnit, p->freq); - v2 = getValOfDiffPrecision(p->keepUnit, p->keep); len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit); + if (size > 1) { + len += sprintf(p1 + len, ","); + p = taosArrayGet(pRetension, 1); + + v1 = getValOfDiffPrecision(p->freqUnit, p->freq); + v2 = getValOfDiffPrecision(p->keepUnit, p->keep); + len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit); + } + + if (size > 2) { + len += sprintf(p1 + len, ","); + p = taosArrayGet(pRetension, 2); + + v1 = getValOfDiffPrecision(p->freqUnit, p->freq); + v2 = getValOfDiffPrecision(p->keepUnit, p->keep); + len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit); + } + varDataSetLen(p1, len); return p1; } @@ -1742,3 +1643,4 @@ static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index aeff018aa8..c7e347fb8b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -254,7 +254,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { return sdbGetSize(pSdb, SDB_DNODE); } -bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { +bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs) { int64_t interval = TABS(pDnode->lastAccessTime - curMs); if (interval > 5000 * tsStatusInterval) { if (pDnode->rebootTime > 0) { @@ -393,7 +393,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE); int64_t curMs = taosGetTimestampMs(); - bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); + bool online = mndIsDnodeOnline(pDnode, curMs); bool dnodeChanged = (statusReq.dnodeVer != dnodeVer); bool reboot = (pDnode->rebootTime != statusReq.rebootTime); bool needCheck = !online || dnodeChanged || reboot; @@ -542,7 +542,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { goto CREATE_DNODE_OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto CREATE_DNODE_OVER; } @@ -559,30 +559,36 @@ CREATE_DNODE_OVER: return code; } -static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); - if (pTrans == NULL) { - mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr()); - return -1; - } +static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMnodeObj *pMObj) { + int32_t code = -1; + SSdbRaw *pRaw = NULL; + STrans *pTrans = NULL; + + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); mDebug("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id); - SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + pRaw = mndDnodeActionEncode(pDnode); + if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; + sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING); + pRaw = NULL; - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + pRaw = mndDnodeActionEncode(pDnode); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); + pRaw = NULL; + if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj) != 0) goto _OVER; + if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: mndTransDrop(pTrans); - return 0; + sdbFreeRaw(pRaw); + return code; } static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { @@ -595,42 +601,53 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto DROP_DNODE_OVER; + goto _OVER; } mDebug("dnode:%d, start to drop", dropReq.dnodeId); if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_MND_INVALID_DNODE_ID; - goto DROP_DNODE_OVER; + goto _OVER; } pDnode = mndAcquireDnode(pMnode, dropReq.dnodeId); if (pDnode == NULL) { terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - goto DROP_DNODE_OVER; + goto _OVER; + } + + if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { + terrno = TSDB_CODE_NODE_OFFLINE; + goto _OVER; } pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId); if (pMObj != NULL) { - terrno = TSDB_CODE_MND_MNODE_NOT_EXIST; - goto DROP_DNODE_OVER; + if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) { + terrno = TSDB_CODE_MND_TOO_FEW_MNODES; + goto _OVER; + } + if (pMnode->selfDnodeId == dropReq.dnodeId) { + terrno = TSDB_CODE_MND_CANT_DROP_MASTER; + goto _OVER; + } } pUser = mndAcquireUser(pMnode, pReq->conn.user); if (pUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto DROP_DNODE_OVER; + goto _OVER; } - if (mndCheckNodeAuth(pUser)) { - goto DROP_DNODE_OVER; + if (mndCheckNodeAuth(pUser) != 0) { + goto _OVER; } - code = mndDropDnode(pMnode, pReq, pDnode); + code = mndDropDnode(pMnode, pReq, pDnode, pMObj); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; -DROP_DNODE_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } @@ -638,7 +655,6 @@ DROP_DNODE_OVER: mndReleaseDnode(pMnode, pDnode); mndReleaseUser(pMnode, pUser); mndReleaseMnode(pMnode, pMObj); - return code; } @@ -736,7 +752,7 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_DNODE, pShow->pIter, (void **)&pDnode); if (pShow->pIter == NULL) break; - bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); + bool online = mndIsDnodeOnline(pDnode, curMs); cols = 0; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index d989ba6ec3..69300c0889 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -529,7 +529,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr SMonDnodeDesc desc = {0}; desc.dnode_id = pObj->id; tstrncpy(desc.dnode_ep, pObj->ep, sizeof(desc.dnode_ep)); - if (mndIsDnodeOnline(pMnode, pObj, ms)) { + if (mndIsDnodeOnline(pObj, ms)) { tstrncpy(desc.status, "ready", sizeof(desc.status)); } else { tstrncpy(desc.status, "offline", sizeof(desc.status)); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 1cc832b3af..072ad05f5a 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -358,9 +358,9 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) goto _OVER; - - mDebug("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); mndTransSetSerial(pTrans); + mDebug("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); + if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; @@ -408,7 +408,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (!mndIsDnodeOnline(pMnode, pDnode, taosGetTimestampMs())) { + if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { terrno = TSDB_CODE_NODE_OFFLINE; goto _OVER; } @@ -419,7 +419,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } @@ -535,18 +535,25 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode return 0; } +int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { + if (pObj == NULL) return 0; + if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; + if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; + if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1; + if (mndTransAppendNullLog(pTrans) != 0) return -1; + return 0; +} + static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) { int32_t code = -1; + STrans *pTrans = NULL; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) goto _OVER; - - mDebug("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id); mndTransSetSerial(pTrans); - if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) goto _OVER; - if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) goto _OVER; - if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) goto _OVER; - if (mndTransAppendNullLog(pTrans) != 0) goto _OVER; + mDebug("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id); + + if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -596,7 +603,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } @@ -642,7 +649,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB if (pObj->id == pMnode->selfDnodeId) { roles = syncStr(TAOS_SYNC_STATE_LEADER); } - if (pObj->pDnode && mndIsDnodeOnline(pMnode, pObj->pDnode, curMs)) { + if (pObj->pDnode && mndIsDnodeOnline(pObj->pDnode, curMs)) { roles = syncStr(pObj->state); } char b2[12 + VARSTR_HEADER_SIZE] = {0}; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 4fb9226144..aac6eaba47 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -301,7 +301,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } @@ -411,7 +411,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index b390a7fe4a..8497a57edd 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -190,7 +190,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p sdbRelease(pSdb, pVgroup); continue; } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewSStreamTask(pStream->uid, 0); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -230,7 +230,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { ASSERT(pStream->fixedSinkVgId != 0); SArray* tasks = taosArrayGetP(pStream->tasks, 0); - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewSStreamTask(pStream->uid, 0); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -322,7 +322,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { sdbRelease(pSdb, pVgroup); continue; } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewSStreamTask(pStream->uid, 0); // source part pTask->sourceType = TASK_SOURCE__SCAN; pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK; @@ -387,7 +387,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // else, assign to vnode ASSERT(plan->subplanType == SUBPLAN_TYPE_MERGE); - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewSStreamTask(pStream->uid, 0); // source part, currently only support multi source pTask->sourceType = TASK_SOURCE__PIPE; @@ -477,7 +477,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { sdbRelease(pSdb, pVgroup); continue; } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewSStreamTask(pStream->uid, 0); // source part pTask->sourceType = TASK_SOURCE__MERGE; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index e8e23ccc2a..bcc413c1de 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -36,6 +36,7 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw); static int32_t mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma); static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSpSmatb); static int32_t mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew); +static int32_t mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups); static int32_t mndProcessMCreateSmaReq(SRpcMsg *pReq); static int32_t mndProcessMDropSmaReq(SRpcMsg *pReq); static int32_t mndProcessGetSmaReq(SRpcMsg *pReq); @@ -262,7 +263,9 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm req.sliding = pSma->sliding; req.expr = pSma->expr; req.tagsFilter = pSma->tagsFilter; - + req.numOfVgroups = pSma->numOfVgroups; + req.pVgEpSet = pSma->pVgEpSet; + // get length int32_t ret = 0; tEncodeSize(tEncodeSVCreateTSmaReq, &req, contLen, ret); @@ -420,6 +423,15 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, mndReleaseDnode(pMnode, pDnode); // todo add sma info here + SVgEpSet *pVgEpSet = NULL; + int32_t numOfVgroups = 0; + if (mndSmaGetVgEpSet(pMnode, pDb, &pVgEpSet, &numOfVgroups) != 0) { + return -1; + } + + pSma->pVgEpSet = pVgEpSet; + pSma->numOfVgroups = numOfVgroups; + int32_t smaContLen = 0; void *pSmaReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &smaContLen); if (pSmaReq == NULL) return -1; @@ -510,10 +522,9 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; - - mDebug("trans:%d, used to create sma:%s", pTrans->id, pCreate->name); mndTransSetDbName(pTrans, pDb->name); mndTransSetSerial(pTrans); + mDebug("trans:%d, used to create sma:%s", pTrans->id, pCreate->name); if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; @@ -964,3 +975,52 @@ static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +static int32_t mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + SVgEpSet *pVgEpSet = NULL; + int32_t nAllocVgs = 16; + int32_t nVgs = 0; + + pVgEpSet = taosMemoryCalloc(nAllocVgs, sizeof(SVgEpSet)); + if (!pVgEpSet) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) { + sdbRelease(pSdb, pVgroup); + continue; + } + + if (nVgs >= nAllocVgs) { + void *p = taosMemoryRealloc(pVgEpSet, nAllocVgs * 2 * sizeof(SVgEpSet)); + if (!p) { + taosMemoryFree(pVgEpSet); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pVgEpSet = (SVgEpSet *)p; + nAllocVgs *= 2; + } + + (pVgEpSet + nVgs)->vgId = pVgroup->vgId; + (pVgEpSet + nVgs)->epSet = mndGetVgroupEpset(pMnode, pVgroup); + + ++nVgs; + + sdbRelease(pSdb, pVgroup); + } + + *ppVgEpSet = pVgEpSet; + *numOfVgroups = nVgs; + + return 0; +} diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index f5a1e76723..7d21528260 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -307,7 +307,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } @@ -419,7 +419,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 175878959d..9981dc8530 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -23,6 +23,7 @@ #include "mndPerfSchema.h" #include "mndScheduler.h" #include "mndShow.h" +#include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" @@ -394,14 +395,14 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.pRSmaParam.xFilesFactor = pStb->xFilesFactor; req.pRSmaParam.delay = pStb->delay; if (pStb->ast1Len > 0) { - if (mndConvertRSmaTask(pStb->pAst1, pStb->uid, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, req.pRSmaParam.xFilesFactor) != - TSDB_CODE_SUCCESS) { + if (mndConvertRSmaTask(pStb->pAst1, pStb->uid, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, + req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { return NULL; } } if (pStb->ast2Len > 0) { - if (mndConvertRSmaTask(pStb->pAst2, pStb->uid, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, req.pRSmaParam.xFilesFactor) != - TSDB_CODE_SUCCESS) { + if (mndConvertRSmaTask(pStb->pAst2, pStb->uid, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, + req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { return NULL; } } @@ -949,13 +950,18 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p return 0; } -static int32_t mndDropSuperTableTag(const SStbObj *pOld, SStbObj *pNew, const char *tagName) { +static int32_t mndDropSuperTableTag(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const char *tagName) { int32_t tag = mndFindSuperTableTagIndex(pOld, tagName); if (tag < 0) { terrno = TSDB_CODE_MND_TAG_NOT_EXIST; return -1; } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -968,7 +974,7 @@ static int32_t mndDropSuperTableTag(const SStbObj *pOld, SStbObj *pNew, const ch return 0; } -static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pFields) { +static int32_t mndAlterStbTagName(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, SArray *pFields) { if ((int32_t)taosArrayGetSize(pFields) != 2) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; @@ -986,6 +992,11 @@ static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pF return -1; } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndFindSuperTableTagIndex(pOld, newTagName) >= 0) { terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; return -1; @@ -1008,13 +1019,18 @@ static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pF return 0; } -static int32_t mndAlterStbTagBytes(const SStbObj *pOld, SStbObj *pNew, const SField *pField) { +static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const SField *pField) { int32_t tag = mndFindSuperTableTagIndex(pOld, pField->name); if (tag < 0) { terrno = TSDB_CODE_MND_TAG_NOT_EXIST; return -1; } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -1075,7 +1091,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray return 0; } -static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const char *colName) { +static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const char *colName) { int32_t col = mndFindSuperTableColumnIndex(pOld, colName); if (col < 0) { terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST; @@ -1092,6 +1108,11 @@ static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const return -1; } + col_id_t colId = pOld->pColumns[col].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -1104,7 +1125,7 @@ static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const return 0; } -static int32_t mndAlterStbColumnBytes(const SStbObj *pOld, SStbObj *pNew, const SField *pField) { +static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const SField *pField) { int32_t col = mndFindSuperTableColumnIndex(pOld, pField->name); if (col < 0) { terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST; @@ -1121,6 +1142,11 @@ static int32_t mndAlterStbColumnBytes(const SStbObj *pOld, SStbObj *pNew, const return -1; } + col_id_t colId = pOld->pColumns[col].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -1199,7 +1225,6 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } - static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableMetaRsp *pRsp) { taosRLockLatch(&pStb->lock); @@ -1269,13 +1294,13 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return code; } - -static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, int32_t *pLen) { +static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, + int32_t *pLen) { int ret; SEncoder ec = {0}; - uint32_t contLen = 0; + uint32_t contLen = 0; SMAlterStbRsp alterRsp = {0}; - SName name = {0}; + SName name = {0}; tNameFromString(&name, pAlter->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); alterRsp.pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); @@ -1283,20 +1308,20 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, S terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - + ret = mndBuildStbSchemaImp(pDb, pObj, name.tname, alterRsp.pMeta); if (ret) { tFreeSMAlterStbRsp(&alterRsp); return ret; } - + tEncodeSize(tEncodeSMAlterStbRsp, &alterRsp, contLen, ret); if (ret) { tFreeSMAlterStbRsp(&alterRsp); return ret; } - void* cont = taosMemoryMalloc(contLen); + void *cont = taosMemoryMalloc(contLen); tEncoderInit(&ec, cont, contLen); tEncodeSMAlterStbRsp(&ec, &alterRsp); tEncoderClear(&ec); @@ -1305,24 +1330,24 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, S *pCont = cont; *pLen = contLen; - + return 0; } static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { - bool needRsp = true; + bool needRsp = true; + int32_t code = -1; + STrans *pTrans = NULL; + SField *pField0 = NULL; + SStbObj stbObj = {0}; taosRLockLatch(&pOld->lock); memcpy(&stbObj, pOld, sizeof(SStbObj)); + taosRUnLockLatch(&pOld->lock); stbObj.pColumns = NULL; stbObj.pTags = NULL; stbObj.updateTime = taosGetTimestampMs(); stbObj.lock = 0; - taosRUnLockLatch(&pOld->lock); - - int32_t code = -1; - STrans *pTrans = NULL; - SField *pField0 = NULL; switch (pAlter->alterType) { case TSDB_ALTER_TABLE_ADD_TAG: @@ -1330,25 +1355,25 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p break; case TSDB_ALTER_TABLE_DROP_TAG: pField0 = taosArrayGet(pAlter->pFields, 0); - code = mndDropSuperTableTag(pOld, &stbObj, pField0->name); + code = mndDropSuperTableTag(pMnode, pOld, &stbObj, pField0->name); break; case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: - code = mndAlterStbTagName(pOld, &stbObj, pAlter->pFields); + code = mndAlterStbTagName(pMnode, pOld, &stbObj, pAlter->pFields); break; case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: pField0 = taosArrayGet(pAlter->pFields, 0); - code = mndAlterStbTagBytes(pOld, &stbObj, pField0); + code = mndAlterStbTagBytes(pMnode, pOld, &stbObj, pField0); break; case TSDB_ALTER_TABLE_ADD_COLUMN: code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); break; case TSDB_ALTER_TABLE_DROP_COLUMN: pField0 = taosArrayGet(pAlter->pFields, 0); - code = mndDropSuperTableColumn(pOld, &stbObj, pField0->name); + code = mndDropSuperTableColumn(pMnode, pOld, &stbObj, pField0->name); break; case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: pField0 = taosArrayGet(pAlter->pFields, 0); - code = mndAlterStbColumnBytes(pOld, &stbObj, pField0); + code = mndAlterStbColumnBytes(pMnode, pOld, &stbObj, pField0); break; case TSDB_ALTER_TABLE_UPDATE_OPTIONS: needRsp = false; @@ -1370,12 +1395,12 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p mndTransSetDbName(pTrans, pDb->name); if (needRsp) { - void* pCont = NULL; + void *pCont = NULL; int32_t contLen = 0; - if (mndBuildSMAlterStbRsp(pDb, pAlter, &stbObj, &pCont, &contLen)) goto _OVER; + if (mndBuildSMAlterStbRsp(pDb, pAlter, &stbObj, &pCont, &contLen) != 0) goto _OVER; mndTransSetRpcRsp(pTrans, pCont, contLen); } - + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 95f22a33db..605e446e2e 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -71,7 +71,7 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]) { return strchr(topic, '.') + 1; } -bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *colAndTagIds) { +int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; bool found = false; @@ -91,7 +91,7 @@ bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *col } SHashObj *pColHash = NULL; - SNodeList *pNodeList; + SNodeList *pNodeList = NULL; nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList); SNode *pNode = NULL; FOREACH(pNode, pNodeList) { @@ -103,22 +103,24 @@ bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *col if (pCol->colId > 0) { taosHashPut(pColHash, &pCol->colId, sizeof(int16_t), NULL, 0); } + mTrace("topic:%s, colId:%d is used", pTopic->name, pCol->colId); } - for (int32_t i = 0; i < taosArrayGetSize(colAndTagIds); i++) { - int16_t *pColId = taosArrayGet(colAndTagIds, i); - if (taosHashGet(pColHash, pColId, sizeof(int16_t)) != NULL) { - found = true; - goto NEXT; - } + if (taosHashGet(pColHash, &colId, sizeof(int16_t)) != NULL) { + found = true; + goto NEXT; } NEXT: sdbRelease(pSdb, pTopic); nodesDestroyNode(pAst); - if (found) return false; + if (found) { + terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; + return -1; + } } - return true; + + return 0; } SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 4f09eda733..5244bc657b 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -15,11 +15,13 @@ #define _DEFAULT_SOURCE #include "mndVgroup.h" +#include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" +#include "mndUser.h" #define VGROUP_VER_NUMBER 1 #define VGROUP_RESERVE_SIZE 64 @@ -34,6 +36,10 @@ static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); +static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq); +static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq); +static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq); + int32_t mndInitVgroup(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_VGROUP, @@ -344,9 +350,14 @@ static bool mndResetDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SDnodeObj *pDnode = pObj; SArray *pArray = p1; + int32_t exceptDnodeId = *(int32_t *)p2; + + if (exceptDnodeId == pDnode->id) { + return true; + } int64_t curMs = taosGetTimestampMs(); - bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); + bool online = mndIsDnodeOnline(pDnode, curMs); bool isMnode = mndIsMnode(pMnode, pDnode->id); pDnode->numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); @@ -363,7 +374,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 return true; } -SArray *mndBuildDnodesArray(SMnode *pMnode) { +SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) { SSdb *pSdb = pMnode->pSdb; int32_t numOfDnodes = mndGetDnodeSize(pMnode); @@ -374,7 +385,7 @@ SArray *mndBuildDnodesArray(SMnode *pMnode) { } sdbTraverse(pSdb, SDB_DNODE, mndResetDnodesArrayFp, NULL, NULL, NULL); - sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, NULL, NULL); + sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, &exceptDnodeId, NULL); return pArray; } @@ -422,7 +433,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr } int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup) { - SArray *pArray = mndBuildDnodesArray(pMnode); + SArray *pArray = mndBuildDnodesArray(pMnode, 0); if (pArray == NULL) return -1; pVgroup->vgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); @@ -451,7 +462,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { goto _OVER; } - pArray = mndBuildDnodesArray(pMnode); + pArray = mndBuildDnodesArray(pMnode, 0); if (pArray == NULL) goto _OVER; mInfo("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), @@ -501,86 +512,6 @@ _OVER: return code; } -int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { - taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); - for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - SDnodeObj *pDnode = taosArrayGet(pArray, i); - mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); - } - - SVnodeGid *pVgid = &pVgroup->vnodeGid[pVgroup->replica]; - for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) { - SDnodeObj *pDnode = taosArrayGet(pArray, d); - - bool used = false; - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) { - used = true; - break; - } - } - if (used) continue; - - if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) { - terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - return -1; - } - - pVgid->dnodeId = pDnode->id; - pVgid->role = TAOS_SYNC_STATE_ERROR; - mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is added", pVgroup->dbName, pVgroup->vgId, pVgroup->replica, - pVgid->dnodeId); - - pVgroup->replica++; - pDnode->numOfVnodes++; - return 0; - } - - terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - mError("db:%s, failed to add vnode to vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); - return -1; -} - -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid) { - taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); - for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - SDnodeObj *pDnode = taosArrayGet(pArray, i); - mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); - } - - int32_t code = -1; - for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) { - SDnodeObj *pDnode = taosArrayGet(pArray, d); - - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; - if (pVgid->dnodeId == pDnode->id) { - mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); - pDnode->numOfVnodes--; - pVgroup->replica--; - *pDelVgid = *pVgid; - *pVgid = pVgroup->vnodeGid[pVgroup->replica]; - memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid)); - code = 0; - goto _OVER; - } - } - } - -_OVER: - if (code != 0) { - terrno = TSDB_CODE_APP_ERROR; - mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); - return -1; - } - - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; - mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); - } - return 0; -} - SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) { SEpSet epset = {0}; @@ -678,7 +609,7 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p bool online = false; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgroup->vnodeGid[i].dnodeId); if (pDnode != NULL) { - online = mndIsDnodeOnline(pMnode, pDnode, curMs); + online = mndIsDnodeOnline(pDnode, curMs); mndReleaseDnode(pMnode, pDnode); } @@ -797,3 +728,597 @@ static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + } + + SVnodeGid *pVgid = &pVgroup->vnodeGid[pVgroup->replica]; + for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) { + SDnodeObj *pDnode = taosArrayGet(pArray, d); + + bool used = false; + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) { + used = true; + break; + } + } + if (used) continue; + + if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } + + pVgid->dnodeId = pDnode->id; + pVgid->role = TAOS_SYNC_STATE_ERROR; + mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is added", pVgroup->dbName, pVgroup->vgId, pVgroup->replica, pVgid->dnodeId); + + pVgroup->replica++; + pDnode->numOfVnodes++; + return 0; + } + + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + mError("db:%s, failed to add vnode to vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); + return -1; +} + +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid) { + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + } + + int32_t code = -1; + for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) { + SDnodeObj *pDnode = taosArrayGet(pArray, d); + + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + if (pVgid->dnodeId == pDnode->id) { + mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); + pDnode->numOfVnodes--; + pVgroup->replica--; + *pDelVgid = *pVgid; + *pVgid = pVgroup->vnodeGid[pVgroup->replica]; + memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid)); + code = 0; + goto _OVER; + } + } + } + +_OVER: + if (code != 0) { + terrno = TSDB_CODE_APP_ERROR; + mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); + return -1; + } + + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); + } + return 0; +} + +int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool standby) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen, standby); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_VNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + +int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + + int32_t contLen = sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryMalloc(contLen); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + action.pCont = pHead; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_CONFIRM; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pHead); + return -1; + } + + return 0; +} + +int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType) { + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeReq(pMnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = msgType; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + +int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_DROP_VNODE; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + +int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vnIndex, + SArray *pArray) { + SVgObj newVg = {0}; + memcpy(&newVg, pVgroup, sizeof(SVgObj)); + + mInfo("vgId:%d, vgroup info before move, replica:%d", newVg.vgId, newVg.replica); + for (int32_t i = 0; i < newVg.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); + } + + mInfo("vgId:%d, will add 1 vnodes", pVgroup->vgId); + if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[1], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; + + mInfo("vgId:%d, will remove 1 vnodes", pVgroup->vgId); + newVg.replica--; + SVnodeGid del = newVg.vnodeGid[vnIndex]; + newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica]; + memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid)); + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; + + mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica); + for (int32_t i = 0; i < newVg.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); + } + return 0; +} + +int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t delDnodeId) { + SArray *pArray = mndBuildDnodesArray(pMnode, delDnodeId); + if (pArray == NULL) return -1; + + void *pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + int32_t vnIndex = -1; + for (int32_t i = 0; i < pVgroup->replica; ++i) { + if (pVgroup->vnodeGid[i].dnodeId == delDnodeId) { + vnIndex = i; + break; + } + } + + if (vnIndex != -1) { + mInfo("vgId:%d, vnode:%d will be removed from dnode:%d", pVgroup->vgId, vnIndex, delDnodeId); + SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); + mndSetMoveVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, vnIndex, pArray); + mndReleaseDb(pMnode, pDb); + } + + sdbRelease(pMnode->pSdb, pVgroup); + } + + taosArrayDestroy(pArray); + return 0; +} + +static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, + int32_t newDnodeId) { + mDebug("vgId:%d, will add 1 vnode, replica:%d, dnode:%d", pVgroup->vgId, pVgroup->replica, newDnodeId); + + SVnodeGid *pGid = &pVgroup->vnodeGid[pVgroup->replica]; + pVgroup->replica++; + pGid->dnodeId = newDnodeId; + pGid->role = TAOS_SYNC_STATE_ERROR; + + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pGid, true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1; + + return 0; +} + +static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, + int32_t delDnodeId) { + mDebug("vgId:%d, will remove 1 vnode, replica:%d, dnode:%d", pVgroup->vgId, pVgroup->replica, delDnodeId); + + SVnodeGid *pGid = NULL; + SVnodeGid delGid = {0}; + for (int32_t i = 0; i < pVgroup->replica; ++i) { + if (pVgroup->vnodeGid[i].dnodeId == delDnodeId) { + pGid = &pVgroup->vnodeGid[i]; + break; + } + } + + if (pGid == NULL) return 0; + + memcpy(&delGid, pGid, sizeof(SVnodeGid)); + memcpy(pGid, &pVgroup->vnodeGid[pVgroup->replica], sizeof(SVnodeGid)); + memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid)); + pVgroup->replica--; + + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1; + + return 0; +} + +static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup, SDnodeObj *pNew1, + SDnodeObj *pOld1, SDnodeObj *pNew2, SDnodeObj *pOld2, SDnodeObj *pNew3, + SDnodeObj *pOld3) { + int32_t code = -1; + SSdbRaw *pRaw = NULL; + STrans *pTrans = NULL; + + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); + mDebug("trans:%d, used to drop redistribute vgId:%d", pTrans->id, pVgroup->vgId); + + SVgObj newVg = {0}; + memcpy(&newVg, pVgroup, sizeof(SVgObj)); + mInfo("vgId:%d, vgroup info before redistribute, replica:%d", newVg.vgId, newVg.replica); + for (int32_t i = 0; i < newVg.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); + } + + if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew1->id) != 0) goto _OVER; + if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld1->id) != 0) goto _OVER; + if (pNew2 != NULL) { + if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew2->id) != 0) goto _OVER; + if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld2->id) != 0) goto _OVER; + if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew3->id) != 0) goto _OVER; + if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld3->id) != 0) goto _OVER; + } + + pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + + mInfo("vgId:%d, vgroup info after redistribute, replica:%d", newVg.vgId, newVg.replica); + for (int32_t i = 0; i < newVg.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); + } + + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + code = 0; + +_OVER: + mndTransDrop(pTrans); + sdbFreeRaw(pRaw); + mndReleaseDb(pMnode, pDb); + return code; +} + +static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SUserObj *pUser = NULL; + SDnodeObj *pNew1 = NULL; + SDnodeObj *pNew2 = NULL; + SDnodeObj *pNew3 = NULL; + SDnodeObj *pOld1 = NULL; + SDnodeObj *pOld2 = NULL; + SDnodeObj *pOld3 = NULL; + SVgObj *pVgroup = NULL; + SDbObj *pDb = NULL; + int32_t code = -1; + int64_t curMs = taosGetTimestampMs(); + SMDropMnodeReq redReq = {0}; + +#if 0 + if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } +#endif + + mDebug("vgId:%d, start to redistribute", 2); + pUser = mndAcquireUser(pMnode, pReq->conn.user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto _OVER; + } + + if (mndCheckNodeAuth(pUser) != 0) { + goto _OVER; + } + + pVgroup = mndAcquireVgroup(pMnode, 2); + if (pVgroup == NULL) goto _OVER; + + pDb = mndAcquireDb(pMnode, pVgroup->dbName); + if (pDb == NULL) goto _OVER; + + if (pVgroup->replica == 1) { + pNew1 = mndAcquireDnode(pMnode, 1); + pOld1 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[0].dnodeId); + if (pNew1 == NULL || pOld1 == NULL) goto _OVER; + if (!mndIsDnodeOnline(pNew1, curMs) || !mndIsDnodeOnline(pOld1, curMs)) { + terrno = TSDB_CODE_NODE_OFFLINE; + goto _OVER; + } + if (pNew1 == pOld1) { + terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + goto _OVER; + } + if (mndRedistributeVgroup(pMnode, pReq, pDb, pVgroup, pNew1, pOld1, NULL, NULL, NULL, NULL) != 0) goto _OVER; + } + + if (pVgroup->replica == 3) { + pNew1 = mndAcquireDnode(pMnode, 1); + pNew2 = mndAcquireDnode(pMnode, 2); + pNew3 = mndAcquireDnode(pMnode, 3); + pOld1 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[0].dnodeId); + pOld2 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[1].dnodeId); + pOld3 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[2].dnodeId); + if (pNew1 == NULL || pOld1 == NULL || pNew2 == NULL || pOld2 == NULL || pNew3 == NULL || pOld3 == NULL) goto _OVER; + if (!mndIsDnodeOnline(pNew1, curMs) || !mndIsDnodeOnline(pOld1, curMs) || !mndIsDnodeOnline(pNew2, curMs) || + !mndIsDnodeOnline(pOld2, curMs) || !mndIsDnodeOnline(pNew3, curMs) || !mndIsDnodeOnline(pOld3, curMs)) { + terrno = TSDB_CODE_NODE_OFFLINE; + goto _OVER; + } + bool changed = true; + if (pNew1 != pOld1 || pNew1 != pOld2 || pNew1 != pOld3) changed = true; + if (pNew2 != pOld1 || pNew2 != pOld2 || pNew2 != pOld3) changed = true; + if (pNew3 != pOld1 || pNew3 != pOld2 || pNew3 != pOld3) changed = true; + if (!changed) { + terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + goto _OVER; + } + if (mndRedistributeVgroup(pMnode, pReq, pDb, pVgroup, pNew1, pOld1, pNew2, pOld2, pNew3, pOld3) != 0) goto _OVER; + } + + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + +_OVER: + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mDebug("vgId:%d, failed to redistribute since %s", 1, terrstr()); + } + + mndReleaseDnode(pMnode, pNew1); + mndReleaseDnode(pMnode, pNew2); + mndReleaseDnode(pMnode, pNew3); + mndReleaseDnode(pMnode, pOld1); + mndReleaseDnode(pMnode, pOld2); + mndReleaseDnode(pMnode, pOld3); + mndReleaseUser(pMnode, pUser); + mndReleaseVgroup(pMnode, pVgroup); + mndReleaseDb(pMnode, pDb); + + return code; +} + +static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { return 0; } + +static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, + SDnodeObj *pSrc, SDnodeObj *pDst) { + SVgObj newVg = {0}; + memcpy(&newVg, pVgroup, sizeof(SVgObj)); + mInfo("vgId:%d, vgroup info before balance, replica:%d", newVg.vgId, newVg.replica); + for (int32_t i = 0; i < newVg.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); + } + + if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pDst->id) != 0) return -1; + if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pSrc->id) != 0) return -1; + + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + + mInfo("vgId:%d, vgroup info after balance, replica:%d", newVg.vgId, newVg.replica); + for (int32_t i = 0; i < newVg.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); + } + return 0; +} + +static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst) { + void *pIter = NULL; + int32_t code = -1; + + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + bool existInSrc = false; + bool existInDst = false; + for (int32_t i = 0; i < pVgroup->replica; ++i) { + SVnodeGid *pGid = &pVgroup->vnodeGid[i]; + if (pGid->dnodeId == pSrc->id) existInSrc = true; + if (pGid->dnodeId == pDst->id) existInDst = true; + } + + if (!existInSrc || existInDst) { + sdbRelease(pMnode->pSdb, pVgroup); + } + + SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); + code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst); + mndReleaseDb(pMnode, pDb); + sdbRelease(pMnode->pSdb, pVgroup); + break; + } + + return code; +} + +static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { + int32_t code = -1; + int32_t numOfVgroups = 0; + STrans *pTrans = NULL; + + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); + mDebug("trans:%d, used to balance vgroup", pTrans->id); + + while (1) { + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + SDnodeObj *pSrc = taosArrayGet(pArray, 0); + SDnodeObj *pDst = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1); + + float srcScore = (float)(pSrc->numOfVnodes - 1) / pSrc->numOfSupportVnodes; + float dstScore = (float)(pDst->numOfVnodes + 1) / pDst->numOfSupportVnodes; + if (srcScore + 0.0001 < dstScore) { + mDebug("trans:%d, balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id); + code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst); + if (code == 0) { + numOfVgroups++; + continue; + } else { + mError("trans:%d, failed to balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id); + return -1; + } + } else { + mDebug("trans:%d, no vgroup need to balance vgroup any more", pTrans->id); + break; + } + } + + if (numOfVgroups <= 0) { + mDebug("no need to balance vgroup"); + code = 0; + } else { + mDebug("start to balance vgroup, numOfVgroups:%d", numOfVgroups); + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + code = TSDB_CODE_ACTION_IN_PROGRESS; + } + +_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SUserObj *pUser = NULL; + SArray *pArray = NULL; + void *pIter = NULL; + int64_t curMs = taosGetTimestampMs(); + + mDebug("start to balance vgroup"); + pUser = mndAcquireUser(pMnode, pReq->conn.user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto _OVER; + } + + if (mndCheckNodeAuth(pUser) != 0) goto _OVER; + + while (1) { + SDnodeObj *pDnode = NULL; + pIter = sdbFetch(pMnode->pSdb, SDB_DNODE, pIter, (void **)&pDnode); + if (pIter == NULL) break; + if (!mndIsDnodeOnline(pDnode, curMs)) { + terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + mError("failed to balance vgroup since %s, dnode:%d", terrstr(), pDnode->id); + sdbRelease(pMnode->pSdb, pDnode); + goto _OVER; + } + + sdbRelease(pMnode->pSdb, pDnode); + } + + pArray = mndBuildDnodesArray(pMnode, 0); + if (pArray == NULL) goto _OVER; + + if (taosArrayGetSize(pArray) < 2) { + mDebug("no need to balance vgroup since dnode num less than 2"); + code = 0; + } else { + code = mndBalanceVgroup(pMnode, pReq, pArray); + } + +_OVER: + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("failed to balance vgroup since %s", terrstr()); + } + + mndReleaseUser(pMnode, pUser); + taosArrayDestroy(pArray); + return code; +} \ No newline at end of file diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index 90643a54a9..7720a8e88a 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -42,8 +42,10 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { } void sdbFreeRaw(SSdbRaw *pRaw) { - mTrace("raw:%p, is freed", pRaw); - taosMemoryFree(pRaw); + if (pRaw != NULL) { + mTrace("raw:%p, is freed", pRaw); + taosMemoryFree(pRaw); + } } int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) { diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 7cd82b0ac3..e7a744748b 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -78,10 +78,10 @@ typedef struct { tmr_h timerId; int8_t tmrStopped; // exec - int8_t inputStatus; - int8_t execStatus; - SStreamQ inputQ; - SRWLatch lock; + int8_t inputStatus; + int8_t execStatus; + SStreamQueue inputQ; + SRWLatch lock; } STqPushHandle; // tqExec @@ -107,7 +107,7 @@ typedef struct { STqExecCol execCol; STqExecTb execTb; STqExecDb execDb; - } exec; + }; } STqExecHandle; typedef struct { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 4f4a940bc7..c6a6dd8147 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -103,7 +103,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid); indexMultiTermDestroy(terms); #endif - return -1; + return 0; } int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 310b59b2e8..38482ccad5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -266,7 +266,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pHandle->execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); } if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - pHandle->execHandle.exec.execCol.qmsg = req.qmsg; + pHandle->execHandle.execCol.qmsg = req.qmsg; req.qmsg = NULL; for (int32_t i = 0; i < 5; i++) { SReadHandle handle = { @@ -274,15 +274,14 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { .meta = pTq->pVnode->pMeta, .pMsgCb = &pTq->pVnode->msgCb, }; - pHandle->execHandle.exec.execCol.task[i] = - qCreateStreamExecTaskInfo(pHandle->execHandle.exec.execCol.qmsg, &handle); - ASSERT(pHandle->execHandle.exec.execCol.task[i]); + pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle); + ASSERT(pHandle->execHandle.execCol.task[i]); } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { - pHandle->execHandle.exec.execDb.pFilterOutTbUid = + pHandle->execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { - pHandle->execHandle.exec.execTb.suid = req.suid; + pHandle->execHandle.execTb.suid = req.suid; SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); tsdbGetCtbIdList(pTq->pVnode->pMeta, req.suid, tbUidList); tqDebug("vg %d, tq try get suid: %ld", pTq->pVnode->config.vgId, req.suid); @@ -296,17 +295,20 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { taosArrayDestroy(tbUidList); } taosHashPut(pTq->handles, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); + if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { + // TODO + } } else { /*ASSERT(pExec->consumerId == req.oldConsumerId);*/ // TODO handle qmsg and exec modification atomic_store_32(&pHandle->epoch, -1); atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_add_fetch_32(&pHandle->epoch, 1); + if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { + // TODO + } } - if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { - // TODO - } return 0; } @@ -323,16 +325,13 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { tDecoderClear(&decoder); pTask->status = TASK_STATUS__IDLE; + + pTask->inputQueue = streamQueueOpen(); + pTask->outputQueue = streamQueueOpen(); pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; - pTask->inputQ = taosOpenQueue(); - pTask->outputQ = taosOpenQueue(); - pTask->inputQAll = taosAllocateQall(); - pTask->outputQAll = taosAllocateQall(); - - if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL) - goto FAIL; + if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) goto FAIL; // exec if (pTask->execType != TASK_EXEC__NONE) { @@ -369,10 +368,8 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { return 0; FAIL: - if (pTask->inputQ) taosCloseQueue(pTask->inputQ); - if (pTask->outputQ) taosCloseQueue(pTask->outputQ); - if (pTask->inputQAll) taosFreeQall(pTask->inputQAll); - if (pTask->outputQAll) taosFreeQall(pTask->outputQAll); + if (pTask->inputQueue) streamQueueClose(pTask->inputQueue); + if (pTask->outputQueue) streamQueueClose(pTask->outputQueue); if (pTask) taosMemoryFree(pTask); return -1; } @@ -393,38 +390,16 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) { SStreamTask* pTask = (SStreamTask*)pIter; if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue; - int8_t inputStatus = atomic_load_8(&pTask->inputStatus); - if (inputStatus == TASK_INPUT_STATUS__NORMAL) { - if (failed) { - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); + if (!failed) { + if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { continue; } - SStreamDataSubmit* pSubmitClone = streamSubmitRefClone(pSubmit); - if (pSubmitClone == NULL) { - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); + if (streamTriggerByWrite(pTask, pTq->pVnode->config.vgId, &pTq->pVnode->msgCb) < 0) { continue; } - taosWriteQitem(pTask->inputQ, pSubmitClone); - - int8_t execStatus = atomic_load_8(&pTask->status); - if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) { - SStreamTaskRunReq* pRunReq = taosMemoryMalloc(sizeof(SStreamTaskRunReq)); - if (pRunReq == NULL) continue; - // TODO: do we need htonl? - pRunReq->head.vgId = pTq->pVnode->config.vgId; - pRunReq->streamId = pTask->streamId; - pRunReq->taskId = pTask->taskId; - SRpcMsg msg = { - .msgType = TDMT_VND_TASK_RUN, - .pCont = pRunReq, - .contLen = sizeof(SStreamTaskRunReq), - }; - tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &msg); - } - } else { - // blocked or stopped, do nothing + streamTaskInputFail(pTask); } } diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index b8fec34b57..31acc60006 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -59,7 +59,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqD int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId) { if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) { - qTaskInfo_t task = pExec->exec.execCol.task[workerId]; + qTaskInfo_t task = pExec->execCol.task[workerId]; ASSERT(task); qSetStreamInput(task, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK, false); while (1) { @@ -101,7 +101,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR pRsp->withSchema = 1; STqReadHandle* pReader = pExec->pExecReader[workerId]; tqReadHandleSetMsg(pReader, pReq, 0); - while (tqNextDataBlockFilterOut(pReader, pExec->exec.execDb.pFilterOutTbUid)) { + while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { SSDataBlock block = {0}; if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, &block.info.numOfCols) < 0) { diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 9447c4007b..398a09ecbc 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -22,7 +22,7 @@ static int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) { if (tEncodeI32(pEncoder, pHandle->epoch) < 0) return -1; if (tEncodeI8(pEncoder, pHandle->execHandle.subType) < 0) return -1; if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - if (tEncodeCStr(pEncoder, pHandle->execHandle.exec.execCol.qmsg) < 0) return -1; + if (tEncodeCStr(pEncoder, pHandle->execHandle.execCol.qmsg) < 0) return -1; } tEndEncode(pEncoder); return pEncoder->pos; @@ -35,7 +35,7 @@ static int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { if (tDecodeI32(pDecoder, &pHandle->epoch) < 0) return -1; if (tDecodeI8(pDecoder, &pHandle->execHandle.subType) < 0) return -1; if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.exec.execCol.qmsg) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execCol.qmsg) < 0) return -1; } tEndDecode(pDecoder); return 0; @@ -88,12 +88,11 @@ int32_t tqMetaOpen(STQ* pTq) { .meta = pTq->pVnode->pMeta, .pMsgCb = &pTq->pVnode->msgCb, }; - handle.execHandle.exec.execCol.task[i] = - qCreateStreamExecTaskInfo(handle.execHandle.exec.execCol.qmsg, &reader); - ASSERT(handle.execHandle.exec.execCol.task[i]); + handle.execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(handle.execHandle.execCol.qmsg, &reader); + ASSERT(handle.execHandle.execCol.task[i]); } } else { - handle.execHandle.exec.execDb.pFilterOutTbUid = + handle.execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); } taosHashPut(pTq->handles, pKey, kLen, &handle, sizeof(STqHandle)); diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index d94c3e387a..9be94eb5b6 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -29,13 +29,13 @@ static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubm } // update processed atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); - streamQSetSuccess(&pHandle->pushHandle.inputQ); + streamQueueProcessSuccess(&pHandle->pushHandle.inputQ); streamDataSubmitRefDec(pSubmit); if (pRsp->blockNum > 0) { *ppSubmit = pSubmit; return 0; } else { - pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); + pSubmit = streamQueueNextItem(&pHandle->pushHandle.inputQ); } } *ppSubmit = pSubmit; @@ -52,14 +52,14 @@ int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { // 2. check processedVer // 2.1. if not missed, get msg from queue // 2.2. if missed, scan wal - pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); + pSubmit = streamQueueNextItem(&pHandle->pushHandle.inputQ); while (pHandle->pushHandle.processedVer <= pSubmit->ver) { // read from wal } while (pHandle->pushHandle.processedVer > pSubmit->ver + 1) { - streamQSetSuccess(&pHandle->pushHandle.inputQ); + streamQueueProcessSuccess(&pHandle->pushHandle.inputQ); streamDataSubmitRefDec(pSubmit); - pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); + pSubmit = streamQueueNextItem(&pHandle->pushHandle.inputQ); if (pSubmit == NULL) break; } // 3. exec, after each success, update processed ver diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 8909a00c72..43f49b4cc2 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -307,7 +307,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { STqHandle* pExec = (STqHandle*)pIter; if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { for (int32_t i = 0; i < 5; i++) { - int32_t code = qUpdateQualifiedTableId(pExec->execHandle.exec.execCol.task[i], tbUidList, isAdd); + int32_t code = qUpdateQualifiedTableId(pExec->execHandle.execCol.task[i], tbUidList, isAdd); ASSERT(code == 0); } } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) { @@ -315,7 +315,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { int32_t sz = taosArrayGetSize(tbUidList); for (int32_t i = 0; i < sz; i++) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - taosHashPut(pExec->execHandle.exec.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0); + taosHashPut(pExec->execHandle.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0); } } } else { diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 9219a382e4..1eec92f47e 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -518,6 +518,7 @@ int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); +char *ctgTaskTypeStr(CTG_TASK_TYPE type); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 6998d8c778..f43e559244 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -44,7 +44,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, task.type, name->tname); + qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; } @@ -67,7 +67,7 @@ int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:%" PRIx64 " task %d type %d initialized, dbFName:%s", pJob->queryId, taskIdx, task.type, dbFName); + qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; } @@ -90,7 +90,7 @@ int32_t ctgInitGetDbCfgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:%" PRIx64 " task %d type %d initialized, dbFName:%s", pJob->queryId, taskIdx, task.type, dbFName); + qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; } @@ -113,7 +113,7 @@ int32_t ctgInitGetDbInfoTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:%" PRIx64 " task %d type %d initialized, dbFName:%s", pJob->queryId, taskIdx, task.type, dbFName); + qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; } @@ -143,7 +143,7 @@ int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, task.type, name->tname); + qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; } @@ -158,7 +158,7 @@ int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:%" PRIx64 " task %d type %d initialized", pJob->queryId, taskIdx, task.type); + qDebug("QID:%" PRIx64 " the %d task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -181,7 +181,7 @@ int32_t ctgInitGetIndexTask(SCtgJob *pJob, int32_t taskIdx, char *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:%" PRIx64 " task %d type %d initialized, indexFName:%s", pJob->queryId, taskIdx, task.type, name); + qDebug("QID:%" PRIx64 " the %d task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); return TSDB_CODE_SUCCESS; } @@ -204,7 +204,7 @@ int32_t ctgInitGetUdfTask(SCtgJob *pJob, int32_t taskIdx, char *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:%" PRIx64 " task %d type %d initialized, udfName:%s", pJob->queryId, taskIdx, task.type, name); + qDebug("QID:%" PRIx64 " the %d task type %s initialized, udfName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); return TSDB_CODE_SUCCESS; } @@ -227,11 +227,96 @@ int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, SUserAuthInfo *user) taosArrayPush(pJob->pTasks, &task); - qDebug("QID:%" PRIx64 " task %d type %d initialized, user:%s", pJob->queryId, taskIdx, task.type, user->user); + qDebug("QID:%" PRIx64 " the %d task type %s initialized, user:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), user->user); return TSDB_CODE_SUCCESS; } +int32_t ctgHandleForceUpdate(SCatalog* pCtg, SCtgJob *pJob, const SCatalogReq* pReq) { + int32_t dbNum = pJob->dbCfgNum + pJob->dbVgNum + pJob->dbInfoNum; + if (dbNum > 0) { + if (dbNum > pJob->dbCfgNum && dbNum > pJob->dbVgNum && dbNum > pJob->dbInfoNum) { + SHashObj* pDb = taosHashInit(dbNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pDb) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + for (int32_t i = 0; i < pJob->dbVgNum; ++i) { + char* dbFName = taosArrayGet(pReq->pDbVgroup, i); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } + + for (int32_t i = 0; i < pJob->dbCfgNum; ++i) { + char* dbFName = taosArrayGet(pReq->pDbCfg, i); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } + + for (int32_t i = 0; i < pJob->dbInfoNum; ++i) { + char* dbFName = taosArrayGet(pReq->pDbInfo, i); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } + + char* dbFName = taosHashIterate(pDb, NULL); + while (dbFName) { + ctgDropDbVgroupEnqueue(pCtg, dbFName, true); + dbFName = taosHashIterate(pDb, dbFName); + } + + taosHashCleanup(pDb); + } else { + for (int32_t i = 0; i < pJob->dbVgNum; ++i) { + char* dbFName = taosArrayGet(pReq->pDbVgroup, i); + CTG_ERR_RET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true)); + } + + for (int32_t i = 0; i < pJob->dbCfgNum; ++i) { + char* dbFName = taosArrayGet(pReq->pDbCfg, i); + CTG_ERR_RET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true)); + } + + for (int32_t i = 0; i < pJob->dbInfoNum; ++i) { + char* dbFName = taosArrayGet(pReq->pDbInfo, i); + CTG_ERR_RET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true)); + } + } + } + + int32_t tbNum = pJob->tbMetaNum + pJob->tbHashNum; + if (tbNum > 0) { + if (tbNum > pJob->tbMetaNum && tbNum > pJob->tbHashNum) { + SHashObj* pTb = taosHashInit(tbNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { + SName* name = taosArrayGet(pReq->pTableMeta, i); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } + + for (int32_t i = 0; i < pJob->tbHashNum; ++i) { + SName* name = taosArrayGet(pReq->pTableHash, i); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } + + SName* name = taosHashIterate(pTb, NULL); + while (name) { + catalogRemoveTableMeta(pCtg, name); + name = taosHashIterate(pTb, name); + } + + taosHashCleanup(pTb); + } else { + for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { + SName* name = taosArrayGet(pReq->pTableMeta, i); + catalogRemoveTableMeta(pCtg, name); + } + + for (int32_t i = 0; i < pJob->tbHashNum; ++i) { + SName* name = taosArrayGet(pReq->pTableHash, i); + catalogRemoveTableMeta(pCtg, name); + } + } + } + + return TSDB_CODE_SUCCESS; +} int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int32_t* taskNum) { int32_t code = 0; @@ -283,12 +368,13 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + if (pReq->forceUpdate) { + CTG_ERR_JRET(ctgHandleForceUpdate(pCtg, pJob, pReq)); + } + int32_t taskIdx = 0; for (int32_t i = 0; i < dbVgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbVgroup, i); - if (pReq->forceUpdate) { - ctgDropDbVgroupEnqueue(pCtg, dbFName, true); - } CTG_ERR_JRET(ctgInitGetDbVgTask(pJob, taskIdx++, dbFName)); } @@ -304,9 +390,6 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* for (int32_t i = 0; i < tbMetaNum; ++i) { SName* name = taosArrayGet(pReq->pTableMeta, i); - if (pReq->forceUpdate) { - catalogRemoveTableMeta(pCtg, name); - } CTG_ERR_JRET(ctgInitGetTbMetaTask(pJob, taskIdx++, name)); } @@ -342,7 +425,7 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); - qDebug("QID:%" PRIx64 ", job %" PRIx64 " initialized, task num %d", pJob->queryId, pJob->refId, *taskNum); + qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d", pJob->queryId, pJob->refId, *taskNum, pReq->forceUpdate); return TSDB_CODE_SUCCESS; @@ -1108,7 +1191,7 @@ int32_t ctgLaunchJob(SCtgJob *pJob) { for (int32_t i = 0; i < taskNum; ++i) { SCtgTask *pTask = taosArrayGet(pJob->pTasks, i); - qDebug("QID:%" PRIx64 " start to launch task %d", pJob->queryId, pTask->taskId); + qDebug("QID:0x%" PRIx64 " start to launch task %d", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); } diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index b16a082f75..188f5b44c0 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -177,7 +177,7 @@ int32_t ctgHandleMsgCallback(void *param, const SDataBuf *pMsg, int32_t rspCode) SCtgTask *pTask = taosArrayGet(pJob->pTasks, cbParam->taskId); - qDebug("QID:%" PRIx64 " task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); + qDebug("QID:0x%" PRIx64 " task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, cbParam->reqType, pMsg, rspCode)); @@ -244,7 +244,7 @@ int32_t ctgAsyncSendMsg(CTG_PARAMS, SCtgTask* pTask, int32_t msgType, void *msg, CTG_ERR_JRET(code); } - ctgDebug("req msg sent, reqId:%" PRIx64 ", msg type:%d, %s", pTask->pJob->queryId, msgType, TMSG_INFO(msgType)); + ctgDebug("req msg sent, reqId:0x%" PRIx64 ", msg type:%d, %s", pTask->pJob->queryId, msgType, TMSG_INFO(msgType)); return TSDB_CODE_SUCCESS; _return: @@ -565,7 +565,7 @@ int32_t ctgGetTbMetaFromVnode(CTG_PARAMS, const SName* pTableName, SVgroupInfo * } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); - CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pTrans, &vgroupInfo->epSet, pTask, reqType, msg, msgLen)); } SRpcMsg rpcMsg = { diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 4625203dd8..962edf9af1 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -19,6 +19,31 @@ #include "catalogInt.h" #include "systable.h" +char *ctgTaskTypeStr(CTG_TASK_TYPE type) { + switch (type) { + case CTG_TASK_GET_QNODE: + return "[get qnode list]"; + case CTG_TASK_GET_DB_VGROUP: + return "[get db vgroup]"; + case CTG_TASK_GET_DB_CFG: + return "[get db cfg]"; + case CTG_TASK_GET_DB_INFO: + return "[get db info]"; + case CTG_TASK_GET_TB_META: + return "[get table meta]"; + case CTG_TASK_GET_TB_HASH: + return "[get table hash]"; + case CTG_TASK_GET_INDEX: + return "[get index]"; + case CTG_TASK_GET_UDF: + return "[get udf]"; + case CTG_TASK_GET_USER: + return "[get user]"; + default: + return "unknown"; + } +} + void ctgFreeSMetaData(SMetaData* pData) { taosArrayDestroy(pData->pTableMeta); pData->pTableMeta = NULL; @@ -477,6 +502,9 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pVgroup = *vgInfo; + ctgDebug("Got tb %s hash vgroup, vgId %d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, vgInfo->epSet.numOfEps, + vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); + CTG_RET(code); } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 5439ba0c89..4804984397 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -524,6 +524,17 @@ typedef struct SProjectOperatorInfo { int64_t curOutput; } SProjectOperatorInfo; +typedef struct SIndefOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pPseudoColInfo; + + SExprInfo* pScalarExpr; + int32_t numOfScalarExpr; + SqlFunctionCtx* pScalarCtx; + int32_t* rowCellInfoOffset; +} SIndefOperatorInfo; + typedef struct SFillOperatorInfo { struct SFillInfo* pFillInfo; SSDataBlock* pRes; @@ -770,10 +781,13 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, SNode* pCondition, SExecTaskInfo* pTaskInfo); SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pIndexMap, SExecTaskInfo* pTaskInfo); - +SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, + SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0b738d829c..9d7f939bb0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -108,8 +108,6 @@ static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutpu static void releaseQueryBuf(size_t numOfTables); -static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); - static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput); static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); @@ -2522,8 +2520,8 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray } int32_t setDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, - SArray* pColList) { + int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, + SArray* pColList) { if (pColList == NULL) { // data from other sources blockCompressDecode(pRes, numOfOutput, numOfRows, pData); } else { // extract data according to pColList @@ -2677,7 +2675,7 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; code = setDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); + pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); goto _error; @@ -2790,7 +2788,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; int32_t code = setDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); + pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64 @@ -2856,7 +2854,7 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { return seqLoadRemoteData(pOperator); } else { return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); -// return concurrentlyLoadRemoteData(pOperator); + // return concurrentlyLoadRemoteData(pOperator); } } @@ -2911,18 +2909,18 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode goto _error; } - pInfo->seqLoadData = false; + pInfo->seqLoadData = false; pInfo->pTransporter = pTransporter; - pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); tsem_init(&pInfo->ready, 0, 0); - pOperator->name = "ExchangeOperator"; + pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfExprs = pInfo->pResult->info.numOfCols; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfExprs = pInfo->pResult->info.numOfCols; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, NULL, NULL, NULL); @@ -3636,18 +3634,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { break; } -#if 0 - // Return result of the previous group in the firstly. - if (false) { - if (pRes->info.rows > 0) { - pProjectInfo->existDataBlock = pBlock; - break; - } else { // init output buffer for a new group data - initCtxOutputBuffer(pInfo->pCtx, pOperator->numOfExprs); - } - } -#endif - // the pDataBlock are always the same one, no need to call this again int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag); if (code != TSDB_CODE_SUCCESS) { @@ -3793,6 +3779,17 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { } } +static void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { + for (int32_t i = 0; i < numOfExprs; ++i) { + SExprInfo* pExprInfo = &pExpr[i]; + if (pExprInfo->pExpr->nodeType == QUERY_NODE_COLUMN) { + taosMemoryFree(pExprInfo->base.pParam[0].pCol); + } + taosMemoryFree(pExprInfo->base.pParam); + taosMemoryFree(pExprInfo->pExpr); + } +} + static void destroyOperatorInfo(SOperatorInfo* pOperator) { if (pOperator == NULL) { return; @@ -3812,14 +3809,7 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { } if (pOperator->pExpr != NULL) { - for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { - SExprInfo* pExprInfo = &pOperator->pExpr[i]; - if (pExprInfo->pExpr->nodeType == QUERY_NODE_COLUMN) { - taosMemoryFree(pExprInfo->base.pParam[0].pCol); - } - taosMemoryFree(pExprInfo->base.pParam); - taosMemoryFree(pExprInfo->pExpr); - } + destroyExprInfo(pOperator->pExpr, pOperator->numOfExprs); } taosMemoryFreeClear(pOperator->pExpr); @@ -4008,6 +3998,19 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pInfo->pPseudoColInfo); } +static void destroyIndefinitOperatorInfo(void* param, int32_t numOfOutput) { + SIndefOperatorInfo* pInfo = (SIndefOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + + taosArrayDestroy(pInfo->pPseudoColInfo); + cleanupAggSup(&pInfo->aggSup); + + destroySqlFunctionCtx(pInfo->pScalarCtx, numOfOutput); + destroyExprInfo(pInfo->pScalarExpr, pInfo->numOfScalarExpr); + + taosMemoryFree(pInfo->rowCellInfoOffset); +} + void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { SExchangeInfo* pExInfo = (SExchangeInfo*)param; taosArrayDestroy(pExInfo->pSources); @@ -4085,6 +4088,136 @@ _error: return NULL; } +static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { + SIndefOperatorInfo* pIndefInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pIndefInfo->binfo; + + SSDataBlock* pRes = pInfo->pRes; + blockDataCleanup(pRes); + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + int64_t st = 0; + int32_t order = 0; + int32_t scanFlag = 0; + + if (pOperator->cost.openCost == 0) { + st = taosGetTimestampUs(); + } + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + // The downstream exec may change the value of the newgroup, so use a local variable instead. + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + doSetOperatorCompleted(pOperator); + break; + } + + // the pDataBlock are always the same one, no need to call this again + int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + // there is an scalar expression that needs to be calculated before apply the group aggregation. + if (pIndefInfo->pScalarExpr != NULL) { + code = projectApplyFunctions(pIndefInfo->pScalarExpr, pBlock, pBlock, pIndefInfo->pScalarCtx, + pIndefInfo->numOfScalarExpr, pIndefInfo->pPseudoColInfo); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + } + + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false); + blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); + + code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs, + pIndefInfo->pPseudoColInfo); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + } + + size_t rows = pInfo->pRes->info.rows; + pOperator->resultInfo.totalRows += rows; + + if (pOperator->cost.openCost == 0) { + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; + } + + return (rows > 0) ? pInfo->pRes : NULL; +} + +SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, + SExecTaskInfo* pTaskInfo) { + SIndefOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIndefOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + SIndefRowsFuncPhysiNode* pPhyNode = (SIndefRowsFuncPhysiNode*)pNode; + + int32_t numOfExpr = 0; + SExprInfo* pExprInfo = createExprInfo(pPhyNode->pVectorFuncs, NULL, &numOfExpr); + + int32_t numOfScalarExpr = 0; + if (pPhyNode->pExprs != NULL) { + pInfo->pScalarExpr = createExprInfo(pPhyNode->pExprs, NULL, &numOfScalarExpr); + pInfo->pScalarCtx = createSqlFunctionCtx(pInfo->pScalarExpr, numOfScalarExpr, &pInfo->rowCellInfoOffset); + } + + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->node.pOutputDataBlockDesc); + ; + + int32_t numOfRows = 4096; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + // Make sure the size of SSDataBlock will never exceed the size of 2MB. + int32_t TWOMB = 2 * 1024 * 1024; + if (numOfRows * pResBlock->info.rowSize > TWOMB) { + numOfRows = TWOMB / pResBlock->info.rowSize; + } + initResultSizeInfo(pOperator, numOfRows); + + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfExpr, pResBlock, keyBufSize, pTaskInfo->id.str); + setFunctionResultOutput(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr, pTaskInfo); + + pInfo->binfo.pRes = pResBlock; + pInfo->numOfScalarExpr = numOfScalarExpr; + pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pInfo->binfo.pCtx, numOfExpr); + + pOperator->name = "IndefinitOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pExpr = pExprInfo; + pOperator->numOfExprs = numOfExpr; + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, NULL, + destroyIndefinitOperatorInfo, NULL, NULL, NULL); + + int32_t code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + +_error: + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; +} + static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pValNode); @@ -4389,7 +4522,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STimeWindowAggSupp twSup = { - .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; + .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; tsdbReaderT pDataReader = NULL; if (pHandle->vnode) { pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond); @@ -4517,6 +4650,18 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID); pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) { + SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode; + + SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc; + SSDataBlock* pResBlock = createResDataBlock(pDescNode); + + SArray* sortInfo = createSortInfo(pMergePhyNode->pMergeKeys); + int32_t numOfOutputCols = 0; + SArray* pColList = + extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID); + + pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pResBlock, sortInfo, pColList, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; @@ -4577,6 +4722,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SInterval* pInterval = &((SIntervalAggOperatorInfo*)ops[0]->info)->interval; pOptr = createFillOperatorInfo(ops[0], pExprInfo, num, pInterval, &pFillNode->timeRange, pResBlock, pFillNode->mode, (SNodeListNode*)pFillNode->pValues, false, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) { + pOptr = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else { ASSERT(0); } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 5e1b47cc91..c84d4491af 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -13,17 +13,18 @@ * along with this program. If not, see . */ -#include "tdatablock.h" #include "executorimpl.h" +#include "tdatablock.h" static SSDataBlock* doSort(SOperatorInfo* pOperator); -static int32_t doOpenSortOperator(SOperatorInfo* pOperator); -static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); +static int32_t doOpenSortOperator(SOperatorInfo* pOperator); +static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, - SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, + SExprInfo* pExprInfo, int32_t numOfCols, SArray* pColMatchColInfo, + SExecTaskInfo* pTaskInfo) { SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); int32_t rowSize = pResBlock->info.rowSize; @@ -32,33 +33,33 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR goto _error; } - pOperator->pExpr = pExprInfo; + pOperator->pExpr = pExprInfo; pOperator->numOfExprs = numOfCols; - pInfo->binfo.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = pResBlock; + pInfo->binfo.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = pResBlock; initResultSizeInfo(pOperator, 1024); - pInfo->pSortInfo = pSortInfo; - pInfo->pColMatchInfo= pColMatchColInfo; - pOperator->name = "SortOperator"; + pInfo->pSortInfo = pSortInfo; + pInfo->pColMatchInfo = pColMatchColInfo; + pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; // lazy evaluation for the following parameter since the input datablock is not known till now. -// pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header -// pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer + // pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + + // header pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = - createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, getExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, + getExplainExecInfo); int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pInfo); taosMemoryFree(pOperator); @@ -68,7 +69,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - bool isNull = tsortIsNullVal(pTupleHandle, i); + bool isNull = tsortIsNullVal(pTupleHandle, i); if (isNull) { colDataAppendNULL(pColInfo, pBlock->info.rows); } else { @@ -80,7 +81,8 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { pBlock->info.rows += 1; } -SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo) { +SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, + SArray* pColMatchInfo) { blockDataCleanup(pDataBlock); ASSERT(taosArrayGetSize(pColMatchInfo) == pDataBlock->info.numOfCols); @@ -129,10 +131,11 @@ SSDataBlock* loadNextDataBlock(void* param) { // todo refactor: merged with fetch fp void applyScalarFunction(SSDataBlock* pBlock, void* param) { - SOperatorInfo* pOperator = param; + SOperatorInfo* pOperator = param; SSortOperatorInfo* pSort = pOperator->info; if (pOperator->pExpr != NULL) { - int32_t code = projectApplyFunctions(pOperator->pExpr, pBlock, pBlock, pSort->binfo.pCtx, pOperator->numOfExprs, NULL); + int32_t code = + projectApplyFunctions(pOperator->pExpr, pBlock, pBlock, pSort->binfo.pCtx, pOperator->numOfExprs, NULL); if (code != TSDB_CODE_SUCCESS) { longjmp(pOperator->pTaskInfo->env, code); } @@ -141,7 +144,7 @@ void applyScalarFunction(SSDataBlock* pBlock, void* param) { int32_t doOpenSortOperator(SOperatorInfo* pOperator) { SSortOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (OPTR_IS_OPENED(pOperator)) { return TSDB_CODE_SUCCESS; @@ -150,8 +153,8 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) { pInfo->startTs = taosGetTimestampUs(); // pInfo->binfo.pRes is not equalled to the input datablock. - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT, - -1, -1, NULL, pTaskInfo->id.str); + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT, -1, -1, + NULL, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator); @@ -166,7 +169,7 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, terrno); } - pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs)/1000.0; + pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs) / 1000.0; pOperator->status = OP_RES_TO_RETURN; OPTR_SET_OPENED(pOperator); @@ -186,7 +189,8 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, code); } - SSDataBlock* pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); + SSDataBlock* pBlock = + getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; @@ -208,10 +212,147 @@ int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* ASSERT(pOptr != NULL); SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); - SSortOperatorInfo *pOperatorInfo = (SSortOperatorInfo*)pOptr->info; + SSortOperatorInfo* pOperatorInfo = (SSortOperatorInfo*)pOptr->info; *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle); *pOptrExplain = pInfo; *len = sizeof(SSortExecInfo); return TSDB_CODE_SUCCESS; } + +typedef struct SMultiwaySortMergeOperatorInfo { + SOptrBasicInfo binfo; + + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + + SArray* pSortInfo; + SSortHandle* pSortHandle; + SArray* pColMatchInfo; // for index map from table scan output + + int64_t startTs; // sort start time +} SMultiwaySortMergeOperatorInfo; + +int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { + SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + pInfo->startTs = taosGetTimestampUs(); + + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_MULTISOURCE_MERGE, + pInfo->bufPageSize, numOfBufPage, NULL, pTaskInfo->id.str); + + tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); + + for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + SSortSource ps = {0}; + ps.param = pOperator->pDownstream[i]; + tsortAddSource(pInfo->pSortHandle, &ps); + } + + int32_t code = tsortOpen(pInfo->pSortHandle); + + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, terrno); + } + + pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs) / 1000.0; + pOperator->status = OP_RES_TO_RETURN; + + OPTR_SET_OPENED(pOperator); + return TSDB_CODE_SUCCESS; +} + +SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; + + int32_t code = pOperator->fpSet._openFn(pOperator); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + SSDataBlock* pBlock = + getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); + + if (pBlock != NULL) { + pOperator->resultInfo.totalRows += pBlock->info.rows; + } else { + doSetOperatorCompleted(pOperator); + } + return pBlock; +} + +void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) { + SMultiwaySortMergeOperatorInfo * pInfo = (SMultiwaySortMergeOperatorInfo*)param; + pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); + + taosArrayDestroy(pInfo->pSortInfo); + taosArrayDestroy(pInfo->pColMatchInfo); +} + +int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { + ASSERT(pOptr != NULL); + SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); + + SMultiwaySortMergeOperatorInfo* pOperatorInfo = (SMultiwaySortMergeOperatorInfo*)pOptr->info; + + *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle); + *pOptrExplain = pInfo; + *len = sizeof(SSortExecInfo); + return TSDB_CODE_SUCCESS; +} + +SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, + SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo, + SExecTaskInfo* pTaskInfo) { + SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + int32_t rowSize = pResBlock->info.rowSize; + + if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { + goto _error; + } + + pInfo->binfo.pRes = pResBlock; + + initResultSizeInfo(pOperator, 1024); + + pInfo->pSortInfo = pSortInfo; + pInfo->pColMatchInfo = pColMatchColInfo; + pOperator->name = "MultiwaySortMerge"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + + pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2; + pInfo->sortBufSize = pInfo->bufPageSize * 16; + + pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = + createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL, + destroyMultiwaySortMergeOperatorInfo, NULL, NULL, getMultiwaySortMergeExplainExecInfo); + + int32_t code = appendDownstream(pOperator, downStreams, numStreams); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + return pOperator; + +_error: + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + return NULL; +} \ No newline at end of file diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index c9c63169c9..61944705c5 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -177,13 +177,6 @@ static int32_t translatePercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - // param0 - SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0); - if (nodeType(pParamNode0) != QUERY_NODE_COLUMN) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The first parameter of PERCENTILE function can only be column"); - } - // param1 SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); @@ -218,13 +211,6 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - // param0 - SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0); - if (nodeType(pParamNode0) != QUERY_NODE_COLUMN) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The first parameter of APERCENTILE function can only be column"); - } - // param1 SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 1); if (nodeType(pParamNode1) != QUERY_NODE_VALUE) { @@ -284,13 +270,6 @@ static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - // param0 - SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0); - if (nodeType(pParamNode0) != QUERY_NODE_COLUMN) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The first parameter of TOP/BOTTOM function can only be column"); - } - // param1 SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 1); if (nodeType(pParamNode1) != QUERY_NODE_VALUE) { @@ -338,13 +317,6 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - // param0 - SNode* pParaNode0 = nodesListGetNode(pFunc->pParameterList, 0); - if (QUERY_NODE_COLUMN != nodeType(pParaNode0)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The first parameter of ELAPSED function can only be column"); - } - uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; if (TSDB_DATA_TYPE_TIMESTAMP != paraType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); @@ -410,13 +382,6 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - // param0 - SNode* pParaNode0 = nodesListGetNode(pFunc->pParameterList, 0); - if (QUERY_NODE_COLUMN != nodeType(pParaNode0)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The first parameter of HISTOGRAM function can only be column"); - } - uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; if (!IS_NUMERIC_TYPE(colType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); @@ -449,12 +414,6 @@ static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); - if (QUERY_NODE_COLUMN != nodeType(pPara)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The input parameter of HYPERLOGLOG function can only be column"); - } - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } @@ -474,12 +433,6 @@ static int32_t translateStateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - // param0 - SNode* pParaNode0 = nodesListGetNode(pFunc->pParameterList, 0); - if (QUERY_NODE_COLUMN != nodeType(pParaNode0)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The input parameter of STATECOUNT function can only be column"); - } uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; if (!IS_NUMERIC_TYPE(colType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); @@ -520,12 +473,6 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32 return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - // param0 - SNode* pParaNode0 = nodesListGetNode(pFunc->pParameterList, 0); - if (QUERY_NODE_COLUMN != nodeType(pParaNode0)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The input parameter of STATEDURATION function can only be column"); - } uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; if (!IS_NUMERIC_TYPE(colType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); @@ -573,12 +520,6 @@ static int32_t translateCsum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); - if (QUERY_NODE_COLUMN != nodeType(pPara)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The input parameter of CSUM function can only be column"); - } - uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; uint8_t resType; if (!IS_NUMERIC_TYPE(colType)) { @@ -604,13 +545,6 @@ static int32_t translateMavg(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - // param0 - SNode* pParaNode0 = nodesListGetNode(pFunc->pParameterList, 0); - if (QUERY_NODE_COLUMN != nodeType(pParaNode0)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The first parameter of MAVG function can only be column"); - } - uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; // param1 @@ -640,13 +574,6 @@ static int32_t translateSample(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - // param0 - SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0); - if (QUERY_NODE_COLUMN != nodeType(pParamNode0)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The first parameter of SAMPLE function can only be column"); - } - SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); uint8_t colType = pCol->resType.type; @@ -684,12 +611,6 @@ static int32_t translateTail(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - // param0 - SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); - if (QUERY_NODE_COLUMN != nodeType(pPara)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The first parameter of TAIL function can only be column"); - } SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); uint8_t colType = pCol->resType.type; @@ -766,13 +687,6 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - // param0 - SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0); - if (nodeType(pParamNode0) != QUERY_NODE_COLUMN) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The first parameter of DIFF function can only be column"); - } - uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 34251ab918..db0d097d29 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -38,7 +38,7 @@ typedef struct SIFParam { col_id_t colId; int64_t suid; // add later char dbName[TSDB_DB_NAME_LEN]; - char colName[TSDB_COL_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN * 2 + 4]; SIndexMetaArg arg; } SIFParam; @@ -171,7 +171,10 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { param->colId = l->colId; param->colValType = l->node.resType.type; memcpy(param->dbName, l->dbName, sizeof(l->dbName)); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-overflow" sprintf(param->colName, "%s_%s", l->colName, r->literal); +#pragma GCC diagnostic pop param->colValType = r->typeData; return 0; // memcpy(param->colName, l->colName, sizeof(l->colName)); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index f277d30eb1..41ba7a7459 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -318,6 +318,7 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { CLONE_NODE_LIST_FIELD(pScanPseudoCols); COPY_SCALAR_FIELD(tableType); COPY_SCALAR_FIELD(tableId); + COPY_SCALAR_FIELD(stableId); CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone); COPY_SCALAR_FIELD(scanType); COPY_OBJECT_FIELD(scanSeq[0], sizeof(uint8_t) * 2); @@ -370,7 +371,7 @@ static SNode* logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModif COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); COPY_SCALAR_FIELD(modifyType); COPY_SCALAR_FIELD(msgType); - CLONE_NODE_FIELD(pModifyRows); + CLONE_NODE_FIELD(pAffectedRows); COPY_SCALAR_FIELD(tableId); COPY_SCALAR_FIELD(tableType); COPY_CHAR_ARRAY_FIELD(tableFName); @@ -387,6 +388,7 @@ static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNo static SNode* logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pMergeKeys); + CLONE_NODE_LIST_FIELD(pInputs); COPY_SCALAR_FIELD(numOfChannels); COPY_SCALAR_FIELD(srcGroupId); return (SNode*)pDst; @@ -432,6 +434,12 @@ static SNode* logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLogi return (SNode*)pDst; } +static SNode* logicIndefRowsFuncCopy(const SIndefRowsFuncLogicNode* pSrc, SIndefRowsFuncLogicNode* pDst) { + COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + CLONE_NODE_LIST_FIELD(pVectorFuncs); + return (SNode*)pDst; +} + static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) { COPY_OBJECT_FIELD(id, sizeof(SSubplanId)); CLONE_NODE_FIELD(pNode); @@ -563,6 +571,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return logicSortCopy((const SSortLogicNode*)pNode, (SSortLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_PARTITION: return logicPartitionCopy((const SPartitionLogicNode*)pNode, (SPartitionLogicNode*)pDst); + case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: + return logicIndefRowsFuncCopy((const SIndefRowsFuncLogicNode*)pNode, (SIndefRowsFuncLogicNode*)pDst); case QUERY_NODE_LOGIC_SUBPLAN: return logicSubplanCopy((const SLogicSubplan*)pNode, (SLogicSubplan*)pDst); default: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c54289bc56..2feb25d2bb 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -202,6 +202,8 @@ const char* nodesNodeName(ENodeType type) { return "LogicSort"; case QUERY_NODE_LOGIC_PLAN_PARTITION: return "LogicPartition"; + case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: + return "LogicIndefRowsFunc"; case QUERY_NODE_LOGIC_SUBPLAN: return "LogicSubplan"; case QUERY_NODE_LOGIC_PLAN: @@ -230,6 +232,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiSort"; case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return "PhysiHashInterval"; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: + return "PhysiMergeInterval"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: return "PhysiStreamInterval"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: @@ -248,6 +252,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiStreamStateWindow"; case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return "PhysiPartition"; + case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: + return "PhysiIndefRowsFunc"; case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return "PhysiDispatch"; case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -611,7 +617,7 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { static const char* jkVnodeModifyLogicPlanModifyType = "ModifyType"; static const char* jkVnodeModifyLogicPlanMsgType = "MsgType"; -static const char* jkVnodeModifyLogicPlanModifyRows = "ModifyRows"; +static const char* jkVnodeModifyLogicPlanAffectedRows = "AffectedRows"; static int32_t logicVnodeModifyNodeToJson(const void* pObj, SJson* pJson) { const SVnodeModifyLogicNode* pNode = (const SVnodeModifyLogicNode*)pObj; @@ -624,7 +630,7 @@ static int32_t logicVnodeModifyNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddIntegerToObject(pJson, jkVnodeModifyLogicPlanMsgType, pNode->msgType); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkVnodeModifyLogicPlanModifyRows, nodeToJson, pNode->pModifyRows); + code = tjsonAddObject(pJson, jkVnodeModifyLogicPlanAffectedRows, nodeToJson, pNode->pAffectedRows); } return code; @@ -641,7 +647,7 @@ static int32_t jsonToLogicVnodeModifyNode(const SJson* pJson, void* pObj) { code = tjsonGetIntValue(pJson, jkVnodeModifyLogicPlanMsgType, &pNode->msgType); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkVnodeModifyLogicPlanModifyRows, &pNode->pModifyRows); + code = jsonToNodeObject(pJson, jkVnodeModifyLogicPlanAffectedRows, &pNode->pAffectedRows); } return code; @@ -911,6 +917,30 @@ static int32_t jsonToLogicPartitionNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkIndefRowsFuncLogicPlanVectorFuncs = "VectorFuncs"; + +static int32_t logicIndefRowsFuncNodeToJson(const void* pObj, SJson* pJson) { + const SIndefRowsFuncLogicNode* pNode = (const SIndefRowsFuncLogicNode*)pObj; + + int32_t code = logicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkIndefRowsFuncLogicPlanVectorFuncs, pNode->pVectorFuncs); + } + + return code; +} + +static int32_t jsonToLogicIndefRowsFuncNode(const SJson* pJson, void* pObj) { + SIndefRowsFuncLogicNode* pNode = (SIndefRowsFuncLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkIndefRowsFuncLogicPlanVectorFuncs, &pNode->pVectorFuncs); + } + + return code; +} + static const char* jkSubplanIdQueryId = "QueryId"; static const char* jkSubplanIdGroupId = "GroupId"; static const char* jkSubplanIdSubplanId = "SubplanId"; @@ -1249,6 +1279,7 @@ static int32_t jsonToName(const SJson* pJson, void* pObj) { static const char* jkScanPhysiPlanScanCols = "ScanCols"; static const char* jkScanPhysiPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanPhysiPlanTableId = "TableId"; +static const char* jkScanPhysiPlanSTableId = "STableId"; static const char* jkScanPhysiPlanTableType = "TableType"; static const char* jkScanPhysiPlanTableName = "TableName"; @@ -1265,6 +1296,9 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableId, pNode->uid); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanSTableId, pNode->suid); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableType, pNode->tableType); } @@ -1288,6 +1322,9 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUBigIntValue(pJson, jkScanPhysiPlanTableId, &pNode->uid); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUBigIntValue(pJson, jkScanPhysiPlanSTableId, &pNode->suid); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkScanPhysiPlanTableType, &pNode->tableType); } @@ -1644,6 +1681,7 @@ static int32_t jsonToPhysiExchangeNode(const SJson* pJson, void* pObj) { } static const char* jkMergePhysiPlanMergeKeys = "MergeKeys"; +static const char* jkMergePhysiPlanTargets = "Targets"; static const char* jkMergePhysiPlanNumOfChannels = "NumOfChannels"; static const char* jkMergePhysiPlanSrcGroupId = "SrcGroupId"; @@ -1654,6 +1692,9 @@ static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkMergePhysiPlanMergeKeys, pNode->pMergeKeys); } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkMergePhysiPlanTargets, pNode->pTargets); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkMergePhysiPlanNumOfChannels, pNode->numOfChannels); } @@ -1671,6 +1712,9 @@ static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkMergePhysiPlanMergeKeys, &pNode->pMergeKeys); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkMergePhysiPlanTargets, &pNode->pTargets); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkMergePhysiPlanNumOfChannels, &pNode->numOfChannels); } @@ -1979,6 +2023,37 @@ static int32_t jsonToPhysiPartitionNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkIndefRowsFuncPhysiPlanExprs = "Exprs"; +static const char* jkIndefRowsFuncPhysiPlanVectorFuncs = "VectorFuncs"; + +static int32_t physiIndefRowsFuncNodeToJson(const void* pObj, SJson* pJson) { + const SIndefRowsFuncPhysiNode* pNode = (const SIndefRowsFuncPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkIndefRowsFuncPhysiPlanExprs, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkIndefRowsFuncPhysiPlanVectorFuncs, pNode->pVectorFuncs); + } + + return code; +} + +static int32_t jsonToPhysiIndefRowsFuncNode(const SJson* pJson, void* pObj) { + SIndefRowsFuncPhysiNode* pNode = (SIndefRowsFuncPhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkIndefRowsFuncPhysiPlanExprs, &pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkIndefRowsFuncPhysiPlanVectorFuncs, &pNode->pVectorFuncs); + } + + return code; +} + static const char* jkDataSinkInputDataBlockDesc = "InputDataBlockDesc"; static int32_t physicDataSinkNodeToJson(const void* pObj, SJson* pJson) { @@ -2000,6 +2075,7 @@ static const char* jkDeletePhysiPlanTableType = "TableType"; static const char* jkDeletePhysiPlanTableFName = "TableFName"; static const char* jkDeletePhysiPlanDeleteTimeRangeStartKey = "DeleteTimeRangeStartKey"; static const char* jkDeletePhysiPlanDeleteTimeRangeEndKey = "DeleteTimeRangeEndKey"; +static const char* jkDeletePhysiPlanAffectedRows = "AffectedRows"; static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) { const SDataDeleterNode* pNode = (const SDataDeleterNode*)pObj; @@ -2020,6 +2096,9 @@ static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanDeleteTimeRangeEndKey, pNode->deleteTimeRange.ekey); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDeletePhysiPlanAffectedRows, nodeToJson, pNode->pAffectedRows); + } return code; } @@ -2043,6 +2122,9 @@ static int32_t jsonToPhysiDeleteNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkDeletePhysiPlanDeleteTimeRangeEndKey, &pNode->deleteTimeRange.ekey); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDeletePhysiPlanAffectedRows, &pNode->pAffectedRows); + } return code; } @@ -3777,6 +3859,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return logicSortNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_PARTITION: return logicPartitionNodeToJson(pObj, pJson); + case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: + return logicIndefRowsFuncNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_SUBPLAN: return logicSubplanToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN: @@ -3802,6 +3886,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_SORT: return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: @@ -3816,6 +3901,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiStateWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return physiPartitionNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: + return physiIndefRowsFuncNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return physiDispatchNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -3905,6 +3992,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicSortNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_PARTITION: return jsonToLogicPartitionNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: + return jsonToLogicIndefRowsFuncNode(pJson, pObj); case QUERY_NODE_LOGIC_SUBPLAN: return jsonToLogicSubplan(pJson, pObj); case QUERY_NODE_LOGIC_PLAN: @@ -3930,6 +4019,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_SORT: return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: @@ -3944,6 +4034,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiStateWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return jsonToPhysiPartitionNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: + return jsonToPhysiIndefRowsFuncNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return jsonToPhysiDispatchNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_DELETE: diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index d9b61bb002..b0169b7fd1 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -350,6 +350,7 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa case SQL_CLAUSE_GROUP_BY: nodesWalkExpr(pSelect->pHaving, walker, pContext); case SQL_CLAUSE_HAVING: + case SQL_CLAUSE_SELECT: case SQL_CLAUSE_DISTINCT: nodesWalkExprs(pSelect->pOrderByList, walker, pContext); case SQL_CLAUSE_ORDER_BY: @@ -382,6 +383,7 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit case SQL_CLAUSE_GROUP_BY: nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext); case SQL_CLAUSE_HAVING: + case SQL_CLAUSE_SELECT: case SQL_CLAUSE_DISTINCT: nodesRewriteExprs(pSelect->pOrderByList, rewriter, pContext); case SQL_CLAUSE_ORDER_BY: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 16ce4d7a48..d8914fc7e6 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -232,6 +232,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSortLogicNode)); case QUERY_NODE_LOGIC_PLAN_PARTITION: return makeNode(type, sizeof(SPartitionLogicNode)); + case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: + return makeNode(type, sizeof(SIndefRowsFuncLogicNode)); case QUERY_NODE_LOGIC_SUBPLAN: return makeNode(type, sizeof(SLogicSubplan)); case QUERY_NODE_LOGIC_PLAN: @@ -260,8 +262,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL: - return makeNode(type, sizeof(SSortMergeIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: + return makeNode(type, sizeof(SMergeIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: return makeNode(type, sizeof(SStreamIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: @@ -280,6 +282,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SStreamStateWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return makeNode(type, sizeof(SPartitionPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: + return makeNode(type, sizeof(SIndefRowsFuncPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return makeNode(type, sizeof(SDataDispatcherNode)); case QUERY_NODE_PHYSICAL_PLAN_INSERT: diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 6c44ce4c3c..c79820a950 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -258,6 +258,17 @@ static int32_t collectMetaKeyFromExplain(SCollectMetaKeyCxt* pCxt, SExplainStmt* return collectMetaKeyFromQuery(pCxt, pStmt->pQuery); } +static int32_t collectMetaKeyFromDescribe(SCollectMetaKeyCxt* pCxt, SDescribeStmt* pStmt) { + SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; + strcpy(name.dbname, pStmt->dbName); + strcpy(name.tname, pStmt->tableName); + int32_t code = catalogRemoveTableMeta(pCxt->pParseCxt->pCatalog, &name); + if (TSDB_CODE_SUCCESS == code) { + code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); + } + return code; +} + static int32_t collectMetaKeyFromCreateStream(SCollectMetaKeyCxt* pCxt, SCreateStreamStmt* pStmt) { return collectMetaKeyFromQuery(pCxt, pStmt->pQuery); } @@ -381,6 +392,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromCreateTopic(pCxt, (SCreateTopicStmt*)pStmt); case QUERY_NODE_EXPLAIN_STMT: return collectMetaKeyFromExplain(pCxt, (SExplainStmt*)pStmt); + case QUERY_NODE_DESCRIBE_STMT: + return collectMetaKeyFromDescribe(pCxt, (SDescribeStmt*)pStmt); case QUERY_NODE_CREATE_STREAM_STMT: return collectMetaKeyFromCreateStream(pCxt, (SCreateStreamStmt*)pStmt); case QUERY_NODE_SHOW_DNODES_STMT: diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index efe0bd75b2..8f17d500ab 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1006,7 +1006,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; - char* tmpTokenBuf = taosMemoryCalloc(1, sToken.n); // this can be optimize with parse column + char* tmpTokenBuf = taosMemoryCalloc(1, sToken.n); //todo this can be optimize with parse column code = checkAndTrimValue(&sToken, tmpTokenBuf, &pCxt->msg); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(tmpTokenBuf); @@ -1018,7 +1018,11 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint taosMemoryFree(tmpTokenBuf); goto end; } - code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg); + if(isNullStr(&sToken)) { + code = tTagNew(pTagVals, 1, true, &pTag); + } else { + code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg); + } taosMemoryFree(tmpTokenBuf); if (code != TSDB_CODE_SUCCESS) { goto end; diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 685c299c01..0cc9bd7126 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -64,6 +64,7 @@ static SKeyword keywordTable[] = { {"CONSUMER", TK_CONSUMER}, {"COUNT", TK_COUNT}, {"CREATE", TK_CREATE}, + {"CONTAINS", TK_CONTAINS}, {"DATABASE", TK_DATABASE}, {"DATABASES", TK_DATABASES}, {"DAYS", TK_DAYS}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c385f26591..d81c6760e8 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -847,8 +847,12 @@ static EDealRes translateJsonOperator(STranslateContext* pCxt, SOperatorNode* pO if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } - pOp->node.resType.type = TSDB_DATA_TYPE_JSON; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes; + if(pOp->opType == OP_TYPE_JSON_GET_VALUE){ + pOp->node.resType.type = TSDB_DATA_TYPE_JSON; + }else if(pOp->opType == OP_TYPE_JSON_CONTAINS){ + pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; + } + pOp->node.resType.bytes = tDataTypes[pOp->node.resType.type].bytes; return DEAL_RES_CONTINUE; } @@ -2903,7 +2907,6 @@ static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, } } - taosArrayDestroy(dbCfg.pRetensions); return code; } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 8fb28ce395..c7f0990132 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -207,6 +207,13 @@ int32_t __catalogGetUdfInfo(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps return g_mockCatalogService->catalogGetUdfInfo(funcName, pInfo); } +int32_t __catalogRefreshGetTableMeta(SCatalog* pCatalog, void* pTransporter, const SEpSet* pMgmtEps, + const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { + return g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta); +} + +int32_t __catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { return 0; } + void initMetaDataEnv() { g_mockCatalogService.reset(new MockCatalogService()); @@ -221,6 +228,8 @@ void initMetaDataEnv() { stub.set(catalogGetDBCfg, __catalogGetDBCfg); stub.set(catalogChkAuth, __catalogChkAuth); stub.set(catalogGetUdfInfo, __catalogGetUdfInfo); + stub.set(catalogRefreshGetTableMeta, __catalogRefreshGetTableMeta); + stub.set(catalogRemoveTableMeta, __catalogRemoveTableMeta); // { // AddrAny any("libcatalog.so"); // std::map result; diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp index d4f9a540ae..8562926789 100644 --- a/source/libs/parser/test/parInitialDTest.cpp +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -21,7 +21,7 @@ namespace ParserTest { class ParserInitialDTest : public ParserDdlTest {}; -// DELETE FROM tb_name [WHERE condition] +// DELETE FROM table_name [WHERE condition] TEST_F(ParserInitialDTest, delete) { useDb("root", "test"); @@ -40,7 +40,15 @@ TEST_F(ParserInitialDTest, deleteSemanticCheck) { run("DELETE FROM t1 WHERE c1 > 10", TSDB_CODE_PAR_INVALID_DELETE_WHERE, PARSER_STAGE_TRANSLATE); } -// todo desc +// DESC table_name +TEST_F(ParserInitialDTest, describe) { + useDb("root", "test"); + + run("DESC t1"); + + run("DESCRIBE st1"); +} + // todo describe // todo DROP account @@ -51,7 +59,7 @@ TEST_F(ParserInitialDTest, dropBnode) { } // DROP CONSUMER GROUP [ IF EXISTS ] cgroup_name ON topic_name -TEST_F(ParserInitialDTest, dropCGroup) { +TEST_F(ParserInitialDTest, dropConsumerGroup) { useDb("root", "test"); SMDropCgroupReq expect = {0}; diff --git a/source/libs/parser/test/parTestUtil.h b/source/libs/parser/test/parTestUtil.h index 44be7a2474..07f3d3cece 100644 --- a/source/libs/parser/test/parTestUtil.h +++ b/source/libs/parser/test/parTestUtil.h @@ -50,11 +50,13 @@ class ParserDdlTest : public ParserTestBase { virtual void checkDdl(const SQuery* pQuery, ParserStage stage) { ASSERT_NE(pQuery, nullptr); - ASSERT_EQ(pQuery->haveResultSet, false); ASSERT_NE(pQuery->pRoot, nullptr); - ASSERT_EQ(pQuery->numOfResCols, 0); - ASSERT_EQ(pQuery->pResSchema, nullptr); - ASSERT_EQ(pQuery->precision, 0); + if (QUERY_EXEC_MODE_RPC == pQuery->execMode) { + ASSERT_EQ(pQuery->haveResultSet, false); + ASSERT_EQ(pQuery->numOfResCols, 0); + ASSERT_EQ(pQuery->pResSchema, nullptr); + ASSERT_EQ(pQuery->precision, 0); + } if (nullptr != checkDdl_) { checkDdl_(pQuery, stage); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c3aad632c7..a9f3909af6 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -219,9 +219,9 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT return TSDB_CODE_OUT_OF_MEMORY; } - // TSWAP(pScan->pMeta, pRealTable->pMeta); TSWAP(pScan->pVgroupList, pRealTable->pVgroupList); pScan->tableId = pRealTable->pMeta->uid; + pScan->stableId = pRealTable->pMeta->suid; pScan->tableType = pRealTable->pMeta->tableType; pScan->scanSeq[0] = hasRepeatScanFuncs ? 2 : 1; pScan->scanSeq[1] = 0; @@ -456,6 +456,37 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, return code; } +static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { + // top/bottom are both an aggregate function and a indefinite rows function + if (!pSelect->hasIndefiniteRowsFunc || pSelect->hasAggFuncs || NULL != pSelect->pWindow) { + return TSDB_CODE_SUCCESS; + } + + SIndefRowsFuncLogicNode* pIdfRowsFunc = + (SIndefRowsFuncLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC); + if (NULL == pIdfRowsFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pVectorFuncs); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExprForSelect(pIdfRowsFunc->pVectorFuncs, pSelect, SQL_CLAUSE_SELECT); + } + + // set the output + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pIdfRowsFunc->pVectorFuncs, &pIdfRowsFunc->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pIdfRowsFunc; + } else { + nodesDestroyNode(pIdfRowsFunc); + } + + return code; +} + static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) { int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, fmIsWindowClauseFunc, &pWindow->pFuncs); @@ -772,6 +803,9 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele if (TSDB_CODE_SUCCESS == code) { code = createSelectRootLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot); } + if (TSDB_CODE_SUCCESS == code) { + code = createSelectRootLogicNode(pCxt, pSelect, createIndefRowsFuncLogicNode, &pRoot); + } if (TSDB_CODE_SUCCESS == code) { code = createSelectRootLogicNode(pCxt, pSelect, createDistinctLogicNode, &pRoot); } @@ -1057,8 +1091,8 @@ static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDelet snprintf(pModify->tableFName, sizeof(pModify->tableFName), "%d.%s.%s", pCxt->pPlanCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName); pModify->deleteTimeRange = pDelete->timeRange; - pModify->pModifyRows = nodesCloneNode(pDelete->pCountFunc); - if (NULL == pModify->pModifyRows) { + pModify->pAffectedRows = nodesCloneNode(pDelete->pCountFunc); + if (NULL == pModify->pAffectedRows) { nodesDestroyNode(pModify); return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e306db48e8..6b3d82b644 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -425,6 +425,7 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS if (TSDB_CODE_SUCCESS == code) { pScanPhysiNode->uid = pScanLogicNode->tableId; + pScanPhysiNode->suid = pScanLogicNode->stableId; pScanPhysiNode->tableType = pScanLogicNode->tableType; memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); if (NULL != pScanLogicNode->pTagCond) { @@ -790,6 +791,43 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, return code; } +static int32_t createIndefRowsFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, + SIndefRowsFuncLogicNode* pFuncLogicNode, SPhysiNode** pPhyNode) { + SIndefRowsFuncPhysiNode* pIdfRowsFunc = (SIndefRowsFuncPhysiNode*)makePhysiNode( + pCxt, (SLogicNode*)pFuncLogicNode, QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC); + if (NULL == pIdfRowsFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNodeList* pPrecalcExprs = NULL; + SNodeList* pVectorFuncs = NULL; + int32_t code = rewritePrecalcExprs(pCxt, pFuncLogicNode->pVectorFuncs, &pPrecalcExprs, &pVectorFuncs); + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + // push down expression to pOutputDataBlockDesc of child node + if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pIdfRowsFunc->pExprs); + if (TSDB_CODE_SUCCESS == code) { + code = pushdownDataBlockSlots(pCxt, pIdfRowsFunc->pExprs, pChildTupe); + } + } + + if (TSDB_CODE_SUCCESS == code && NULL != pVectorFuncs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pVectorFuncs, &pIdfRowsFunc->pVectorFuncs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pIdfRowsFunc->pVectorFuncs, pIdfRowsFunc->node.pOutputDataBlockDesc); + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pIdfRowsFunc; + } else { + nodesDestroyNode(pIdfRowsFunc); + } + + return code; +} + static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) { SProjectPhysiNode* pProject = @@ -923,8 +961,8 @@ static ENodeType getIntervalOperatorType(EIntervalAlgorithm intervalAlgo) { switch (intervalAlgo) { case INTERVAL_ALGO_HASH: return QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; - case INTERVAL_ALGO_SORT_MERGE: - return QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL; + case INTERVAL_ALGO_MERGE: + return QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL; case INTERVAL_ALGO_STREAM_FINAL: return QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL; case INTERVAL_ALGO_STREAM_SEMI: @@ -1155,6 +1193,8 @@ static int32_t createExchangePhysiNodeByMerge(SMergePhysiNode* pMerge) { nodesDestroyNode(pExchange); return TSDB_CODE_OUT_OF_MEMORY; } + SNode* pSlot = NULL; + FOREACH(pSlot, pExchange->node.pOutputDataBlockDesc->pSlots) { ((SSlotDescNode*)pSlot)->output = true; } return nodesListMakeStrictAppend(&pMerge->node.pChildren, pExchange); } @@ -1168,12 +1208,14 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM pMerge->numOfChannels = pMergeLogicNode->numOfChannels; pMerge->srcGroupId = pMergeLogicNode->srcGroupId; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = addDataBlockSlots(pCxt, pMergeLogicNode->pInputs, pMerge->node.pOutputDataBlockDesc); - for (int32_t i = 0; i < pMerge->numOfChannels; ++i) { - code = createExchangePhysiNodeByMerge(pMerge); - if (TSDB_CODE_SUCCESS != code) { - break; + if (TSDB_CODE_SUCCESS == code) { + for (int32_t i = 0; i < pMerge->numOfChannels; ++i) { + code = createExchangePhysiNodeByMerge(pMerge); + if (TSDB_CODE_SUCCESS != code) { + break; + } } } @@ -1182,6 +1224,14 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM &pMerge->pMergeKeys); } + if (TSDB_CODE_SUCCESS == code) { + code = setListSlotId(pCxt, pMerge->node.pOutputDataBlockDesc->dataBlockId, -1, pMergeLogicNode->node.pTargets, + &pMerge->pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pMerge->pTargets, pMerge->node.pOutputDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pMerge; } else { @@ -1212,6 +1262,8 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode return createPartitionPhysiNode(pCxt, pChildren, (SPartitionLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_FILL: return createFillPhysiNode(pCxt, pChildren, (SFillLogicNode*)pLogicNode, pPhyNode); + case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: + return createIndefRowsFuncPhysiNode(pCxt, pChildren, (SIndefRowsFuncLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_MERGE: return createMergePhysiNode(pCxt, (SMergeLogicNode*)pLogicNode, pPhyNode); default: @@ -1319,13 +1371,21 @@ static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* strcpy(pDeleter->tableFName, pModify->tableFName); pDeleter->deleteTimeRange = pModify->deleteTimeRange; - pDeleter->sink.pInputDataBlockDesc = nodesCloneNode(pRoot->pOutputDataBlockDesc); - if (NULL == pDeleter->sink.pInputDataBlockDesc) { - nodesDestroyNode(pDeleter); - return TSDB_CODE_OUT_OF_MEMORY; + int32_t code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pAffectedRows, + &pDeleter->pAffectedRows); + if (TSDB_CODE_SUCCESS == code) { + pDeleter->sink.pInputDataBlockDesc = nodesCloneNode(pRoot->pOutputDataBlockDesc); + if (NULL == pDeleter->sink.pInputDataBlockDesc) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pSink = (SDataSinkNode*)pDeleter; + } else { + nodesDestroyNode(pDeleter); } - *pSink = (SDataSinkNode*)pDeleter; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 9e6c25ce78..7be33d54e3 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -80,30 +80,36 @@ static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pChild, SE return TSDB_CODE_SUCCESS; } -static int32_t splCreateExchangeNodeForSubplan(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, - ESubplanType subplanType) { - SExchangeLogicNode* pExchange = NULL; - if (TSDB_CODE_SUCCESS != splCreateExchangeNode(pCxt, pSplitNode, &pExchange)) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pSubplan->subplanType = subplanType; - - if (NULL == pSplitNode->pParent) { - pSubplan->pNode = (SLogicNode*)pExchange; +static int32_t splReplaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew) { + if (NULL == pOld->pParent) { + pSubplan->pNode = (SLogicNode*)pNew; return TSDB_CODE_SUCCESS; } SNode* pNode; - FOREACH(pNode, pSplitNode->pParent->pChildren) { - if (nodesEqualNode(pNode, pSplitNode)) { - REPLACE_NODE(pExchange); - pExchange->node.pParent = pSplitNode->pParent; + FOREACH(pNode, pOld->pParent->pChildren) { + if (nodesEqualNode(pNode, pOld)) { + REPLACE_NODE(pNew); + pNew->pParent = pOld->pParent; return TSDB_CODE_SUCCESS; } } - nodesDestroyNode(pExchange); - return TSDB_CODE_FAILED; + return TSDB_CODE_PLAN_INTERNAL_ERROR; +} + +static int32_t splCreateExchangeNodeForSubplan(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, + ESubplanType subplanType) { + SExchangeLogicNode* pExchange = NULL; + int32_t code = splCreateExchangeNode(pCxt, pSplitNode, &pExchange); + if (TSDB_CODE_SUCCESS == code) { + code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pExchange); + } + if (TSDB_CODE_SUCCESS == code) { + pSubplan->subplanType = subplanType; + } else { + nodesDestroyNode(pExchange); + } + return code; } static bool splMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan, int32_t flag, FSplFindSplitNode func, void* pInfo) { @@ -295,24 +301,34 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic return code; } -static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicNode* pParent, SNodeList* pMergeKeys, - SLogicNode* pPartChild) { +static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, + SNodeList* pMergeKeys, SLogicNode* pPartChild) { SMergeLogicNode* pMerge = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); if (NULL == pMerge) { return TSDB_CODE_OUT_OF_MEMORY; } pMerge->numOfChannels = ((SScanLogicNode*)nodesListGetNode(pPartChild->pChildren, 0))->pVgroupList->numOfVgroups; pMerge->srcGroupId = pCxt->groupId; - pMerge->node.pParent = pParent; pMerge->node.precision = pPartChild->precision; pMerge->pMergeKeys = pMergeKeys; - pMerge->node.pTargets = nodesCloneList(pPartChild->pTargets); - if (NULL == pMerge->node.pTargets) { - nodesDestroyNode(pMerge); - return TSDB_CODE_OUT_OF_MEMORY; - } - return nodesListMakeAppend(&pParent->pChildren, pMerge); + int32_t code = TSDB_CODE_SUCCESS; + pMerge->pInputs = nodesCloneList(pPartChild->pTargets); + pMerge->node.pTargets = nodesCloneList(pSplitNode->pTargets); + if (NULL == pMerge->node.pTargets || NULL == pMerge->pInputs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + if (NULL == pSubplan) { + code = nodesListMakeAppend(&pSplitNode->pChildren, pMerge); + } else { + code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); + } + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pMerge); + } + return code; } static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent, SLogicNode* pPartChild) { @@ -329,8 +345,15 @@ static int32_t stbSplSplitWindowNodeForBatch(SSplitContext* pCxt, SStableSplitIn int32_t code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow); if (TSDB_CODE_SUCCESS == code) { ((SWindowLogicNode*)pPartWindow)->intervalAlgo = INTERVAL_ALGO_HASH; - ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_SORT_MERGE; - code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow); + ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_MERGE; + SNodeList* pMergeKeys = NULL; + code = nodesListMakeStrictAppend(&pMergeKeys, nodesCloneNode(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk)); + if (TSDB_CODE_SUCCESS == code) { + code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow); + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pMergeKeys); + } } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, @@ -424,37 +447,99 @@ static int32_t stbSplSplitAggNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) return code; } -static int32_t stbSplCreatePartSortNode(SSortLogicNode* pMergeSort, SLogicNode** pOutput) { - SNodeList* pSortKeys = pMergeSort->pSortKeys; - pMergeSort->pSortKeys = NULL; - SNodeList* pTargets = pMergeSort->node.pTargets; - pMergeSort->node.pTargets = NULL; - SNodeList* pChildren = pMergeSort->node.pChildren; - pMergeSort->node.pChildren = NULL; +static SNode* stbSplCreateColumnNode(SExprNode* pExpr) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return NULL; + } + if (QUERY_NODE_COLUMN == nodeType(pExpr)) { + strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias); + } + strcpy(pCol->colName, pExpr->aliasName); + strcpy(pCol->node.aliasName, pExpr->aliasName); + pCol->node.resType = pExpr->resType; + return (SNode*)pCol; +} + +static SNode* stbSplCreateOrderByExpr(SOrderByExprNode* pSortKey, SNode* pCol) { + SOrderByExprNode* pOutput = nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + if (NULL == pOutput) { + return NULL; + } + pOutput->pExpr = nodesCloneNode(pCol); + if (NULL == pOutput->pExpr) { + nodesDestroyNode(pOutput); + return NULL; + } + pOutput->order = pSortKey->order; + pOutput->nullOrder = pSortKey->nullOrder; + return (SNode*)pOutput; +} + +static int32_t stbSplCreateMergeKeys(SNodeList* pSortKeys, SNodeList* pTargets, SNodeList** pOutput) { + int32_t code = TSDB_CODE_SUCCESS; + SNodeList* pMergeKeys = NULL; + SNode* pNode = NULL; + FOREACH(pNode, pSortKeys) { + SOrderByExprNode* pSortKey = (SOrderByExprNode*)pNode; + SNode* pTarget = NULL; + bool found = false; + FOREACH(pTarget, pTargets) { + if (0 == strcmp(((SExprNode*)pSortKey->pExpr)->aliasName, ((SColumnNode*)pTarget)->colName)) { + code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pTarget)); + if (TSDB_CODE_SUCCESS != code) { + break; + } + found = true; + } + } + if (TSDB_CODE_SUCCESS == code && !found) { + SNode* pCol = stbSplCreateColumnNode((SExprNode*)pSortKey->pExpr); + code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pCol)); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pTargets, pCol); + } else { + nodesDestroyNode(pCol); + } + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pMergeKeys; + } else { + nodesDestroyList(pMergeKeys); + } + return code; +} + +static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOutputPartSort, + SNodeList** pOutputMergeKeys) { + SNodeList* pSortKeys = pSort->pSortKeys; + pSort->pSortKeys = NULL; + SNodeList* pChildren = pSort->node.pChildren; + pSort->node.pChildren = NULL; int32_t code = TSDB_CODE_SUCCESS; - SSortLogicNode* pPartSort = nodesCloneNode(pMergeSort); + SSortLogicNode* pPartSort = nodesCloneNode(pSort); if (NULL == pPartSort) { code = TSDB_CODE_OUT_OF_MEMORY; } - pMergeSort->node.pTargets = pTargets; - pPartSort->node.pChildren = pChildren; + SNodeList* pMergeKeys = NULL; if (TSDB_CODE_SUCCESS == code) { + pPartSort->node.pChildren = pChildren; pPartSort->pSortKeys = pSortKeys; - code = createColumnByRewriteExps(pPartSort->pSortKeys, &pPartSort->node.pTargets); - } - if (TSDB_CODE_SUCCESS == code) { - pMergeSort->pSortKeys = nodesCloneList(pPartSort->node.pTargets); - if (NULL == pMergeSort->pSortKeys) { - code = TSDB_CODE_OUT_OF_MEMORY; - } + code = stbSplCreateMergeKeys(pPartSort->pSortKeys, pPartSort->node.pTargets, &pMergeKeys); } if (TSDB_CODE_SUCCESS == code) { - *pOutput = (SLogicNode*)pPartSort; + *pOutputPartSort = (SLogicNode*)pPartSort; + *pOutputMergeKeys = pMergeKeys; } else { nodesDestroyNode(pPartSort); + nodesDestroyList(pMergeKeys); } return code; @@ -462,17 +547,10 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pMergeSort, SLogicNode** static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { SLogicNode* pPartSort = NULL; - int32_t code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort); + SNodeList* pMergeKeys = NULL; + int32_t code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - SNodeList* pMergeKeys = nodesCloneList(((SSortLogicNode*)pInfo->pSplitNode)->pSortKeys); - if (NULL != pMergeKeys) { - code = stbSplCreateMergeNode(pCxt, pInfo->pSplitNode, pMergeKeys, pPartSort); - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyList(pMergeKeys); - } - } else { - code = TSDB_CODE_OUT_OF_MEMORY; - } + code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, diff --git a/source/libs/planner/test/planIntervalTest.cpp b/source/libs/planner/test/planIntervalTest.cpp index a04f47741e..edd735922b 100644 --- a/source/libs/planner/test/planIntervalTest.cpp +++ b/source/libs/planner/test/planIntervalTest.cpp @@ -50,6 +50,8 @@ TEST_F(PlanIntervalTest, selectFunc) { run("SELECT MAX(c1), MIN(c1) FROM t1 INTERVAL(10s)"); // select function along with the columns of select row, and with INTERVAL clause run("SELECT MAX(c1), c2 FROM t1 INTERVAL(10s)"); + + run("SELECT TOP(c1, 1) FROM t1 INTERVAL(10s) ORDER BY c1"); } TEST_F(PlanIntervalTest, stable) { diff --git a/source/libs/planner/test/planOrderByTest.cpp b/source/libs/planner/test/planOrderByTest.cpp index ef6f438b55..851eda81b5 100644 --- a/source/libs/planner/test/planOrderByTest.cpp +++ b/source/libs/planner/test/planOrderByTest.cpp @@ -46,4 +46,7 @@ TEST_F(PlanOrderByTest, stable) { // ORDER BY key is in the projection list run("SELECT c1 FROM st1 ORDER BY c1"); + + // ORDER BY key is not in the projection list + run("SELECT c2 FROM st1 ORDER BY c1"); } diff --git a/source/libs/planner/test/planProjectTest.cpp b/source/libs/planner/test/planProjectTest.cpp new file mode 100644 index 0000000000..3ef0038ae0 --- /dev/null +++ b/source/libs/planner/test/planProjectTest.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" + +using namespace std; + +class PlanProjectTest : public PlannerTestBase {}; + +TEST_F(PlanProjectTest, basic) { + useDb("root", "test"); + + run("SELECT CEIL(c1) FROM t1"); +} + +TEST_F(PlanProjectTest, indefiniteRowsFunc) { + useDb("root", "test"); + + run("SELECT MAVG(c1, 10) FROM t1"); + + run("SELECT MAVG(CEIL(c1), 20) + 2 FROM t1"); +} diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index e2082d4936..f5c8b58e43 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -81,6 +81,8 @@ int32_t getLogLevel() { return g_logLevel; } class PlannerTestBaseImpl { public: + PlannerTestBaseImpl() : sqlNo_(0) {} + void useDb(const string& acctId, const string& db) { caseEnv_.acctId_ = acctId; caseEnv_.db_ = db; @@ -88,6 +90,7 @@ class PlannerTestBaseImpl { } void run(const string& sql) { + ++sqlNo_; if (caseEnv_.nsql_ > 0) { --(caseEnv_.nsql_); return; @@ -187,6 +190,8 @@ class PlannerTestBaseImpl { string acctId_; string db_; int32_t nsql_; + + caseEnv() : nsql_(0) {} }; struct stmtEnv { @@ -194,6 +199,7 @@ class PlannerTestBaseImpl { array msgBuf_; SQuery* pQuery_; + stmtEnv() : pQuery_(nullptr) {} ~stmtEnv() { qDestroyQuery(pQuery_); } }; @@ -229,7 +235,7 @@ class PlannerTestBaseImpl { return; } - cout << "==========================================sql : [" << stmtEnv_.sql_ << "]" << endl; + cout << "========================================== " << sqlNo_ << " sql : [" << stmtEnv_.sql_ << "]" << endl; if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) { if (res_.prepareAst_.empty()) { @@ -382,6 +388,7 @@ class PlannerTestBaseImpl { caseEnv caseEnv_; stmtEnv stmtEnv_; stmtRes res_; + int32_t sqlNo_; }; PlannerTestBase::PlannerTestBase() : impl_(new PlannerTestBaseImpl()) {} diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index e77f2b0ca4..949a58c2fa 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -37,6 +37,8 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup->vgVersion = usedbRsp->vgVersion; pOut->dbVgroup->hashMethod = usedbRsp->hashMethod; + qDebug("Got %d vgroup for db %s", usedbRsp->vgNum, usedbRsp->db); + if (usedbRsp->vgNum <= 0) { return TSDB_CODE_SUCCESS; } @@ -50,6 +52,8 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { for (int32_t i = 0; i < usedbRsp->vgNum; ++i) { SVgroupInfo *pVgInfo = taosArrayGet(usedbRsp->pVgroupInfos, i); pOut->dbVgroup->numOfTable += pVgInfo->numOfTable; + qDebug("the %dth vgroup, id %d, epNum %d, current %s port %d", i, pVgInfo->vgId, pVgInfo->epSet.numOfEps, + pVgInfo->epSet.eps[pVgInfo->epSet.inUse].fqdn, pVgInfo->epSet.eps[pVgInfo->epSet.inUse].port); if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 5fdbbfe810..12e9ec5aea 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -168,7 +168,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8, - compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch, compareJsonContainsKey + compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch }; int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index d9f582e669..747da673d1 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -88,7 +88,7 @@ void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int } } -void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){ +void convertNcharToDouble(const void *inData, void *outData){ char *tmp = taosMemoryMalloc(varDataTLen(inData)); int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); if (len < 0) { @@ -97,13 +97,24 @@ void convertStringToDouble(const void *inData, void *outData, int8_t inType, int tmp[len] = 0; - ASSERT(outType == TSDB_DATA_TYPE_DOUBLE); double value = taosStr2Double(tmp, NULL); *((double *)outData) = value; taosMemoryFreeClear(tmp); } +void convertBinaryToDouble(const void *inData, void *outData){ + char *tmp = taosMemoryCalloc(1, varDataTLen(inData)); + if(tmp == NULL){ + *((double *)outData) = 0.; + return; + } + memcpy(tmp, varDataVal(inData), varDataLen(inData)); + double ret = taosStr2Double(tmp, NULL); + taosMemoryFree(tmp); + *((double *)outData) = ret; +} + typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index); int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) { @@ -147,7 +158,7 @@ int64_t getVectorBigintValue_JSON(void *src, int32_t index){ if (*data == TSDB_DATA_TYPE_NULL){ return 0; } else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY - convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + convertNcharToDouble(data+CHAR_BYTES, &out); } else { convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); } @@ -445,14 +456,30 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){ if (*data == TSDB_DATA_TYPE_NULL){ return out; } else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY - convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + convertNcharToDouble(data+CHAR_BYTES, &out); } else { convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); } return out; } -bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){ +void* ncharTobinary(void *buf){ // todo need to remove , if tobinary is nchar + int32_t inputLen = varDataLen(buf); + + void* t = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); + if (len < 0) { + sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + (char*)varDataVal(buf)); + taosMemoryFree(t); + return NULL; + } + varDataSetLen(t, len); + return t; +} + +bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, + void *pLeftOut, void *pRightOut, bool *isNull, bool *freeLeft, bool *freeRight){ if(optr == OP_TYPE_JSON_CONTAINS) { return true; } @@ -489,21 +516,41 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t *fp = filterGetCompFunc(type, optr); - if(typeLeft == TSDB_DATA_TYPE_NCHAR) { - convertStringToDouble(*pLeftData, pLeftOut, typeLeft, type); - *pLeftData = pLeftOut; - } else if(typeLeft != type) { - convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type); - *pLeftData = pLeftOut; + if(IS_NUMERIC_TYPE(type) || IS_FLOAT_TYPE(type)){ + if(typeLeft == TSDB_DATA_TYPE_NCHAR) { + convertNcharToDouble(*pLeftData, pLeftOut); + *pLeftData = pLeftOut; + } else if(typeLeft == TSDB_DATA_TYPE_BINARY) { + convertBinaryToDouble(*pLeftData, pLeftOut); + *pLeftData = pLeftOut; + } else if(typeLeft != type) { + convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type); + *pLeftData = pLeftOut; + } + + if(typeRight == TSDB_DATA_TYPE_NCHAR) { + convertNcharToDouble(*pRightData, pRightOut); + *pRightData = pRightOut; + } else if(typeRight == TSDB_DATA_TYPE_BINARY) { + convertBinaryToDouble(*pRightData, pRightOut); + *pRightData = pRightOut; + } else if(typeRight != type) { + convertNumberToNumber(*pRightData, pRightOut, typeRight, type); + *pRightData = pRightOut; + } + }else if(type == TSDB_DATA_TYPE_BINARY){ + if(typeLeft == TSDB_DATA_TYPE_NCHAR){ + *pLeftData = ncharTobinary(*pLeftData); + *freeLeft = true; + } + if(typeRight == TSDB_DATA_TYPE_NCHAR){ + *pRightData = ncharTobinary(*pRightData); + *freeRight = true; + } + }else{ + ASSERT(0); } - if(typeRight == TSDB_DATA_TYPE_NCHAR) { - convertStringToDouble(*pRightData, pRightOut, typeRight, type); - *pRightData = pRightOut; - } else if(typeRight != type) { - convertNumberToNumber(*pRightData, pRightOut, typeRight, type); - *pRightData = pRightOut; - } return true; } @@ -935,44 +982,6 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { } } -STagVal getJsonValue(char *json, char *key, bool *isExist) { - STagVal val = {.pKey = key}; - bool find = tTagGet(((const STag *)json), &val); // json value is null and not exist is different - if(isExist){ - *isExist = find; - } - return val; -} - -void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { - SColumnInfoData *pOutputCol = pOut->columnData; - - int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1; - - pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); - - char *pRightData = colDataGetVarData(pRight->columnData, 0); - char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1); - memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData)); - for (; i >= 0 && i < pLeft->numOfRows; i += step) { - if (colDataIsNull_var(pLeft->columnData, i)) { - colDataSetNull_var(pOutputCol, i); - pOutputCol->hasNull = true; - continue; - } - char *pLeftData = colDataGetVarData(pLeft->columnData, i); - bool isExist = false; - STagVal value = getJsonValue(pLeftData, jsonKey, &isExist); - char *data = isExist ? tTagValToData(&value, true) : NULL; - colDataAppend(pOutputCol, i, data, data == NULL); - if(isExist && IS_VAR_DATA_TYPE(value.type) && data){ - taosMemoryFree(data); - } - } - taosMemoryFree(jsonKey); -} - void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { SColumnInfoData *pOutputCol = pOut->columnData; @@ -1510,6 +1519,38 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, doReleaseVec(pRightCol, rightConvert); } +#define VEC_COM_INNER(pCol, index1, index2) \ + for (; i < pCol->numOfRows && i >= 0; i += step) {\ + if (IS_HELPER_NULL(pLeft->columnData, index1) || IS_HELPER_NULL(pRight->columnData, index2)) {\ + bool res = false;\ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);\ + continue;\ + }\ + char *pLeftData = colDataGetData(pLeft->columnData, index1);\ + char *pRightData = colDataGetData(pRight->columnData, index2);\ + int64_t leftOut = 0;\ + int64_t rightOut = 0;\ + bool freeLeft = false;\ + bool freeRight = false;\ + bool isJsonnull = false;\ + bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight),\ + &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight);\ + if(isJsonnull){\ + ASSERT(0);\ + }\ + if(!pLeftData || !pRightData){\ + result = false;\ + }\ + if(!result){\ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);\ + }else{\ + bool res = filterDoCompare(fp, optr, pLeftData, pRightData);\ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);\ + }\ + if(freeLeft) taosMemoryFreeClear(pLeftData);\ + if(freeRight) taosMemoryFreeClear(pRightData);\ + } + void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -1533,79 +1574,11 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * } if (pLeft->numOfRows == pRight->numOfRows) { - for (; i < pRight->numOfRows && i >= 0; i += step) { - if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, i)) { - bool res = false; - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); - continue; // TODO set null or ignore - } - - char *pLeftData = colDataGetData(pLeft->columnData, i); - char *pRightData = colDataGetData(pRight->columnData, i); - - int64_t leftOut = 0; - int64_t rightOut = 0; - bool isJsonnull = false; - bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); - if(isJsonnull){ - ASSERT(0); - } - if(!result){ - colDataAppendInt8(pOut->columnData, i, (int8_t*)&result); - }else{ - bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); - } - } + VEC_COM_INNER(pLeft, i, i) } else if (pRight->numOfRows == 1) { - ASSERT(pLeft->pHashFilter == NULL); - for (; i >= 0 && i < pLeft->numOfRows; i += step) { - if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, 0)) { - bool res = false; - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); - continue; - } - - char *pLeftData = colDataGetData(pLeft->columnData, i); - char *pRightData = colDataGetData(pRight->columnData, 0); - int64_t leftOut = 0; - int64_t rightOut = 0; - bool isJsonnull = false; - bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); - if(isJsonnull){ - ASSERT(0); - } - if(!result){ - colDataAppendInt8(pOut->columnData, i, (int8_t*)&result); - }else{ - bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); - } - } + VEC_COM_INNER(pLeft, i, 0) } else if (pLeft->numOfRows == 1) { - for (; i >= 0 && i < pRight->numOfRows; i += step) { - if (IS_HELPER_NULL(pRight->columnData, i) || IS_HELPER_NULL(pLeft->columnData, 0)) { - bool res = false; - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); - continue; - } - - char *pLeftData = colDataGetData(pLeft->columnData, 0); - char *pRightData = colDataGetData(pLeft->columnData, i); - int64_t leftOut = 0; - int64_t rightOut = 0; - bool isJsonnull = false; - bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); - if(isJsonnull){ - ASSERT(0); - } - if(!result){ - colDataAppendInt8(pOut->columnData, i, (int8_t*)&result); - }else{ - bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); - } - } + VEC_COM_INNER(pRight, 0, i) } } @@ -1683,10 +1656,6 @@ void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH); } -void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { - vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_JSON_CONTAINS); -} - void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0; @@ -1707,6 +1676,69 @@ void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, vectorConvertImpl(pLeft, pOut); } +STagVal getJsonValue(char *json, char *key, bool *isExist) { + STagVal val = {.pKey = key}; + bool find = tTagGet(((const STag *)json), &val); // json value is null and not exist is different + if(isExist){ + *isExist = find; + } + return val; +} + +void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + SColumnInfoData *pOutputCol = pOut->columnData; + + int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1; + + pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + + char *pRightData = colDataGetVarData(pRight->columnData, 0); + char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1); + memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData)); + for (; i >= 0 && i < pLeft->numOfRows; i += step) { + bool isExist = false; + + if (!colDataIsNull_var(pLeft->columnData, i)) { + char *pLeftData = colDataGetVarData(pLeft->columnData, i); + getJsonValue(pLeftData, jsonKey, &isExist); + } + + colDataAppend(pOutputCol, i, (const char*)(&isExist), false); + + } + taosMemoryFree(jsonKey); +} + +void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + SColumnInfoData *pOutputCol = pOut->columnData; + + int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1; + + pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + + char *pRightData = colDataGetVarData(pRight->columnData, 0); + char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1); + memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData)); + for (; i >= 0 && i < pLeft->numOfRows; i += step) { + if (colDataIsNull_var(pLeft->columnData, i)) { + colDataSetNull_var(pOutputCol, i); + pOutputCol->hasNull = true; + continue; + } + char *pLeftData = colDataGetVarData(pLeft->columnData, i); + bool isExist = false; + STagVal value = getJsonValue(pLeftData, jsonKey, &isExist); + char *data = isExist ? tTagValToData(&value, true) : NULL; + colDataAppend(pOutputCol, i, data, data == NULL); + if(isExist && IS_VAR_DATA_TYPE(value.type) && data){ + taosMemoryFree(data); + } + } + taosMemoryFree(jsonKey); +} + _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { switch (binFunctionId) { case OP_TYPE_ADD: diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index b51ce8e1c1..09f2efc7e2 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -108,7 +108,7 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *pTrans, SArray *pN pJob->refId = refId; - SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId); + SCH_JOB_DLOG("job refId:0x%" PRIx64" created", pJob->refId); pJob->status = JOB_TASK_STATUS_NOT_START; diff --git a/source/libs/stream/inc/tstreamInc.h b/source/libs/stream/inc/streamInc.h similarity index 74% rename from source/libs/stream/inc/tstreamInc.h rename to source/libs/stream/inc/streamInc.h index c96901e567..604539f16a 100644 --- a/source/libs/stream/inc/tstreamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -13,15 +13,21 @@ * along with this program. If not, see . */ +#ifndef _STREAM_INC_H_ +#define _STREAM_INC_H_ + +#include "executor.h" +#include "tstream.h" + #ifdef __cplusplus extern "C" { #endif -#ifndef _TSTREAM_H_ -#define _TSTREAM_H_ +int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb); +int32_t streamSink1(SStreamTask* pTask, SMsgCb* pMsgCb); #ifdef __cplusplus } #endif -#endif /* ifndef _TSTREAM_H_ */ +#endif /* ifndef _STREAM_INC_H_ */ diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c new file mode 100644 index 0000000000..d89f2ed57d --- /dev/null +++ b/source/libs/stream/src/stream.c @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "streamInc.h" + +int32_t streamTriggerByWrite(SStreamTask* pTask, int32_t vgId, SMsgCb* pMsgCb) { + int8_t execStatus = atomic_load_8(&pTask->status); + if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) { + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) return -1; + + // TODO: do we need htonl? + pRunReq->head.vgId = vgId; + pRunReq->streamId = pTask->streamId; + pRunReq->taskId = pTask->taskId; + SRpcMsg msg = { + .msgType = TDMT_VND_TASK_RUN, + .pCont = pRunReq, + .contLen = sizeof(SStreamTaskRunReq), + }; + tmsgPutToQueue(pMsgCb, FETCH_QUEUE, &msg); + } + return 0; +} + +#if 1 +int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp) { + SStreamDataBlock* pBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + int8_t status; + + // enqueue + if (pBlock != NULL) { + pBlock->type = STREAM_DATA_TYPE_SSDATA_BLOCK; + pBlock->sourceVg = pReq->sourceVg; + pBlock->blocks = pReq->data; + /*pBlock->sourceVer = pReq->sourceVer;*/ + if (streamTaskInput(pTask, (SStreamQueueItem*)pBlock) == 0) { + status = TASK_INPUT_STATUS__NORMAL; + } else { + status = TASK_INPUT_STATUS__FAILED; + } + } else { + streamTaskInputFail(pTask); + status = TASK_INPUT_STATUS__FAILED; + } + + // rsp by input status + SStreamDispatchRsp* pCont = rpcMallocCont(sizeof(SStreamDispatchRsp)); + pCont->inputStatus = status; + pCont->streamId = pReq->streamId; + pCont->taskId = pReq->sourceTaskId; + pRsp->pCont = pCont; + pRsp->contLen = sizeof(SStreamDispatchRsp); + tmsgSendRsp(pRsp); + return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; +} +#endif + +int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pRsp) { + // 1. handle input + streamTaskEnqueue(pTask, pReq, pRsp); + + // 2. try exec + // 2.1. idle: exec + // 2.2. executing: return + // 2.3. closing: keep trying + streamExec(pTask, pMsgCb); + + // 3. handle output + // 3.1 check and set status + // 3.2 dispatch / sink + streamSink1(pTask, pMsgCb); + + return 0; +} + +int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp) { + if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { + // TODO: init recover timer + } + // continue dispatch + streamSink1(pTask, pMsgCb); + return 0; +} + +int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb) { + streamExec(pTask, pMsgCb); + streamSink1(pTask, pMsgCb); + return 0; +} + +int32_t streamProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg) { + // + return 0; +} + +int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) { + // + return 0; +} diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c new file mode 100644 index 0000000000..95c0290058 --- /dev/null +++ b/source/libs/stream/src/streamData.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tstream.h" + +#if 0 +int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput) { + int32_t tlen = 0; + tlen += taosEncodeFixedI8(buf, pOutput->type); + tlen += taosEncodeFixedI32(buf, pOutput->sourceVg); + tlen += taosEncodeFixedI64(buf, pOutput->sourceVer); + ASSERT(pOutput->type == STREAM_INPUT__DATA_BLOCK); + tlen += tEncodeDataBlocks(buf, pOutput->blocks); + return tlen; +} + +void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) { + buf = taosDecodeFixedI8(buf, &pInput->type); + buf = taosDecodeFixedI32(buf, &pInput->sourceVg); + buf = taosDecodeFixedI64(buf, &pInput->sourceVer); + ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK); + buf = tDecodeDataBlocks(buf, &pInput->blocks); + return (void*)buf; +} +#endif + +SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) { + SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); + if (pDataSubmit == NULL) return NULL; + pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t)); + if (pDataSubmit->dataRef == NULL) goto FAIL; + pDataSubmit->data = pReq; + *pDataSubmit->dataRef = 1; + pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT; + return pDataSubmit; +FAIL: + taosFreeQitem(pDataSubmit); + return NULL; +} + +static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) { + // + atomic_add_fetch_32(pDataSubmit->dataRef, 1); +} + +SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit) { + SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); + if (pSubmitClone == NULL) { + return NULL; + } + streamDataSubmitRefInc(pSubmit); + memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit)); + return pSubmitClone; +} + +#if 0 +int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pReq->streamId); + tlen += taosEncodeFixedI32(buf, pReq->taskId); + tlen += tEncodeDataBlocks(buf, pReq->data); + return tlen; +} + +void* tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq) { + buf = taosDecodeFixedI64(buf, &pReq->streamId); + buf = taosDecodeFixedI32(buf, &pReq->taskId); + buf = tDecodeDataBlocks(buf, &pReq->data); + return (void*)buf; +} + +void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) { taosArrayDestroy(pReq->data); } +#endif diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c new file mode 100644 index 0000000000..4b88cf503e --- /dev/null +++ b/source/libs/stream/src/streamExec.c @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "executor.h" +#include "tstream.h" + +static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) { + void* exec = pTask->exec.executor; + + // set input + if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; + ASSERT(pSubmit->type == STREAM_INPUT__DATA_SUBMIT); + + qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false); + } else if (pTask->inputType == STREAM_INPUT__DATA_BLOCK) { + SStreamDataBlock* pBlock = (SStreamDataBlock*)data; + ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); + + SArray* blocks = pBlock->blocks; + qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false); + } + + // exec + while (1) { + SSDataBlock* output = NULL; + uint64_t ts = 0; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) break; + // TODO: do we need free memory? + SSDataBlock* outputCopy = createOneDataBlock(output, true); + taosArrayPush(pRes, outputCopy); + } + return 0; +} + +static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { + while (1) { + void* data = streamQueueNextItem(pTask->inputQueue); + if (data == NULL) break; + + streamTaskExecImpl(pTask, data, pRes); + + if (taosArrayGetSize(pRes) != 0) { + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + if (qRes == NULL) { + streamQueueProcessFail(pTask->inputQueue); + taosArrayDestroy(pRes); + return NULL; + } + qRes->type = STREAM_INPUT__DATA_BLOCK; + qRes->blocks = pRes; + if (streamTaskOutput(pTask, qRes) < 0) { + streamQueueProcessFail(pTask->inputQueue); + taosArrayDestroy(pRes); + taosFreeQitem(qRes); + return NULL; + } + + if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { + streamDataSubmitRefDec((SStreamDataSubmit*)data); + taosFreeQitem(data); + } else { + taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock); + taosFreeQitem(data); + } + streamQueueProcessSuccess(pTask->inputQueue); + return taosArrayInit(0, sizeof(SSDataBlock)); + } + } + return pRes; +} + +// TODO: handle version +int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) { + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) return -1; + while (1) { + int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); + if (execStatus == TASK_STATUS__IDLE) { + // first run + pRes = streamExecForQall(pTask, pRes); + if (pRes == NULL) goto FAIL; + + // set status closing + atomic_store_8(&pTask->status, TASK_STATUS__CLOSING); + + // second run, make sure inputQ and qall are cleared + pRes = streamExecForQall(pTask, pRes); + if (pRes == NULL) goto FAIL; + + break; + } else if (execStatus == TASK_STATUS__CLOSING) { + continue; + } else if (execStatus == TASK_STATUS__EXECUTING) { + break; + } else { + ASSERT(0); + } + } + if (pRes) taosArrayDestroy(pRes); + atomic_store_8(&pTask->status, TASK_STATUS__IDLE); + return 0; +FAIL: + if (pRes) taosArrayDestroy(pRes); + atomic_store_8(&pTask->status, TASK_STATUS__IDLE); + return -1; +} + diff --git a/source/libs/stream/src/streamMsg.c b/source/libs/stream/src/streamMsg.c new file mode 100644 index 0000000000..81f18fea8d --- /dev/null +++ b/source/libs/stream/src/streamMsg.c @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tstream.h" + +int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->sourceTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->sourceVg) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->sourceChildId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->blockNum) < 0) return -1; + ASSERT(taosArrayGetSize(pReq->data) == pReq->blockNum); + ASSERT(taosArrayGetSize(pReq->dataLen) == pReq->blockNum); + for (int32_t i = 0; i < pReq->blockNum; i++) { + int32_t len = *(int32_t*)taosArrayGet(pReq->dataLen, i); + void* data = taosArrayGetP(pReq->data, i); + if (tEncodeI32(pEncoder, len) < 0) return -1; + if (tEncodeBinary(pEncoder, data, len) < 0) return -1; + } + tEndEncode(pEncoder); + return 0; +} + +int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->sourceTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->sourceVg) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->sourceChildId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->blockNum) < 0) return -1; + ASSERT(pReq->blockNum > 0); + pReq->data = taosArrayInit(pReq->blockNum, sizeof(void*)); + pReq->dataLen = taosArrayInit(pReq->blockNum, sizeof(int32_t)); + for (int32_t i = 0; i < pReq->blockNum; i++) { + int32_t len1; + uint64_t len2; + void* data; + if (tDecodeI32(pDecoder, &len1) < 0) return -1; + if (tDecodeBinaryAlloc(pDecoder, &data, &len2) < 0) return -1; + ASSERT(len1 == len2); + taosArrayPush(pReq->dataLen, &len1); + taosArrayPush(pReq->data, &data); + } + tEndDecode(pDecoder); + return 0; +} + +int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { + SStreamDispatchReq req = { + .streamId = pTask->streamId, + .data = data, + }; + return 0; +} + +static int32_t streamBuildExecMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { + SStreamTaskExecReq req = { + .streamId = pTask->streamId, + .data = data, + }; + + int32_t tlen = sizeof(SMsgHead) + tEncodeSStreamTaskExecReq(NULL, &req); + void* buf = rpcMallocCont(tlen); + + if (buf == NULL) { + return -1; + } + + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + ((SMsgHead*)buf)->vgId = 0; + req.taskId = pTask->inplaceDispatcher.taskId; + + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + ((SMsgHead*)buf)->vgId = htonl(pTask->fixedEpDispatcher.nodeId); + *ppEpSet = &pTask->fixedEpDispatcher.epSet; + req.taskId = pTask->fixedEpDispatcher.taskId; + + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + // TODO use general name rule of schemaless + char ctbName[TSDB_TABLE_FNAME_LEN + 22] = {0}; + // all groupId must be the same in an array + SSDataBlock* pBlock = taosArrayGet(data, 0); + sprintf(ctbName, "%s:%ld", pTask->shuffleDispatcher.stbFullName, pBlock->info.groupId); + + // TODO: get hash function by hashMethod + + // get groupId, compute hash value + uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName)); + + // get node + // TODO: optimize search process + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t sz = taosArrayGetSize(vgInfo); + int32_t nodeId = 0; + for (int32_t i = 0; i < sz; i++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); + if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { + nodeId = pVgInfo->vgId; + req.taskId = pVgInfo->taskId; + *ppEpSet = &pVgInfo->epSet; + break; + } + } + ASSERT(nodeId != 0); + ((SMsgHead*)buf)->vgId = htonl(nodeId); + } + + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncodeSStreamTaskExecReq(&abuf, &req); + + pMsg->pCont = buf; + pMsg->contLen = tlen; + pMsg->code = 0; + pMsg->msgType = pTask->dispatchMsgType; + pMsg->info.noResp = 1; + + return 0; +} + +static int32_t streamShuffleDispatch(SStreamTask* pTask, SMsgCb* pMsgCb, SHashObj* data) { + void* pIter = NULL; + while (1) { + pIter = taosHashIterate(data, pIter); + if (pIter == NULL) return 0; + SArray* pData = *(SArray**)pIter; + SRpcMsg dispatchMsg = {0}; + SEpSet* pEpSet; + if (streamBuildExecMsg(pTask, pData, &dispatchMsg, &pEpSet) < 0) { + ASSERT(0); + return -1; + } + tmsgSendReq(pEpSet, &dispatchMsg); + } + return 0; +} diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c new file mode 100644 index 0000000000..f82ef1b42f --- /dev/null +++ b/source/libs/stream/src/streamQueue.c @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tstream.h" + +SStreamQueue* streamQueueOpen() { + SStreamQueue* pQueue = taosMemoryCalloc(1, sizeof(SStreamQueue)); + if (pQueue == NULL) return NULL; + pQueue->queue = taosOpenQueue(); + pQueue->qall = taosAllocateQall(); + if (pQueue->queue == NULL || pQueue->qall == NULL) { + goto FAIL; + } + pQueue->status = STREAM_QUEUE__SUCESS; + return pQueue; +FAIL: + if (pQueue->queue) taosCloseQueue(pQueue->queue); + if (pQueue->qall) taosFreeQall(pQueue->qall); + taosMemoryFree(pQueue); + return NULL; +} + +void streamQueueClose(SStreamQueue* queue) { + while (1) { + void* qItem = streamQueueNextItem(queue); + if (qItem) + taosFreeQitem(qItem); + else + return; + } +} diff --git a/source/libs/stream/src/streamSink.c b/source/libs/stream/src/streamSink.c new file mode 100644 index 0000000000..6acdd1064c --- /dev/null +++ b/source/libs/stream/src/streamSink.c @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "executor.h" +#include "tstream.h" + +int32_t streamSink1(SStreamTask* pTask, SMsgCb* pMsgCb) { + SStreamQueue* queue; + if (pTask->execType == TASK_EXEC__NONE) { + queue = pTask->inputQueue; + } else { + queue = pTask->outputQueue; + } + /*if (streamDequeueBegin(queue) == true) {*/ + /*return -1;*/ + /*}*/ + + if (pTask->sinkType == TASK_SINK__TABLE || pTask->sinkType == TASK_SINK__SMA) { + ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); + while (1) { + SStreamDataBlock* pBlock = streamQueueNextItem(queue); + if (pBlock == NULL) break; + ASSERT(pBlock->type == STREAM_DATA_TYPE_SSDATA_BLOCK); + + // local sink + if (pTask->sinkType == TASK_SINK__TABLE) { + pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks); + } else if (pTask->sinkType == TASK_SINK__SMA) { + pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pBlock->blocks); + } + + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + ASSERT(queue == pTask->outputQueue); + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + ASSERT(queue == pTask->outputQueue); + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + ASSERT(queue == pTask->outputQueue); + } + + streamQueueProcessSuccess(queue); + } + } + + return 0; +} + +#if 0 +int32_t streamSink(SStreamTask* pTask, SMsgCb* pMsgCb) { + bool firstRun = 1; + while (1) { + SStreamDataBlock* pBlock = NULL; + if (!firstRun) { + taosReadAllQitems(pTask->outputQ, pTask->outputQAll); + } + taosGetQitem(pTask->outputQAll, (void**)&pBlock); + if (pBlock == NULL) { + if (firstRun) { + firstRun = 0; + continue; + } else { + break; + } + } + + SArray* pRes = pBlock->blocks; + + // sink + if (pTask->sinkType == TASK_SINK__TABLE) { + // blockDebugShowData(pRes); + pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pRes); + } else if (pTask->sinkType == TASK_SINK__SMA) { + pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes); + // + } else if (pTask->sinkType == TASK_SINK__FETCH) { + // + } else { + ASSERT(pTask->sinkType == TASK_SINK__NONE); + } + + // dispatch + // TODO dispatch guard + int8_t outputStatus = atomic_load_8(&pTask->outputStatus); + if (outputStatus == TASK_OUTPUT_STATUS__NORMAL) { + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + SRpcMsg dispatchMsg = {0}; + if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, NULL) < 0) { + ASSERT(0); + return -1; + } + + int32_t qType; + if (pTask->dispatchMsgType == TDMT_VND_TASK_DISPATCH || pTask->dispatchMsgType == TDMT_SND_TASK_DISPATCH) { + qType = FETCH_QUEUE; + /*} else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC ||*/ + /*pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) {*/ + /*qType = MERGE_QUEUE;*/ + /*} else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) {*/ + /*qType = WRITE_QUEUE;*/ + } else { + ASSERT(0); + } + tmsgPutToQueue(pMsgCb, qType, &dispatchMsg); + + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + SRpcMsg dispatchMsg = {0}; + SEpSet* pEpSet = NULL; + if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) { + ASSERT(0); + return -1; + } + + tmsgSendReq(pEpSet, &dispatchMsg); + + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + SHashObj* pShuffleRes = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (pShuffleRes == NULL) { + return -1; + } + + int32_t sz = taosArrayGetSize(pRes); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pRes, i); + SArray* pArray = taosHashGet(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t)); + if (pArray == NULL) { + pArray = taosArrayInit(0, sizeof(SSDataBlock)); + if (pArray == NULL) { + return -1; + } + taosHashPut(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t), &pArray, sizeof(void*)); + } + taosArrayPush(pArray, pDataBlock); + } + + if (streamShuffleDispatch(pTask, pMsgCb, pShuffleRes) < 0) { + return -1; + } + + } else { + ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); + } + } + } + return 0; +} +#endif diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c new file mode 100644 index 0000000000..890ae1f3b5 --- /dev/null +++ b/source/libs/stream/src/streamTask.c @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "executor.h" +#include "tstream.h" + +SStreamTask* tNewSStreamTask(int64_t streamId, int32_t childId) { + SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); + if (pTask == NULL) { + return NULL; + } + pTask->taskId = tGenIdPI32(); + pTask->streamId = streamId; + pTask->childId = childId; + pTask->status = TASK_STATUS__IDLE; + pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; + pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; + + return pTask; +} + +int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { + /*if (tStartEncode(pEncoder) < 0) return -1;*/ + if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1; + if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; + + if (tEncodeI32(pEncoder, pTask->childId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; + + if (pTask->execType != TASK_EXEC__NONE) { + if (tEncodeI8(pEncoder, pTask->exec.parallelizable) < 0) return -1; + if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1; + } + + if (pTask->sinkType == TASK_SINK__TABLE) { + if (tEncodeI64(pEncoder, pTask->tbSink.stbUid) < 0) return -1; + if (tEncodeCStr(pEncoder, pTask->tbSink.stbFullName) < 0) return -1; + if (tEncodeSSchemaWrapper(pEncoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; + } else if (pTask->sinkType == TASK_SINK__SMA) { + if (tEncodeI64(pEncoder, pTask->smaSink.smaId) < 0) return -1; + } else if (pTask->sinkType == TASK_SINK__FETCH) { + if (tEncodeI8(pEncoder, pTask->fetchSink.reserved) < 0) return -1; + } else { + ASSERT(pTask->sinkType == TASK_SINK__NONE); + } + + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + if (tEncodeI32(pEncoder, pTask->inplaceDispatcher.taskId) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + if (tSerializeSUseDbRspImp(pEncoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1; + /*if (tEncodeI8(pEncoder, pTask->shuffleDispatcher.hashMethod) < 0) return -1;*/ + } + + /*tEndEncode(pEncoder);*/ + return pEncoder->pos; +} + +int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { + /*if (tStartDecode(pDecoder) < 0) return -1;*/ + if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1; + if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; + + if (tDecodeI32(pDecoder, &pTask->childId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; + + if (pTask->execType != TASK_EXEC__NONE) { + if (tDecodeI8(pDecoder, &pTask->exec.parallelizable) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1; + } + + if (pTask->sinkType == TASK_SINK__TABLE) { + if (tDecodeI64(pDecoder, &pTask->tbSink.stbUid) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pTask->tbSink.stbFullName) < 0) return -1; + pTask->tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); + if (pTask->tbSink.pSchemaWrapper == NULL) return -1; + if (tDecodeSSchemaWrapper(pDecoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; + } else if (pTask->sinkType == TASK_SINK__SMA) { + if (tDecodeI64(pDecoder, &pTask->smaSink.smaId) < 0) return -1; + } else if (pTask->sinkType == TASK_SINK__FETCH) { + if (tDecodeI8(pDecoder, &pTask->fetchSink.reserved) < 0) return -1; + } else { + ASSERT(pTask->sinkType == TASK_SINK__NONE); + } + + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + if (tDecodeI32(pDecoder, &pTask->inplaceDispatcher.taskId) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + /*if (tDecodeI8(pDecoder, &pTask->shuffleDispatcher.hashMethod) < 0) return -1;*/ + if (tDeserializeSUseDbRspImp(pDecoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1; + } + + /*tEndDecode(pDecoder);*/ + return 0; +} + +void tFreeSStreamTask(SStreamTask* pTask) { + streamQueueClose(pTask->inputQueue); + streamQueueClose(pTask->outputQueue); + if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg); + qDestroyTask(pTask->exec.executor); + taosMemoryFree(pTask); +} diff --git a/source/libs/stream/src/tstreamUpdate.c b/source/libs/stream/src/streamUpdate.c similarity index 100% rename from source/libs/stream/src/tstreamUpdate.c rename to source/libs/stream/src/streamUpdate.c diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c deleted file mode 100644 index 7d406a7144..0000000000 --- a/source/libs/stream/src/tstream.c +++ /dev/null @@ -1,594 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tstream.h" -#include "executor.h" - -int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput) { - int32_t tlen = 0; - tlen += taosEncodeFixedI8(buf, pOutput->type); - tlen += taosEncodeFixedI32(buf, pOutput->sourceVg); - tlen += taosEncodeFixedI64(buf, pOutput->sourceVer); - ASSERT(pOutput->type == STREAM_INPUT__DATA_BLOCK); - tlen += tEncodeDataBlocks(buf, pOutput->blocks); - return tlen; -} - -void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) { - buf = taosDecodeFixedI8(buf, &pInput->type); - buf = taosDecodeFixedI32(buf, &pInput->sourceVg); - buf = taosDecodeFixedI64(buf, &pInput->sourceVer); - ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK); - buf = tDecodeDataBlocks(buf, &pInput->blocks); - return (void*)buf; -} - -SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit) { - SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); - if (pSubmitClone == NULL) { - return NULL; - } - streamDataSubmitRefInc(pSubmit); - memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit)); - return pSubmitClone; -} - -static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { - SStreamDispatchReq req = { - .streamId = pTask->streamId, - .data = data, - }; - return 0; -} - -static int32_t streamBuildExecMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { - SStreamTaskExecReq req = { - .streamId = pTask->streamId, - .data = data, - }; - - int32_t tlen = sizeof(SMsgHead) + tEncodeSStreamTaskExecReq(NULL, &req); - void* buf = rpcMallocCont(tlen); - - if (buf == NULL) { - return -1; - } - - if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { - ((SMsgHead*)buf)->vgId = 0; - req.taskId = pTask->inplaceDispatcher.taskId; - - } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { - ((SMsgHead*)buf)->vgId = htonl(pTask->fixedEpDispatcher.nodeId); - *ppEpSet = &pTask->fixedEpDispatcher.epSet; - req.taskId = pTask->fixedEpDispatcher.taskId; - - } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { - // TODO use general name rule of schemaless - char ctbName[TSDB_TABLE_FNAME_LEN + 22] = {0}; - // all groupId must be the same in an array - SSDataBlock* pBlock = taosArrayGet(data, 0); - sprintf(ctbName, "%s:%ld", pTask->shuffleDispatcher.stbFullName, pBlock->info.groupId); - - // TODO: get hash function by hashMethod - - // get groupId, compute hash value - uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName)); - - // get node - // TODO: optimize search process - SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t sz = taosArrayGetSize(vgInfo); - int32_t nodeId = 0; - for (int32_t i = 0; i < sz; i++) { - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { - nodeId = pVgInfo->vgId; - req.taskId = pVgInfo->taskId; - *ppEpSet = &pVgInfo->epSet; - break; - } - } - ASSERT(nodeId != 0); - ((SMsgHead*)buf)->vgId = htonl(nodeId); - } - - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncodeSStreamTaskExecReq(&abuf, &req); - - pMsg->pCont = buf; - pMsg->contLen = tlen; - pMsg->code = 0; - pMsg->msgType = pTask->dispatchMsgType; - pMsg->info.noResp = 1; - - return 0; -} - -static int32_t streamShuffleDispatch(SStreamTask* pTask, SMsgCb* pMsgCb, SHashObj* data) { - void* pIter = NULL; - while (1) { - pIter = taosHashIterate(data, pIter); - if (pIter == NULL) return 0; - SArray* pData = *(SArray**)pIter; - SRpcMsg dispatchMsg = {0}; - SEpSet* pEpSet; - if (streamBuildExecMsg(pTask, pData, &dispatchMsg, &pEpSet) < 0) { - ASSERT(0); - return -1; - } - tmsgSendReq(pEpSet, &dispatchMsg); - } - return 0; -} - -int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input) { - ASSERT(pTask->inputType == TASK_INPUT_TYPE__SUMBIT_BLOCK); - int8_t inputStatus = atomic_load_8(&pTask->inputStatus); - if (inputStatus == TASK_INPUT_STATUS__NORMAL) { - streamDataSubmitRefInc(input); - taosWriteQitem(pTask->inputQ, input); - } - return inputStatus; -} - -int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) { - ASSERT(pTask->inputType == TASK_INPUT_TYPE__DATA_BLOCK); - taosWriteQitem(pTask->inputQ, input); - int8_t inputStatus = atomic_load_8(&pTask->inputStatus); - return inputStatus; -} - -static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) { - void* exec = pTask->exec.executor; - - // set input - if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; - ASSERT(pSubmit->type == STREAM_INPUT__DATA_SUBMIT); - - qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false); - } else if (pTask->inputType == STREAM_INPUT__DATA_BLOCK) { - SStreamDataBlock* pBlock = (SStreamDataBlock*)data; - ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - - SArray* blocks = pBlock->blocks; - qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false); - } - - // exec - while (1) { - SSDataBlock* output = NULL; - uint64_t ts = 0; - if (qExecTask(exec, &output, &ts) < 0) { - ASSERT(false); - } - if (output == NULL) break; - // TODO: do we need free memory? - SSDataBlock* outputCopy = createOneDataBlock(output, true); - taosArrayPush(pRes, outputCopy); - } - - // destroy - if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { - streamDataSubmitRefDec((SStreamDataSubmit*)data); - taosFreeQitem(data); - } else { - taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock); - taosFreeQitem(data); - } - return 0; -} - -static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { - while (1) { - void* data = NULL; - taosGetQitem(pTask->inputQAll, &data); - if (data == NULL) break; - - streamTaskExecImpl(pTask, data, pRes); - - if (taosArrayGetSize(pRes) != 0) { - SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); - qRes->type = STREAM_INPUT__DATA_BLOCK; - qRes->blocks = pRes; - taosWriteQitem(pTask->outputQ, qRes); - return taosArrayInit(0, sizeof(SSDataBlock)); - } - } - return pRes; -} - -// TODO: handle version -int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) { - SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - if (pRes == NULL) return -1; - while (1) { - int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); - if (execStatus == TASK_STATUS__IDLE) { - // first run, from qall, handle failure from last exec - pRes = streamExecForQall(pTask, pRes); - if (pRes == NULL) goto FAIL; - - // second run, from inputQ - taosReadAllQitems(pTask->inputQ, pTask->inputQAll); - pRes = streamExecForQall(pTask, pRes); - if (pRes == NULL) goto FAIL; - - // set status closing - atomic_store_8(&pTask->status, TASK_STATUS__CLOSING); - - // third run, make sure inputQ and qall are cleared - taosReadAllQitems(pTask->inputQ, pTask->inputQAll); - pRes = streamExecForQall(pTask, pRes); - if (pRes == NULL) goto FAIL; - - atomic_store_8(&pTask->status, TASK_STATUS__IDLE); - break; - } else if (execStatus == TASK_STATUS__CLOSING) { - continue; - } else if (execStatus == TASK_STATUS__EXECUTING) { - break; - } else { - ASSERT(0); - } - } - return 0; -FAIL: - atomic_store_8(&pTask->status, TASK_STATUS__IDLE); - return -1; -} - -int32_t streamSink(SStreamTask* pTask, SMsgCb* pMsgCb) { - bool firstRun = 1; - while (1) { - SStreamDataBlock* pBlock = NULL; - if (!firstRun) { - taosReadAllQitems(pTask->outputQ, pTask->outputQAll); - } - taosGetQitem(pTask->outputQAll, (void**)&pBlock); - if (pBlock == NULL) { - if (firstRun) { - firstRun = 0; - continue; - } else { - break; - } - } - - SArray* pRes = pBlock->blocks; - - // sink - if (pTask->sinkType == TASK_SINK__TABLE) { - // blockDebugShowData(pRes); - pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pRes); - } else if (pTask->sinkType == TASK_SINK__SMA) { - pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes); - // - } else if (pTask->sinkType == TASK_SINK__FETCH) { - // - } else { - ASSERT(pTask->sinkType == TASK_SINK__NONE); - } - - // dispatch - // TODO dispatch guard - int8_t outputStatus = atomic_load_8(&pTask->outputStatus); - if (outputStatus == TASK_OUTPUT_STATUS__NORMAL) { - if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { - SRpcMsg dispatchMsg = {0}; - if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, NULL) < 0) { - ASSERT(0); - return -1; - } - - int32_t qType; - if (pTask->dispatchMsgType == TDMT_VND_TASK_DISPATCH || pTask->dispatchMsgType == TDMT_SND_TASK_DISPATCH) { - qType = FETCH_QUEUE; - /*} else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC ||*/ - /*pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) {*/ - /*qType = MERGE_QUEUE;*/ - /*} else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) {*/ - /*qType = WRITE_QUEUE;*/ - } else { - ASSERT(0); - } - tmsgPutToQueue(pMsgCb, qType, &dispatchMsg); - - } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { - SRpcMsg dispatchMsg = {0}; - SEpSet* pEpSet = NULL; - if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) { - ASSERT(0); - return -1; - } - - tmsgSendReq(pEpSet, &dispatchMsg); - - } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { - SHashObj* pShuffleRes = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - if (pShuffleRes == NULL) { - return -1; - } - - int32_t sz = taosArrayGetSize(pRes); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pRes, i); - SArray* pArray = taosHashGet(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t)); - if (pArray == NULL) { - pArray = taosArrayInit(0, sizeof(SSDataBlock)); - if (pArray == NULL) { - return -1; - } - taosHashPut(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t), &pArray, sizeof(void*)); - } - taosArrayPush(pArray, pDataBlock); - } - - if (streamShuffleDispatch(pTask, pMsgCb, pShuffleRes) < 0) { - return -1; - } - - } else { - ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); - } - } - } - return 0; -} - -int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp) { - SStreamDataBlock* pBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); - int8_t status; - - // 1.1 update status - // TODO cal backpressure - if (pBlock == NULL) { - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); - status = TASK_INPUT_STATUS__FAILED; - } else { - status = atomic_load_8(&pTask->inputStatus); - } - - // 1.2 enqueue - pBlock->type = STREAM_DATA_TYPE_SSDATA_BLOCK; - pBlock->sourceVg = pReq->sourceVg; - /*pBlock->sourceVer = pReq->sourceVer;*/ - taosWriteQitem(pTask->inputQ, pBlock); - - // 1.3 rsp by input status - SStreamDispatchRsp* pCont = rpcMallocCont(sizeof(SStreamDispatchRsp)); - pCont->inputStatus = status; - pCont->streamId = pReq->streamId; - pCont->taskId = pReq->sourceTaskId; - pRsp->pCont = pCont; - pRsp->contLen = sizeof(SStreamDispatchRsp); - tmsgSendRsp(pRsp); - - return 0; -} - -int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pRsp) { - // 1. handle input - streamTaskEnqueue(pTask, pReq, pRsp); - - // 2. try exec - // 2.1. idle: exec - // 2.2. executing: return - // 2.3. closing: keep trying - streamExec(pTask, pMsgCb); - - // 3. handle output - // 3.1 check and set status - // 3.2 dispatch / sink - streamSink(pTask, pMsgCb); - - return 0; -} - -int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp) { - atomic_store_8(&pTask->inputStatus, pRsp->inputStatus); - if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { - // TODO: init recover timer - } - // continue dispatch - streamSink(pTask, pMsgCb); - return 0; -} - -int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb) { - streamExec(pTask, pMsgCb); - streamSink(pTask, pMsgCb); - return 0; -} - -int32_t streamProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg) { - // - return 0; -} - -int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) { - // - return 0; -} - -int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* pReq) { - if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->sourceTaskId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->sourceVg) < 0) return -1; - tEndEncode(pEncoder); - return 0; -} - -int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { - if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->sourceTaskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->sourceVg) < 0) return -1; - tEndDecode(pDecoder); - return 0; -} - -int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->streamId); - tlen += taosEncodeFixedI32(buf, pReq->taskId); - tlen += tEncodeDataBlocks(buf, pReq->data); - return tlen; -} - -void* tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq) { - buf = taosDecodeFixedI64(buf, &pReq->streamId); - buf = taosDecodeFixedI32(buf, &pReq->taskId); - buf = tDecodeDataBlocks(buf, &pReq->data); - return (void*)buf; -} - -void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) { taosArrayDestroy(pReq->data); } - -SStreamTask* tNewSStreamTask(int64_t streamId) { - SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); - if (pTask == NULL) { - return NULL; - } - pTask->taskId = tGenIdPI32(); - pTask->streamId = streamId; - pTask->status = TASK_STATUS__IDLE; - - return pTask; -} - -int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { - /*if (tStartEncode(pEncoder) < 0) return -1;*/ - if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1; - if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; - - if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; - if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; - - if (pTask->execType != TASK_EXEC__NONE) { - if (tEncodeI8(pEncoder, pTask->exec.parallelizable) < 0) return -1; - if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1; - } - - if (pTask->sinkType == TASK_SINK__TABLE) { - if (tEncodeI64(pEncoder, pTask->tbSink.stbUid) < 0) return -1; - if (tEncodeCStr(pEncoder, pTask->tbSink.stbFullName) < 0) return -1; - if (tEncodeSSchemaWrapper(pEncoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; - } else if (pTask->sinkType == TASK_SINK__SMA) { - if (tEncodeI64(pEncoder, pTask->smaSink.smaId) < 0) return -1; - } else if (pTask->sinkType == TASK_SINK__FETCH) { - if (tEncodeI8(pEncoder, pTask->fetchSink.reserved) < 0) return -1; - } else { - ASSERT(pTask->sinkType == TASK_SINK__NONE); - } - - if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { - if (tEncodeI32(pEncoder, pTask->inplaceDispatcher.taskId) < 0) return -1; - } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { - if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.taskId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1; - if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; - } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { - if (tSerializeSUseDbRspImp(pEncoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1; - /*if (tEncodeI8(pEncoder, pTask->shuffleDispatcher.hashMethod) < 0) return -1;*/ - } - - /*tEndEncode(pEncoder);*/ - return pEncoder->pos; -} - -int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { - /*if (tStartDecode(pDecoder) < 0) return -1;*/ - if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1; - if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; - - if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; - if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; - - if (pTask->execType != TASK_EXEC__NONE) { - if (tDecodeI8(pDecoder, &pTask->exec.parallelizable) < 0) return -1; - if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1; - } - - if (pTask->sinkType == TASK_SINK__TABLE) { - if (tDecodeI64(pDecoder, &pTask->tbSink.stbUid) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pTask->tbSink.stbFullName) < 0) return -1; - pTask->tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); - if (pTask->tbSink.pSchemaWrapper == NULL) return -1; - if (tDecodeSSchemaWrapper(pDecoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; - } else if (pTask->sinkType == TASK_SINK__SMA) { - if (tDecodeI64(pDecoder, &pTask->smaSink.smaId) < 0) return -1; - } else if (pTask->sinkType == TASK_SINK__FETCH) { - if (tDecodeI8(pDecoder, &pTask->fetchSink.reserved) < 0) return -1; - } else { - ASSERT(pTask->sinkType == TASK_SINK__NONE); - } - - if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { - if (tDecodeI32(pDecoder, &pTask->inplaceDispatcher.taskId) < 0) return -1; - } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { - if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.taskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1; - if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; - } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { - /*if (tDecodeI8(pDecoder, &pTask->shuffleDispatcher.hashMethod) < 0) return -1;*/ - if (tDeserializeSUseDbRspImp(pDecoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1; - } - - /*tEndDecode(pDecoder);*/ - return 0; -} - -void tFreeSStreamTask(SStreamTask* pTask) { - taosCloseQueue(pTask->inputQ); - taosCloseQueue(pTask->outputQ); - // TODO - if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg); - qDestroyTask(pTask->exec.executor); - taosMemoryFree(pTask); -} - -#if 0 -int32_t tEncodeSStreamTaskExecReq(SCoder* pEncoder, const SStreamTaskExecReq* pReq) { - if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; - /*if (tEncodeDataBlocks(buf, pReq->streamId) < 0) return -1;*/ - return pEncoder->size; -} -int32_t tDecodeSStreamTaskExecReq(SCoder* pDecoder, SStreamTaskExecReq* pReq) { - return pEncoder->size; -} -void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) { - taosArrayDestroyEx(pReq->data, tDeleteSSDataBlock); -} -#endif diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index b660927899..11a1cc1c71 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -222,11 +222,6 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) { return compareLenPrefixedWStr(pRight, pLeft); } -int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) { - if(pLeft) return 0; - return 1; -} - // string > number > bool > null // ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison int32_t compareJsonVal(const void *pLeft, const void *pRight) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 0b1de5dd89..c14c279d81 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -222,17 +222,19 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_INDEX_NOT_EXIST, "Index not exist") // mnode-vgroup TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, "Vgroup already in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, "Vgroup not in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "Vgroup does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_UN_CHANGED, "Vgroup distribution has not changed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_HAS_OFFLINE_DNODE, "Offline dnode exists") // mnode-stable -TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "Stable already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_NOT_EXIST, "Stable not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC, "Stable confilct with topic") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "STable already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_NOT_EXIST, "STable not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC, "STable confilct with topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_STBS, "Too many stables") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB, "Invalid stable name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_OPTION, "Invalid stable options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_ALTER_OPTION, "Invalid stable alter options") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED, "Stable option unchanged") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED, "STable option unchanged") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ROW_BYTES, "Invalid row bytes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREADY_EXIST, "Tag already exists") @@ -240,6 +242,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, "Tag does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC,"Field used by topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SINGLE_STB_MODE_DB, "Database is single stable mode") // mnode-infoSchema diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 37935087fa..94311bc435 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -34,11 +34,11 @@ typedef struct STaosQnode { } STaosQnode; typedef struct STaosQueue { - STaosQnode * head; - STaosQnode * tail; - STaosQueue * next; // for queue set - STaosQset * qset; // for queue set - void * ahandle; // for queue set + STaosQnode *head; + STaosQnode *tail; + STaosQueue *next; // for queue set + STaosQset *qset; // for queue set + void *ahandle; // for queue set FItem itemFp; FItems itemsFp; TdThreadMutex mutex; @@ -47,8 +47,8 @@ typedef struct STaosQueue { } STaosQueue; typedef struct STaosQset { - STaosQueue * head; - STaosQueue * current; + STaosQueue *head; + STaosQueue *current; TdThreadMutex mutex; tsem_t sem; int32_t numOfQueues; @@ -86,7 +86,7 @@ void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp) { void taosCloseQueue(STaosQueue *queue) { if (queue == NULL) return; STaosQnode *pTemp; - STaosQset * qset; + STaosQset *qset; taosThreadMutexLock(&queue->mutex); STaosQnode *pNode = queue->head; @@ -282,6 +282,8 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) { *ppItem = pNode->item; num = 1; uTrace("item:%p is fetched", *ppItem); + } else { + *ppItem = NULL; } return num; diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index b6ffda5c86..63b841cfc5 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -388,14 +388,14 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) else: - self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)\ntdDnodes.startWithoutSleep(%d)"%(self.index, self.index)) + self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].deployed=1\ntdDnodes.dnodes[%d].logDir=\"%%s/sim/dnode%%d/log\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.dnodes[%d].cfgDir=\"%%s/sim/dnode%%d/cfg\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.startWithoutSleep(%d)"%(self.index-1,self.index-1,self.index-1,self.index,self.index-1,self.index-1,self.index,self.index)) self.running = 1 tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) def stop(self): if (not self.remoteIP == ""): - self.remoteExec(self.cfgDict, "tdDnodes.stop(%d)"%self.index) + self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1)) tdLog.info("stop dnode%d"%self.index) return if self.valgrind == 0: @@ -426,7 +426,7 @@ class TDDnode: def forcestop(self): if (not self.remoteIP == ""): - self.remoteExec(self.cfgDict, "tdDnodes.forcestop(%d)"%self.index) + self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].forcestop()"%(self.index-1,self.index-1)) return if self.valgrind == 0: toBeKilled = "taosd" @@ -497,46 +497,12 @@ class TDDnodes: self.killValgrind = 1 def init(self, path, remoteIP = ""): - psCmd = "ps -ef|grep -w taosd| grep -v grep| grep -v defunct | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -9 %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - if self.killValgrind == 1: - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -9 %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - binPath = self.dnodes[0].getPath() + "/../../../" # tdLog.debug("binPath %s" % (binPath)) binPath = os.path.realpath(binPath) # tdLog.debug("binPath real path %s" % (binPath)) - # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) - # tdLog.debug(cmd) - # os.system(cmd) - - # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - if path == "": - # self.path = os.path.expanduser('~') self.path = os.path.abspath(binPath + "../../") else: self.path = os.path.realpath(path) diff --git a/tests/script/tsim/dnode/basic1.sim b/tests/script/tsim/dnode/basic1.sim index a5b5427e03..8a28897d5a 100644 --- a/tests/script/tsim/dnode/basic1.sim +++ b/tests/script/tsim/dnode/basic1.sim @@ -183,15 +183,15 @@ if $rows != 15 then endi print =============== drop dnode -sql drop dnode 2; -sql show dnodes; -if $rows != 1 then - return -1 -endi +#sql drop dnode 2; +#sql show dnodes; +#if $rows != 1 then +# return -1 +#endi -if $data00 != 1 then - return -1 -endi +#if $data00 != 1 then +# return -1 +#endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/stable/column_drop.sim b/tests/script/tsim/stable/column_drop.sim index 3401465103..63ac44eccd 100644 --- a/tests/script/tsim/stable/column_drop.sim +++ b/tests/script/tsim/stable/column_drop.sim @@ -118,6 +118,7 @@ if $data[0][5] != 5 then return -1 endi if $data[0][6] != 101 then + print expect 101, actual: $data06 return -1 endi diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index c73efde9d5..1f258028cc 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -7,57 +7,57 @@ #run tsim/table/basic1.sim #run tsim/trans/lossdata1.sim #run tsim/trans/create_db.sim -##run tsim/stable/alter_metrics.sim -##run tsim/stable/tag_modify.sim -##run tsim/stable/alter_comment.sim -##run tsim/stable/column_drop.sim -##run tsim/stable/column_modify.sim -##run tsim/stable/tag_rename.sim +#run tsim/stable/alter_metrics.sim +#run tsim/stable/tag_modify.sim +#run tsim/stable/alter_comment.sim +#run tsim/stable/column_drop.sim +#run tsim/stable/column_modify.sim +#run tsim/stable/tag_rename.sim #run tsim/stable/vnode3.sim #run tsim/stable/metrics.sim -##run tsim/stable/alter_insert2.sim +#run tsim/stable/alter_insert2.sim #run tsim/stable/show.sim #run tsim/stable/alter_import.sim -##run tsim/stable/tag_add.sim +#run tsim/stable/tag_add.sim #run tsim/stable/tag_drop.sim #run tsim/stable/column_add.sim #run tsim/stable/alter_count.sim #run tsim/stable/values.sim -#run tsim/stable/dnode3.sim +run tsim/stable/dnode3.sim #run tsim/stable/alter_insert1.sim #run tsim/stable/refcount.sim #run tsim/stable/disk.sim -#run tsim/db/basic1.sim -#run tsim/db/basic3.sim -##run tsim/db/basic7.sim -#run tsim/db/basic6.sim -#run tsim/db/create_all_options.sim -#run tsim/db/basic2.sim -#run tsim/db/error1.sim -#run tsim/db/taosdlog.sim -#run tsim/db/alter_option.sim -#run tsim/mnode/basic1.sim -#run tsim/mnode/basic3.sim -#run tsim/mnode/basic2.sim -#run tsim/parser/fourArithmetic-basic.sim -#run tsim/parser/groupby-basic.sim -#run tsim/snode/basic1.sim -#run tsim/query/time_process.sim -#run tsim/query/stddev.sim -#run tsim/query/interval-offset.sim -#run tsim/query/charScalarFunction.sim -#run tsim/query/complex_select.sim -#run tsim/query/explain.sim -#run tsim/query/crash_sql.sim -#run tsim/query/diff.sim -#run tsim/query/complex_limit.sim -#run tsim/query/complex_having.sim -#run tsim/query/udf.sim -#run tsim/query/complex_group.sim -#run tsim/query/interval.sim -#run tsim/query/session.sim -#run tsim/query/scalarFunction.sim -##run tsim/query/scalarNull.sim +run tsim/db/basic1.sim +run tsim/db/basic3.sim +#run tsim/db/basic7.sim +run tsim/db/basic6.sim +run tsim/db/create_all_options.sim +run tsim/db/basic2.sim +run tsim/db/error1.sim +run tsim/db/taosdlog.sim +run tsim/db/alter_option.sim +run tsim/mnode/basic1.sim +run tsim/mnode/basic3.sim +run tsim/mnode/basic2.sim +run tsim/parser/fourArithmetic-basic.sim +run tsim/parser/groupby-basic.sim +run tsim/snode/basic1.sim +run tsim/query/time_process.sim +run tsim/query/stddev.sim +run tsim/query/interval-offset.sim +run tsim/query/charScalarFunction.sim +run tsim/query/complex_select.sim +run tsim/query/explain.sim +run tsim/query/crash_sql.sim +run tsim/query/diff.sim +run tsim/query/complex_limit.sim +run tsim/query/complex_having.sim +run tsim/query/udf.sim +run tsim/query/complex_group.sim +run tsim/query/interval.sim +run tsim/query/session.sim +run tsim/query/scalarFunction.sim +#run tsim/query/scalarNull.sim run tsim/query/complex_where.sim run tsim/tmq/basic1.sim run tsim/tmq/basic4.sim diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py index 3c99ddb8d6..22c9c8c0c5 100644 --- a/tests/system-test/0-others/taosShellNetChk.py +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -187,50 +187,51 @@ class TDTestCase: # stop taosd tdDnodes.stop(1) - role = 'server' - if platform.system().lower() == 'windows': - taosCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\taos.exe -c ' + keyDict['c'] - taosCmd = taosCmd.replace('\\','\\\\') - taosCmd = taosCmd + ' -n ' + role - else: - taosCmd = 'nohup ' + buildPath + '/build/bin/taos -c ' + keyDict['c'] - taosCmd = taosCmd + ' -n ' + role + ' > /dev/null 2>&1 &' - print (taosCmd) - os.system(taosCmd) + try: + role = 'server' + if platform.system().lower() == 'windows': + taosCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\taos.exe -c ' + keyDict['c'] + taosCmd = taosCmd.replace('\\','\\\\') + taosCmd = taosCmd + ' -n ' + role + else: + taosCmd = 'nohup ' + buildPath + '/build/bin/taos -c ' + keyDict['c'] + taosCmd = taosCmd + ' -n ' + role + ' > /dev/null 2>&1 &' + print (taosCmd) + os.system(taosCmd) - pktLen = '2000' - pktNum = '10' - role = 'client' - if platform.system().lower() == 'windows': - taosCmd = buildPath + '\\build\\bin\\taos.exe -h 127.0.0.1 -c ' + keyDict['c'] - taosCmd = taosCmd.replace('\\','\\\\') - else: - taosCmd = buildPath + '/build/bin/taos -c ' + keyDict['c'] - taosCmd = taosCmd + ' -n ' + role + ' -l ' + pktLen + ' -N ' + pktNum - print (taosCmd) - child = taosExpect.spawn(taosCmd, timeout=3) - i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=6) + pktLen = '2000' + pktNum = '10' + role = 'client' + if platform.system().lower() == 'windows': + taosCmd = buildPath + '\\build\\bin\\taos.exe -h 127.0.0.1 -c ' + keyDict['c'] + taosCmd = taosCmd.replace('\\','\\\\') + else: + taosCmd = buildPath + '/build/bin/taos -c ' + keyDict['c'] + taosCmd = taosCmd + ' -n ' + role + ' -l ' + pktLen + ' -N ' + pktNum + print (taosCmd) + child = taosExpect.spawn(taosCmd, timeout=3) + i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=6) - if platform.system().lower() == 'windows': - retResult = child.before - else: - retResult = child.before.decode() - print("expect() return code: %d, content:\n %s\n"%(i, retResult)) - #print(child.after.decode()) - if i == 0: - tdLog.exit('taos -n server fail!') - - expectString1 = 'response is received, size:' + pktLen - expectSTring2 = pktNum + '/' + pktNum - if expectString1 in retResult and expectSTring2 in retResult: - tdLog.info("taos -n client success") - else: - tdLog.exit('taos -n client fail!') - - if platform.system().lower() == 'windows': - os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9') - else: - os.system('pkill taos') + if platform.system().lower() == 'windows': + retResult = child.before + else: + retResult = child.before.decode() + print("expect() return code: %d, content:\n %s\n"%(i, retResult)) + #print(child.after.decode()) + if i == 0: + tdLog.exit('taos -n server fail!') + + expectString1 = 'response is received, size:' + pktLen + expectSTring2 = pktNum + '/' + pktNum + if expectString1 in retResult and expectSTring2 in retResult: + tdLog.info("taos -n client success") + else: + tdLog.exit('taos -n client fail!') + finally: + if platform.system().lower() == 'windows': + os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9') + else: + os.system('pkill taos') def stop(self): tdSql.close() diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index a219c54e59..4c5a434f0c 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -8,6 +8,7 @@ import http.server import gzip import threading import json +import pickle from util.log import * from util.sql import * @@ -15,206 +16,203 @@ from util.cases import * from util.dnodes import * telemetryPort = '6043' +serverPort = '7080' +hostname = socket.gethostname() +class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): + hostPort = hostname + ":" + serverPort -def telemetryInfoCheck(infoDict=''): + def telemetryInfoCheck(self, infoDict=''): + if "ts" not in infoDict or len(infoDict["ts"]) == 0: + tdLog.exit("ts is null!") - hostname = socket.gethostname() - serverPort = 7080 + if "dnode_id" not in infoDict or infoDict["dnode_id"] != 1: + tdLog.exit("dnode_id is null!") - if "ts" not in infoDict or len(infoDict["ts"]) == 0: - tdLog.exit("ts is null!") + if "dnode_ep" not in infoDict: + tdLog.exit("dnode_ep is null!") - if "dnode_id" not in infoDict or infoDict["dnode_id"] != 1: - tdLog.exit("dnode_id is null!") + if "cluster_id" not in infoDict: + tdLog.exit("cluster_id is null!") - if "dnode_ep" not in infoDict: - tdLog.exit("dnode_ep is null!") + if "protocol" not in infoDict or infoDict["protocol"] != 1: + tdLog.exit("protocol is null!") - if "cluster_id" not in infoDict: - tdLog.exit("cluster_id is null!") + if "cluster_info" not in infoDict : + tdLog.exit("cluster_info is null!") - if "protocol" not in infoDict or infoDict["protocol"] != 1: - tdLog.exit("protocol is null!") + # cluster_info ==================================== - if "cluster_info" not in infoDict : - tdLog.exit("cluster_info is null!") + if "first_ep" not in infoDict["cluster_info"] or infoDict["cluster_info"]["first_ep"] == None: + tdLog.exit("first_ep is null!") - # cluster_info ==================================== + if "first_ep_dnode_id" not in infoDict["cluster_info"] or infoDict["cluster_info"]["first_ep_dnode_id"] != 1: + tdLog.exit("first_ep_dnode_id is null!") - if "first_ep" not in infoDict["cluster_info"] or infoDict["cluster_info"]["first_ep"] == None: - tdLog.exit("first_ep is null!") - - if "first_ep_dnode_id" not in infoDict["cluster_info"] or infoDict["cluster_info"]["first_ep_dnode_id"] != 1: - tdLog.exit("first_ep_dnode_id is null!") - - if "version" not in infoDict["cluster_info"] or infoDict["cluster_info"]["version"] == None: - tdLog.exit("first_ep_dnode_id is null!") - - if "master_uptime" not in infoDict["cluster_info"] or infoDict["cluster_info"]["master_uptime"] == None: - tdLog.exit("master_uptime is null!") - - if "monitor_interval" not in infoDict["cluster_info"] or infoDict["cluster_info"]["monitor_interval"] !=5: - tdLog.exit("monitor_interval is null!") - - if "vgroups_total" not in infoDict["cluster_info"] or infoDict["cluster_info"]["vgroups_total"] < 0: - tdLog.exit("vgroups_total is null!") - - if "vgroups_alive" not in infoDict["cluster_info"] or infoDict["cluster_info"]["vgroups_alive"] < 0: - tdLog.exit("vgroups_alive is null!") - - if "connections_total" not in infoDict["cluster_info"] or infoDict["cluster_info"]["connections_total"] < 0 : - tdLog.exit("connections_total is null!") - - if "dnodes" not in infoDict["cluster_info"] or infoDict["cluster_info"]["dnodes"] == None : - tdLog.exit("dnodes is null!") - - dnodes_info = { "dnode_id": 1,"dnode_ep": f"{hostname}:{serverPort}","status":"ready"} - - for k ,v in dnodes_info.items(): - if k not in infoDict["cluster_info"]["dnodes"][0] or v != infoDict["cluster_info"]["dnodes"][0][k] : - tdLog.exit("dnodes info is null!") - - mnodes_info = { "mnode_id":1, "mnode_ep":f"{hostname}:{serverPort}","role": "leader" } - - for k ,v in mnodes_info.items(): - if k not in infoDict["cluster_info"]["mnodes"][0] or v != infoDict["cluster_info"]["mnodes"][0][k] : - tdLog.exit("mnodes info is null!") - - # vgroup_infos ==================================== - - if "vgroup_infos" not in infoDict or infoDict["vgroup_infos"]== None: - tdLog.exit("vgroup_infos is null!") - - vgroup_infos_nums = len(infoDict["vgroup_infos"]) - - for index in range(vgroup_infos_nums): - if "vgroup_id" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["vgroup_id"]<0: - tdLog.exit("vgroup_id is null!") - if "database_name" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["database_name"]) < 0: - tdLog.exit("database_name is null!") - if "tables_num" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["tables_num"]!= 0: - tdLog.exit("tables_num is null!") - if "status" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["status"]) < 0 : - tdLog.exit("status is null!") - if "vnodes" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["vnodes"] ==None : - tdLog.exit("vnodes is null!") - if "dnode_id" not in infoDict["vgroup_infos"][index]["vnodes"][0] or infoDict["vgroup_infos"][index]["vnodes"][0]["dnode_id"] < 0 : - tdLog.exit("vnodes is null!") - - # grant_info ==================================== - - if "grant_info" not in infoDict or infoDict["grant_info"]== None: - tdLog.exit("grant_info is null!") - - if "expire_time" not in infoDict["grant_info"] or not infoDict["grant_info"]["expire_time"] > 0: - tdLog.exit("expire_time is null!") - - if "timeseries_used" not in infoDict["grant_info"] or not infoDict["grant_info"]["timeseries_used"] > 0: - tdLog.exit("timeseries_used is null!") - - if "timeseries_total" not in infoDict["grant_info"] or not infoDict["grant_info"]["timeseries_total"] > 0: - tdLog.exit("timeseries_total is null!") + if "version" not in infoDict["cluster_info"] or infoDict["cluster_info"]["version"] == None: + tdLog.exit("first_ep_dnode_id is null!") - # dnode_info ==================================== + if "master_uptime" not in infoDict["cluster_info"] or infoDict["cluster_info"]["master_uptime"] == None: + tdLog.exit("master_uptime is null!") - if "dnode_info" not in infoDict or infoDict["dnode_info"]== None: - tdLog.exit("dnode_info is null!") + if "monitor_interval" not in infoDict["cluster_info"] or infoDict["cluster_info"]["monitor_interval"] !=5: + tdLog.exit("monitor_interval is null!") - dnode_infos = ['uptime', 'cpu_engine', 'cpu_system', 'cpu_cores', 'mem_engine', 'mem_system', 'mem_total', 'disk_engine', - 'disk_used', 'disk_total', 'net_in', 'net_out', 'io_read', 'io_write', 'io_read_disk', 'io_write_disk', 'req_select', - 'req_select_rate', 'req_insert', 'req_insert_success', 'req_insert_rate', 'req_insert_batch', 'req_insert_batch_success', - 'req_insert_batch_rate', 'errors', 'vnodes_num', 'masters', 'has_mnode', 'has_qnode', 'has_snode', 'has_bnode'] - for elem in dnode_infos: - if elem not in infoDict["dnode_info"] or infoDict["dnode_info"][elem] < 0: - tdLog.exit(f"{elem} is null!") + if "vgroups_total" not in infoDict["cluster_info"] or infoDict["cluster_info"]["vgroups_total"] < 0: + tdLog.exit("vgroups_total is null!") - # dnode_info ==================================== + if "vgroups_alive" not in infoDict["cluster_info"] or infoDict["cluster_info"]["vgroups_alive"] < 0: + tdLog.exit("vgroups_alive is null!") - if "disk_infos" not in infoDict or infoDict["disk_infos"]== None: - tdLog.exit("disk_infos is null!") - - # bug for data_dir - if "datadir" not in infoDict["disk_infos"] or len(infoDict["disk_infos"]["datadir"]) <=0 : - tdLog.exit("datadir is null!") + if "connections_total" not in infoDict["cluster_info"] or infoDict["cluster_info"]["connections_total"] < 0 : + tdLog.exit("connections_total is null!") - if "name" not in infoDict["disk_infos"]["datadir"][0] or len(infoDict["disk_infos"]["datadir"][0]["name"]) <= 0: - tdLog.exit("name is null!") + if "dnodes" not in infoDict["cluster_info"] or infoDict["cluster_info"]["dnodes"] == None : + tdLog.exit("dnodes is null!") + + dnodes_info = { "dnode_id": 1,"dnode_ep": self.hostPort,"status":"ready"} + + for k ,v in dnodes_info.items(): + if k not in infoDict["cluster_info"]["dnodes"][0] or v != infoDict["cluster_info"]["dnodes"][0][k] : + tdLog.exit("dnodes info is null!") + + mnodes_info = { "mnode_id":1, "mnode_ep": self.hostPort,"role": "leader" } - if "level" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["level"] < 0: - tdLog.exit("level is null!") + for k ,v in mnodes_info.items(): + if k not in infoDict["cluster_info"]["mnodes"][0] or v != infoDict["cluster_info"]["mnodes"][0][k] : + tdLog.exit("mnodes info is null!") - if "avail" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["avail"] <= 0: - tdLog.exit("avail is null!") + # vgroup_infos ==================================== - if "used" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["used"] <= 0: - tdLog.exit("used is null!") + if "vgroup_infos" not in infoDict or infoDict["vgroup_infos"]== None: + tdLog.exit("vgroup_infos is null!") + + vgroup_infos_nums = len(infoDict["vgroup_infos"]) - if "total" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["total"] <= 0: - tdLog.exit("total is null!") + for index in range(vgroup_infos_nums): + if "vgroup_id" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["vgroup_id"]<0: + tdLog.exit("vgroup_id is null!") + if "database_name" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["database_name"]) < 0: + tdLog.exit("database_name is null!") + if "tables_num" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["tables_num"]!= 0: + tdLog.exit("tables_num is null!") + if "status" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["status"]) < 0 : + tdLog.exit("status is null!") + if "vnodes" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["vnodes"] ==None : + tdLog.exit("vnodes is null!") + if "dnode_id" not in infoDict["vgroup_infos"][index]["vnodes"][0] or infoDict["vgroup_infos"][index]["vnodes"][0]["dnode_id"] < 0 : + tdLog.exit("vnodes is null!") + + # grant_info ==================================== + + if "grant_info" not in infoDict or infoDict["grant_info"]== None: + tdLog.exit("grant_info is null!") + + if "expire_time" not in infoDict["grant_info"] or not infoDict["grant_info"]["expire_time"] > 0: + tdLog.exit("expire_time is null!") + + if "timeseries_used" not in infoDict["grant_info"] or not infoDict["grant_info"]["timeseries_used"] > 0: + tdLog.exit("timeseries_used is null!") + + if "timeseries_total" not in infoDict["grant_info"] or not infoDict["grant_info"]["timeseries_total"] > 0: + tdLog.exit("timeseries_total is null!") + + # dnode_info ==================================== + + if "dnode_info" not in infoDict or infoDict["dnode_info"]== None: + tdLog.exit("dnode_info is null!") + + dnode_infos = ['uptime', 'cpu_engine', 'cpu_system', 'cpu_cores', 'mem_engine', 'mem_system', 'mem_total', 'disk_engine', + 'disk_used', 'disk_total', 'net_in', 'net_out', 'io_read', 'io_write', 'io_read_disk', 'io_write_disk', 'req_select', + 'req_select_rate', 'req_insert', 'req_insert_success', 'req_insert_rate', 'req_insert_batch', 'req_insert_batch_success', + 'req_insert_batch_rate', 'errors', 'vnodes_num', 'masters', 'has_mnode', 'has_qnode', 'has_snode', 'has_bnode'] + for elem in dnode_infos: + if elem not in infoDict["dnode_info"] or infoDict["dnode_info"][elem] < 0: + tdLog.exit(f"{elem} is null!") + + # dnode_info ==================================== + + if "disk_infos" not in infoDict or infoDict["disk_infos"]== None: + tdLog.exit("disk_infos is null!") + + # bug for data_dir + if "datadir" not in infoDict["disk_infos"] or len(infoDict["disk_infos"]["datadir"]) <=0 : + tdLog.exit("datadir is null!") + + if "name" not in infoDict["disk_infos"]["datadir"][0] or len(infoDict["disk_infos"]["datadir"][0]["name"]) <= 0: + tdLog.exit("name is null!") + + if "level" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["level"] < 0: + tdLog.exit("level is null!") + + if "avail" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["avail"] <= 0: + tdLog.exit("avail is null!") + + if "used" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["used"] <= 0: + tdLog.exit("used is null!") + + if "total" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["total"] <= 0: + tdLog.exit("total is null!") - if "logdir" not in infoDict["disk_infos"] or infoDict["disk_infos"]["logdir"]== None: - tdLog.exit("logdir is null!") + if "logdir" not in infoDict["disk_infos"] or infoDict["disk_infos"]["logdir"]== None: + tdLog.exit("logdir is null!") - if "name" not in infoDict["disk_infos"]["logdir"] or len(infoDict["disk_infos"]["logdir"]["name"]) <= 0: - tdLog.exit("name is null!") + if "name" not in infoDict["disk_infos"]["logdir"] or len(infoDict["disk_infos"]["logdir"]["name"]) <= 0: + tdLog.exit("name is null!") - if "avail" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["avail"] <= 0: - tdLog.exit("avail is null!") + if "avail" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["avail"] <= 0: + tdLog.exit("avail is null!") - if "used" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["used"] <= 0: - tdLog.exit("used is null!") + if "used" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["used"] <= 0: + tdLog.exit("used is null!") - if "total" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["total"] <= 0: - tdLog.exit("total is null!") + if "total" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["total"] <= 0: + tdLog.exit("total is null!") + if "tempdir" not in infoDict["disk_infos"] or infoDict["disk_infos"]["tempdir"]== None: + tdLog.exit("tempdir is null!") + + if "name" not in infoDict["disk_infos"]["tempdir"] or len(infoDict["disk_infos"]["tempdir"]["name"]) <= 0: + tdLog.exit("name is null!") + + if "avail" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["avail"] <= 0: + tdLog.exit("avail is null!") + + if "used" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["used"] <= 0: + tdLog.exit("used is null!") + + if "total" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["total"] <= 0: + tdLog.exit("total is null!") + + + # log_infos ==================================== + + if "log_infos" not in infoDict or infoDict["log_infos"]== None: + tdLog.exit("log_infos is null!") + + if "logs" not in infoDict["log_infos"] or len(infoDict["log_infos"]["logs"])!= 10: + tdLog.exit("logs is null!") + + if "ts" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 10: + tdLog.exit("ts is null!") + + if "level" not in infoDict["log_infos"]["logs"][0] or infoDict["log_infos"]["logs"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: + tdLog.exit("level is null!") + + if "content" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 1: + tdLog.exit("content is null!") + + if "summary" not in infoDict["log_infos"] or len(infoDict["log_infos"]["summary"])!= 4: + tdLog.exit("summary is null!") + + + if "total" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["total"] < 0 : + tdLog.exit("total is null!") + + if "level" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: + tdLog.exit("level is null!") - - if "tempdir" not in infoDict["disk_infos"] or infoDict["disk_infos"]["tempdir"]== None: - tdLog.exit("tempdir is null!") - - if "name" not in infoDict["disk_infos"]["tempdir"] or len(infoDict["disk_infos"]["tempdir"]["name"]) <= 0: - tdLog.exit("name is null!") - - if "avail" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["avail"] <= 0: - tdLog.exit("avail is null!") - - if "used" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["used"] <= 0: - tdLog.exit("used is null!") - - if "total" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["total"] <= 0: - tdLog.exit("total is null!") - - - # log_infos ==================================== - - if "log_infos" not in infoDict or infoDict["log_infos"]== None: - tdLog.exit("log_infos is null!") - - if "logs" not in infoDict["log_infos"] or len(infoDict["log_infos"]["logs"])!= 10: - tdLog.exit("logs is null!") - - if "ts" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 10: - tdLog.exit("ts is null!") - - if "level" not in infoDict["log_infos"]["logs"][0] or infoDict["log_infos"]["logs"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: - tdLog.exit("level is null!") - - if "content" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 1: - tdLog.exit("content is null!") - - if "summary" not in infoDict["log_infos"] or len(infoDict["log_infos"]["summary"])!= 4: - tdLog.exit("summary is null!") - - - if "total" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["total"] < 0 : - tdLog.exit("total is null!") - - if "level" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: - tdLog.exit("level is null!") - -class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): def do_GET(self): """ process GET request @@ -245,17 +243,23 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): infoDict = json.loads(plainText) #print("================") # print(infoDict) - telemetryInfoCheck(infoDict) + self.telemetryInfoCheck(infoDict) # 4. shutdown the server and exit case - assassin = threading.Thread(target=httpServer.shutdown) + assassin = threading.Thread(target=self.server.shutdown) assassin.daemon = True assassin.start() print ("==== shutdown http server ====") class TDTestCase: - hostname = socket.gethostname() - serverPort = '7080' + global hostname + global serverPort + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + try: + config = eval(tdDnodes.dnodes[0].remoteIP ) + hostname = config["host"] + except Exception: + hostname = tdDnodes.dnodes[0].remoteIP rpcDebugFlagVal = '143' clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} clientCfgDict["serverPort"] = serverPort @@ -291,21 +295,19 @@ class TDTestCase: sql = "create database db3 vgroups " + vgroups tdSql.query(sql) - # loop to wait request - httpServer.serve_forever() + # create http server: bing ip/port , and request processor + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + RequestHandlerImplStr = base64.b64encode(pickle.dumps(RequestHandlerImpl)).decode() + cmdStr = "import pickle\nimport http\nRequestHandlerImpl=pickle.loads(base64.b64decode(\"%s\".encode()))\nclass NewRequestHandlerImpl(RequestHandlerImpl):\n hostPort = \'%s\'\nhttp.server.HTTPServer((\"\", %s), NewRequestHandlerImpl).serve_forever()"%(RequestHandlerImplStr,hostname+":"+serverPort,telemetryPort) + tdDnodes.dnodes[0].remoteExec({}, cmdStr) + else: + serverAddress = ("", int(telemetryPort)) + http.server.HTTPServer(serverAddress, RequestHandlerImpl).serve_forever() def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") -# create http server: bing ip/port , and request processor -serverAddress = ("", int(telemetryPort)) -httpServer = http.server.HTTPServer(serverAddress, RequestHandlerImpl) - tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) - - - - diff --git a/tests/system-test/0-others/telemetry.py b/tests/system-test/0-others/telemetry.py index 203f87c085..4483e113bf 100644 --- a/tests/system-test/0-others/telemetry.py +++ b/tests/system-test/0-others/telemetry.py @@ -8,6 +8,7 @@ import http.server import gzip import threading import json +import pickle from util.log import * from util.sql import * @@ -136,13 +137,19 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): telemetryInfoCheck(infoDict) # 4. shutdown the server and exit case - assassin = threading.Thread(target=httpServer.shutdown) + assassin = threading.Thread(target=self.server.shutdown) assassin.daemon = True assassin.start() print ("==== shutdown http server ====") class TDTestCase: hostname = socket.gethostname() + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + try: + config = eval(tdDnodes.dnodes[0].remoteIP) + hostname = config["host"] + except Exception: + hostname = tdDnodes.dnodes[0].remoteIP serverPort = '7080' rpcDebugFlagVal = '143' clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} @@ -177,17 +184,20 @@ class TDTestCase: sql = "create database db3 vgroups " + vgroups tdSql.query(sql) - # loop to wait request - httpServer.serve_forever() + # create http server: bing ip/port , and request processor + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + RequestHandlerImplStr = base64.b64encode(pickle.dumps(RequestHandlerImpl)).decode() + telemetryInfoCheckStr = base64.b64encode(pickle.dumps(telemetryInfoCheck)).decode() + cmdStr = "import pickle\nimport http\ntelemetryInfoCheck=pickle.loads(base64.b64decode(\"%s\".encode()))\nRequestHandlerImpl=pickle.loads(base64.b64decode(\"%s\".encode()))\nhttp.server.HTTPServer((\"\", %d), RequestHandlerImpl).serve_forever()"%(telemetryInfoCheckStr,RequestHandlerImplStr,int(telemetryPort)) + tdDnodes.dnodes[0].remoteExec({}, cmdStr) + else: + serverAddress = ("", int(telemetryPort)) + http.server.HTTPServer(serverAddress, RequestHandlerImpl).serve_forever() def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") -# create http server: bing ip/port , and request processor -serverAddress = ("", int(telemetryPort)) -httpServer = http.server.HTTPServer(serverAddress, RequestHandlerImpl) - tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index 88ed667161..0d24b09616 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -9,6 +9,8 @@ from util.sql import * from util.cases import * from util.dnodes import * import subprocess +# import win32gui +# import threading class TDTestCase: @@ -533,8 +535,17 @@ class TDTestCase: return udf1_sqls ,udf2_sqls + # def checkRunTimeError(self): + # while 1: + # time.sleep(1) + # hwnd = win32gui.FindWindow(None, "Microsoft Visual C++ Runtime Library") + # if hwnd: + # os.system("TASKKILL /F /IM udfd.exe") def unexpected_create(self): + # if (platform.system().lower() == 'windows' and tdDnodes.dnodes[0].remoteIP == ""): + # checkErrorThread = threading.Thread(target=self.checkRunTimeError,daemon=True) + # checkErrorThread.start() tdLog.info(" create function with out bufsize ") tdSql.query("drop function udf1 ") diff --git a/tests/system-test/1-insert/create_table_comment.py b/tests/system-test/1-insert/create_table_comment.py new file mode 100644 index 0000000000..92ea083c5a --- /dev/null +++ b/tests/system-test/1-insert/create_table_comment.py @@ -0,0 +1,113 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def get_long_name(self, length, mode="mixed"): + """ + generate long name + mode could be numbers/letters/letters_mixed/mixed + """ + if mode == "numbers": + population = string.digits + elif mode == "letters": + population = string.ascii_letters.lower() + elif mode == "letters_mixed": + population = string.ascii_letters.upper() + string.ascii_letters.lower() + else: + population = string.ascii_letters.lower() + string.digits + return "".join(random.choices(population, k=length)) + + def __create_tb(self,dbname,stbname,tbname,comment): + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute( + f'create table {stbname} (ts timestamp,c0 int) tags(t0 int) ') + tdSql.execute( + f'create table {tbname} using {stbname} tags(1) comment "{comment}"') + def __create_normaltb(self,dbname,tbname,comment): + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute( + f'create table {tbname} (ts timestamp,c0 int) comment "{comment}"') + + def check_comment(self): + dbname = self.get_long_name(length=10, mode="letters") + ntbname = self.get_long_name(length=5, mode="letters") + + # create normal table with comment + comment = self.get_long_name(length=10, mode="letters") + self.__create_normaltb(dbname,ntbname,comment) + ntb_kv_list = tdSql.getResult("show tables") + print(ntb_kv_list) + tdSql.checkEqual(ntb_kv_list[0][8], comment) + tdSql.error('alter table {ntbname} comment "test1"') + tdSql.execute(f'drop database {dbname}') + + # max length(1024) + comment = self.get_long_name(length=1024, mode="letters") + self.__create_normaltb(dbname,ntbname,comment) + ntb_kv_list = tdSql.getResult("show tables") + tdSql.checkEqual(ntb_kv_list[0][8], comment) + tdSql.execute(f'drop database {dbname}') + + # error overlength + comment = self.get_long_name(length=1025, mode="letters") + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.error(f"create table ntb (ts timestamp,c0 int) comment '{comment}'") + tdSql.execute(f'drop database {dbname}') + + # create child table with comment + comment = self.get_long_name(length=10, mode="letters") + stbname = self.get_long_name(length=5, mode="letters") + tbname = self.get_long_name(length=3, mode="letters") + self.__create_tb(dbname,stbname,tbname,comment) + ntb_kv_list = tdSql.getResult("show tables") + tdSql.checkEqual(ntb_kv_list[0][8], comment) + tdSql.error(f'alter table {tbname} comment "test1"') + tdSql.execute(f'drop database {dbname}') + + # max length 1024 + comment = self.get_long_name(length=1024, mode="letters") + self.__create_tb(dbname,ntbname,comment) + ntb_kv_list = tdSql.getResult("show tables") + tdSql.checkEqual(ntb_kv_list[0][8], comment) + tdSql.execute(f'drop database {dbname}') + + # error overlength + comment = self.get_long_name(length=1025, mode="letters") + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute(f"create table stb (ts timestamp,c0 int) tags(t0 int)") + tdSql.error(f'create table stb_1 us stb tags(1) comment "{comment}"') + tdSql.execute(f'drop database {dbname}') + + def run(self): + self.check_comment() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/csum.py b/tests/system-test/2-query/csum.py index a331311fd2..5c2de5ea14 100644 --- a/tests/system-test/2-query/csum.py +++ b/tests/system-test/2-query/csum.py @@ -240,7 +240,7 @@ class TDTestCase: tdSql.error("select csum(c1) t1") # no from tdSql.error("select csum( c1 ) from ") # no table_expr # tdSql.error(self.csum_query_form(col="st1")) # tag col - tdSql.error(self.csum_query_form(col=1)) # col is a value + # tdSql.error(self.csum_query_form(col=1)) # col is a value tdSql.error(self.csum_query_form(col="'c1'")) # col is a string tdSql.error(self.csum_query_form(col=None)) # col is NULL 1 tdSql.error(self.csum_query_form(col="NULL")) # col is NULL 2 @@ -407,6 +407,14 @@ class TDTestCase: tdDnodes.start(index) self.csum_current_query() self.csum_error_query() + tdSql.query("select csum(1) from t1 ") + tdSql.checkRows(7) + tdSql.checkData(0,0,1) + tdSql.checkData(1,0,2) + tdSql.checkData(2,0,3) + tdSql.checkData(3,0,4) + tdSql.query("select csum(abs(c1))+2 from t1 ") + tdSql.checkRows(4) def run(self): import traceback diff --git a/tests/system-test/2-query/elapsed.py b/tests/system-test/2-query/elapsed.py index 017090128d..1553e06914 100644 --- a/tests/system-test/2-query/elapsed.py +++ b/tests/system-test/2-query/elapsed.py @@ -141,7 +141,7 @@ class TDTestCase: tablenames = ["sub_table1_1","sub_table1_2","sub_table1_3","sub_table2_1","sub_table2_2","sub_table2_3","regular_table_1","regular_table_2","regular_table_3"] abnormal_list = ["()","(NULL)","(*)","(abc)","( , )","(NULL,*)","( ,NULL)","(%)","(+)","(*,)","(*, /)","(ts,*)" "(ts,tbname*10)","(ts,tagname)", - "(ts,2d+3m-2s,NULL)","(ts+1d,10s)","(ts+10d,NULL)" ,"(ts,now -1m%1d)","(ts+10d)","(ts+10d,_c0)","(ts+10d,)","(ts,%)","(ts, , m)","(ts,abc)","(ts,/)","(ts,*)","(ts,1s,100)", + "(ts,2d+3m-2s,NULL)","(ts+10d,NULL)" ,"(ts,now -1m%1d)","(ts+10d,_c0)","(ts+10d,)","(ts,%)","(ts, , m)","(ts,abc)","(ts,/)","(ts,*)","(ts,1s,100)", "(ts,1s,abc)","(ts,1s,_c0)","(ts,1s,*)","(ts,1s,NULL)","(ts,,_c0)","(ts,tbname,ts)","(ts,0,tbname)","('2021-11-18 00:00:10')","('2021-11-18 00:00:10', 1s)", "('2021-11-18T00:00:10+0800', '1s')","('2021-11-18T00:00:10Z', '1s')","('2021-11-18T00:00:10+0800', 10000000d,)","('ts', ,2021-11-18T00:00:10+0800, )"] diff --git a/tests/system-test/2-query/function_stateduration.py b/tests/system-test/2-query/function_stateduration.py index b25a658469..9aa6fdaefa 100644 --- a/tests/system-test/2-query/function_stateduration.py +++ b/tests/system-test/2-query/function_stateduration.py @@ -85,8 +85,8 @@ class TDTestCase: "select stateduration(c1 ,'GT','*',1s) from t1", "select stateduration(c1 ,'GT',ts,1s) from t1", "select stateduration(c1 ,'GT',max(c1),1s) from t1", - "select stateduration(abs(c1) ,'GT',1,1s) from t1", - "select stateduration(c1+2 ,'GT',1,1s) from t1", + # "select stateduration(abs(c1) ,'GT',1,1s) from t1", + # "select stateduration(c1+2 ,'GT',1,1s) from t1", "select stateduration(c1 ,'GT',1,1u) from t1", "select stateduration(c1 ,'GT',1,now) from t1", "select stateduration(c1 ,'GT','1',1s) from t1", @@ -323,6 +323,11 @@ class TDTestCase: tdSql.checkData(0, 0, None) tdSql.checkData(1, 0, 0.000000000) tdSql.checkData(3, 0, -86404.000000000) + + tdSql.query("select stateduration(abs(c1) ,'GT',1,1s) from t1") + tdSql.checkRows(12) + tdSql.query("select stateduration(c1+2 ,'GT',1,1s) from t1") + tdSql.checkRows(12) # bug for stable diff --git a/tests/system-test/2-query/hyperloglog.py b/tests/system-test/2-query/hyperloglog.py index 35703e441d..b20ec35f07 100644 --- a/tests/system-test/2-query/hyperloglog.py +++ b/tests/system-test/2-query/hyperloglog.py @@ -223,7 +223,7 @@ class TDTestCase: tdLog.printNoPrefix("===step 0: err case, must return err") tdSql.error( "select hyperloglog() from ct1" ) tdSql.error( "select hyperloglog(c1, c2) from ct2" ) - tdSql.error( "select hyperloglog(1) from ct2" ) + # tdSql.error( "select hyperloglog(1) from ct2" ) tdSql.error( f"select hyperloglog({NUM_COL[0]}, {NUM_COL[1]}) from ct4" ) tdSql.error( ''' select hyperloglog(['c1 + c1', 'c1 + c2', 'c1 + c3', 'c1 + c4', 'c1 + c5', 'c1 + c6', 'c1 + c7', 'c1 + c8', 'c1 + c9', 'c1 + c10']) from ct1 diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 6a855ebd4b..7838b5c3e4 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -61,6 +61,8 @@ class TDTestCase: # test invalidate json tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('\"efwewf\"')") tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('3333')") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags(76)") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags(hell)") tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('33.33')") tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('false')") tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('[1,true]')") @@ -116,7 +118,7 @@ class TDTestCase: tdSql.error("select jtag->location from jsons1") tdSql.error("select jtag contains location from jsons1") tdSql.error("select * from jsons1 where jtag contains location") - tdSql.error("select * from jsons1 where jtag contains''") + #tdSql.error("select * from jsons1 where jtag contains''") tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'") # # # test function error @@ -127,8 +129,43 @@ class TDTestCase: tdSql.error("select ceil(jtag->'tag1') from jsons1") tdSql.error("select ceil(jtag) from jsons1") # + + #test scalar operation + tdSql.query("select jtag contains 'tag1',jtag->'tag1' from jsons1 order by jtag->'tag1'") + tdSql.checkRows(13) + tdSql.checkData(0, 0, False) + tdSql.checkData(5, 0, True) + tdSql.checkData(12, 0, True) + tdSql.query("select jtag->'tag1' like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'") + tdSql.checkRows(13) + tdSql.checkData(10, 0, False) + tdSql.checkData(11, 0, False) + tdSql.checkData(12, 0, True) + tdSql.query("select jtag->'tag1' not like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'") + tdSql.checkRows(13) + tdSql.checkData(10, 0, False) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, False) + tdSql.query("select jtag->'tag1' match 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'") + tdSql.checkRows(13) + tdSql.checkData(10, 0, False) + tdSql.checkData(11, 0, False) + tdSql.checkData(12, 0, True) + tdSql.query("select jtag->'tag1' nmatch 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'") + tdSql.checkRows(13) + tdSql.checkData(10, 0, False) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, False) + tdSql.query("select jtag->'tag1',jtag->'tag1'>='a' from jsons1 order by jtag->'tag1'") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, False) + tdSql.checkData(7, 0, "false") + tdSql.checkData(7, 1, True) + tdSql.checkData(12, 1, True) + # # test select normal column - tdSql.query("select dataint from jsons1") + tdSql.query("select dataint from jsons1 order by dataint") tdSql.checkRows(9) tdSql.checkData(1, 0, 1) @@ -137,9 +174,9 @@ class TDTestCase: tdSql.checkRows(9) tdSql.query("select jtag from jsons1") tdSql.checkRows(13) - tdSql.query("select jtag from jsons1 where jtag is null") + # tdSql.query("select jtag from jsons1 where jtag is null") # tdSql.checkRows(5) - tdSql.query("select jtag from jsons1 where jtag is not null") + # tdSql.query("select jtag from jsons1 where jtag is not null") # tdSql.checkRows(8) # test jtag is NULL @@ -259,12 +296,6 @@ class TDTestCase: # tdSql.query("select * from jsons1 where jtag->'tag1'=null") # only json suport =null. This synatx will change later. # tdSql.checkRows(1) # - # # where json is null - tdSql.query("select * from jsons1 where jtag is null") - # tdSql.checkRows(1) - tdSql.query("select * from jsons1 where jtag is not null") - # tdSql.checkRows(8) - # # # where json key is null # tdSql.query("select * from jsons1 where jtag->'tag_no_exist'=3") # tdSql.checkRows(0) @@ -358,25 +389,28 @@ class TDTestCase: # tdSql.checkRows(0) # # # test join - # tdSql.execute("create table if not exists jsons2(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") - # tdSql.execute("insert into jsons2_1 using jsons2 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 2, false, 'json2', '你是2')") - # tdSql.execute("insert into jsons2_2 using jsons2 tags('{\"tag1\":5,\"tag2\":null}') values (1591060628000, 2, true, 'json2', 'sss')") - # - # tdSql.execute("create table if not exists jsons3(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") - # tdSql.execute("insert into jsons3_1 using jsons3 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 3, false, 'json3', '你是3')") - # tdSql.execute("insert into jsons3_2 using jsons3 tags('{\"tag1\":5,\"tag2\":\"beijing\"}') values (1591060638000, 2, true, 'json3', 'sss')") - # tdSql.query("select 'sss',33,a.jtag->'tag3' from jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") - # tdSql.checkData(0, 0, "sss") - # tdSql.checkData(0, 2, "true") - # - # res = tdSql.getColNameList("select 'sss',33,a.jtag->'tag3' from jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") - # cname_list = [] - # cname_list.append("sss") - # cname_list.append("33") - # cname_list.append("a.jtag->'tag3'") - # tdSql.checkColNameList(res, cname_list) + tdSql.execute("create table if not exists jsons2(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + tdSql.execute("insert into jsons2_1 using jsons2 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 2, false, 'json2', '你是2')") + tdSql.execute("insert into jsons2_2 using jsons2 tags('{\"tag1\":5,\"tag2\":null}') values (1591060628000, 2, true, 'json2', 'sss')") + + tdSql.execute("create table if not exists jsons3(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + tdSql.execute("insert into jsons3_1 using jsons3 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 3, false, 'json3', '你是3')") + tdSql.execute("insert into jsons3_2 using jsons3 tags('{\"tag1\":5,\"tag2\":\"beijing\"}') values (1591060638000, 2, true, 'json3', 'sss')") + tdSql.query("select 'sss',33,a.jtag->'tag3' from jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") + tdSql.checkData(0, 0, "sss") + tdSql.checkData(0, 2, "true") + + res = tdSql.getColNameList("select 'sss',33,a.jtag->'tag3' from jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") + cname_list = [] + cname_list.append("'sss'") + cname_list.append("33") + cname_list.append("a.jtag->'tag3'") + tdSql.checkColNameList(res, cname_list) # # test group by & order by json tag + # tdSql.error("select count(*) from jsons1 group by jtag") + # tdSql.error("select count(*) from jsons1 partition by jtag") + # tdSql.error("select count(*) from jsons1 group by jtag order by jtag") tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag2'") tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag") tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") @@ -432,14 +466,14 @@ class TDTestCase: tdSql.checkData(10, 1, '"femail"') # test having - # tdSql.query("select stddev(dataint) from jsons1 group by jtag->'tag1' having stddev(dataint) > 0") - # tdSql.checkRows(2) + tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1") + tdSql.checkRows(3) # subquery with json tag tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint") tdSql.checkRows(11) tdSql.checkData(1, 1, 1) - tdSql.checkData(2, 0, '{"tag1":5,"tag2":"beijing"}') + tdSql.checkData(5, 0, '{"tag1":false,"tag2":"beijing"}') # tdSql.query("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)") # tdSql.checkRows(11) @@ -457,16 +491,18 @@ class TDTestCase: # tdSql.checkData(1, 2, '"femail"') # # # union all - # tdSql.error("select jtag->'tag1' from jsons1 union all select jtag->'tag2' from jsons2") - # tdSql.error("select jtag->'tag1' from jsons1_1 union all select jtag->'tag2' from jsons2_1") - # - # tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag1' from jsons2_1") - # tdSql.checkRows(2) - # tdSql.query("select dataint,jtag->'tag1',tbname from jsons1 union all select dataint,jtag->'tag1',tbname from jsons2") - # tdSql.checkRows(13) - # tdSql.query("select dataint,jtag,tbname from jsons1 union all select dataint,jtag,tbname from jsons2") - # tdSql.checkRows(13) - # + tdSql.query("select jtag->'tag1' from jsons1 union all select jtag->'tag2' from jsons2") + tdSql.checkRows(17) + tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag2' from jsons2_1") + tdSql.checkRows(2) + + tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag1' from jsons2_1") + tdSql.checkRows(2) + tdSql.query("select dataint,jtag->'tag1',tbname from jsons1 union all select dataint,jtag->'tag1',tbname from jsons2") + tdSql.checkRows(13) + tdSql.query("select dataint,jtag,tbname from jsons1 union all select dataint,jtag,tbname from jsons2") + tdSql.checkRows(13) + # #show create table # tdSql.query("show create table jsons1") # tdSql.checkData(0, 1, 'CREATE TABLE `jsons1` (`ts` TIMESTAMP,`dataint` INT,`databool` BOOL,`datastr` NCHAR(50),`datastrbin` BINARY(150)) TAGS (`jtag` JSON)') @@ -528,9 +564,9 @@ class TDTestCase: tdSql.execute("CREATE TABLE if not exists jsons1_20 using jsons1 tags(NULL)") tdSql.query("select jtag from jsons1_20") tdSql.checkData(0, 0, None) - # tdSql.execute("insert into jsons1_21 using jsons1 tags(NULL) values(1591061628000, 11, false, '你就会','')") - # tdSql.query("select jtag from jsons1_21") - # tdSql.checkData(0, 0, None) + tdSql.execute("insert into jsons1_21 using jsons1 tags(NULL) values(1591061628000, 11, false, '你就会','')") + tdSql.query("select jtag from jsons1_21") + tdSql.checkData(0, 0, None) # # #test TD-12389 tdSql.query("describe jsons1") diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index 6e4c292183..907ba329ee 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -66,6 +66,37 @@ class TDTestCase: ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ''' ) + + + def check_result_auto_log(self ,origin_query , log_query): + + log_result = tdSql.getResult(log_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >0: + elem = math.log(elem) + elif elem <=0: + elem = None + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(log_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("log function value has not as expected , sql is \"%s\" "%log_query ) + sys.exit(1) + else: + tdLog.info("log value check pass , it work as expected ,sql is \"%s\" "%log_query ) def check_result_auto_log2(self ,origin_query , log_query): @@ -270,6 +301,7 @@ class TDTestCase: tdSql.checkRows(25) # used for empty table , ct3 is empty + tdSql.query("select log(c1 ,2) from ct3") tdSql.checkRows(0) tdSql.query("select log(c2 ,2) from ct3") @@ -291,6 +323,13 @@ class TDTestCase: tdSql.checkData(3 , 0, 1.584962501) tdSql.checkData(5 , 0, None) + tdSql.query("select log(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 0.000000000) + tdSql.checkData(2 , 0, 0.693147181) + tdSql.checkData(3 , 0, 1.098612289) + tdSql.checkData(4 , 0, 1.386294361) + tdSql.query("select c1, c2, c3 , c4, c5 from t1") tdSql.checkData(1, 4, 1.11000) tdSql.checkData(3, 3, 33) @@ -301,6 +340,7 @@ class TDTestCase: tdSql.checkData(3, 4, 33) tdSql.checkData(5, 5, None) + self.check_result_auto_log( "select c1, c2, c3 , c4, c5 from t1", "select log(c1), log(c2) ,log(c3), log(c4), log(c5) from t1") self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,2), log(c2 ,2) ,log(c3, 2), log(c4 ,2), log(c5 ,2) from t1") self.check_result_auto_log1( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,1), log(c2 ,1) ,log(c3, 1), log(c4 ,1), log(c5 ,1) from t1") self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,-10), log(c2 ,-10) ,log(c3, -10), log(c4 ,-10), log(c5 ,-10) from t1") @@ -321,7 +361,7 @@ class TDTestCase: tdSql.checkData(3 , 2, 0.147315235) tdSql.checkData(4 , 2, None) - + self.check_result_auto_log( "select c1, c2, c3 , c4, c5 from ct1", "select log(c1), log(c2) ,log(c3), log(c4), log(c5) from ct1") self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 from ct1", "select log(c1,2), log(c2,2) ,log(c3,2), log(c4,2), log(c5,2) from ct1") self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 from ct1", "select log(c1,-10), log(c2,-10) ,log(c3,-10), log(c4,-10), log(c5,-10) from ct1") @@ -569,10 +609,12 @@ class TDTestCase: tdSql.error( f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) + self.check_result_auto_log( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1), log(c2) ,log(c3), log(c4), log(c5) ,log(c6) from sub1_bound") self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1,2), log(c2,2) ,log(c3,2), log(c4,2), log(c5,2) ,log(c6,2) from sub1_bound") self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1,-10), log(c2,-10) ,log(c3,-10), log(c4,-10), log(c5,-10) ,log(c6,-10) from sub1_bound") self.check_result_auto_log2( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select log(c1,2), log(c2,2) ,log(c3,2), log(c3,2), log(c2,2) ,log(c1,2) from sub1_bound") + self.check_result_auto_log( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select log(c1), log(c2) ,log(c3), log(c3), log(c2) ,log(c1) from sub1_bound") self.check_result_auto_log2("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select log(abs(c1) ,2) from sub1_bound" ) @@ -598,6 +640,29 @@ class TDTestCase: tdSql.checkData(3,4,math.log(339999995214436424907732413799364296704.00000,2)) tdSql.checkData(3,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000,2)) + # check basic elem for table per row + tdSql.query("select log(abs(c1)) ,log(abs(c2)) , log(abs(c3)) , log(abs(c4)), log(abs(c5)), log(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.log(2147483647)) + tdSql.checkData(0,1,math.log(9223372036854775807)) + tdSql.checkData(0,2,math.log(32767)) + tdSql.checkData(0,3,math.log(127)) + tdSql.checkData(0,4,math.log(339999995214436424907732413799364296704.00000)) + tdSql.checkData(0,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000 )) + tdSql.checkData(1,0,math.log(2147483647)) + tdSql.checkData(1,1,math.log(9223372036854775807)) + tdSql.checkData(1,2,math.log(32767)) + tdSql.checkData(1,3,math.log(127)) + tdSql.checkData(1,4,math.log(339999995214436424907732413799364296704.00000 )) + tdSql.checkData(1,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)) + tdSql.checkData(3,0,math.log(2147483646)) + tdSql.checkData(3,1,math.log(9223372036854775806)) + tdSql.checkData(3,2,math.log(32766)) + tdSql.checkData(3,3,math.log(126)) + tdSql.checkData(3,4,math.log(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)) + + + # check + - * / in functions tdSql.query("select log(abs(c1+1) ,2) ,log(abs(c2),2) , log(abs(c3*1),2) , log(abs(c4/2),2), log(abs(c5) ,2)/2, log(abs(c6) ,2) from sub1_bound ") tdSql.checkData(0,0,math.log(2147483648.000000000,2)) diff --git a/tests/system-test/2-query/mavg.py b/tests/system-test/2-query/mavg.py index 1d92964615..13d2b4d420 100644 --- a/tests/system-test/2-query/mavg.py +++ b/tests/system-test/2-query/mavg.py @@ -417,8 +417,8 @@ class TDTestCase: # err9 = {"col": "st1"} # self.checkmavg(**err9) # col: tag - err10 = {"col": 1} - self.checkmavg(**err10) # col: value + # err10 = {"col": 1} + # self.checkmavg(**err10) # col: value err11 = {"col": "NULL"} self.checkmavg(**err11) # col: NULL err12 = {"col": "%_"} @@ -660,6 +660,14 @@ class TDTestCase: tdDnodes.start(index) self.mavg_current_query() self.mavg_error_query() + tdSql.query("select mavg(1,1) from t1") + tdSql.checkRows(7) + tdSql.checkData(0,0,1.000000000) + tdSql.checkData(1,0,1.000000000) + tdSql.checkData(5,0,1.000000000) + + tdSql.query("select mavg(abs(c1),1) from t1") + tdSql.checkRows(4) def run(self): import traceback diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index 94e06347d2..4b651b56e7 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -427,10 +427,10 @@ class TDTestCase: # err9 = {"col": "st1"} # self.checksample(**err9) # col: tag tdSql.query(" select sample(st1 ,1) from t1 ") - err10 = {"col": 1} - self.checksample(**err10) # col: value - err11 = {"col": "NULL"} - self.checksample(**err11) # col: NULL + # err10 = {"col": 1} + # self.checksample(**err10) # col: value + # err11 = {"col": "NULL"} + # self.checksample(**err11) # col: NULL err12 = {"col": "%_"} self.checksample(**err12) # col: %_ err13 = {"col": "c3"} @@ -445,12 +445,12 @@ class TDTestCase: self.checksample(**err17) # nchar col err18 = {"col": "c6"} self.checksample(**err18) # bool col - err19 = {"col": "'c1'"} - self.checksample(**err19) # col: string + # err19 = {"col": "'c1'"} + # self.checksample(**err19) # col: string err20 = {"col": None} self.checksample(**err20) # col: None - err21 = {"col": "''"} - self.checksample(**err21) # col: '' + # err21 = {"col": "''"} + # self.checksample(**err21) # col: '' err22 = {"col": "tt1.c1"} self.checksample(**err22) # not table_expr col err23 = {"col": "t1"} @@ -459,10 +459,10 @@ class TDTestCase: self.checksample(**err24) # stbname err25 = {"col": "db"} self.checksample(**err25) # datbasename - err26 = {"col": "True"} - self.checksample(**err26) # col: BOOL 1 - err27 = {"col": True} - self.checksample(**err27) # col: BOOL 2 + # err26 = {"col": "True"} + # self.checksample(**err26) # col: BOOL 1 + # err27 = {"col": True} + # self.checksample(**err27) # col: BOOL 2 err28 = {"col": "*"} self.checksample(**err28) # col: all col err29 = {"func": "sample[", "r_comm": "]"} @@ -678,7 +678,7 @@ class TDTestCase: tdSql.error(" select sample(c1,tbname) from t1 ") tdSql.error(" select sample(c1,ts) from t1 ") tdSql.error(" select sample(c1,false) from t1 ") - tdSql.error(" select sample(123,1) from t1 ") + tdSql.query(" select sample(123,1) from t1 ") tdSql.query(" select sample(c1,2) from t1 ") tdSql.checkRows(2) diff --git a/tests/system-test/2-query/statecount.py b/tests/system-test/2-query/statecount.py index 2634d9a9ab..ccda55f7ba 100644 --- a/tests/system-test/2-query/statecount.py +++ b/tests/system-test/2-query/statecount.py @@ -85,8 +85,8 @@ class TDTestCase: "select statecount(c1 ,'GT','*') from t1", "select statecount(c1 ,'GT',ts) from t1", "select statecount(c1 ,'GT',max(c1)) from t1", - "select statecount(abs(c1) ,'GT',1) from t1", - "select statecount(c1+2 ,'GT',1) from t1", + # "select statecount(abs(c1) ,'GT',1) from t1", + # "select statecount(c1+2 ,'GT',1) from t1", "select statecount(c1 ,'GT',1,1u) from t1", "select statecount(c1 ,'GT',1,now) from t1", "select statecount(c1 ,'GT','1') from t1", diff --git a/tests/system-test/7-tmq/schema.py b/tests/system-test/7-tmq/schema.py index 633a097db6..51c8da2413 100644 --- a/tests/system-test/7-tmq/schema.py +++ b/tests/system-test/7-tmq/schema.py @@ -282,7 +282,7 @@ class TDTestCase: tdSql.execute("create topic %s as select ts, c1, c2, t1, t2 from %s.%s" %(columnTopicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(columnTopicFromNtb, parameterDict['dbName'], ntbName)) - tsLog.info("======== super table test:") + tdLog.info("======== super table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -301,13 +301,13 @@ class TDTestCase: tdSql.query("alter table %s.%s modify column c4 binary(60)"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s modify tag t4 binary(60)"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s rename column c3 c3new"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s rename column c4 c4new"%(parameterDict['dbName'], parameterDict['stbName'])) + # tdSql.query("alter table %s.%s rename column c3 c3new"%(parameterDict['dbName'], parameterDict['stbName'])) + # tdSql.query("alter table %s.%s rename column c4 c4new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s rename tag t3 t3new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s rename tag t4 t4new"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s drop column c3new"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s drop column c4new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c3"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c4"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s drop tag t3new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s drop tag t4new"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -316,7 +316,7 @@ class TDTestCase: tdSql.query("alter table %s.%s add tag t3 int"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s add tag t4 float"%(parameterDict['dbName'], parameterDict['stbName'])) - tsLog.info("======== normal table test:") + tdLog.info("======== normal table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], ntbName)) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], ntbName)) @@ -408,6 +408,7 @@ class TDTestCase: parameterDict['cfg'] = cfgPath # tdLog.info("create database, super table, child table, normal table") + # self.create_database(tdSql, parameterDict["dbName"]) ntbName = 'ntb2' tdSql.query("create table %s.%s (ts timestamp, c1 int, c2 binary(32), c3 double, c4 binary(32), c5 nchar(10)) tags (t1 int, t2 binary(32), t3 double, t4 binary(32), t5 nchar(10))"%(parameterDict["dbName"],parameterDict["stbName"])) tdSql.query("create table %s.%s (ts timestamp, c1 int, c2 binary(32), c3 double, c4 binary(32), c5 nchar(10))"%(parameterDict["dbName"],ntbName)) @@ -419,7 +420,7 @@ class TDTestCase: tdSql.execute("create topic %s as select ts, c1, c2, t1, t2 from %s.%s where c3 > 3 and c4 like 'abc' and t3 = 5 and t4 = 'beijing'" %(columnTopicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s where c3 > 3 and c4 like 'abc'" %(columnTopicFromNtb, parameterDict['dbName'], ntbName)) - tsLog.info("======== super table test:") + tdLog.info("======== super table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -457,7 +458,7 @@ class TDTestCase: tdSql.query("alter table %s.%s add column c5 int"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s add tag t5 float"%(parameterDict['dbName'], parameterDict['stbName'])) - tsLog.info("======== normal table test:") + tdLog.info("======== normal table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], ntbName)) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], ntbName)) @@ -566,7 +567,7 @@ class TDTestCase: tdSql.execute("create topic %s as select * from %s.%s" %(columnTopicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select * from %s.%s " %(columnTopicFromNtb, parameterDict['dbName'], ntbName)) - tsLog.info("======== super table test:") + tdLog.info("======== super table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -601,7 +602,7 @@ class TDTestCase: tdSql.query("alter table %s.%s add column c6 int"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s add tag t6 float"%(parameterDict['dbName'], parameterDict['stbName'])) - tsLog.info("======== normal table test:") + tdLog.info("======== normal table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], ntbName)) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], ntbName)) @@ -688,7 +689,7 @@ class TDTestCase: self.tmqCase1(cfgPath, buildPath) self.tmqCase2(cfgPath, buildPath) - self.tmqCase3(cfgPath, buildPath) + # self.tmqCase3(cfgPath, buildPath) def stop(self): tdSql.close() diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat index ea57bc7427..6ef66c58c2 100644 --- a/tests/system-test/fulltest.bat +++ b/tests/system-test/fulltest.bat @@ -2,6 +2,102 @@ python3 .\test.py -f 0-others\taosShell.py python3 .\test.py -f 0-others\taosShellError.py python3 .\test.py -f 0-others\taosShellNetChk.py +python3 .\test.py -f 0-others\telemetry.py +python3 .\test.py -f 0-others\taosdMonitor.py python3 .\test.py -f 0-others\udfTest.py python3 .\test.py -f 0-others\udf_create.py -python3 .\test.py -f 0-others\udf_restart_taosd.py \ No newline at end of file +python3 .\test.py -f 0-others\udf_restart_taosd.py +@REM python3 .\test.py -f 0-others\cachelast.py + +@REM python3 .\test.py -f 0-others\user_control.py +@REM python3 .\test.py -f 0-others\fsync.py + +@REM python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py +@REM python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py +@REM python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py +@REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py +@REM python3 .\test.py -f 1-insert\alter_stable.py +@REM python3 .\test.py -f 1-insert\alter_table.py +@REM python3 .\test.py -f 2-query\between.py +@REM python3 .\test.py -f 2-query\distinct.py +@REM python3 .\test.py -f 2-query\varchar.py +@REM python3 .\test.py -f 2-query\ltrim.py +@REM python3 .\test.py -f 2-query\rtrim.py +@REM python3 .\test.py -f 2-query\length.py +@REM python3 .\test.py -f 2-query\char_length.py +@REM python3 .\test.py -f 2-query\upper.py +@REM python3 .\test.py -f 2-query\lower.py +@REM python3 .\test.py -f 2-query\join.py +@REM python3 .\test.py -f 2-query\join2.py +@REM python3 .\test.py -f 2-query\cast.py +@REM python3 .\test.py -f 2-query\union.py +@REM python3 .\test.py -f 2-query\union1.py +@REM python3 .\test.py -f 2-query\concat.py +@REM python3 .\test.py -f 2-query\concat2.py +@REM python3 .\test.py -f 2-query\concat_ws.py +@REM python3 .\test.py -f 2-query\concat_ws2.py +@REM python3 .\test.py -f 2-query\check_tsdb.py +@REM python3 .\test.py -f 2-query\spread.py +@REM python3 .\test.py -f 2-query\hyperloglog.py + + +@REM python3 .\test.py -f 2-query\timezone.py +@REM python3 .\test.py -f 2-query\Now.py +@REM python3 .\test.py -f 2-query\Today.py +@REM python3 .\test.py -f 2-query\max.py +@REM python3 .\test.py -f 2-query\min.py +@REM python3 .\test.py -f 2-query\count.py +@REM python3 .\test.py -f 2-query\last.py +@REM python3 .\test.py -f 2-query\first.py +@REM python3 .\test.py -f 2-query\To_iso8601.py +@REM python3 .\test.py -f 2-query\To_unixtimestamp.py +@REM python3 .\test.py -f 2-query\timetruncate.py +@REM python3 .\test.py -f 2-query\diff.py +@REM python3 .\test.py -f 2-query\Timediff.py + +@REM python3 .\test.py -f 2-query\top.py +@REM python3 .\test.py -f 2-query\bottom.py +@REM python3 .\test.py -f 2-query\percentile.py +@REM python3 .\test.py -f 2-query\apercentile.py +@REM python3 .\test.py -f 2-query\abs.py +@REM python3 .\test.py -f 2-query\ceil.py +@REM python3 .\test.py -f 2-query\floor.py +@REM python3 .\test.py -f 2-query\round.py +@REM python3 .\test.py -f 2-query\log.py +@REM python3 .\test.py -f 2-query\pow.py +@REM python3 .\test.py -f 2-query\sqrt.py +@REM python3 .\test.py -f 2-query\sin.py +@REM python3 .\test.py -f 2-query\cos.py +@REM python3 .\test.py -f 2-query\tan.py +@REM python3 .\test.py -f 2-query\arcsin.py +@REM python3 .\test.py -f 2-query\arccos.py +@REM python3 .\test.py -f 2-query\arctan.py +@REM python3 .\test.py -f 2-query\query_cols_tags_and_or.py +@REM # python3 .\test.py -f 2-query\nestedQuery.py +@REM # TD-15983 subquery output duplicate name column. +@REM # Please Xiangyang Guo modify the following script +@REM # python3 .\test.py -f 2-query\nestedQuery_str.py + +@REM python3 .\test.py -f 2-query\avg.py +@REM python3 .\test.py -f 2-query\elapsed.py +@REM python3 .\test.py -f 2-query\csum.py +@REM python3 .\test.py -f 2-query\mavg.py +@REM python3 .\test.py -f 2-query\diff.py +@REM python3 .\test.py -f 2-query\sample.py +@REM python3 .\test.py -f 2-query\function_diff.py +@REM python3 .\test.py -f 2-query\unique.py +@REM python3 .\test.py -f 2-query\stateduration.py +@REM python3 .\test.py -f 2-query\function_stateduration.py +@REM python3 .\test.py -f 2-query\statecount.py + +@REM python3 .\test.py -f 7-tmq\basic5.py +@REM python3 .\test.py -f 7-tmq\subscribeDb.py +@REM python3 .\test.py -f 7-tmq\subscribeDb0.py +@REM python3 .\test.py -f 7-tmq\subscribeDb1.py +@REM python3 .\test.py -f 7-tmq\subscribeStb.py +@REM python3 .\test.py -f 7-tmq\subscribeStb0.py +@REM python3 .\test.py -f 7-tmq\subscribeStb1.py +@REM python3 .\test.py -f 7-tmq\subscribeStb2.py +@REM python3 .\test.py -f 7-tmq\subscribeStb3.py +@REM python3 .\test.py -f 7-tmq\subscribeStb4.py +@REM python3 .\test.py -f 7-tmq\db.py \ No newline at end of file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 3991654350..abc8a81248 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -21,6 +21,7 @@ python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py #python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py python3 ./test.py -f 1-insert/alter_stable.py python3 ./test.py -f 1-insert/alter_table.py +# python3 ./test.py -f 1-inerst/create_table_comment.py python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py @@ -57,6 +58,7 @@ python3 ./test.py -f 2-query/To_unixtimestamp.py python3 ./test.py -f 2-query/timetruncate.py python3 ./test.py -f 2-query/diff.py python3 ./test.py -f 2-query/Timediff.py +python3 ./test.py -f 2-query/json_tag.py python3 ./test.py -f 2-query/top.py python3 ./test.py -f 2-query/bottom.py diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index ae6c98b06f..baafbc9e08 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -11,10 +11,12 @@ set /a a=0 @REM ) echo Linux Taosd Test for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( - echo Processing %%i - set /a a+=1 - call %%i ARG1 -m %1 > result_!a!.txt 2>error_!a!.txt - if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) + for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( + echo Processing %%i + set /a a+=1 + call %%i ARG1 -m %1 > result_!a!.txt 2>error_!a!.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && echo result: && cat result_!a!.txt && echo error: && cat error_!a!.txt && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) + ) ) exit diff --git a/tests/system-test/test.py b/tests/system-test/test.py index a11085708c..d9ae792901 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -114,6 +114,7 @@ if __name__ == "__main__": if not execCmd == "": tdDnodes.init(deployPath) + print(execCmd) exec(execCmd) quit() diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 851d9a2070..329842aeb3 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1003,7 +1003,7 @@ int32_t shellExecute() { taosSetSignal(SIGINT, shellSigintHandler); - shellGetGrantInfo(shell.conn); + shellGetGrantInfo(); while (1) { taosThreadCreate(&shell.pid, NULL, shellThreadLoop, shell.conn);