diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 39dc5361e6..47c26ed431 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -177,8 +177,8 @@ typedef struct { typedef struct SField { char name[TSDB_COL_NAME_LEN]; uint8_t type; - int32_t bytes; int8_t flags; + int32_t bytes; } SField; typedef struct SRetention { @@ -1122,13 +1122,13 @@ typedef struct { SSchema* pSchemas; } STableMetaRsp; -typedef struct { +typedef struct { STableMetaRsp* pMeta; } SMAlterStbRsp; -int32_t tEncodeSMAlterStbRsp(SEncoder *pEncoder, const SMAlterStbRsp *pRsp); -int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp); -void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp); +int32_t tEncodeSMAlterStbRsp(SEncoder* pEncoder, const SMAlterStbRsp* pRsp); +int32_t tDecodeSMAlterStbRsp(SDecoder* pDecoder, SMAlterStbRsp* pRsp); +void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp); int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); @@ -2303,23 +2303,23 @@ typedef struct { } SVgEpSet; typedef struct { - int8_t version; // for compatibility(default 0) - int8_t intervalUnit; // MACRO: TIME_UNIT_XXX - int8_t slidingUnit; // MACRO: TIME_UNIT_XXX - int8_t timezoneInt; // sma data expired if timezone changes. - int32_t dstVgId; - char indexName[TSDB_INDEX_NAME_LEN]; - int32_t exprLen; - int32_t tagsFilterLen; - int32_t numOfVgroups; - int64_t indexUid; - tb_uid_t tableUid; // super/child/common table uid - int64_t interval; - int64_t offset; // use unit by precision of DB - int64_t sliding; - char* expr; // sma expression - char* tagsFilter; - SVgEpSet vgEpSet[]; + int8_t version; // for compatibility(default 0) + int8_t intervalUnit; // MACRO: TIME_UNIT_XXX + int8_t slidingUnit; // MACRO: TIME_UNIT_XXX + int8_t timezoneInt; // sma data expired if timezone changes. + int32_t dstVgId; + char indexName[TSDB_INDEX_NAME_LEN]; + int32_t exprLen; + int32_t tagsFilterLen; + int32_t numOfVgroups; + int64_t indexUid; + tb_uid_t tableUid; // super/child/common table uid + int64_t interval; + int64_t offset; // use unit by precision of DB + int64_t sliding; + char* expr; // sma expression + char* tagsFilter; + SVgEpSet* pVgEpSet; } STSma; // Time-range-wise SMA typedef STSma SVCreateTSmaReq; @@ -2405,7 +2405,7 @@ static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) { } typedef struct { - int64_t indexUid; + int64_t indexUid; STimeWindow queryWindow; } SVGetTsmaExpWndsReq; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 29fabb5418..6f52f8dab0 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -211,7 +211,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_MERGE, QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, - QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 38c3f150ef..5811b76588 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -42,6 +42,7 @@ typedef struct SScanLogicNode { SNodeList* pScanPseudoCols; int8_t tableType; uint64_t tableId; + uint64_t stableId; SVgroupsInfo* pVgroupList; EScanType scanType; uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count @@ -94,7 +95,7 @@ typedef struct SVnodeModifyLogicNode { int32_t msgType; SArray* pDataBlocks; SVgDataBlocks* pVgDataBlocks; - SNode* pModifyRows; // SColumnNode + SNode* pAffectedRows; // SColumnNode uint64_t tableId; int8_t tableType; // table type char tableFName[TSDB_TABLE_FNAME_LEN]; @@ -109,6 +110,7 @@ typedef struct SExchangeLogicNode { typedef struct SMergeLogicNode { SLogicNode node; SNodeList* pMergeKeys; + SNodeList* pInputs; int32_t numOfChannels; int32_t srcGroupId; } SMergeLogicNode; @@ -117,7 +119,7 @@ typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW typedef enum EIntervalAlgorithm { INTERVAL_ALGO_HASH = 1, - INTERVAL_ALGO_SORT_MERGE, + INTERVAL_ALGO_MERGE, INTERVAL_ALGO_STREAM_FINAL, INTERVAL_ALGO_STREAM_SEMI, INTERVAL_ALGO_STREAM_SINGLE, @@ -220,6 +222,7 @@ typedef struct SScanPhysiNode { SNodeList* pScanCols; SNodeList* pScanPseudoCols; uint64_t uid; // unique id of the table + uint64_t suid; int8_t tableType; SName tableName; } SScanPhysiNode; @@ -296,6 +299,7 @@ typedef struct SExchangePhysiNode { typedef struct SMergePhysiNode { SPhysiNode node; SNodeList* pMergeKeys; + SNodeList* pTargets; int32_t numOfChannels; int32_t srcGroupId; } SMergePhysiNode; @@ -319,7 +323,7 @@ typedef struct SIntervalPhysiNode { int8_t slidingUnit; } SIntervalPhysiNode; -typedef SIntervalPhysiNode SSortMergeIntervalPhysiNode; +typedef SIntervalPhysiNode SMergeIntervalPhysiNode; typedef SIntervalPhysiNode SStreamIntervalPhysiNode; typedef SIntervalPhysiNode SStreamFinalIntervalPhysiNode; typedef SIntervalPhysiNode SStreamSemiIntervalPhysiNode; @@ -388,6 +392,7 @@ typedef struct SDataDeleterNode { int8_t tableType; // table type char tableFName[TSDB_TABLE_FNAME_LEN]; STimeWindow deleteTimeRange; + SNode* pAffectedRows; } SDataDeleterNode; typedef struct SSubplan { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index cbef0b46a7..17851bc4a8 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -218,6 +218,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0390) #define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0391) #define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0392) +#define TSDB_CODE_MND_VGROUP_UN_CHANGED TAOS_DEF_ERROR_CODE(0, 0x0393) // mnode-stable #define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0) @@ -235,6 +236,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x03AC) #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AD) #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE) +#define TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03AF) #define TSDB_CODE_MND_SINGLE_STB_MODE_DB TAOS_DEF_ERROR_CODE(0, 0x03B0) // mnode-infoSchema diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c615af705a..d9293433ea 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1590,7 +1590,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks for (int32_t k = 0; k < colNum; ++k) { // iterate by column SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); STColumn* pCol = &pTSchema->columns[k]; - ASSERT(pCol->type == pColInfoData->info.type); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: @@ -1614,20 +1613,39 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_MEDIUMBLOB: uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); TASSERT(0); break; default: if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pColInfoData->info.type, TD_VTYPE_NORM, var, true, offset, k); + char tv[8] = {0}; + if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, k); } else { uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); TASSERT(0); } break; } - offset += TYPE_BYTES[pColInfoData->info.type]; + offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation } dataLen += TD_ROW_LEN(rb.pBuf); #ifdef TD_DEBUG_PRINT_ROW diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9c9f33ac96..7041c9478e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -694,7 +694,6 @@ void tFreeSMAltertbReq(SMAlterStbReq *pReq) { pReq->pFields = NULL; } - int32_t tSerializeSEpSet(void *buf, int32_t bufLen, const SEpSet *pEpset) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3674,12 +3673,12 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1; } for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { - if (tEncodeI32(pCoder, pSma->vgEpSet[v].vgId) < 0) return -1; - if (tEncodeI8(pCoder, pSma->vgEpSet[v].epSet.inUse) < 0) return -1; - int8_t numOfEps = pSma->vgEpSet[v].epSet.numOfEps; + if (tEncodeI32(pCoder, pSma->pVgEpSet[v].vgId) < 0) return -1; + if (tEncodeI8(pCoder, pSma->pVgEpSet[v].epSet.inUse) < 0) return -1; + int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps; if (tEncodeI8(pCoder, numOfEps) < 0) return -1; for (int32_t n = 0; n < numOfEps; ++n) { - const SEp *pEp = &pSma->vgEpSet[v].epSet.eps[n]; + const SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n]; if (tEncodeCStr(pCoder, pEp->fqdn) < 0) return -1; if (tEncodeU16(pCoder, pEp->port) < 0) return -1; } @@ -3712,15 +3711,25 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { } else { pSma->tagsFilter = NULL; } - for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { - if (tDecodeI32(pCoder, &pSma->vgEpSet[v].vgId) < 0) return -1; - if (tDecodeI8(pCoder, &pSma->vgEpSet[v].epSet.inUse) < 0) return -1; - if (tDecodeI8(pCoder, &pSma->vgEpSet[v].epSet.numOfEps) < 0) return -1; - int8_t numOfEps = pSma->vgEpSet[v].epSet.numOfEps; - for (int32_t n = 0; n < numOfEps; ++n) { - SEp *pEp = &pSma->vgEpSet[v].epSet.eps[n]; - if (tDecodeCStrTo(pCoder, pEp->fqdn) < 0) return -1; - if (tDecodeU16(pCoder, &pEp->port) < 0) return -1; + if (pSma->numOfVgroups > 0) { + pSma->pVgEpSet = (SVgEpSet *)tDecoderMalloc(pCoder, pSma->numOfVgroups * sizeof(SVgEpSet)); + if (!pSma->pVgEpSet) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + memset(pSma->pVgEpSet, 0, pSma->numOfVgroups * sizeof(SVgEpSet)); + + for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { + if (tDecodeI32(pCoder, &pSma->pVgEpSet[v].vgId) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.inUse) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.numOfEps) < 0) return -1; + int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps; + for (int32_t n = 0; n < numOfEps; ++n) { + SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n]; + if (tDecodeCStrTo(pCoder, pEp->fqdn) < 0) return -1; + if (tDecodeU16(pCoder, &pEp->port) < 0) return -1; + } } } @@ -3765,7 +3774,7 @@ int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { return 0; } -int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder* pCoder, const SVGetTsmaExpWndsReq* pReq) { +int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder *pCoder, const SVGetTsmaExpWndsReq *pReq) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1; @@ -3773,10 +3782,10 @@ int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder* pCoder, const SVGetTsmaExpWndsReq* if (tEncodeI64(pCoder, pReq->queryWindow.ekey) < 0) return -1; tEndEncode(pCoder); - return 0; + return 0; } -int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder* pCoder, SVGetTsmaExpWndsReq* pReq) { +int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder *pCoder, SVGetTsmaExpWndsReq *pReq) { if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; @@ -3787,7 +3796,7 @@ int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder* pCoder, SVGetTsmaExpWndsReq* pReq) return 0; } -int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder* pCoder, const SVGetTsmaExpWndsRsp* pReq) { +int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder *pCoder, const SVGetTsmaExpWndsRsp *pReq) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1; @@ -3814,7 +3823,7 @@ int32_t tDecodeSVGetTsmaExpWndsRsp(SDecoder *pCoder, SVGetTsmaExpWndsRsp *pReq) return 0; } -int32_t tEncodeSVDeleteReq(SEncoder* pCoder, const SVDeleteReq* pReq) { +int32_t tEncodeSVDeleteReq(SEncoder *pCoder, const SVDeleteReq *pReq) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeI64(pCoder, pReq->delUid) < 0) return -1; @@ -3832,7 +3841,7 @@ int32_t tEncodeSVDeleteReq(SEncoder* pCoder, const SVDeleteReq* pReq) { return 0; } -int32_t tDecodeSVDeleteReq(SDecoder* pCoder, SVDeleteReq* pReq) { +int32_t tDecodeSVDeleteReq(SDecoder *pCoder, SVDeleteReq *pReq) { if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI64(pCoder, &pReq->delUid) < 0) return -1; @@ -3850,7 +3859,7 @@ int32_t tDecodeSVDeleteReq(SDecoder* pCoder, SVDeleteReq* pReq) { return 0; } -int32_t tEncodeSVDeleteRsp(SEncoder* pCoder, const SVDeleteRsp* pReq) { +int32_t tEncodeSVDeleteRsp(SEncoder *pCoder, const SVDeleteRsp *pReq) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeI32(pCoder, pReq->code) < 0) return -1; @@ -3860,7 +3869,7 @@ int32_t tEncodeSVDeleteRsp(SEncoder* pCoder, const SVDeleteRsp* pReq) { return 0; } -int32_t tDecodeSVDeleteRsp(SDecoder* pCoder, SVDeleteRsp* pReq) { +int32_t tDecodeSVDeleteRsp(SDecoder *pCoder, SVDeleteRsp *pReq) { if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI32(pCoder, &pReq->code) < 0) return -1; @@ -4502,7 +4511,7 @@ int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) { } int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) { - int32_t meta = 0; + int32_t meta = 0; SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -4543,7 +4552,7 @@ int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp) { } int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) { - int32_t meta = 0; + int32_t meta = 0; SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -4559,7 +4568,7 @@ int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp return 0; } -void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp) { +void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) { if (NULL == pRsp) { return; } @@ -4569,6 +4578,3 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp) { taosMemoryFree(pRsp->pMeta); } } - - - diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 83a36f4b0d..8ea172ef94 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -298,28 +298,30 @@ typedef struct { } SVgObj; typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; - char stb[TSDB_TABLE_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createdTime; - int64_t uid; - int64_t stbUid; - int64_t dbUid; - int8_t intervalUnit; - int8_t slidingUnit; - int8_t timezone; - int32_t dstVgId; // for stream - int64_t interval; - int64_t offset; - int64_t sliding; - int32_t exprLen; // strlen + 1 - int32_t tagsFilterLen; - int32_t sqlLen; - int32_t astLen; - char* expr; - char* tagsFilter; - char* sql; - char* ast; + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createdTime; + int64_t uid; + int64_t stbUid; + int64_t dbUid; + int8_t intervalUnit; + int8_t slidingUnit; + int8_t timezone; + int32_t dstVgId; // for stream + int64_t interval; + int64_t offset; + int64_t sliding; + int32_t exprLen; // strlen + 1 + int32_t tagsFilterLen; + int32_t sqlLen; + int32_t astLen; + int32_t numOfVgroups; + char* expr; + char* tagsFilter; + char* sql; + char* ast; + SVgEpSet* pVgEpSet; } SSmaObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h index c76186c0a2..cf1e7422be 100644 --- a/source/dnode/mnode/impl/inc/mndDnode.h +++ b/source/dnode/mnode/impl/inc/mndDnode.h @@ -28,7 +28,7 @@ SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId); void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode); SEpSet mndGetDnodeEpset(SDnodeObj *pDnode); int32_t mndGetDnodeSize(SMnode *pMnode); -bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs); +bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndMnode.h b/source/dnode/mnode/impl/inc/mndMnode.h index fd62b3ce75..a433af9947 100644 --- a/source/dnode/mnode/impl/inc/mndMnode.h +++ b/source/dnode/mnode/impl/inc/mndMnode.h @@ -28,6 +28,7 @@ SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId); void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj); bool mndIsMnode(SMnode *pMnode, int32_t dnodeId); void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet); +int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index 4aa18ea591..4becad6da2 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -37,7 +37,7 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); -bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *colIds); +int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 2119320de5..89f93d30b5 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -30,15 +30,21 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); -int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup); -int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); -SArray *mndBuildDnodesArray(SMnode *pMnode); -int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray); -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pVgId); +SArray *mndBuildDnodesArray(SMnode *, int32_t exceptDnodeId); +int32_t mndAllocSmaVgroup(SMnode *, SDbObj *pDb, SVgObj *pVgroup); +int32_t mndAllocVgroup(SMnode *, SDbObj *pDb, SVgObj **ppVgroups); +int32_t mndAddVnodeToVgroup(SMnode *, SVgObj *pVgroup, SArray *pArray); +int32_t mndRemoveVnodeFromVgroup(SMnode *, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid); +int32_t mndAddCreateVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool standby); +int32_t mndAddAlterVnodeConfirmAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup); +int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType); +int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo); +int32_t mndSetMoveVgroupInfoToTrans(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vn, SArray *pArray); +int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId); -void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen, bool standby); -void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *cntlen, bool standby); +void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildAlterVnodeReq(SMnode *, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 38a166900d..ed07e15c63 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -299,7 +299,7 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } @@ -409,7 +409,7 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 1ce1a2590a..d2b468d588 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -263,111 +263,6 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, - bool standby) { - STransAction action = {0}; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen, standby); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_CREATE_VNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - - return 0; -} - -static int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { - STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgroup); - - int32_t contLen = sizeof(SMsgHead); - SMsgHead *pHead = taosMemoryMalloc(contLen); - if (pHead == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pHead->contLen = htonl(contLen); - pHead->vgId = htonl(pVgroup->vgId); - - action.pCont = pHead; - action.contLen = contLen; - action.msgType = TDMT_VND_ALTER_CONFIRM; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pHead); - return -1; - } - - return 0; -} - -static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType) { - STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgroup); - - int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeReq(pMnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = msgType; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - - return 0; -} - -static int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, - bool isRedo) { - STransAction action = {0}; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - - if (isRedo) { - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } else { - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - return 0; -} - static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { char *pos = strstr(dbName, TS_PATH_DELIMITER); if (pos == NULL) { @@ -795,7 +690,7 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; - SArray *pArray = mndBuildDnodesArray(pMnode); + SArray *pArray = mndBuildDnodesArray(pMnode, 0); while (1) { SVgObj *pVgroup = NULL; @@ -1742,3 +1637,4 @@ static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index aeff018aa8..c7e347fb8b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -254,7 +254,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { return sdbGetSize(pSdb, SDB_DNODE); } -bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { +bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs) { int64_t interval = TABS(pDnode->lastAccessTime - curMs); if (interval > 5000 * tsStatusInterval) { if (pDnode->rebootTime > 0) { @@ -393,7 +393,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE); int64_t curMs = taosGetTimestampMs(); - bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); + bool online = mndIsDnodeOnline(pDnode, curMs); bool dnodeChanged = (statusReq.dnodeVer != dnodeVer); bool reboot = (pDnode->rebootTime != statusReq.rebootTime); bool needCheck = !online || dnodeChanged || reboot; @@ -542,7 +542,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { goto CREATE_DNODE_OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto CREATE_DNODE_OVER; } @@ -559,30 +559,36 @@ CREATE_DNODE_OVER: return code; } -static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); - if (pTrans == NULL) { - mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr()); - return -1; - } +static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMnodeObj *pMObj) { + int32_t code = -1; + SSdbRaw *pRaw = NULL; + STrans *pTrans = NULL; + + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); mDebug("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id); - SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + pRaw = mndDnodeActionEncode(pDnode); + if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; + sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING); + pRaw = NULL; - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + pRaw = mndDnodeActionEncode(pDnode); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); + pRaw = NULL; + if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj) != 0) goto _OVER; + if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: mndTransDrop(pTrans); - return 0; + sdbFreeRaw(pRaw); + return code; } static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { @@ -595,42 +601,53 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto DROP_DNODE_OVER; + goto _OVER; } mDebug("dnode:%d, start to drop", dropReq.dnodeId); if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_MND_INVALID_DNODE_ID; - goto DROP_DNODE_OVER; + goto _OVER; } pDnode = mndAcquireDnode(pMnode, dropReq.dnodeId); if (pDnode == NULL) { terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - goto DROP_DNODE_OVER; + goto _OVER; + } + + if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { + terrno = TSDB_CODE_NODE_OFFLINE; + goto _OVER; } pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId); if (pMObj != NULL) { - terrno = TSDB_CODE_MND_MNODE_NOT_EXIST; - goto DROP_DNODE_OVER; + if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) { + terrno = TSDB_CODE_MND_TOO_FEW_MNODES; + goto _OVER; + } + if (pMnode->selfDnodeId == dropReq.dnodeId) { + terrno = TSDB_CODE_MND_CANT_DROP_MASTER; + goto _OVER; + } } pUser = mndAcquireUser(pMnode, pReq->conn.user); if (pUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto DROP_DNODE_OVER; + goto _OVER; } - if (mndCheckNodeAuth(pUser)) { - goto DROP_DNODE_OVER; + if (mndCheckNodeAuth(pUser) != 0) { + goto _OVER; } - code = mndDropDnode(pMnode, pReq, pDnode); + code = mndDropDnode(pMnode, pReq, pDnode, pMObj); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; -DROP_DNODE_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } @@ -638,7 +655,6 @@ DROP_DNODE_OVER: mndReleaseDnode(pMnode, pDnode); mndReleaseUser(pMnode, pUser); mndReleaseMnode(pMnode, pMObj); - return code; } @@ -736,7 +752,7 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_DNODE, pShow->pIter, (void **)&pDnode); if (pShow->pIter == NULL) break; - bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); + bool online = mndIsDnodeOnline(pDnode, curMs); cols = 0; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index d989ba6ec3..69300c0889 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -529,7 +529,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr SMonDnodeDesc desc = {0}; desc.dnode_id = pObj->id; tstrncpy(desc.dnode_ep, pObj->ep, sizeof(desc.dnode_ep)); - if (mndIsDnodeOnline(pMnode, pObj, ms)) { + if (mndIsDnodeOnline(pObj, ms)) { tstrncpy(desc.status, "ready", sizeof(desc.status)); } else { tstrncpy(desc.status, "offline", sizeof(desc.status)); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 1cc832b3af..072ad05f5a 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -358,9 +358,9 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) goto _OVER; - - mDebug("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); mndTransSetSerial(pTrans); + mDebug("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); + if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; @@ -408,7 +408,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (!mndIsDnodeOnline(pMnode, pDnode, taosGetTimestampMs())) { + if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { terrno = TSDB_CODE_NODE_OFFLINE; goto _OVER; } @@ -419,7 +419,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } @@ -535,18 +535,25 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode return 0; } +int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { + if (pObj == NULL) return 0; + if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; + if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; + if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1; + if (mndTransAppendNullLog(pTrans) != 0) return -1; + return 0; +} + static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) { int32_t code = -1; + STrans *pTrans = NULL; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) goto _OVER; - - mDebug("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id); mndTransSetSerial(pTrans); - if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) goto _OVER; - if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) goto _OVER; - if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) goto _OVER; - if (mndTransAppendNullLog(pTrans) != 0) goto _OVER; + mDebug("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id); + + if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -596,7 +603,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } @@ -642,7 +649,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB if (pObj->id == pMnode->selfDnodeId) { roles = syncStr(TAOS_SYNC_STATE_LEADER); } - if (pObj->pDnode && mndIsDnodeOnline(pMnode, pObj->pDnode, curMs)) { + if (pObj->pDnode && mndIsDnodeOnline(pObj->pDnode, curMs)) { roles = syncStr(pObj->state); } char b2[12 + VARSTR_HEADER_SIZE] = {0}; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 4fb9226144..aac6eaba47 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -301,7 +301,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } @@ -411,7 +411,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index e8e23ccc2a..bcc413c1de 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -36,6 +36,7 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw); static int32_t mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma); static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSpSmatb); static int32_t mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew); +static int32_t mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups); static int32_t mndProcessMCreateSmaReq(SRpcMsg *pReq); static int32_t mndProcessMDropSmaReq(SRpcMsg *pReq); static int32_t mndProcessGetSmaReq(SRpcMsg *pReq); @@ -262,7 +263,9 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm req.sliding = pSma->sliding; req.expr = pSma->expr; req.tagsFilter = pSma->tagsFilter; - + req.numOfVgroups = pSma->numOfVgroups; + req.pVgEpSet = pSma->pVgEpSet; + // get length int32_t ret = 0; tEncodeSize(tEncodeSVCreateTSmaReq, &req, contLen, ret); @@ -420,6 +423,15 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, mndReleaseDnode(pMnode, pDnode); // todo add sma info here + SVgEpSet *pVgEpSet = NULL; + int32_t numOfVgroups = 0; + if (mndSmaGetVgEpSet(pMnode, pDb, &pVgEpSet, &numOfVgroups) != 0) { + return -1; + } + + pSma->pVgEpSet = pVgEpSet; + pSma->numOfVgroups = numOfVgroups; + int32_t smaContLen = 0; void *pSmaReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &smaContLen); if (pSmaReq == NULL) return -1; @@ -510,10 +522,9 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; - - mDebug("trans:%d, used to create sma:%s", pTrans->id, pCreate->name); mndTransSetDbName(pTrans, pDb->name); mndTransSetSerial(pTrans); + mDebug("trans:%d, used to create sma:%s", pTrans->id, pCreate->name); if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; @@ -964,3 +975,52 @@ static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +static int32_t mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + SVgEpSet *pVgEpSet = NULL; + int32_t nAllocVgs = 16; + int32_t nVgs = 0; + + pVgEpSet = taosMemoryCalloc(nAllocVgs, sizeof(SVgEpSet)); + if (!pVgEpSet) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) { + sdbRelease(pSdb, pVgroup); + continue; + } + + if (nVgs >= nAllocVgs) { + void *p = taosMemoryRealloc(pVgEpSet, nAllocVgs * 2 * sizeof(SVgEpSet)); + if (!p) { + taosMemoryFree(pVgEpSet); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pVgEpSet = (SVgEpSet *)p; + nAllocVgs *= 2; + } + + (pVgEpSet + nVgs)->vgId = pVgroup->vgId; + (pVgEpSet + nVgs)->epSet = mndGetVgroupEpset(pMnode, pVgroup); + + ++nVgs; + + sdbRelease(pSdb, pVgroup); + } + + *ppVgEpSet = pVgEpSet; + *numOfVgroups = nVgs; + + return 0; +} diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index f5a1e76723..7d21528260 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -307,7 +307,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } @@ -419,7 +419,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 175878959d..9981dc8530 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -23,6 +23,7 @@ #include "mndPerfSchema.h" #include "mndScheduler.h" #include "mndShow.h" +#include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" @@ -394,14 +395,14 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.pRSmaParam.xFilesFactor = pStb->xFilesFactor; req.pRSmaParam.delay = pStb->delay; if (pStb->ast1Len > 0) { - if (mndConvertRSmaTask(pStb->pAst1, pStb->uid, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, req.pRSmaParam.xFilesFactor) != - TSDB_CODE_SUCCESS) { + if (mndConvertRSmaTask(pStb->pAst1, pStb->uid, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, + req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { return NULL; } } if (pStb->ast2Len > 0) { - if (mndConvertRSmaTask(pStb->pAst2, pStb->uid, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, req.pRSmaParam.xFilesFactor) != - TSDB_CODE_SUCCESS) { + if (mndConvertRSmaTask(pStb->pAst2, pStb->uid, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, + req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { return NULL; } } @@ -949,13 +950,18 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p return 0; } -static int32_t mndDropSuperTableTag(const SStbObj *pOld, SStbObj *pNew, const char *tagName) { +static int32_t mndDropSuperTableTag(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const char *tagName) { int32_t tag = mndFindSuperTableTagIndex(pOld, tagName); if (tag < 0) { terrno = TSDB_CODE_MND_TAG_NOT_EXIST; return -1; } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -968,7 +974,7 @@ static int32_t mndDropSuperTableTag(const SStbObj *pOld, SStbObj *pNew, const ch return 0; } -static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pFields) { +static int32_t mndAlterStbTagName(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, SArray *pFields) { if ((int32_t)taosArrayGetSize(pFields) != 2) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; @@ -986,6 +992,11 @@ static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pF return -1; } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndFindSuperTableTagIndex(pOld, newTagName) >= 0) { terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; return -1; @@ -1008,13 +1019,18 @@ static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pF return 0; } -static int32_t mndAlterStbTagBytes(const SStbObj *pOld, SStbObj *pNew, const SField *pField) { +static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const SField *pField) { int32_t tag = mndFindSuperTableTagIndex(pOld, pField->name); if (tag < 0) { terrno = TSDB_CODE_MND_TAG_NOT_EXIST; return -1; } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -1075,7 +1091,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray return 0; } -static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const char *colName) { +static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const char *colName) { int32_t col = mndFindSuperTableColumnIndex(pOld, colName); if (col < 0) { terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST; @@ -1092,6 +1108,11 @@ static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const return -1; } + col_id_t colId = pOld->pColumns[col].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -1104,7 +1125,7 @@ static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const return 0; } -static int32_t mndAlterStbColumnBytes(const SStbObj *pOld, SStbObj *pNew, const SField *pField) { +static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const SField *pField) { int32_t col = mndFindSuperTableColumnIndex(pOld, pField->name); if (col < 0) { terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST; @@ -1121,6 +1142,11 @@ static int32_t mndAlterStbColumnBytes(const SStbObj *pOld, SStbObj *pNew, const return -1; } + col_id_t colId = pOld->pColumns[col].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -1199,7 +1225,6 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } - static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableMetaRsp *pRsp) { taosRLockLatch(&pStb->lock); @@ -1269,13 +1294,13 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return code; } - -static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, int32_t *pLen) { +static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, + int32_t *pLen) { int ret; SEncoder ec = {0}; - uint32_t contLen = 0; + uint32_t contLen = 0; SMAlterStbRsp alterRsp = {0}; - SName name = {0}; + SName name = {0}; tNameFromString(&name, pAlter->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); alterRsp.pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); @@ -1283,20 +1308,20 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, S terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - + ret = mndBuildStbSchemaImp(pDb, pObj, name.tname, alterRsp.pMeta); if (ret) { tFreeSMAlterStbRsp(&alterRsp); return ret; } - + tEncodeSize(tEncodeSMAlterStbRsp, &alterRsp, contLen, ret); if (ret) { tFreeSMAlterStbRsp(&alterRsp); return ret; } - void* cont = taosMemoryMalloc(contLen); + void *cont = taosMemoryMalloc(contLen); tEncoderInit(&ec, cont, contLen); tEncodeSMAlterStbRsp(&ec, &alterRsp); tEncoderClear(&ec); @@ -1305,24 +1330,24 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, S *pCont = cont; *pLen = contLen; - + return 0; } static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { - bool needRsp = true; + bool needRsp = true; + int32_t code = -1; + STrans *pTrans = NULL; + SField *pField0 = NULL; + SStbObj stbObj = {0}; taosRLockLatch(&pOld->lock); memcpy(&stbObj, pOld, sizeof(SStbObj)); + taosRUnLockLatch(&pOld->lock); stbObj.pColumns = NULL; stbObj.pTags = NULL; stbObj.updateTime = taosGetTimestampMs(); stbObj.lock = 0; - taosRUnLockLatch(&pOld->lock); - - int32_t code = -1; - STrans *pTrans = NULL; - SField *pField0 = NULL; switch (pAlter->alterType) { case TSDB_ALTER_TABLE_ADD_TAG: @@ -1330,25 +1355,25 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p break; case TSDB_ALTER_TABLE_DROP_TAG: pField0 = taosArrayGet(pAlter->pFields, 0); - code = mndDropSuperTableTag(pOld, &stbObj, pField0->name); + code = mndDropSuperTableTag(pMnode, pOld, &stbObj, pField0->name); break; case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: - code = mndAlterStbTagName(pOld, &stbObj, pAlter->pFields); + code = mndAlterStbTagName(pMnode, pOld, &stbObj, pAlter->pFields); break; case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: pField0 = taosArrayGet(pAlter->pFields, 0); - code = mndAlterStbTagBytes(pOld, &stbObj, pField0); + code = mndAlterStbTagBytes(pMnode, pOld, &stbObj, pField0); break; case TSDB_ALTER_TABLE_ADD_COLUMN: code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); break; case TSDB_ALTER_TABLE_DROP_COLUMN: pField0 = taosArrayGet(pAlter->pFields, 0); - code = mndDropSuperTableColumn(pOld, &stbObj, pField0->name); + code = mndDropSuperTableColumn(pMnode, pOld, &stbObj, pField0->name); break; case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: pField0 = taosArrayGet(pAlter->pFields, 0); - code = mndAlterStbColumnBytes(pOld, &stbObj, pField0); + code = mndAlterStbColumnBytes(pMnode, pOld, &stbObj, pField0); break; case TSDB_ALTER_TABLE_UPDATE_OPTIONS: needRsp = false; @@ -1370,12 +1395,12 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p mndTransSetDbName(pTrans, pDb->name); if (needRsp) { - void* pCont = NULL; + void *pCont = NULL; int32_t contLen = 0; - if (mndBuildSMAlterStbRsp(pDb, pAlter, &stbObj, &pCont, &contLen)) goto _OVER; + if (mndBuildSMAlterStbRsp(pDb, pAlter, &stbObj, &pCont, &contLen) != 0) goto _OVER; mndTransSetRpcRsp(pTrans, pCont, contLen); } - + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 95f22a33db..605e446e2e 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -71,7 +71,7 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]) { return strchr(topic, '.') + 1; } -bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *colAndTagIds) { +int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; bool found = false; @@ -91,7 +91,7 @@ bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *col } SHashObj *pColHash = NULL; - SNodeList *pNodeList; + SNodeList *pNodeList = NULL; nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList); SNode *pNode = NULL; FOREACH(pNode, pNodeList) { @@ -103,22 +103,24 @@ bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *col if (pCol->colId > 0) { taosHashPut(pColHash, &pCol->colId, sizeof(int16_t), NULL, 0); } + mTrace("topic:%s, colId:%d is used", pTopic->name, pCol->colId); } - for (int32_t i = 0; i < taosArrayGetSize(colAndTagIds); i++) { - int16_t *pColId = taosArrayGet(colAndTagIds, i); - if (taosHashGet(pColHash, pColId, sizeof(int16_t)) != NULL) { - found = true; - goto NEXT; - } + if (taosHashGet(pColHash, &colId, sizeof(int16_t)) != NULL) { + found = true; + goto NEXT; } NEXT: sdbRelease(pSdb, pTopic); nodesDestroyNode(pAst); - if (found) return false; + if (found) { + terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; + return -1; + } } - return true; + + return 0; } SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 4f09eda733..d8f2dae234 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -15,11 +15,13 @@ #define _DEFAULT_SOURCE #include "mndVgroup.h" +#include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" +#include "mndUser.h" #define VGROUP_VER_NUMBER 1 #define VGROUP_RESERVE_SIZE 64 @@ -34,6 +36,9 @@ static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); +static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq); +static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq); + int32_t mndInitVgroup(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_VGROUP, @@ -344,9 +349,14 @@ static bool mndResetDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SDnodeObj *pDnode = pObj; SArray *pArray = p1; + int32_t exceptDnodeId = *(int32_t *)p2; + + if (exceptDnodeId == pDnode->id) { + return true; + } int64_t curMs = taosGetTimestampMs(); - bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); + bool online = mndIsDnodeOnline(pDnode, curMs); bool isMnode = mndIsMnode(pMnode, pDnode->id); pDnode->numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); @@ -363,7 +373,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 return true; } -SArray *mndBuildDnodesArray(SMnode *pMnode) { +SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) { SSdb *pSdb = pMnode->pSdb; int32_t numOfDnodes = mndGetDnodeSize(pMnode); @@ -374,7 +384,7 @@ SArray *mndBuildDnodesArray(SMnode *pMnode) { } sdbTraverse(pSdb, SDB_DNODE, mndResetDnodesArrayFp, NULL, NULL, NULL); - sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, NULL, NULL); + sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, &exceptDnodeId, NULL); return pArray; } @@ -422,7 +432,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr } int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup) { - SArray *pArray = mndBuildDnodesArray(pMnode); + SArray *pArray = mndBuildDnodesArray(pMnode, 0); if (pArray == NULL) return -1; pVgroup->vgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); @@ -451,7 +461,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { goto _OVER; } - pArray = mndBuildDnodesArray(pMnode); + pArray = mndBuildDnodesArray(pMnode, 0); if (pArray == NULL) goto _OVER; mInfo("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), @@ -501,86 +511,6 @@ _OVER: return code; } -int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { - taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); - for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - SDnodeObj *pDnode = taosArrayGet(pArray, i); - mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); - } - - SVnodeGid *pVgid = &pVgroup->vnodeGid[pVgroup->replica]; - for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) { - SDnodeObj *pDnode = taosArrayGet(pArray, d); - - bool used = false; - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) { - used = true; - break; - } - } - if (used) continue; - - if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) { - terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - return -1; - } - - pVgid->dnodeId = pDnode->id; - pVgid->role = TAOS_SYNC_STATE_ERROR; - mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is added", pVgroup->dbName, pVgroup->vgId, pVgroup->replica, - pVgid->dnodeId); - - pVgroup->replica++; - pDnode->numOfVnodes++; - return 0; - } - - terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - mError("db:%s, failed to add vnode to vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); - return -1; -} - -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid) { - taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); - for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - SDnodeObj *pDnode = taosArrayGet(pArray, i); - mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); - } - - int32_t code = -1; - for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) { - SDnodeObj *pDnode = taosArrayGet(pArray, d); - - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; - if (pVgid->dnodeId == pDnode->id) { - mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); - pDnode->numOfVnodes--; - pVgroup->replica--; - *pDelVgid = *pVgid; - *pVgid = pVgroup->vnodeGid[pVgroup->replica]; - memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid)); - code = 0; - goto _OVER; - } - } - } - -_OVER: - if (code != 0) { - terrno = TSDB_CODE_APP_ERROR; - mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); - return -1; - } - - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; - mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); - } - return 0; -} - SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) { SEpSet epset = {0}; @@ -678,7 +608,7 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p bool online = false; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgroup->vnodeGid[i].dnodeId); if (pDnode != NULL) { - online = mndIsDnodeOnline(pMnode, pDnode, curMs); + online = mndIsDnodeOnline(pDnode, curMs); mndReleaseDnode(pMnode, pDnode); } @@ -797,3 +727,442 @@ static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + } + + SVnodeGid *pVgid = &pVgroup->vnodeGid[pVgroup->replica]; + for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) { + SDnodeObj *pDnode = taosArrayGet(pArray, d); + + bool used = false; + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) { + used = true; + break; + } + } + if (used) continue; + + if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } + + pVgid->dnodeId = pDnode->id; + pVgid->role = TAOS_SYNC_STATE_ERROR; + mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is added", pVgroup->dbName, pVgroup->vgId, pVgroup->replica, pVgid->dnodeId); + + pVgroup->replica++; + pDnode->numOfVnodes++; + return 0; + } + + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + mError("db:%s, failed to add vnode to vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); + return -1; +} + +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid) { + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + } + + int32_t code = -1; + for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) { + SDnodeObj *pDnode = taosArrayGet(pArray, d); + + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + if (pVgid->dnodeId == pDnode->id) { + mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); + pDnode->numOfVnodes--; + pVgroup->replica--; + *pDelVgid = *pVgid; + *pVgid = pVgroup->vnodeGid[pVgroup->replica]; + memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid)); + code = 0; + goto _OVER; + } + } + } + +_OVER: + if (code != 0) { + terrno = TSDB_CODE_APP_ERROR; + mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); + return -1; + } + + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); + } + return 0; +} + +int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool standby) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen, standby); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_VNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + +int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + + int32_t contLen = sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryMalloc(contLen); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + action.pCont = pHead; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_CONFIRM; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pHead); + return -1; + } + + return 0; +} + +int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType) { + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeReq(pMnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = msgType; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + +int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_DROP_VNODE; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + +int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vnIndex, + SArray *pArray) { + SVgObj newVg = {0}; + memcpy(&newVg, pVgroup, sizeof(SVgObj)); + + mInfo("vgId:%d, vgroup info before move, replica:%d", newVg.vgId, newVg.replica); + for (int32_t i = 0; i < newVg.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); + } + + mInfo("vgId:%d, will add 1 vnodes", pVgroup->vgId); + if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[1], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; + + mInfo("vgId:%d, will remove 1 vnodes", pVgroup->vgId); + newVg.replica--; + SVnodeGid del = newVg.vnodeGid[vnIndex]; + newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica]; + memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid)); + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; + + mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica); + for (int32_t i = 0; i < newVg.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); + } + return 0; +} + +int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t delDnodeId) { + SArray *pArray = mndBuildDnodesArray(pMnode, delDnodeId); + if (pArray == NULL) return -1; + + void *pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + int32_t vnIndex = -1; + for (int32_t i = 0; i < pVgroup->replica; ++i) { + if (pVgroup->vnodeGid[i].dnodeId == delDnodeId) { + vnIndex = i; + break; + } + } + + if (vnIndex != -1) { + mInfo("vgId:%d, vnode:%d will be removed from dnode:%d", pVgroup->vgId, vnIndex, delDnodeId); + SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); + mndSetMoveVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, vnIndex, pArray); + mndReleaseDb(pMnode, pDb); + } + + sdbRelease(pMnode->pSdb, pVgroup); + } + + taosArrayDestroy(pArray); + return 0; +} + +static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, + int32_t newDnodeId) { + mDebug("vgId:%d, will add 1 vnode, replica:%d, dnode:%d", pVgroup->vgId, pVgroup->replica, newDnodeId); + + SVnodeGid *pGid = &pVgroup->vnodeGid[pVgroup->replica]; + pVgroup->replica++; + pGid->dnodeId = newDnodeId; + pGid->role = TAOS_SYNC_STATE_ERROR; + + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pGid, true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1; + + return 0; +} + +static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, + int32_t delDnodeId) { + mDebug("vgId:%d, will remove 1 vnode, replica:%d, dnode:%d", pVgroup->vgId, pVgroup->replica, delDnodeId); + + SVnodeGid *pGid = NULL; + SVnodeGid delGid = {0}; + for (int32_t i = 0; i < pVgroup->replica; ++i) { + if (pVgroup->vnodeGid[i].dnodeId == delDnodeId) { + pGid = &pVgroup->vnodeGid[i]; + break; + } + } + + if (pGid == NULL) return 0; + + memcpy(&delGid, pGid, sizeof(SVnodeGid)); + memcpy(pGid, &pVgroup->vnodeGid[pVgroup->replica], sizeof(SVnodeGid)); + memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid)); + pVgroup->replica--; + + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1; + + return 0; +} + +static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup, SDnodeObj *pNew1, + SDnodeObj *pOld1, SDnodeObj *pNew2, SDnodeObj *pOld2, SDnodeObj *pNew3, + SDnodeObj *pOld3) { + int32_t code = -1; + SSdbRaw *pRaw = NULL; + STrans *pTrans = NULL; + + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); + mDebug("trans:%d, used to drop redistribute vgId:%d", pTrans->id, pVgroup->vgId); + + SVgObj newVg = {0}; + memcpy(&newVg, pVgroup, sizeof(SVgObj)); + mInfo("vgId:%d, vgroup info before redistribute, replica:%d", newVg.vgId, newVg.replica); + for (int32_t i = 0; i < newVg.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); + } + + if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew1->id) != 0) goto _OVER; + if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld1->id) != 0) goto _OVER; + if (pNew2 != NULL) { + if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew2->id) != 0) goto _OVER; + if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld2->id) != 0) goto _OVER; + if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew3->id) != 0) goto _OVER; + if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld3->id) != 0) goto _OVER; + } + + pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + + mInfo("vgId:%d, vgroup info after redistribute, replica:%d", newVg.vgId, newVg.replica); + for (int32_t i = 0; i < newVg.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); + } + + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + code = 0; + +_OVER: + mndTransDrop(pTrans); + sdbFreeRaw(pRaw); + mndReleaseDb(pMnode, pDb); + return code; +} + +static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SUserObj *pUser = NULL; + SDnodeObj *pNew1 = NULL; + SDnodeObj *pNew2 = NULL; + SDnodeObj *pNew3 = NULL; + SDnodeObj *pOld1 = NULL; + SDnodeObj *pOld2 = NULL; + SDnodeObj *pOld3 = NULL; + SVgObj *pVgroup = NULL; + SDbObj *pDb = NULL; + int32_t code = -1; + int64_t curMs = taosGetTimestampMs(); + SMDropMnodeReq redReq = {0}; + +#if 0 + if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } +#endif + + mDebug("vgId:%d, start to redistribute", 2); + pUser = mndAcquireUser(pMnode, pReq->conn.user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto _OVER; + } + + if (mndCheckNodeAuth(pUser) != 0) { + goto _OVER; + } + + pVgroup = mndAcquireVgroup(pMnode, 2); + if (pVgroup == NULL) goto _OVER; + + pDb = mndAcquireDb(pMnode, pVgroup->dbName); + if (pDb == NULL) goto _OVER; + + if (pVgroup->replica == 1) { + pNew1 = mndAcquireDnode(pMnode, 1); + pOld1 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[0].dnodeId); + if (pNew1 == NULL || pOld1 == NULL) goto _OVER; + if (!mndIsDnodeOnline(pNew1, curMs) || !mndIsDnodeOnline(pOld1, curMs)) { + terrno = TSDB_CODE_NODE_OFFLINE; + goto _OVER; + } + if (pNew1 == pOld1) { + terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + goto _OVER; + } + if (mndRedistributeVgroup(pMnode, pReq, pDb, pVgroup, pNew1, pOld1, NULL, NULL, NULL, NULL) != 0) goto _OVER; + } + + if (pVgroup->replica == 3) { + pNew1 = mndAcquireDnode(pMnode, 1); + pNew2 = mndAcquireDnode(pMnode, 2); + pNew3 = mndAcquireDnode(pMnode, 3); + pOld1 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[0].dnodeId); + pOld2 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[1].dnodeId); + pOld3 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[2].dnodeId); + if (pNew1 == NULL || pOld1 == NULL || pNew2 == NULL || pOld2 == NULL || pNew3 == NULL || pOld3 == NULL) goto _OVER; + if (!mndIsDnodeOnline(pNew1, curMs) || !mndIsDnodeOnline(pOld1, curMs) || !mndIsDnodeOnline(pNew2, curMs) || + !mndIsDnodeOnline(pOld2, curMs) || !mndIsDnodeOnline(pNew3, curMs) || !mndIsDnodeOnline(pOld3, curMs)) { + terrno = TSDB_CODE_NODE_OFFLINE; + goto _OVER; + } + bool changed = true; + if (pNew1 != pOld1 || pNew1 != pOld2 || pNew1 != pOld3) changed = true; + if (pNew2 != pOld1 || pNew2 != pOld2 || pNew2 != pOld3) changed = true; + if (pNew3 != pOld1 || pNew3 != pOld2 || pNew3 != pOld3) changed = true; + if (!changed) { + terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + goto _OVER; + } + if (mndRedistributeVgroup(pMnode, pReq, pDb, pVgroup, pNew1, pOld1, pNew2, pOld2, pNew3, pOld3) != 0) goto _OVER; + } + + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + +_OVER: + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mDebug("vgId:%d, failed to redistribute since %s", 1, terrstr()); + } + + mndReleaseDnode(pMnode, pNew1); + mndReleaseDnode(pMnode, pNew2); + mndReleaseDnode(pMnode, pNew3); + mndReleaseDnode(pMnode, pOld1); + mndReleaseDnode(pMnode, pOld2); + mndReleaseDnode(pMnode, pOld3); + mndReleaseUser(pMnode, pUser); + mndReleaseVgroup(pMnode, pVgroup); + mndReleaseDb(pMnode, pDb); + + return code; +} + +static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { return 0; } \ No newline at end of file diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index 90643a54a9..7720a8e88a 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -42,8 +42,10 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { } void sdbFreeRaw(SSdbRaw *pRaw) { - mTrace("raw:%p, is freed", pRaw); - taosMemoryFree(pRaw); + if (pRaw != NULL) { + mTrace("raw:%p, is freed", pRaw); + taosMemoryFree(pRaw); + } } int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 4f4a940bc7..c6a6dd8147 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -103,7 +103,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid); indexMultiTermDestroy(terms); #endif - return -1; + return 0; } int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 7d597217ee..17a06a40c1 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -773,7 +773,9 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, SNode* pCondition, SExecTaskInfo* pTaskInfo); SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pIndexMap, SExecTaskInfo* pTaskInfo); - +SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, + SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 92b3195f36..10381f72fd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2522,8 +2522,8 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray } int32_t setDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, - SArray* pColList) { + int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, + SArray* pColList) { if (pColList == NULL) { // data from other sources blockCompressDecode(pRes, numOfOutput, numOfRows, pData); } else { // extract data according to pColList @@ -2677,7 +2677,7 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; code = setDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); + pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); goto _error; @@ -2790,7 +2790,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; int32_t code = setDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); + pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64 @@ -2856,7 +2856,7 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { return seqLoadRemoteData(pOperator); } else { return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); -// return concurrentlyLoadRemoteData(pOperator); + // return concurrentlyLoadRemoteData(pOperator); } } @@ -2911,18 +2911,18 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode goto _error; } - pInfo->seqLoadData = false; + pInfo->seqLoadData = false; pInfo->pTransporter = pTransporter; - pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); tsem_init(&pInfo->ready, 0, 0); - pOperator->name = "ExchangeOperator"; + pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfExprs = pInfo->pResult->info.numOfCols; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfExprs = pInfo->pResult->info.numOfCols; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, NULL, NULL, NULL); @@ -4389,7 +4389,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STimeWindowAggSupp twSup = { - .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; + .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; tsdbReaderT pDataReader = NULL; if (pHandle->vnode) { pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond); @@ -4517,6 +4517,18 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID); pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) { + SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode; + + SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc; + SSDataBlock* pResBlock = createResDataBlock(pDescNode); + + SArray* sortInfo = createSortInfo(pMergePhyNode->pMergeKeys); + int32_t numOfOutputCols = 0; + SArray* pColList = + extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID); + + pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pResBlock, sortInfo, pColList, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 5e1b47cc91..c84d4491af 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -13,17 +13,18 @@ * along with this program. If not, see . */ -#include "tdatablock.h" #include "executorimpl.h" +#include "tdatablock.h" static SSDataBlock* doSort(SOperatorInfo* pOperator); -static int32_t doOpenSortOperator(SOperatorInfo* pOperator); -static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); +static int32_t doOpenSortOperator(SOperatorInfo* pOperator); +static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, - SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, + SExprInfo* pExprInfo, int32_t numOfCols, SArray* pColMatchColInfo, + SExecTaskInfo* pTaskInfo) { SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); int32_t rowSize = pResBlock->info.rowSize; @@ -32,33 +33,33 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR goto _error; } - pOperator->pExpr = pExprInfo; + pOperator->pExpr = pExprInfo; pOperator->numOfExprs = numOfCols; - pInfo->binfo.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = pResBlock; + pInfo->binfo.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = pResBlock; initResultSizeInfo(pOperator, 1024); - pInfo->pSortInfo = pSortInfo; - pInfo->pColMatchInfo= pColMatchColInfo; - pOperator->name = "SortOperator"; + pInfo->pSortInfo = pSortInfo; + pInfo->pColMatchInfo = pColMatchColInfo; + pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; // lazy evaluation for the following parameter since the input datablock is not known till now. -// pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header -// pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer + // pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + + // header pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = - createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, getExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, + getExplainExecInfo); int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pInfo); taosMemoryFree(pOperator); @@ -68,7 +69,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - bool isNull = tsortIsNullVal(pTupleHandle, i); + bool isNull = tsortIsNullVal(pTupleHandle, i); if (isNull) { colDataAppendNULL(pColInfo, pBlock->info.rows); } else { @@ -80,7 +81,8 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { pBlock->info.rows += 1; } -SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo) { +SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, + SArray* pColMatchInfo) { blockDataCleanup(pDataBlock); ASSERT(taosArrayGetSize(pColMatchInfo) == pDataBlock->info.numOfCols); @@ -129,10 +131,11 @@ SSDataBlock* loadNextDataBlock(void* param) { // todo refactor: merged with fetch fp void applyScalarFunction(SSDataBlock* pBlock, void* param) { - SOperatorInfo* pOperator = param; + SOperatorInfo* pOperator = param; SSortOperatorInfo* pSort = pOperator->info; if (pOperator->pExpr != NULL) { - int32_t code = projectApplyFunctions(pOperator->pExpr, pBlock, pBlock, pSort->binfo.pCtx, pOperator->numOfExprs, NULL); + int32_t code = + projectApplyFunctions(pOperator->pExpr, pBlock, pBlock, pSort->binfo.pCtx, pOperator->numOfExprs, NULL); if (code != TSDB_CODE_SUCCESS) { longjmp(pOperator->pTaskInfo->env, code); } @@ -141,7 +144,7 @@ void applyScalarFunction(SSDataBlock* pBlock, void* param) { int32_t doOpenSortOperator(SOperatorInfo* pOperator) { SSortOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (OPTR_IS_OPENED(pOperator)) { return TSDB_CODE_SUCCESS; @@ -150,8 +153,8 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) { pInfo->startTs = taosGetTimestampUs(); // pInfo->binfo.pRes is not equalled to the input datablock. - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT, - -1, -1, NULL, pTaskInfo->id.str); + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT, -1, -1, + NULL, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator); @@ -166,7 +169,7 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, terrno); } - pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs)/1000.0; + pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs) / 1000.0; pOperator->status = OP_RES_TO_RETURN; OPTR_SET_OPENED(pOperator); @@ -186,7 +189,8 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, code); } - SSDataBlock* pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); + SSDataBlock* pBlock = + getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; @@ -208,10 +212,147 @@ int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* ASSERT(pOptr != NULL); SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); - SSortOperatorInfo *pOperatorInfo = (SSortOperatorInfo*)pOptr->info; + SSortOperatorInfo* pOperatorInfo = (SSortOperatorInfo*)pOptr->info; *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle); *pOptrExplain = pInfo; *len = sizeof(SSortExecInfo); return TSDB_CODE_SUCCESS; } + +typedef struct SMultiwaySortMergeOperatorInfo { + SOptrBasicInfo binfo; + + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + + SArray* pSortInfo; + SSortHandle* pSortHandle; + SArray* pColMatchInfo; // for index map from table scan output + + int64_t startTs; // sort start time +} SMultiwaySortMergeOperatorInfo; + +int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { + SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + pInfo->startTs = taosGetTimestampUs(); + + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_MULTISOURCE_MERGE, + pInfo->bufPageSize, numOfBufPage, NULL, pTaskInfo->id.str); + + tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); + + for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + SSortSource ps = {0}; + ps.param = pOperator->pDownstream[i]; + tsortAddSource(pInfo->pSortHandle, &ps); + } + + int32_t code = tsortOpen(pInfo->pSortHandle); + + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, terrno); + } + + pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs) / 1000.0; + pOperator->status = OP_RES_TO_RETURN; + + OPTR_SET_OPENED(pOperator); + return TSDB_CODE_SUCCESS; +} + +SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; + + int32_t code = pOperator->fpSet._openFn(pOperator); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + SSDataBlock* pBlock = + getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); + + if (pBlock != NULL) { + pOperator->resultInfo.totalRows += pBlock->info.rows; + } else { + doSetOperatorCompleted(pOperator); + } + return pBlock; +} + +void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) { + SMultiwaySortMergeOperatorInfo * pInfo = (SMultiwaySortMergeOperatorInfo*)param; + pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); + + taosArrayDestroy(pInfo->pSortInfo); + taosArrayDestroy(pInfo->pColMatchInfo); +} + +int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { + ASSERT(pOptr != NULL); + SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); + + SMultiwaySortMergeOperatorInfo* pOperatorInfo = (SMultiwaySortMergeOperatorInfo*)pOptr->info; + + *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle); + *pOptrExplain = pInfo; + *len = sizeof(SSortExecInfo); + return TSDB_CODE_SUCCESS; +} + +SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, + SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo, + SExecTaskInfo* pTaskInfo) { + SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + int32_t rowSize = pResBlock->info.rowSize; + + if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { + goto _error; + } + + pInfo->binfo.pRes = pResBlock; + + initResultSizeInfo(pOperator, 1024); + + pInfo->pSortInfo = pSortInfo; + pInfo->pColMatchInfo = pColMatchColInfo; + pOperator->name = "MultiwaySortMerge"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + + pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2; + pInfo->sortBufSize = pInfo->bufPageSize * 16; + + pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = + createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL, + destroyMultiwaySortMergeOperatorInfo, NULL, NULL, getMultiwaySortMergeExplainExecInfo); + + int32_t code = appendDownstream(pOperator, downStreams, numStreams); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + return pOperator; + +_error: + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + return NULL; +} \ No newline at end of file diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 34251ab918..db0d097d29 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -38,7 +38,7 @@ typedef struct SIFParam { col_id_t colId; int64_t suid; // add later char dbName[TSDB_DB_NAME_LEN]; - char colName[TSDB_COL_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN * 2 + 4]; SIndexMetaArg arg; } SIFParam; @@ -171,7 +171,10 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { param->colId = l->colId; param->colValType = l->node.resType.type; memcpy(param->dbName, l->dbName, sizeof(l->dbName)); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-overflow" sprintf(param->colName, "%s_%s", l->colName, r->literal); +#pragma GCC diagnostic pop param->colValType = r->typeData; return 0; // memcpy(param->colName, l->colName, sizeof(l->colName)); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index f277d30eb1..7d58b999dd 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -318,6 +318,7 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { CLONE_NODE_LIST_FIELD(pScanPseudoCols); COPY_SCALAR_FIELD(tableType); COPY_SCALAR_FIELD(tableId); + COPY_SCALAR_FIELD(stableId); CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone); COPY_SCALAR_FIELD(scanType); COPY_OBJECT_FIELD(scanSeq[0], sizeof(uint8_t) * 2); @@ -370,7 +371,7 @@ static SNode* logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModif COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); COPY_SCALAR_FIELD(modifyType); COPY_SCALAR_FIELD(msgType); - CLONE_NODE_FIELD(pModifyRows); + CLONE_NODE_FIELD(pAffectedRows); COPY_SCALAR_FIELD(tableId); COPY_SCALAR_FIELD(tableType); COPY_CHAR_ARRAY_FIELD(tableFName); @@ -387,6 +388,7 @@ static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNo static SNode* logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pMergeKeys); + CLONE_NODE_LIST_FIELD(pInputs); COPY_SCALAR_FIELD(numOfChannels); COPY_SCALAR_FIELD(srcGroupId); return (SNode*)pDst; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c54289bc56..8583f72c26 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -230,6 +230,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiSort"; case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return "PhysiHashInterval"; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: + return "PhysiMergeInterval"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: return "PhysiStreamInterval"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: @@ -611,7 +613,7 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { static const char* jkVnodeModifyLogicPlanModifyType = "ModifyType"; static const char* jkVnodeModifyLogicPlanMsgType = "MsgType"; -static const char* jkVnodeModifyLogicPlanModifyRows = "ModifyRows"; +static const char* jkVnodeModifyLogicPlanAffectedRows = "AffectedRows"; static int32_t logicVnodeModifyNodeToJson(const void* pObj, SJson* pJson) { const SVnodeModifyLogicNode* pNode = (const SVnodeModifyLogicNode*)pObj; @@ -624,7 +626,7 @@ static int32_t logicVnodeModifyNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddIntegerToObject(pJson, jkVnodeModifyLogicPlanMsgType, pNode->msgType); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkVnodeModifyLogicPlanModifyRows, nodeToJson, pNode->pModifyRows); + code = tjsonAddObject(pJson, jkVnodeModifyLogicPlanAffectedRows, nodeToJson, pNode->pAffectedRows); } return code; @@ -641,7 +643,7 @@ static int32_t jsonToLogicVnodeModifyNode(const SJson* pJson, void* pObj) { code = tjsonGetIntValue(pJson, jkVnodeModifyLogicPlanMsgType, &pNode->msgType); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkVnodeModifyLogicPlanModifyRows, &pNode->pModifyRows); + code = jsonToNodeObject(pJson, jkVnodeModifyLogicPlanAffectedRows, &pNode->pAffectedRows); } return code; @@ -1249,6 +1251,7 @@ static int32_t jsonToName(const SJson* pJson, void* pObj) { static const char* jkScanPhysiPlanScanCols = "ScanCols"; static const char* jkScanPhysiPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanPhysiPlanTableId = "TableId"; +static const char* jkScanPhysiPlanSTableId = "STableId"; static const char* jkScanPhysiPlanTableType = "TableType"; static const char* jkScanPhysiPlanTableName = "TableName"; @@ -1265,6 +1268,9 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableId, pNode->uid); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanSTableId, pNode->suid); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableType, pNode->tableType); } @@ -1288,6 +1294,9 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUBigIntValue(pJson, jkScanPhysiPlanTableId, &pNode->uid); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUBigIntValue(pJson, jkScanPhysiPlanSTableId, &pNode->suid); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkScanPhysiPlanTableType, &pNode->tableType); } @@ -1644,6 +1653,7 @@ static int32_t jsonToPhysiExchangeNode(const SJson* pJson, void* pObj) { } static const char* jkMergePhysiPlanMergeKeys = "MergeKeys"; +static const char* jkMergePhysiPlanTargets = "Targets"; static const char* jkMergePhysiPlanNumOfChannels = "NumOfChannels"; static const char* jkMergePhysiPlanSrcGroupId = "SrcGroupId"; @@ -1654,6 +1664,9 @@ static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkMergePhysiPlanMergeKeys, pNode->pMergeKeys); } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkMergePhysiPlanTargets, pNode->pTargets); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkMergePhysiPlanNumOfChannels, pNode->numOfChannels); } @@ -1671,6 +1684,9 @@ static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkMergePhysiPlanMergeKeys, &pNode->pMergeKeys); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkMergePhysiPlanTargets, &pNode->pTargets); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkMergePhysiPlanNumOfChannels, &pNode->numOfChannels); } @@ -2000,6 +2016,7 @@ static const char* jkDeletePhysiPlanTableType = "TableType"; static const char* jkDeletePhysiPlanTableFName = "TableFName"; static const char* jkDeletePhysiPlanDeleteTimeRangeStartKey = "DeleteTimeRangeStartKey"; static const char* jkDeletePhysiPlanDeleteTimeRangeEndKey = "DeleteTimeRangeEndKey"; +static const char* jkDeletePhysiPlanAffectedRows = "AffectedRows"; static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) { const SDataDeleterNode* pNode = (const SDataDeleterNode*)pObj; @@ -2020,6 +2037,9 @@ static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanDeleteTimeRangeEndKey, pNode->deleteTimeRange.ekey); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDeletePhysiPlanAffectedRows, nodeToJson, pNode->pAffectedRows); + } return code; } @@ -2043,6 +2063,9 @@ static int32_t jsonToPhysiDeleteNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkDeletePhysiPlanDeleteTimeRangeEndKey, &pNode->deleteTimeRange.ekey); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDeletePhysiPlanAffectedRows, &pNode->pAffectedRows); + } return code; } @@ -3802,6 +3825,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_SORT: return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: @@ -3930,6 +3954,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_SORT: return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 16ce4d7a48..0aec211e4c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -260,8 +260,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL: - return makeNode(type, sizeof(SSortMergeIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: + return makeNode(type, sizeof(SMergeIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: return makeNode(type, sizeof(SStreamIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp index d4f9a540ae..528d465dbd 100644 --- a/source/libs/parser/test/parInitialDTest.cpp +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -51,7 +51,7 @@ TEST_F(ParserInitialDTest, dropBnode) { } // DROP CONSUMER GROUP [ IF EXISTS ] cgroup_name ON topic_name -TEST_F(ParserInitialDTest, dropCGroup) { +TEST_F(ParserInitialDTest, dropConsumerGroup) { useDb("root", "test"); SMDropCgroupReq expect = {0}; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c3aad632c7..ac46531fad 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -219,9 +219,9 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT return TSDB_CODE_OUT_OF_MEMORY; } - // TSWAP(pScan->pMeta, pRealTable->pMeta); TSWAP(pScan->pVgroupList, pRealTable->pVgroupList); pScan->tableId = pRealTable->pMeta->uid; + pScan->stableId = pRealTable->pMeta->suid; pScan->tableType = pRealTable->pMeta->tableType; pScan->scanSeq[0] = hasRepeatScanFuncs ? 2 : 1; pScan->scanSeq[1] = 0; @@ -1057,8 +1057,8 @@ static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDelet snprintf(pModify->tableFName, sizeof(pModify->tableFName), "%d.%s.%s", pCxt->pPlanCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName); pModify->deleteTimeRange = pDelete->timeRange; - pModify->pModifyRows = nodesCloneNode(pDelete->pCountFunc); - if (NULL == pModify->pModifyRows) { + pModify->pAffectedRows = nodesCloneNode(pDelete->pCountFunc); + if (NULL == pModify->pAffectedRows) { nodesDestroyNode(pModify); return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 3c000e3ff7..bf5ff57949 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -425,6 +425,7 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS if (TSDB_CODE_SUCCESS == code) { pScanPhysiNode->uid = pScanLogicNode->tableId; + pScanPhysiNode->suid = pScanLogicNode->stableId; pScanPhysiNode->tableType = pScanLogicNode->tableType; memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); if (NULL != pScanLogicNode->pTagCond) { @@ -923,8 +924,8 @@ static ENodeType getIntervalOperatorType(EIntervalAlgorithm intervalAlgo) { switch (intervalAlgo) { case INTERVAL_ALGO_HASH: return QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; - case INTERVAL_ALGO_SORT_MERGE: - return QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL; + case INTERVAL_ALGO_MERGE: + return QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL; case INTERVAL_ALGO_STREAM_FINAL: return QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL; case INTERVAL_ALGO_STREAM_SEMI: @@ -1155,6 +1156,8 @@ static int32_t createExchangePhysiNodeByMerge(SMergePhysiNode* pMerge) { nodesDestroyNode(pExchange); return TSDB_CODE_OUT_OF_MEMORY; } + SNode* pSlot = NULL; + FOREACH(pSlot, pExchange->node.pOutputDataBlockDesc->pSlots) { ((SSlotDescNode*)pSlot)->output = true; } return nodesListMakeStrictAppend(&pMerge->node.pChildren, pExchange); } @@ -1168,12 +1171,14 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM pMerge->numOfChannels = pMergeLogicNode->numOfChannels; pMerge->srcGroupId = pMergeLogicNode->srcGroupId; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = addDataBlockSlots(pCxt, pMergeLogicNode->pInputs, pMerge->node.pOutputDataBlockDesc); - for (int32_t i = 0; i < pMerge->numOfChannels; ++i) { - code = createExchangePhysiNodeByMerge(pMerge); - if (TSDB_CODE_SUCCESS != code) { - break; + if (TSDB_CODE_SUCCESS == code) { + for (int32_t i = 0; i < pMerge->numOfChannels; ++i) { + code = createExchangePhysiNodeByMerge(pMerge); + if (TSDB_CODE_SUCCESS != code) { + break; + } } } @@ -1182,6 +1187,14 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM &pMerge->pMergeKeys); } + if (TSDB_CODE_SUCCESS == code) { + code = setListSlotId(pCxt, pMerge->node.pOutputDataBlockDesc->dataBlockId, -1, pMergeLogicNode->node.pTargets, + &pMerge->pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pMerge->pTargets, pMerge->node.pOutputDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pMerge; } else { @@ -1319,13 +1332,21 @@ static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* strcpy(pDeleter->tableFName, pModify->tableFName); pDeleter->deleteTimeRange = pModify->deleteTimeRange; - pDeleter->sink.pInputDataBlockDesc = nodesCloneNode(pRoot->pOutputDataBlockDesc); - if (NULL == pDeleter->sink.pInputDataBlockDesc) { - nodesDestroyNode(pDeleter); - return TSDB_CODE_OUT_OF_MEMORY; + int32_t code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pAffectedRows, + &pDeleter->pAffectedRows); + if (TSDB_CODE_SUCCESS == code) { + pDeleter->sink.pInputDataBlockDesc = nodesCloneNode(pRoot->pOutputDataBlockDesc); + if (NULL == pDeleter->sink.pInputDataBlockDesc) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pSink = (SDataSinkNode*)pDeleter; + } else { + nodesDestroyNode(pDeleter); } - *pSink = (SDataSinkNode*)pDeleter; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 9e6c25ce78..7be33d54e3 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -80,30 +80,36 @@ static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pChild, SE return TSDB_CODE_SUCCESS; } -static int32_t splCreateExchangeNodeForSubplan(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, - ESubplanType subplanType) { - SExchangeLogicNode* pExchange = NULL; - if (TSDB_CODE_SUCCESS != splCreateExchangeNode(pCxt, pSplitNode, &pExchange)) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pSubplan->subplanType = subplanType; - - if (NULL == pSplitNode->pParent) { - pSubplan->pNode = (SLogicNode*)pExchange; +static int32_t splReplaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew) { + if (NULL == pOld->pParent) { + pSubplan->pNode = (SLogicNode*)pNew; return TSDB_CODE_SUCCESS; } SNode* pNode; - FOREACH(pNode, pSplitNode->pParent->pChildren) { - if (nodesEqualNode(pNode, pSplitNode)) { - REPLACE_NODE(pExchange); - pExchange->node.pParent = pSplitNode->pParent; + FOREACH(pNode, pOld->pParent->pChildren) { + if (nodesEqualNode(pNode, pOld)) { + REPLACE_NODE(pNew); + pNew->pParent = pOld->pParent; return TSDB_CODE_SUCCESS; } } - nodesDestroyNode(pExchange); - return TSDB_CODE_FAILED; + return TSDB_CODE_PLAN_INTERNAL_ERROR; +} + +static int32_t splCreateExchangeNodeForSubplan(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, + ESubplanType subplanType) { + SExchangeLogicNode* pExchange = NULL; + int32_t code = splCreateExchangeNode(pCxt, pSplitNode, &pExchange); + if (TSDB_CODE_SUCCESS == code) { + code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pExchange); + } + if (TSDB_CODE_SUCCESS == code) { + pSubplan->subplanType = subplanType; + } else { + nodesDestroyNode(pExchange); + } + return code; } static bool splMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan, int32_t flag, FSplFindSplitNode func, void* pInfo) { @@ -295,24 +301,34 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic return code; } -static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicNode* pParent, SNodeList* pMergeKeys, - SLogicNode* pPartChild) { +static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, + SNodeList* pMergeKeys, SLogicNode* pPartChild) { SMergeLogicNode* pMerge = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); if (NULL == pMerge) { return TSDB_CODE_OUT_OF_MEMORY; } pMerge->numOfChannels = ((SScanLogicNode*)nodesListGetNode(pPartChild->pChildren, 0))->pVgroupList->numOfVgroups; pMerge->srcGroupId = pCxt->groupId; - pMerge->node.pParent = pParent; pMerge->node.precision = pPartChild->precision; pMerge->pMergeKeys = pMergeKeys; - pMerge->node.pTargets = nodesCloneList(pPartChild->pTargets); - if (NULL == pMerge->node.pTargets) { - nodesDestroyNode(pMerge); - return TSDB_CODE_OUT_OF_MEMORY; - } - return nodesListMakeAppend(&pParent->pChildren, pMerge); + int32_t code = TSDB_CODE_SUCCESS; + pMerge->pInputs = nodesCloneList(pPartChild->pTargets); + pMerge->node.pTargets = nodesCloneList(pSplitNode->pTargets); + if (NULL == pMerge->node.pTargets || NULL == pMerge->pInputs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + if (NULL == pSubplan) { + code = nodesListMakeAppend(&pSplitNode->pChildren, pMerge); + } else { + code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); + } + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pMerge); + } + return code; } static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent, SLogicNode* pPartChild) { @@ -329,8 +345,15 @@ static int32_t stbSplSplitWindowNodeForBatch(SSplitContext* pCxt, SStableSplitIn int32_t code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow); if (TSDB_CODE_SUCCESS == code) { ((SWindowLogicNode*)pPartWindow)->intervalAlgo = INTERVAL_ALGO_HASH; - ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_SORT_MERGE; - code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow); + ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_MERGE; + SNodeList* pMergeKeys = NULL; + code = nodesListMakeStrictAppend(&pMergeKeys, nodesCloneNode(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk)); + if (TSDB_CODE_SUCCESS == code) { + code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow); + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pMergeKeys); + } } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, @@ -424,37 +447,99 @@ static int32_t stbSplSplitAggNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) return code; } -static int32_t stbSplCreatePartSortNode(SSortLogicNode* pMergeSort, SLogicNode** pOutput) { - SNodeList* pSortKeys = pMergeSort->pSortKeys; - pMergeSort->pSortKeys = NULL; - SNodeList* pTargets = pMergeSort->node.pTargets; - pMergeSort->node.pTargets = NULL; - SNodeList* pChildren = pMergeSort->node.pChildren; - pMergeSort->node.pChildren = NULL; +static SNode* stbSplCreateColumnNode(SExprNode* pExpr) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return NULL; + } + if (QUERY_NODE_COLUMN == nodeType(pExpr)) { + strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias); + } + strcpy(pCol->colName, pExpr->aliasName); + strcpy(pCol->node.aliasName, pExpr->aliasName); + pCol->node.resType = pExpr->resType; + return (SNode*)pCol; +} + +static SNode* stbSplCreateOrderByExpr(SOrderByExprNode* pSortKey, SNode* pCol) { + SOrderByExprNode* pOutput = nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + if (NULL == pOutput) { + return NULL; + } + pOutput->pExpr = nodesCloneNode(pCol); + if (NULL == pOutput->pExpr) { + nodesDestroyNode(pOutput); + return NULL; + } + pOutput->order = pSortKey->order; + pOutput->nullOrder = pSortKey->nullOrder; + return (SNode*)pOutput; +} + +static int32_t stbSplCreateMergeKeys(SNodeList* pSortKeys, SNodeList* pTargets, SNodeList** pOutput) { + int32_t code = TSDB_CODE_SUCCESS; + SNodeList* pMergeKeys = NULL; + SNode* pNode = NULL; + FOREACH(pNode, pSortKeys) { + SOrderByExprNode* pSortKey = (SOrderByExprNode*)pNode; + SNode* pTarget = NULL; + bool found = false; + FOREACH(pTarget, pTargets) { + if (0 == strcmp(((SExprNode*)pSortKey->pExpr)->aliasName, ((SColumnNode*)pTarget)->colName)) { + code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pTarget)); + if (TSDB_CODE_SUCCESS != code) { + break; + } + found = true; + } + } + if (TSDB_CODE_SUCCESS == code && !found) { + SNode* pCol = stbSplCreateColumnNode((SExprNode*)pSortKey->pExpr); + code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pCol)); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pTargets, pCol); + } else { + nodesDestroyNode(pCol); + } + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pMergeKeys; + } else { + nodesDestroyList(pMergeKeys); + } + return code; +} + +static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOutputPartSort, + SNodeList** pOutputMergeKeys) { + SNodeList* pSortKeys = pSort->pSortKeys; + pSort->pSortKeys = NULL; + SNodeList* pChildren = pSort->node.pChildren; + pSort->node.pChildren = NULL; int32_t code = TSDB_CODE_SUCCESS; - SSortLogicNode* pPartSort = nodesCloneNode(pMergeSort); + SSortLogicNode* pPartSort = nodesCloneNode(pSort); if (NULL == pPartSort) { code = TSDB_CODE_OUT_OF_MEMORY; } - pMergeSort->node.pTargets = pTargets; - pPartSort->node.pChildren = pChildren; + SNodeList* pMergeKeys = NULL; if (TSDB_CODE_SUCCESS == code) { + pPartSort->node.pChildren = pChildren; pPartSort->pSortKeys = pSortKeys; - code = createColumnByRewriteExps(pPartSort->pSortKeys, &pPartSort->node.pTargets); - } - if (TSDB_CODE_SUCCESS == code) { - pMergeSort->pSortKeys = nodesCloneList(pPartSort->node.pTargets); - if (NULL == pMergeSort->pSortKeys) { - code = TSDB_CODE_OUT_OF_MEMORY; - } + code = stbSplCreateMergeKeys(pPartSort->pSortKeys, pPartSort->node.pTargets, &pMergeKeys); } if (TSDB_CODE_SUCCESS == code) { - *pOutput = (SLogicNode*)pPartSort; + *pOutputPartSort = (SLogicNode*)pPartSort; + *pOutputMergeKeys = pMergeKeys; } else { nodesDestroyNode(pPartSort); + nodesDestroyList(pMergeKeys); } return code; @@ -462,17 +547,10 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pMergeSort, SLogicNode** static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { SLogicNode* pPartSort = NULL; - int32_t code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort); + SNodeList* pMergeKeys = NULL; + int32_t code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - SNodeList* pMergeKeys = nodesCloneList(((SSortLogicNode*)pInfo->pSplitNode)->pSortKeys); - if (NULL != pMergeKeys) { - code = stbSplCreateMergeNode(pCxt, pInfo->pSplitNode, pMergeKeys, pPartSort); - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyList(pMergeKeys); - } - } else { - code = TSDB_CODE_OUT_OF_MEMORY; - } + code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, diff --git a/source/libs/planner/test/planOrderByTest.cpp b/source/libs/planner/test/planOrderByTest.cpp index ef6f438b55..851eda81b5 100644 --- a/source/libs/planner/test/planOrderByTest.cpp +++ b/source/libs/planner/test/planOrderByTest.cpp @@ -46,4 +46,7 @@ TEST_F(PlanOrderByTest, stable) { // ORDER BY key is in the projection list run("SELECT c1 FROM st1 ORDER BY c1"); + + // ORDER BY key is not in the projection list + run("SELECT c2 FROM st1 ORDER BY c1"); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 0b1de5dd89..8d7bc0851b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -225,14 +225,14 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, "Vgroup not in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") // mnode-stable -TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "Stable already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_NOT_EXIST, "Stable not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC, "Stable confilct with topic") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "STable already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_NOT_EXIST, "STable not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC, "STable confilct with topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_STBS, "Too many stables") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB, "Invalid stable name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_OPTION, "Invalid stable options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_ALTER_OPTION, "Invalid stable alter options") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED, "Stable option unchanged") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED, "STable option unchanged") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ROW_BYTES, "Invalid row bytes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREADY_EXIST, "Tag already exists") @@ -240,6 +240,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, "Tag does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC,"Field used by topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SINGLE_STB_MODE_DB, "Database is single stable mode") // mnode-infoSchema diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index b6ffda5c86..63b841cfc5 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -388,14 +388,14 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) else: - self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)\ntdDnodes.startWithoutSleep(%d)"%(self.index, self.index)) + self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].deployed=1\ntdDnodes.dnodes[%d].logDir=\"%%s/sim/dnode%%d/log\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.dnodes[%d].cfgDir=\"%%s/sim/dnode%%d/cfg\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.startWithoutSleep(%d)"%(self.index-1,self.index-1,self.index-1,self.index,self.index-1,self.index-1,self.index,self.index)) self.running = 1 tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) def stop(self): if (not self.remoteIP == ""): - self.remoteExec(self.cfgDict, "tdDnodes.stop(%d)"%self.index) + self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1)) tdLog.info("stop dnode%d"%self.index) return if self.valgrind == 0: @@ -426,7 +426,7 @@ class TDDnode: def forcestop(self): if (not self.remoteIP == ""): - self.remoteExec(self.cfgDict, "tdDnodes.forcestop(%d)"%self.index) + self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].forcestop()"%(self.index-1,self.index-1)) return if self.valgrind == 0: toBeKilled = "taosd" @@ -497,46 +497,12 @@ class TDDnodes: self.killValgrind = 1 def init(self, path, remoteIP = ""): - psCmd = "ps -ef|grep -w taosd| grep -v grep| grep -v defunct | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -9 %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - if self.killValgrind == 1: - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -9 %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - binPath = self.dnodes[0].getPath() + "/../../../" # tdLog.debug("binPath %s" % (binPath)) binPath = os.path.realpath(binPath) # tdLog.debug("binPath real path %s" % (binPath)) - # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) - # tdLog.debug(cmd) - # os.system(cmd) - - # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - if path == "": - # self.path = os.path.expanduser('~') self.path = os.path.abspath(binPath + "../../") else: self.path = os.path.realpath(path) diff --git a/tests/script/tsim/dnode/basic1.sim b/tests/script/tsim/dnode/basic1.sim index a5b5427e03..8a28897d5a 100644 --- a/tests/script/tsim/dnode/basic1.sim +++ b/tests/script/tsim/dnode/basic1.sim @@ -183,15 +183,15 @@ if $rows != 15 then endi print =============== drop dnode -sql drop dnode 2; -sql show dnodes; -if $rows != 1 then - return -1 -endi +#sql drop dnode 2; +#sql show dnodes; +#if $rows != 1 then +# return -1 +#endi -if $data00 != 1 then - return -1 -endi +#if $data00 != 1 then +# return -1 +#endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py index 3c99ddb8d6..22c9c8c0c5 100644 --- a/tests/system-test/0-others/taosShellNetChk.py +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -187,50 +187,51 @@ class TDTestCase: # stop taosd tdDnodes.stop(1) - role = 'server' - if platform.system().lower() == 'windows': - taosCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\taos.exe -c ' + keyDict['c'] - taosCmd = taosCmd.replace('\\','\\\\') - taosCmd = taosCmd + ' -n ' + role - else: - taosCmd = 'nohup ' + buildPath + '/build/bin/taos -c ' + keyDict['c'] - taosCmd = taosCmd + ' -n ' + role + ' > /dev/null 2>&1 &' - print (taosCmd) - os.system(taosCmd) + try: + role = 'server' + if platform.system().lower() == 'windows': + taosCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\taos.exe -c ' + keyDict['c'] + taosCmd = taosCmd.replace('\\','\\\\') + taosCmd = taosCmd + ' -n ' + role + else: + taosCmd = 'nohup ' + buildPath + '/build/bin/taos -c ' + keyDict['c'] + taosCmd = taosCmd + ' -n ' + role + ' > /dev/null 2>&1 &' + print (taosCmd) + os.system(taosCmd) - pktLen = '2000' - pktNum = '10' - role = 'client' - if platform.system().lower() == 'windows': - taosCmd = buildPath + '\\build\\bin\\taos.exe -h 127.0.0.1 -c ' + keyDict['c'] - taosCmd = taosCmd.replace('\\','\\\\') - else: - taosCmd = buildPath + '/build/bin/taos -c ' + keyDict['c'] - taosCmd = taosCmd + ' -n ' + role + ' -l ' + pktLen + ' -N ' + pktNum - print (taosCmd) - child = taosExpect.spawn(taosCmd, timeout=3) - i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=6) + pktLen = '2000' + pktNum = '10' + role = 'client' + if platform.system().lower() == 'windows': + taosCmd = buildPath + '\\build\\bin\\taos.exe -h 127.0.0.1 -c ' + keyDict['c'] + taosCmd = taosCmd.replace('\\','\\\\') + else: + taosCmd = buildPath + '/build/bin/taos -c ' + keyDict['c'] + taosCmd = taosCmd + ' -n ' + role + ' -l ' + pktLen + ' -N ' + pktNum + print (taosCmd) + child = taosExpect.spawn(taosCmd, timeout=3) + i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=6) - if platform.system().lower() == 'windows': - retResult = child.before - else: - retResult = child.before.decode() - print("expect() return code: %d, content:\n %s\n"%(i, retResult)) - #print(child.after.decode()) - if i == 0: - tdLog.exit('taos -n server fail!') - - expectString1 = 'response is received, size:' + pktLen - expectSTring2 = pktNum + '/' + pktNum - if expectString1 in retResult and expectSTring2 in retResult: - tdLog.info("taos -n client success") - else: - tdLog.exit('taos -n client fail!') - - if platform.system().lower() == 'windows': - os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9') - else: - os.system('pkill taos') + if platform.system().lower() == 'windows': + retResult = child.before + else: + retResult = child.before.decode() + print("expect() return code: %d, content:\n %s\n"%(i, retResult)) + #print(child.after.decode()) + if i == 0: + tdLog.exit('taos -n server fail!') + + expectString1 = 'response is received, size:' + pktLen + expectSTring2 = pktNum + '/' + pktNum + if expectString1 in retResult and expectSTring2 in retResult: + tdLog.info("taos -n client success") + else: + tdLog.exit('taos -n client fail!') + finally: + if platform.system().lower() == 'windows': + os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9') + else: + os.system('pkill taos') def stop(self): tdSql.close() diff --git a/tests/system-test/0-others/telemetry.py b/tests/system-test/0-others/telemetry.py index 203f87c085..4483e113bf 100644 --- a/tests/system-test/0-others/telemetry.py +++ b/tests/system-test/0-others/telemetry.py @@ -8,6 +8,7 @@ import http.server import gzip import threading import json +import pickle from util.log import * from util.sql import * @@ -136,13 +137,19 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): telemetryInfoCheck(infoDict) # 4. shutdown the server and exit case - assassin = threading.Thread(target=httpServer.shutdown) + assassin = threading.Thread(target=self.server.shutdown) assassin.daemon = True assassin.start() print ("==== shutdown http server ====") class TDTestCase: hostname = socket.gethostname() + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + try: + config = eval(tdDnodes.dnodes[0].remoteIP) + hostname = config["host"] + except Exception: + hostname = tdDnodes.dnodes[0].remoteIP serverPort = '7080' rpcDebugFlagVal = '143' clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} @@ -177,17 +184,20 @@ class TDTestCase: sql = "create database db3 vgroups " + vgroups tdSql.query(sql) - # loop to wait request - httpServer.serve_forever() + # create http server: bing ip/port , and request processor + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + RequestHandlerImplStr = base64.b64encode(pickle.dumps(RequestHandlerImpl)).decode() + telemetryInfoCheckStr = base64.b64encode(pickle.dumps(telemetryInfoCheck)).decode() + cmdStr = "import pickle\nimport http\ntelemetryInfoCheck=pickle.loads(base64.b64decode(\"%s\".encode()))\nRequestHandlerImpl=pickle.loads(base64.b64decode(\"%s\".encode()))\nhttp.server.HTTPServer((\"\", %d), RequestHandlerImpl).serve_forever()"%(telemetryInfoCheckStr,RequestHandlerImplStr,int(telemetryPort)) + tdDnodes.dnodes[0].remoteExec({}, cmdStr) + else: + serverAddress = ("", int(telemetryPort)) + http.server.HTTPServer(serverAddress, RequestHandlerImpl).serve_forever() def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") -# create http server: bing ip/port , and request processor -serverAddress = ("", int(telemetryPort)) -httpServer = http.server.HTTPServer(serverAddress, RequestHandlerImpl) - tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index 88ed667161..0d24b09616 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -9,6 +9,8 @@ from util.sql import * from util.cases import * from util.dnodes import * import subprocess +# import win32gui +# import threading class TDTestCase: @@ -533,8 +535,17 @@ class TDTestCase: return udf1_sqls ,udf2_sqls + # def checkRunTimeError(self): + # while 1: + # time.sleep(1) + # hwnd = win32gui.FindWindow(None, "Microsoft Visual C++ Runtime Library") + # if hwnd: + # os.system("TASKKILL /F /IM udfd.exe") def unexpected_create(self): + # if (platform.system().lower() == 'windows' and tdDnodes.dnodes[0].remoteIP == ""): + # checkErrorThread = threading.Thread(target=self.checkRunTimeError,daemon=True) + # checkErrorThread.start() tdLog.info(" create function with out bufsize ") tdSql.query("drop function udf1 ") diff --git a/tests/system-test/1-insert/create_table_comment.py b/tests/system-test/1-insert/create_table_comment.py new file mode 100644 index 0000000000..92ea083c5a --- /dev/null +++ b/tests/system-test/1-insert/create_table_comment.py @@ -0,0 +1,113 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def get_long_name(self, length, mode="mixed"): + """ + generate long name + mode could be numbers/letters/letters_mixed/mixed + """ + if mode == "numbers": + population = string.digits + elif mode == "letters": + population = string.ascii_letters.lower() + elif mode == "letters_mixed": + population = string.ascii_letters.upper() + string.ascii_letters.lower() + else: + population = string.ascii_letters.lower() + string.digits + return "".join(random.choices(population, k=length)) + + def __create_tb(self,dbname,stbname,tbname,comment): + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute( + f'create table {stbname} (ts timestamp,c0 int) tags(t0 int) ') + tdSql.execute( + f'create table {tbname} using {stbname} tags(1) comment "{comment}"') + def __create_normaltb(self,dbname,tbname,comment): + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute( + f'create table {tbname} (ts timestamp,c0 int) comment "{comment}"') + + def check_comment(self): + dbname = self.get_long_name(length=10, mode="letters") + ntbname = self.get_long_name(length=5, mode="letters") + + # create normal table with comment + comment = self.get_long_name(length=10, mode="letters") + self.__create_normaltb(dbname,ntbname,comment) + ntb_kv_list = tdSql.getResult("show tables") + print(ntb_kv_list) + tdSql.checkEqual(ntb_kv_list[0][8], comment) + tdSql.error('alter table {ntbname} comment "test1"') + tdSql.execute(f'drop database {dbname}') + + # max length(1024) + comment = self.get_long_name(length=1024, mode="letters") + self.__create_normaltb(dbname,ntbname,comment) + ntb_kv_list = tdSql.getResult("show tables") + tdSql.checkEqual(ntb_kv_list[0][8], comment) + tdSql.execute(f'drop database {dbname}') + + # error overlength + comment = self.get_long_name(length=1025, mode="letters") + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.error(f"create table ntb (ts timestamp,c0 int) comment '{comment}'") + tdSql.execute(f'drop database {dbname}') + + # create child table with comment + comment = self.get_long_name(length=10, mode="letters") + stbname = self.get_long_name(length=5, mode="letters") + tbname = self.get_long_name(length=3, mode="letters") + self.__create_tb(dbname,stbname,tbname,comment) + ntb_kv_list = tdSql.getResult("show tables") + tdSql.checkEqual(ntb_kv_list[0][8], comment) + tdSql.error(f'alter table {tbname} comment "test1"') + tdSql.execute(f'drop database {dbname}') + + # max length 1024 + comment = self.get_long_name(length=1024, mode="letters") + self.__create_tb(dbname,ntbname,comment) + ntb_kv_list = tdSql.getResult("show tables") + tdSql.checkEqual(ntb_kv_list[0][8], comment) + tdSql.execute(f'drop database {dbname}') + + # error overlength + comment = self.get_long_name(length=1025, mode="letters") + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute(f"create table stb (ts timestamp,c0 int) tags(t0 int)") + tdSql.error(f'create table stb_1 us stb tags(1) comment "{comment}"') + tdSql.execute(f'drop database {dbname}') + + def run(self): + self.check_comment() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index 6e4c292183..907ba329ee 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -66,6 +66,37 @@ class TDTestCase: ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ''' ) + + + def check_result_auto_log(self ,origin_query , log_query): + + log_result = tdSql.getResult(log_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >0: + elem = math.log(elem) + elif elem <=0: + elem = None + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(log_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("log function value has not as expected , sql is \"%s\" "%log_query ) + sys.exit(1) + else: + tdLog.info("log value check pass , it work as expected ,sql is \"%s\" "%log_query ) def check_result_auto_log2(self ,origin_query , log_query): @@ -270,6 +301,7 @@ class TDTestCase: tdSql.checkRows(25) # used for empty table , ct3 is empty + tdSql.query("select log(c1 ,2) from ct3") tdSql.checkRows(0) tdSql.query("select log(c2 ,2) from ct3") @@ -291,6 +323,13 @@ class TDTestCase: tdSql.checkData(3 , 0, 1.584962501) tdSql.checkData(5 , 0, None) + tdSql.query("select log(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 0.000000000) + tdSql.checkData(2 , 0, 0.693147181) + tdSql.checkData(3 , 0, 1.098612289) + tdSql.checkData(4 , 0, 1.386294361) + tdSql.query("select c1, c2, c3 , c4, c5 from t1") tdSql.checkData(1, 4, 1.11000) tdSql.checkData(3, 3, 33) @@ -301,6 +340,7 @@ class TDTestCase: tdSql.checkData(3, 4, 33) tdSql.checkData(5, 5, None) + self.check_result_auto_log( "select c1, c2, c3 , c4, c5 from t1", "select log(c1), log(c2) ,log(c3), log(c4), log(c5) from t1") self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,2), log(c2 ,2) ,log(c3, 2), log(c4 ,2), log(c5 ,2) from t1") self.check_result_auto_log1( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,1), log(c2 ,1) ,log(c3, 1), log(c4 ,1), log(c5 ,1) from t1") self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,-10), log(c2 ,-10) ,log(c3, -10), log(c4 ,-10), log(c5 ,-10) from t1") @@ -321,7 +361,7 @@ class TDTestCase: tdSql.checkData(3 , 2, 0.147315235) tdSql.checkData(4 , 2, None) - + self.check_result_auto_log( "select c1, c2, c3 , c4, c5 from ct1", "select log(c1), log(c2) ,log(c3), log(c4), log(c5) from ct1") self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 from ct1", "select log(c1,2), log(c2,2) ,log(c3,2), log(c4,2), log(c5,2) from ct1") self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 from ct1", "select log(c1,-10), log(c2,-10) ,log(c3,-10), log(c4,-10), log(c5,-10) from ct1") @@ -569,10 +609,12 @@ class TDTestCase: tdSql.error( f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) + self.check_result_auto_log( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1), log(c2) ,log(c3), log(c4), log(c5) ,log(c6) from sub1_bound") self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1,2), log(c2,2) ,log(c3,2), log(c4,2), log(c5,2) ,log(c6,2) from sub1_bound") self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1,-10), log(c2,-10) ,log(c3,-10), log(c4,-10), log(c5,-10) ,log(c6,-10) from sub1_bound") self.check_result_auto_log2( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select log(c1,2), log(c2,2) ,log(c3,2), log(c3,2), log(c2,2) ,log(c1,2) from sub1_bound") + self.check_result_auto_log( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select log(c1), log(c2) ,log(c3), log(c3), log(c2) ,log(c1) from sub1_bound") self.check_result_auto_log2("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select log(abs(c1) ,2) from sub1_bound" ) @@ -598,6 +640,29 @@ class TDTestCase: tdSql.checkData(3,4,math.log(339999995214436424907732413799364296704.00000,2)) tdSql.checkData(3,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000,2)) + # check basic elem for table per row + tdSql.query("select log(abs(c1)) ,log(abs(c2)) , log(abs(c3)) , log(abs(c4)), log(abs(c5)), log(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.log(2147483647)) + tdSql.checkData(0,1,math.log(9223372036854775807)) + tdSql.checkData(0,2,math.log(32767)) + tdSql.checkData(0,3,math.log(127)) + tdSql.checkData(0,4,math.log(339999995214436424907732413799364296704.00000)) + tdSql.checkData(0,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000 )) + tdSql.checkData(1,0,math.log(2147483647)) + tdSql.checkData(1,1,math.log(9223372036854775807)) + tdSql.checkData(1,2,math.log(32767)) + tdSql.checkData(1,3,math.log(127)) + tdSql.checkData(1,4,math.log(339999995214436424907732413799364296704.00000 )) + tdSql.checkData(1,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)) + tdSql.checkData(3,0,math.log(2147483646)) + tdSql.checkData(3,1,math.log(9223372036854775806)) + tdSql.checkData(3,2,math.log(32766)) + tdSql.checkData(3,3,math.log(126)) + tdSql.checkData(3,4,math.log(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)) + + + # check + - * / in functions tdSql.query("select log(abs(c1+1) ,2) ,log(abs(c2),2) , log(abs(c3*1),2) , log(abs(c4/2),2), log(abs(c5) ,2)/2, log(abs(c6) ,2) from sub1_bound ") tdSql.checkData(0,0,math.log(2147483648.000000000,2)) diff --git a/tests/system-test/7-tmq/schema.py b/tests/system-test/7-tmq/schema.py index 633a097db6..51c8da2413 100644 --- a/tests/system-test/7-tmq/schema.py +++ b/tests/system-test/7-tmq/schema.py @@ -282,7 +282,7 @@ class TDTestCase: tdSql.execute("create topic %s as select ts, c1, c2, t1, t2 from %s.%s" %(columnTopicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(columnTopicFromNtb, parameterDict['dbName'], ntbName)) - tsLog.info("======== super table test:") + tdLog.info("======== super table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -301,13 +301,13 @@ class TDTestCase: tdSql.query("alter table %s.%s modify column c4 binary(60)"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s modify tag t4 binary(60)"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s rename column c3 c3new"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s rename column c4 c4new"%(parameterDict['dbName'], parameterDict['stbName'])) + # tdSql.query("alter table %s.%s rename column c3 c3new"%(parameterDict['dbName'], parameterDict['stbName'])) + # tdSql.query("alter table %s.%s rename column c4 c4new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s rename tag t3 t3new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s rename tag t4 t4new"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s drop column c3new"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s drop column c4new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c3"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c4"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s drop tag t3new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s drop tag t4new"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -316,7 +316,7 @@ class TDTestCase: tdSql.query("alter table %s.%s add tag t3 int"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s add tag t4 float"%(parameterDict['dbName'], parameterDict['stbName'])) - tsLog.info("======== normal table test:") + tdLog.info("======== normal table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], ntbName)) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], ntbName)) @@ -408,6 +408,7 @@ class TDTestCase: parameterDict['cfg'] = cfgPath # tdLog.info("create database, super table, child table, normal table") + # self.create_database(tdSql, parameterDict["dbName"]) ntbName = 'ntb2' tdSql.query("create table %s.%s (ts timestamp, c1 int, c2 binary(32), c3 double, c4 binary(32), c5 nchar(10)) tags (t1 int, t2 binary(32), t3 double, t4 binary(32), t5 nchar(10))"%(parameterDict["dbName"],parameterDict["stbName"])) tdSql.query("create table %s.%s (ts timestamp, c1 int, c2 binary(32), c3 double, c4 binary(32), c5 nchar(10))"%(parameterDict["dbName"],ntbName)) @@ -419,7 +420,7 @@ class TDTestCase: tdSql.execute("create topic %s as select ts, c1, c2, t1, t2 from %s.%s where c3 > 3 and c4 like 'abc' and t3 = 5 and t4 = 'beijing'" %(columnTopicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s where c3 > 3 and c4 like 'abc'" %(columnTopicFromNtb, parameterDict['dbName'], ntbName)) - tsLog.info("======== super table test:") + tdLog.info("======== super table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -457,7 +458,7 @@ class TDTestCase: tdSql.query("alter table %s.%s add column c5 int"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s add tag t5 float"%(parameterDict['dbName'], parameterDict['stbName'])) - tsLog.info("======== normal table test:") + tdLog.info("======== normal table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], ntbName)) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], ntbName)) @@ -566,7 +567,7 @@ class TDTestCase: tdSql.execute("create topic %s as select * from %s.%s" %(columnTopicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select * from %s.%s " %(columnTopicFromNtb, parameterDict['dbName'], ntbName)) - tsLog.info("======== super table test:") + tdLog.info("======== super table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -601,7 +602,7 @@ class TDTestCase: tdSql.query("alter table %s.%s add column c6 int"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s add tag t6 float"%(parameterDict['dbName'], parameterDict['stbName'])) - tsLog.info("======== normal table test:") + tdLog.info("======== normal table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], ntbName)) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], ntbName)) @@ -688,7 +689,7 @@ class TDTestCase: self.tmqCase1(cfgPath, buildPath) self.tmqCase2(cfgPath, buildPath) - self.tmqCase3(cfgPath, buildPath) + # self.tmqCase3(cfgPath, buildPath) def stop(self): tdSql.close() diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat index ea57bc7427..c4eaeed2ad 100644 --- a/tests/system-test/fulltest.bat +++ b/tests/system-test/fulltest.bat @@ -1,7 +1,103 @@ -python3 .\test.py -f 0-others\taosShell.py +@REM python3 .\test.py -f 0-others\taosShell.py python3 .\test.py -f 0-others\taosShellError.py python3 .\test.py -f 0-others\taosShellNetChk.py +python3 .\test.py -f 0-others\telemetry.py +@REM python3 .\test.py -f 0-others\taosdMonitor.py python3 .\test.py -f 0-others\udfTest.py python3 .\test.py -f 0-others\udf_create.py -python3 .\test.py -f 0-others\udf_restart_taosd.py \ No newline at end of file +python3 .\test.py -f 0-others\udf_restart_taosd.py +@REM python3 .\test.py -f 0-others\cachelast.py + +@REM python3 .\test.py -f 0-others\user_control.py +@REM python3 .\test.py -f 0-others\fsync.py + +@REM python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py +@REM python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py +@REM python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py +@REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py +@REM python3 .\test.py -f 1-insert\alter_stable.py +@REM python3 .\test.py -f 1-insert\alter_table.py +@REM python3 .\test.py -f 2-query\between.py +@REM python3 .\test.py -f 2-query\distinct.py +@REM python3 .\test.py -f 2-query\varchar.py +@REM python3 .\test.py -f 2-query\ltrim.py +@REM python3 .\test.py -f 2-query\rtrim.py +@REM python3 .\test.py -f 2-query\length.py +@REM python3 .\test.py -f 2-query\char_length.py +@REM python3 .\test.py -f 2-query\upper.py +@REM python3 .\test.py -f 2-query\lower.py +@REM python3 .\test.py -f 2-query\join.py +@REM python3 .\test.py -f 2-query\join2.py +@REM python3 .\test.py -f 2-query\cast.py +@REM python3 .\test.py -f 2-query\union.py +@REM python3 .\test.py -f 2-query\union1.py +@REM python3 .\test.py -f 2-query\concat.py +@REM python3 .\test.py -f 2-query\concat2.py +@REM python3 .\test.py -f 2-query\concat_ws.py +@REM python3 .\test.py -f 2-query\concat_ws2.py +@REM python3 .\test.py -f 2-query\check_tsdb.py +@REM python3 .\test.py -f 2-query\spread.py +@REM python3 .\test.py -f 2-query\hyperloglog.py + + +@REM python3 .\test.py -f 2-query\timezone.py +@REM python3 .\test.py -f 2-query\Now.py +@REM python3 .\test.py -f 2-query\Today.py +@REM python3 .\test.py -f 2-query\max.py +@REM python3 .\test.py -f 2-query\min.py +@REM python3 .\test.py -f 2-query\count.py +@REM python3 .\test.py -f 2-query\last.py +@REM python3 .\test.py -f 2-query\first.py +@REM python3 .\test.py -f 2-query\To_iso8601.py +@REM python3 .\test.py -f 2-query\To_unixtimestamp.py +@REM python3 .\test.py -f 2-query\timetruncate.py +@REM python3 .\test.py -f 2-query\diff.py +@REM python3 .\test.py -f 2-query\Timediff.py + +@REM python3 .\test.py -f 2-query\top.py +@REM python3 .\test.py -f 2-query\bottom.py +@REM python3 .\test.py -f 2-query\percentile.py +@REM python3 .\test.py -f 2-query\apercentile.py +@REM python3 .\test.py -f 2-query\abs.py +@REM python3 .\test.py -f 2-query\ceil.py +@REM python3 .\test.py -f 2-query\floor.py +@REM python3 .\test.py -f 2-query\round.py +@REM python3 .\test.py -f 2-query\log.py +@REM python3 .\test.py -f 2-query\pow.py +@REM python3 .\test.py -f 2-query\sqrt.py +@REM python3 .\test.py -f 2-query\sin.py +@REM python3 .\test.py -f 2-query\cos.py +@REM python3 .\test.py -f 2-query\tan.py +@REM python3 .\test.py -f 2-query\arcsin.py +@REM python3 .\test.py -f 2-query\arccos.py +@REM python3 .\test.py -f 2-query\arctan.py +@REM python3 .\test.py -f 2-query\query_cols_tags_and_or.py +@REM # python3 .\test.py -f 2-query\nestedQuery.py +@REM # TD-15983 subquery output duplicate name column. +@REM # Please Xiangyang Guo modify the following script +@REM # python3 .\test.py -f 2-query\nestedQuery_str.py + +@REM python3 .\test.py -f 2-query\avg.py +@REM python3 .\test.py -f 2-query\elapsed.py +@REM python3 .\test.py -f 2-query\csum.py +@REM python3 .\test.py -f 2-query\mavg.py +@REM python3 .\test.py -f 2-query\diff.py +@REM python3 .\test.py -f 2-query\sample.py +@REM python3 .\test.py -f 2-query\function_diff.py +@REM python3 .\test.py -f 2-query\unique.py +@REM python3 .\test.py -f 2-query\stateduration.py +@REM python3 .\test.py -f 2-query\function_stateduration.py +@REM python3 .\test.py -f 2-query\statecount.py + +@REM python3 .\test.py -f 7-tmq\basic5.py +@REM python3 .\test.py -f 7-tmq\subscribeDb.py +@REM python3 .\test.py -f 7-tmq\subscribeDb0.py +@REM python3 .\test.py -f 7-tmq\subscribeDb1.py +@REM python3 .\test.py -f 7-tmq\subscribeStb.py +@REM python3 .\test.py -f 7-tmq\subscribeStb0.py +@REM python3 .\test.py -f 7-tmq\subscribeStb1.py +@REM python3 .\test.py -f 7-tmq\subscribeStb2.py +@REM python3 .\test.py -f 7-tmq\subscribeStb3.py +@REM python3 .\test.py -f 7-tmq\subscribeStb4.py +@REM python3 .\test.py -f 7-tmq\db.py \ No newline at end of file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 1fbbc9f37e..abc8a81248 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -21,6 +21,7 @@ python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py #python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py python3 ./test.py -f 1-insert/alter_stable.py python3 ./test.py -f 1-insert/alter_table.py +# python3 ./test.py -f 1-inerst/create_table_comment.py python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index ae6c98b06f..baafbc9e08 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -11,10 +11,12 @@ set /a a=0 @REM ) echo Linux Taosd Test for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( - echo Processing %%i - set /a a+=1 - call %%i ARG1 -m %1 > result_!a!.txt 2>error_!a!.txt - if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) + for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( + echo Processing %%i + set /a a+=1 + call %%i ARG1 -m %1 > result_!a!.txt 2>error_!a!.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && echo result: && cat result_!a!.txt && echo error: && cat error_!a!.txt && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) + ) ) exit