diff --git a/deps/arm/dm_static/libdmodule.a b/deps/arm/dm_static/libdmodule.a index 37077ef63b..072e9f857d 100644 Binary files a/deps/arm/dm_static/libdmodule.a and b/deps/arm/dm_static/libdmodule.a differ diff --git a/deps/darwin/arm/dm_static/libdmodule.a b/deps/darwin/arm/dm_static/libdmodule.a index 246b2247af..2bede61caa 100644 Binary files a/deps/darwin/arm/dm_static/libdmodule.a and b/deps/darwin/arm/dm_static/libdmodule.a differ diff --git a/deps/darwin/x64/dm_static/libdmodule.a b/deps/darwin/x64/dm_static/libdmodule.a index 8745f57636..ae1abc5dfc 100644 Binary files a/deps/darwin/x64/dm_static/libdmodule.a and b/deps/darwin/x64/dm_static/libdmodule.a differ diff --git a/deps/mips/dm_static/libdmodule.a b/deps/mips/dm_static/libdmodule.a index 855a6a41d9..868ac62d3e 100644 Binary files a/deps/mips/dm_static/libdmodule.a and b/deps/mips/dm_static/libdmodule.a differ diff --git a/deps/x86/dm_static/libdmodule.a b/deps/x86/dm_static/libdmodule.a index 6a3c0d45c2..658bb95c27 100644 Binary files a/deps/x86/dm_static/libdmodule.a and b/deps/x86/dm_static/libdmodule.a differ diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 2cdd18a86d..139ace0fca 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2366,20 +2366,31 @@ void tFreeSVArbSetAssignedLeaderRsp(SVArbSetAssignedLeaderRsp* pRsp); typedef struct { int32_t dnodeId; char* token; -} SMArbUpdateGroupReqMember; +} SMArbUpdateGroupMember; typedef struct { - int32_t vgId; - int64_t dbUid; - SMArbUpdateGroupReqMember members[2]; - int8_t isSync; - SMArbUpdateGroupReqMember assignedLeader; - int64_t version; -} SMArbUpdateGroupReq; + int32_t dnodeId; + char* token; + int8_t acked; +} SMArbUpdateGroupAssigned; -int32_t tSerializeSMArbUpdateGroupReq(void* buf, int32_t bufLen, SMArbUpdateGroupReq* pReq); -int32_t tDeserializeSMArbUpdateGroupReq(void* buf, int32_t bufLen, SMArbUpdateGroupReq* pReq); -void tFreeSMArbUpdateGroupReq(SMArbUpdateGroupReq* pReq); +typedef struct { + int32_t vgId; + int64_t dbUid; + SMArbUpdateGroupMember members[2]; + int8_t isSync; + int8_t assignedAcked; + SMArbUpdateGroupAssigned assignedLeader; + int64_t version; +} SMArbUpdateGroup; + +typedef struct { + SArray* updateArray; // SMArbUpdateGroup +} SMArbUpdateGroupBatchReq; + +int32_t tSerializeSMArbUpdateGroupBatchReq(void* buf, int32_t bufLen, SMArbUpdateGroupBatchReq* pReq); +int32_t tDeserializeSMArbUpdateGroupBatchReq(void* buf, int32_t bufLen, SMArbUpdateGroupBatchReq* pReq); +void tFreeSMArbUpdateGroupBatchReq(SMArbUpdateGroupBatchReq* pReq); typedef struct { char queryStrId[TSDB_QUERY_ID_LEN]; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index a5a3bd5ee0..6f15d7df70 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -388,7 +388,8 @@ TD_NEW_MSG_SEG(TDMT_MND_ARB_MSG) //9 << 8 TD_DEF_MSG_TYPE(TDMT_MND_ARB_HEARTBEAT_TIMER, "mnd-arb-hb-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ARB_CHECK_SYNC_TIMER, "mnd-arb-check-sync-tmr", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL) // no longer used + TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP_BATCH, "mnd-arb-update-group-batch", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ARB_MAX_MSG, "mnd-arb-max", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_ARB_MSG) diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 82823e3f57..15cb6d59aa 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -44,6 +44,11 @@ int32_t dmRun(); */ void dmStop(); +/** + * for tests + */ +bool dmReadyForTest(); + #ifdef __cplusplus } #endif diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 108e6f18a6..fe96fe1117 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -101,6 +101,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr */ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); +int64_t mndGetRoleTimeMs(SMnode *pMnode); + /** * @brief Process the rpc, sync request. * diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 18bc24d612..a691433ee6 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -183,6 +183,7 @@ typedef struct SProjectLogicNode { char stmtName[TSDB_TABLE_NAME_LEN]; bool ignoreGroupId; bool inputIgnoreGroup; + bool isSetOpProj; } SProjectLogicNode; typedef struct SIndefRowsFuncLogicNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5b889171b3..befd6afce9 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -415,6 +415,7 @@ typedef struct SSelectStmt { int32_t returnRows; // EFuncReturnRows ETimeLineMode timeLineCurMode; ETimeLineMode timeLineResMode; + bool timeLineFromOrderBy; bool isEmptyResult; bool isSubquery; bool hasAggFuncs; @@ -453,6 +454,7 @@ typedef struct SSetOperator { char stmtName[TSDB_TABLE_NAME_LEN]; uint8_t precision; ETimeLineMode timeLineResMode; + bool timeLineFromOrderBy; bool joinContains; } SSetOperator; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index dafdac9649..9ae75bade2 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -327,7 +327,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) // #define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A) #define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B) -#define TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x039C) // mnode-node #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0) @@ -612,6 +611,16 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x0821) #define TSDB_CODE_GRANT_DUPLICATED_ACTIVE TAOS_DEF_ERROR_CODE(0, 0x0822) #define TSDB_CODE_GRANT_VIEW_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0823) +#define TSDB_CODE_GRANT_BASIC_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0824) +#define TSDB_CODE_GRANT_STREAM_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0825) +#define TSDB_CODE_GRANT_SUBSCRIPTION_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0826) +#define TSDB_CODE_GRANT_VIEW_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0827) +#define TSDB_CODE_GRANT_AUDIT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0828) +#define TSDB_CODE_GRANT_CSV_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0829) +#define TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082A) +#define TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082B) +#define TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082C) +#define TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082D) // sync // #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 7d30a19140..1b6cb8fd22 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -92,6 +92,7 @@ static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *bat } if (!pRsp) { releaseTscObj(pReq->connKey.tscRid); + taosHashCancelIterate(hbMgr->activeInfo, pReq); break; } } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index be51c9c5da..cac9ee0e8b 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -76,6 +76,7 @@ static const SSysDbTableSchema arbGroupsSchema[] = { {.name = "is_sync", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "assigned_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "assigned_token", .bytes = TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "assigned_acked", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, }; static const SSysDbTableSchema clusterSchema[] = { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ae3fe6a2a0..8d3db72fb5 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2907,7 +2907,7 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt } } else { if (varDataTLen(data + offset) > bytes) { - uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), + uError("var data length invalid, varDataTLen(data + offset):%d >= bytes:%d", (int)varDataTLen(data + offset), bytes); code = TSDB_CODE_PAR_VALUE_TOO_LONG; goto _exit; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index cf537a0477..84b89883c6 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -6441,21 +6441,33 @@ void tFreeSVArbSetAssignedLeaderRsp(SVArbSetAssignedLeaderRsp *pRsp) { taosMemoryFreeClear(pRsp->memberToken); } -int32_t tSerializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGroupReq *pReq) { +int32_t tSerializeSMArbUpdateGroupBatchReq(void *buf, int32_t bufLen, SMArbUpdateGroupBatchReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; - if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; - for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { - if (tEncodeI32(&encoder, pReq->members[i].dnodeId) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->members[i].token) < 0) return -1; + + int32_t sz = taosArrayGetSize(pReq->updateArray); + if (tEncodeI32(&encoder, sz) < 0) return -1; + + for (int32_t i = 0; i < sz; i++) { + SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i); + if (tEncodeI32(&encoder, pGroup->vgId) < 0) return -1; + if (tEncodeI64(&encoder, pGroup->dbUid) < 0) return -1; + for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { + if (tEncodeI32(&encoder, pGroup->members[i].dnodeId) < 0) return -1; + if (tEncodeCStr(&encoder, pGroup->members[i].token) < 0) return -1; + } + if (tEncodeI8(&encoder, pGroup->isSync) < 0) return -1; + if (tEncodeI32(&encoder, pGroup->assignedLeader.dnodeId) < 0) return -1; + if (tEncodeCStr(&encoder, pGroup->assignedLeader.token) < 0) return -1; + if (tEncodeI64(&encoder, pGroup->version) < 0) return -1; + } + + for (int32_t i = 0; i < sz; i++) { + SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i); + if (tEncodeI8(&encoder, pGroup->assignedLeader.acked) < 0) return -1; } - if (tEncodeI8(&encoder, pReq->isSync) < 0) return -1; - if (tEncodeI32(&encoder, pReq->assignedLeader.dnodeId) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->assignedLeader.token) < 0) return -1; - if (tEncodeI64(&encoder, pReq->version) < 0) return -1; tEndEncode(&encoder); @@ -6464,23 +6476,44 @@ int32_t tSerializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGrou return tlen; } -int32_t tDeserializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGroupReq *pReq) { +int32_t tDeserializeSMArbUpdateGroupBatchReq(void *buf, int32_t bufLen, SMArbUpdateGroupBatchReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; - for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { - if (tDecodeI32(&decoder, &pReq->members[i].dnodeId) < 0) return -1; - pReq->members[i].token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE); - if (tDecodeCStrTo(&decoder, pReq->members[i].token) < 0) return -1; + int32_t sz = 0; + if (tDecodeI32(&decoder, &sz) < 0) return -1; + + SArray *updateArray = taosArrayInit(sz, sizeof(SMArbUpdateGroup)); + if (!updateArray) return -1; + + for (int32_t i = 0; i < sz; i++) { + SMArbUpdateGroup group = {0}; + if (tDecodeI32(&decoder, &group.vgId) < 0) return -1; + if (tDecodeI64(&decoder, &group.dbUid) < 0) return -1; + for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { + if (tDecodeI32(&decoder, &group.members[i].dnodeId) < 0) return -1; + group.members[i].token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE); + if (tDecodeCStrTo(&decoder, group.members[i].token) < 0) return -1; + } + if (tDecodeI8(&decoder, &group.isSync) < 0) return -1; + if (tDecodeI32(&decoder, &group.assignedLeader.dnodeId) < 0) return -1; + group.assignedLeader.token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE); + if (tDecodeCStrTo(&decoder, group.assignedLeader.token) < 0) return -1; + if (tDecodeI64(&decoder, &group.version) < 0) return -1; + group.assignedLeader.acked = false; + + taosArrayPush(updateArray, &group); } - if (tDecodeI8(&decoder, &pReq->isSync) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->assignedLeader.dnodeId) < 0) return -1; - pReq->assignedLeader.token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE); - if (tDecodeCStrTo(&decoder, pReq->assignedLeader.token) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->version) < 0) return -1; + + if (!tDecodeIsEnd(&decoder)) { + for (int32_t i = 0; i < sz; i++) { + SMArbUpdateGroup *pGroup = taosArrayGet(updateArray, i); + if (tDecodeI8(&decoder, &pGroup->assignedLeader.acked) < 0) return -1; + } + } + + pReq->updateArray = updateArray; tEndDecode(&decoder); @@ -6488,14 +6521,20 @@ int32_t tDeserializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGr return 0; } -void tFreeSMArbUpdateGroupReq(SMArbUpdateGroupReq *pReq) { - if (NULL == pReq) { +void tFreeSMArbUpdateGroupBatchReq(SMArbUpdateGroupBatchReq *pReq) { + if (NULL == pReq || NULL == pReq->updateArray) { return; } - for (int i = 0; i < 2; i++) { - taosMemoryFreeClear(pReq->members[i].token); + + int32_t sz = taosArrayGetSize(pReq->updateArray); + for (int32_t i = 0; i < sz; i++) { + SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i); + for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { + taosMemoryFreeClear(pGroup->members[i].token); + } + taosMemoryFreeClear(pGroup->assignedLeader.token); } - taosMemoryFreeClear(pReq->assignedLeader.token); + taosArrayDestroy(pReq->updateArray); } // int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 8e760c28be..5b1f31e6c6 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -415,3 +415,7 @@ void dmReportStartup(const char *pName, const char *pDesc) { } int64_t dmGetClusterId() { return globalDnode.data.clusterId; } + +bool dmReadyForTest() { + return dmInstance()->data.dnodeVer > 0; +} diff --git a/source/dnode/mgmt/test/sut/inc/server.h b/source/dnode/mgmt/test/sut/inc/server.h index 7343276210..78055393b5 100644 --- a/source/dnode/mgmt/test/sut/inc/server.h +++ b/source/dnode/mgmt/test/sut/inc/server.h @@ -20,10 +20,10 @@ class TestServer { public: bool Start(); void Stop(); -bool runnning; + bool running; private: TdThread threadId; }; -#endif /* _TD_TEST_SERVER_H_ */ \ No newline at end of file +#endif /* _TD_TEST_SERVER_H_ */ diff --git a/source/dnode/mgmt/test/sut/src/server.cpp b/source/dnode/mgmt/test/sut/src/server.cpp index 81e6dcf495..2218504df4 100644 --- a/source/dnode/mgmt/test/sut/src/server.cpp +++ b/source/dnode/mgmt/test/sut/src/server.cpp @@ -17,13 +17,11 @@ void* serverLoop(void* param) { TestServer* server = (TestServer*)param; - server->runnning = false; if (dmInit() != 0) { return NULL; } - server->runnning = true; if (dmRun() != 0) { return NULL; } @@ -33,13 +31,18 @@ void* serverLoop(void* param) { } bool TestServer::Start() { + tstrncpy(tsVersionName, "trial", strlen("trial")); + running = false; TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadCreate(&threadId, &thAttr, serverLoop, this); taosThreadAttrDestroy(&thAttr); - taosMsleep(2100); - return runnning; + while (!dmReadyForTest()) { + taosMsleep(500); + } + running = true; + return running; } void TestServer::Stop() { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 2f4efc2db7..1a8155da26 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -180,7 +180,7 @@ typedef struct { tmsg_t originRpcType; char dbname[TSDB_TABLE_FNAME_LEN]; char stbname[TSDB_TABLE_FNAME_LEN]; - int32_t arbGroupId; + SHashObj* arbGroupIds; int32_t startFunc; int32_t stopFunc; int32_t paramLen; @@ -255,6 +255,7 @@ typedef struct { typedef struct { int32_t dnodeId; char token[TSDB_ARB_TOKEN_SIZE]; + int8_t acked; } SArbAssignedLeader; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 8c9ca87fb1..8008eb76e7 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -78,7 +78,7 @@ int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen); void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname); -void mndTransSetArbGroupId(STrans *pTrans, int32_t groupId); +void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId); void mndTransSetSerial(STrans *pTrans); void mndTransSetParallel(STrans *pTrans); void mndTransSetChangeless(STrans *pTrans); diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index d0a86bdde7..6a6b3d2daa 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -25,7 +25,7 @@ #include "mndVgroup.h" #define ARBGROUP_VER_NUMBER 1 -#define ARBGROUP_RESERVE_SIZE 64 +#define ARBGROUP_RESERVE_SIZE 63 static SHashObj *arbUpdateHash = NULL; @@ -39,10 +39,11 @@ static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup); static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew); static int32_t mndPullupArbUpdateGroup(SMnode *pMnode, SArbGroup *pNewGroup); +static int32_t mndPullupArbUpdateGroupBatch(SMnode *pMnode, SArray *newGroupArray); static int32_t mndProcessArbHbTimer(SRpcMsg *pReq); static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq); -static int32_t mndProcessArbUpdateGroupReq(SRpcMsg *pReq); +static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq); static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp); static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp); static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp); @@ -68,7 +69,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_ARB_HEARTBEAT_TIMER, mndProcessArbHbTimer); mndSetMsgHandle(pMnode, TDMT_MND_ARB_CHECK_SYNC_TIMER, mndProcessArbCheckSyncTimer); - mndSetMsgHandle(pMnode, TDMT_MND_ARB_UPDATE_GROUP, mndProcessArbUpdateGroupReq); + mndSetMsgHandle(pMnode, TDMT_MND_ARB_UPDATE_GROUP_BATCH, mndProcessArbUpdateGroupBatchReq); mndSetMsgHandle(pMnode, TDMT_VND_ARB_HEARTBEAT_RSP, mndProcessArbHbRsp); mndSetMsgHandle(pMnode, TDMT_VND_ARB_CHECK_SYNC_RSP, mndProcessArbCheckSyncRsp); mndSetMsgHandle(pMnode, TDMT_SYNC_SET_ASSIGNED_LEADER_RSP, mndProcessArbSetAssignedLeaderRsp); @@ -81,9 +82,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -void mndCleanupArbGroup(SMnode *pMnode) { - taosHashCleanup(arbUpdateHash); -} +void mndCleanupArbGroup(SMnode *pMnode) { taosHashCleanup(arbUpdateHash); } SArbGroup *mndAcquireArbGroup(SMnode *pMnode, int32_t vgId) { SArbGroup *pGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &vgId); @@ -130,6 +129,7 @@ SSdbRaw *mndArbGroupActionEncode(SArbGroup *pGroup) { SDB_SET_INT32(pRaw, dataPos, pLeader->dnodeId, _OVER) SDB_SET_BINARY(pRaw, dataPos, pLeader->token, TSDB_ARB_TOKEN_SIZE, _OVER) SDB_SET_INT64(pRaw, dataPos, pGroup->version, _OVER) + SDB_SET_INT8(pRaw, dataPos, pLeader->acked, _OVER) SDB_SET_RESERVE(pRaw, dataPos, ARBGROUP_RESERVE_SIZE, _OVER) @@ -183,6 +183,7 @@ SSdbRow *mndArbGroupActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pLeader->dnodeId, _OVER) SDB_GET_BINARY(pRaw, dataPos, pLeader->token, TSDB_ARB_TOKEN_SIZE, _OVER) SDB_GET_INT64(pRaw, dataPos, &pGroup->version, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pLeader->acked, _OVER) pGroup->mutexInited = false; @@ -236,6 +237,7 @@ static int32_t mndArbGroupActionUpdate(SSdb *pSdb, SArbGroup *pOld, SArbGroup *p pOld->isSync = pNew->isSync; pOld->assignedLeader.dnodeId = pNew->assignedLeader.dnodeId; memcpy(pOld->assignedLeader.token, pNew->assignedLeader.token, TSDB_ARB_TOKEN_SIZE); + pOld->assignedLeader.acked = pNew->assignedLeader.acked; pOld->version++; _OVER: @@ -541,6 +543,16 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { return -1; } + int64_t roleTimeMs = mndGetRoleTimeMs(pMnode); + int64_t nowMs = taosGetTimestampMs(); + if (nowMs - roleTimeMs < tsArbHeartBeatIntervalSec * 1000 * 2) { + mInfo("arb skip to check sync since mnd had just switch over, roleTime:%" PRId64 " now:%" PRId64, roleTimeMs, + nowMs); + return 0; + } + + SArray *pUpdateArray = taosArrayInit(16, sizeof(SArbGroup)); + while (1) { pIter = sdbFetch(pSdb, SDB_ARBGROUP, pIter, (void **)&pArbGroup); if (pIter == NULL) break; @@ -550,15 +562,14 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { taosThreadMutexUnlock(&pArbGroup->mutex); int32_t vgId = arbGroupDup.vgId; - int64_t nowMs = taosGetTimestampMs(); bool member0IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 0, nowMs); bool member1IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 1, nowMs); SArbAssignedLeader *pAssignedLeader = &arbGroupDup.assignedLeader; int32_t currentAssignedDnodeId = pAssignedLeader->dnodeId; - // 1. has assigned && is sync => send req - if (currentAssignedDnodeId != 0 && arbGroupDup.isSync == true) { + // 1. has assigned && is sync && no response => send req + if (currentAssignedDnodeId != 0 && arbGroupDup.isSync == true && pAssignedLeader->acked == false) { (void)mndSendArbSetAssignedLeaderReq(pMnode, currentAssignedDnodeId, vgId, arbToken, term, pAssignedLeader->token); mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", vgId, currentAssignedDnodeId); @@ -612,40 +623,27 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { SArbGroup newGroup = {0}; mndArbGroupDupObj(&arbGroupDup, &newGroup); mndArbGroupSetAssignedLeader(&newGroup, candidateIndex); - if (mndPullupArbUpdateGroup(pMnode, &newGroup) != 0) { - mError("vgId:%d, arb failed to pullup set assigned leader to dnodeId:%d, since %s", vgId, pMember->info.dnodeId, - terrstr()); - sdbRelease(pSdb, pArbGroup); - return -1; - } - - mInfo("vgId:%d, arb pull up set assigned leader to dnodeId:%d", vgId, pMember->info.dnodeId); + taosArrayPush(pUpdateArray, &newGroup); sdbRelease(pSdb, pArbGroup); } + (void)mndPullupArbUpdateGroupBatch(pMnode, pUpdateArray); + + taosArrayDestroy(pUpdateArray); return 0; } -static void *mndBuildArbUpdateGroupReq(int32_t *pContLen, SArbGroup *pNewGroup) { - SMArbUpdateGroupReq req = {0}; - req.vgId = pNewGroup->vgId; - req.dbUid = pNewGroup->dbUid; - for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { - req.members[i].dnodeId = pNewGroup->members[i].info.dnodeId; - req.members[i].token = pNewGroup->members[i].state.token; - } - req.isSync = pNewGroup->isSync; - req.assignedLeader.dnodeId = pNewGroup->assignedLeader.dnodeId; - req.assignedLeader.token = pNewGroup->assignedLeader.token; - req.version = pNewGroup->version; +static void *mndBuildArbUpdateGroupBatchReq(int32_t *pContLen, SArray *updateArray) { + SMArbUpdateGroupBatchReq req = {0}; + req.updateArray = updateArray; - int32_t contLen = tSerializeSMArbUpdateGroupReq(NULL, 0, &req); + int32_t contLen = tSerializeSMArbUpdateGroupBatchReq(NULL, 0, &req); if (contLen <= 0) return NULL; SMsgHead *pHead = rpcMallocCont(contLen); if (pHead == NULL) return NULL; - if (tSerializeSMArbUpdateGroupReq(pHead, contLen, &req) <= 0) { + if (tSerializeSMArbUpdateGroupBatchReq(pHead, contLen, &req) <= 0) { rpcFreeCont(pHead); return NULL; } @@ -653,60 +651,174 @@ static void *mndBuildArbUpdateGroupReq(int32_t *pContLen, SArbGroup *pNewGroup) return pHead; } +static void mndInitArbUpdateGroup(SArbGroup *pGroup, SMArbUpdateGroup *outGroup) { + outGroup->vgId = pGroup->vgId; + outGroup->dbUid = pGroup->dbUid; + for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { + outGroup->members[i].dnodeId = pGroup->members[i].info.dnodeId; + outGroup->members[i].token = pGroup->members[i].state.token; // just copy the pointer + } + outGroup->isSync = pGroup->isSync; + outGroup->assignedLeader.dnodeId = pGroup->assignedLeader.dnodeId; + outGroup->assignedLeader.token = pGroup->assignedLeader.token; // just copy the pointer + outGroup->assignedLeader.acked = pGroup->assignedLeader.acked; + outGroup->version = pGroup->version; +} + static int32_t mndPullupArbUpdateGroup(SMnode *pMnode, SArbGroup *pNewGroup) { if (taosHashGet(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)) != NULL) { mInfo("vgId:%d, arb skip to pullup arb-update-group request, since it is in process", pNewGroup->vgId); return 0; } - int32_t contLen = 0; - void *pHead = mndBuildArbUpdateGroupReq(&contLen, pNewGroup); - if (!pHead) { - mError("vgId:%d, failed to build arb-update-group request", pNewGroup->vgId); - return -1; - } - SRpcMsg rpcMsg = {.msgType = TDMT_MND_ARB_UPDATE_GROUP, .pCont = pHead, .contLen = contLen, .info.noResp = true}; + int32_t ret = -1; - int32_t ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - if (ret == 0) { - taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0); + SMArbUpdateGroup newGroup = {0}; + mndInitArbUpdateGroup(pNewGroup, &newGroup); + + SArray *pArray = taosArrayInit(1, sizeof(SMArbUpdateGroup)); + taosArrayPush(pArray, &newGroup); + + int32_t contLen = 0; + void *pHead = mndBuildArbUpdateGroupBatchReq(&contLen, pArray); + if (!pHead) { + mError("failed to build arb-update-group request"); + goto _OVER; } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_ARB_UPDATE_GROUP_BATCH, .pCont = pHead, .contLen = contLen, .info.noResp = true}; + ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + if (ret != 0) goto _OVER; + + taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0); + +_OVER: + taosArrayDestroy(pArray); return ret; } -static int32_t mndProcessArbUpdateGroupReq(SRpcMsg *pReq) { - int ret = 0; +static int32_t mndPullupArbUpdateGroupBatch(SMnode *pMnode, SArray *newGroupArray) { + int32_t ret = -1; - SMArbUpdateGroupReq req = {0}; - tDeserializeSMArbUpdateGroupReq(pReq->pCont, pReq->contLen, &req); + size_t sz = taosArrayGetSize(newGroupArray); + SArray *pArray = taosArrayInit(sz, sizeof(SMArbUpdateGroup)); + for (size_t i = 0; i < sz; i++) { + SArbGroup *pNewGroup = taosArrayGet(newGroupArray, i); + if (taosHashGet(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)) != NULL) { + mInfo("vgId:%d, arb skip to pullup arb-update-group request, since it is in process", pNewGroup->vgId); + continue; + } - SArbGroup newGroup = {0}; - newGroup.vgId = req.vgId; - newGroup.dbUid = req.dbUid; - for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { - newGroup.members[i].info.dnodeId = req.members[i].dnodeId; - memcpy(newGroup.members[i].state.token, req.members[i].token, TSDB_ARB_TOKEN_SIZE); + SMArbUpdateGroup newGroup = {0}; + mndInitArbUpdateGroup(pNewGroup, &newGroup); + + taosArrayPush(pArray, &newGroup); + taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0); } - newGroup.isSync = req.isSync; - newGroup.assignedLeader.dnodeId = req.assignedLeader.dnodeId; - memcpy(newGroup.assignedLeader.token, req.assignedLeader.token, TSDB_ARB_TOKEN_SIZE); - newGroup.version = req.version; - - SMnode *pMnode = pReq->info.node; - SArbGroup *pOldGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &newGroup.vgId); - if (!pOldGroup) { - mInfo("vgId:%d, arb skip to update arbgroup, since no obj found", newGroup.vgId); - return 0; - } - sdbRelease(pMnode->pSdb, pOldGroup); - - if (mndArbGroupUpdateTrans(pMnode, &newGroup) != 0) { - mError("vgId:%d, arb failed to update arbgroup, since %s", newGroup.vgId, terrstr()); - ret = -1; + if (taosArrayGetSize(pArray) == 0) { + ret = 0; + goto _OVER; } - tFreeSMArbUpdateGroupReq(&req); + int32_t contLen = 0; + void *pHead = mndBuildArbUpdateGroupBatchReq(&contLen, pArray); + if (!pHead) { + mError("failed to build arb-update-group request"); + goto _OVER; + } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_ARB_UPDATE_GROUP_BATCH, .pCont = pHead, .contLen = contLen, .info.noResp = true}; + ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + +_OVER: + taosArrayDestroy(pArray); + + if (ret != 0) { + for (size_t i = 0; i < sz; i++) { + SArbGroup *pNewGroup = taosArrayGet(newGroupArray, i); + taosHashRemove(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)); + } + } + + return ret; +} + +static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) { + int ret = -1; + size_t sz = 0; + + SMArbUpdateGroupBatchReq req = {0}; + if (tDeserializeSMArbUpdateGroupBatchReq(pReq->pCont, pReq->contLen, &req) != 0) { + mError("arb failed to decode arb-update-group request"); + return -1; + } + + SMnode *pMnode = pReq->info.node; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-arbgroup"); + if (pTrans == NULL) { + mError("failed to update arbgroup in create trans, since %s", terrstr()); + goto _OVER; + } + + sz = taosArrayGetSize(req.updateArray); + for (size_t i = 0; i < sz; i++) { + SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i); + SArbGroup newGroup = {0}; + newGroup.vgId = pUpdateGroup->vgId; + newGroup.dbUid = pUpdateGroup->dbUid; + for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { + newGroup.members[i].info.dnodeId = pUpdateGroup->members[i].dnodeId; + memcpy(newGroup.members[i].state.token, pUpdateGroup->members[i].token, TSDB_ARB_TOKEN_SIZE); + } + + newGroup.isSync = pUpdateGroup->isSync; + newGroup.assignedLeader.dnodeId = pUpdateGroup->assignedLeader.dnodeId; + memcpy(newGroup.assignedLeader.token, pUpdateGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE); + newGroup.assignedLeader.acked = pUpdateGroup->assignedLeader.acked; + newGroup.version = pUpdateGroup->version; + + SArbGroup *pOldGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &newGroup.vgId); + if (!pOldGroup) { + mInfo("vgId:%d, arb skip to update arbgroup, since no obj found", newGroup.vgId); + taosHashRemove(arbUpdateHash, &newGroup.vgId, sizeof(int32_t)); + continue; + } + + mndTransAddArbGroupId(pTrans, newGroup.vgId); + + if (mndSetCreateArbGroupCommitLogs(pTrans, &newGroup) != 0) { + mError("failed to update arbgroup in set commit log, vgId:%d, trans:%d, since %s", newGroup.vgId, pTrans->id, + terrstr()); + goto _OVER; + } + + mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s][%d]", + pTrans->id, newGroup.vgId, newGroup.members[0].info.dnodeId, newGroup.members[0].state.token, + newGroup.members[1].info.dnodeId, newGroup.members[1].state.token, newGroup.isSync, + newGroup.assignedLeader.dnodeId, newGroup.assignedLeader.token, newGroup.assignedLeader.acked); + + sdbRelease(pMnode->pSdb, pOldGroup); + } + + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + ret = 0; + +_OVER: + if (ret != 0) { + // failed to update arbgroup + for (size_t i = 0; i < sz; i++) { + SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i); + taosHashRemove(arbUpdateHash, &pUpdateGroup->vgId, sizeof(int32_t)); + } + } + + mndTransDrop(pTrans); + tFreeSMArbUpdateGroupBatchReq(&req); return ret; } @@ -719,11 +831,13 @@ static void mndArbGroupSetAssignedLeader(SArbGroup *pGroup, int32_t index) { pGroup->assignedLeader.dnodeId = pMember->info.dnodeId; strncpy(pGroup->assignedLeader.token, pMember->state.token, TSDB_ARB_TOKEN_SIZE); + pGroup->assignedLeader.acked = false; } static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup) { pGroup->assignedLeader.dnodeId = 0; memset(pGroup->assignedLeader.token, 0, TSDB_ARB_TOKEN_SIZE); + pGroup->assignedLeader.acked = false; } static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) { @@ -734,12 +848,12 @@ static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) { goto _OVER; } - mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s]", + mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s][%d]", pTrans->id, pNew->vgId, pNew->members[0].info.dnodeId, pNew->members[0].state.token, pNew->members[1].info.dnodeId, pNew->members[1].state.token, pNew->isSync, pNew->assignedLeader.dnodeId, - pNew->assignedLeader.token); + pNew->assignedLeader.token, pNew->assignedLeader.acked); - mndTransSetArbGroupId(pTrans, pNew->vgId); + mndTransAddArbGroupId(pTrans, pNew->vgId); if (mndTransCheckConflict(pMnode, pTrans) != 0) { ret = -1; goto _OVER; @@ -816,10 +930,10 @@ _OVER: } static int32_t mndUpdateArbHeartBeat(SMnode *pMnode, int32_t dnodeId, SArray *memberArray) { - int ret = 0; int64_t nowMs = taosGetTimestampMs(); + size_t size = taosArrayGetSize(memberArray); + SArray *pUpdateArray = taosArrayInit(size, sizeof(SArbGroup)); - size_t size = taosArrayGetSize(memberArray); for (size_t i = 0; i < size; i++) { SVArbHbRspMember *pRspMember = taosArrayGet(memberArray, i); @@ -832,17 +946,16 @@ static int32_t mndUpdateArbHeartBeat(SMnode *pMnode, int32_t dnodeId, SArray *me bool updateToken = mndUpdateArbGroupByHeartBeat(pGroup, pRspMember, nowMs, dnodeId, &newGroup); if (updateToken) { - ret = mndPullupArbUpdateGroup(pMnode, &newGroup); - if (ret != 0) { - mInfo("failed to pullup update arb token, vgId:%d, since %s", pRspMember->vgId, terrstr()); - } + taosArrayPush(pUpdateArray, &newGroup); } sdbRelease(pMnode->pSdb, pGroup); - if (ret != 0) break; } - return ret; + (void)mndPullupArbUpdateGroupBatch(pMnode, pUpdateArray); + + taosArrayDestroy(pUpdateArray); + return 0; } bool mndUpdateArbGroupByCheckSync(SArbGroup *pGroup, int32_t vgId, char *member0Token, char *member1Token, @@ -900,6 +1013,11 @@ static int32_t mndUpdateArbSync(SMnode *pMnode, int32_t vgId, char *member0Token } static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) { + if (pRsp->contLen == 0) { + mDebug("arb hb-rsp contLen is 0"); + return 0; + } + int32_t ret = -1; SMnode *pMnode = pRsp->info.node; @@ -914,6 +1032,7 @@ static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) { SVArbHeartBeatRsp arbHbRsp = {0}; if (tDeserializeSVArbHeartBeatRsp(pRsp->pCont, pRsp->contLen, &arbHbRsp) != 0) { + mInfo("arb hb-rsp des failed, since:%s", tstrerror(pRsp->code)); terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -934,6 +1053,11 @@ _OVER: } static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) { + if (pRsp->contLen == 0) { + mDebug("arb check-sync-rsp contLen is 0"); + return 0; + } + int32_t ret = -1; SMnode *pMnode = pRsp->info.node; @@ -948,7 +1072,7 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) { SVArbCheckSyncRsp syncRsp = {0}; if (tDeserializeSVArbCheckSyncRsp(pRsp->pCont, pRsp->contLen, &syncRsp) != 0) { - mInfo("arb sync check failed, since:%s", tstrerror(pRsp->code)); + mInfo("arb check-sync-rsp des failed, since:%s", tstrerror(pRsp->code)); if (pRsp->code == TSDB_CODE_MND_ARB_TOKEN_MISMATCH) { terrno = TSDB_CODE_SUCCESS; return 0; @@ -993,11 +1117,12 @@ bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char goto _OVER; } - if (pGroup->isSync) { + if (pGroup->assignedLeader.acked == false) { mndArbGroupDupObj(pGroup, pNewGroup); pNewGroup->isSync = false; + pNewGroup->assignedLeader.acked = true; - mInfo("vgId:%d, arb isSync is setting to false", vgId); + mInfo("vgId:%d, arb received assigned ack", vgId); updateAssigned = true; goto _OVER; } @@ -1008,6 +1133,11 @@ _OVER: } static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) { + if (pRsp->contLen == 0) { + mDebug("arb set-assigned-rsp contLen is 0"); + return 0; + } + int32_t ret = -1; SMnode *pMnode = pRsp->info.node; @@ -1022,8 +1152,8 @@ static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) { SVArbSetAssignedLeaderRsp setAssignedRsp = {0}; if (tDeserializeSVArbSetAssignedLeaderRsp(pRsp->pCont, pRsp->contLen, &setAssignedRsp) != 0) { + mInfo("arb set-assigned-rsp des failed, since:%s", tstrerror(pRsp->code)); terrno = TSDB_CODE_INVALID_MSG; - mInfo("arb set assigned failed, des failed since:%s", tstrerror(pRsp->code)); return -1; } @@ -1102,12 +1232,18 @@ static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock STR_WITH_MAXSIZE_TO_VARSTR(token, pGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)token, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&pGroup->assignedLeader.acked, false); } else { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetNULL(pColInfo, numOfRows); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetNULL(pColInfo, numOfRows); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, numOfRows); } taosThreadMutexUnlock(&pGroup->mutex); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f254fb16a5..ad596289de 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -821,8 +821,7 @@ static int32_t mndCheckDbEncryptKey(SMnode *pMnode, SCreateDbReq *pReq) { #ifdef TD_ENTERPRISE if (pReq->encryptAlgorithm == TSDB_ENCRYPT_ALGO_NONE) goto _exit; - if (grantCheck(TSDB_GRANT_DB_ENCRYPTION) != 0) { - code = TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED; + if ((code = grantCheck(TSDB_GRANT_DB_ENCRYPTION)) != 0) { goto _exit; } if (tsEncryptionKeyStat != ENCRYPT_KEY_STAT_LOADED) { @@ -1226,7 +1225,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (terrno != 0) code = terrno; - mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); + mError("db:%s, failed to alter since %s", alterReq.db, tstrerror(code)); } mndReleaseDb(pMnode, pDb); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index a78edcb05e..850c527a14 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -334,6 +334,8 @@ static int32_t minCronTime() { min = TMIN(min, tsStreamCheckpointInterval); min = TMIN(min, 6); // checkpointRemain min = TMIN(min, tsStreamNodeCheckInterval); + min = TMIN(min, tsArbHeartBeatIntervalSec); + min = TMIN(min, tsArbCheckSyncIntervalSec); int64_t telemInt = TMIN(60, (tsTelemInterval - 1)); min = TMIN(min, telemInt); @@ -390,6 +392,18 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { if (sec % tsUptimeInterval == 0) { mndIncreaseUpTime(pMnode); } + + if (sec % (tsArbHeartBeatIntervalSec) == 0) { + if (mndPullupArbHeartbeat(pMnode) != 0) { + mError("failed to pullup arb heartbeat, since:%s", terrstr()); + } + } + + if (sec % (tsArbCheckSyncIntervalSec) == 0) { + if (mndPullupArbCheckSync(pMnode) != 0) { + mError("failed to pullup arb check sync, since:%s", terrstr()); + } + } } void mndDoTimerCheckTask(SMnode *pMnode, int64_t sec) { if (sec % (tsStatusInterval * 5) == 0) { @@ -421,18 +435,6 @@ static void *mndThreadFp(void *param) { continue; } mndDoTimerPullupTask(pMnode, sec); - - if (sec % (tsArbHeartBeatIntervalSec) == 0) { - if (mndPullupArbHeartbeat(pMnode) != 0) { - mError("failed to pullup arb heartbeat, since:%s", terrstr()); - } - } - - if (sec % (tsArbCheckSyncIntervalSec) == 0) { - if (mndPullupArbCheckSync(pMnode) != 0) { - mError("failed to pullup arb check sync, since:%s", terrstr()); - } - } } return NULL; @@ -1076,6 +1078,11 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { return 0; } +int64_t mndGetRoleTimeMs(SMnode *pMnode) { + SSyncState state = syncGetState(pMnode->syncMgmt.sync); + return state.roleTimeMs; +} + void mndSetRestored(SMnode *pMnode, bool restored) { if (restored) { taosThreadRwlockWrlock(&pMnode->lock); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index fb15e4b857..48549fce42 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1639,11 +1639,11 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS // info if (pTask->info.taskLevel == TASK_LEVEL__SINK) { const char *sinkStr = "%.2fMiB"; - sprintf(buf, sinkStr, pe->sinkDataSize); + snprintf(buf, tListLen(buf), sinkStr, pe->sinkDataSize); } else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { // offset info const char *offsetStr = "%" PRId64 " [%" PRId64 ", %" PRId64 "]"; - sprintf(buf, offsetStr, pe->processedVer, pe->verRange.minVer, pe->verRange.maxVer); + snprintf(buf, tListLen(buf), offsetStr, pe->processedVer, pe->verRange.minVer, pe->verRange.maxVer); } STR_TO_VARSTR(vbuf, buf); @@ -1847,8 +1847,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; - if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) { - terrno = TSDB_CODE_GRANT_EXPIRED; + if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index a81a391c3d..9bd7b3b18f 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -225,7 +225,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SArray *pFailedTasks = NULL; SArray *pOrphanTasks = NULL; - if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) { + if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if (suspendAllStreams(pMnode, &pReq->info) < 0) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 84940e01d4..8b01d296a3 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -26,7 +26,7 @@ #define TRANS_VER1_NUMBER 1 #define TRANS_VER2_NUMBER 2 #define TRANS_ARRAY_SIZE 8 -#define TRANS_RESERVE_SIZE 48 +#define TRANS_RESERVE_SIZE 44 static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld); @@ -196,10 +196,21 @@ SSdbRaw *mndTransEncode(STrans *pTrans) { } SDB_SET_BINARY(pRaw, dataPos, pTrans->opername, TSDB_TRANS_OPER_LEN, _OVER) + + int32_t arbGroupNum = taosHashGetSize(pTrans->arbGroupIds); + SDB_SET_INT32(pRaw, dataPos, arbGroupNum, _OVER) + void *pIter = NULL; + pIter = taosHashIterate(pTrans->arbGroupIds, NULL); + while (pIter) { + int32_t arbGroupId = *(int32_t *)pIter; + SDB_SET_INT32(pRaw, dataPos, arbGroupId, _OVER) + pIter = taosHashIterate(pTrans->arbGroupIds, pIter); + } + SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) - terrno = 0; + terrno = 0; _OVER: if (terrno != 0) { @@ -279,6 +290,7 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) { int32_t undoActionNum = 0; int32_t commitActionNum = 0; int32_t dataPos = 0; + int32_t arbgroupIdNum = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; @@ -350,6 +362,16 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) { } SDB_GET_BINARY(pRaw, dataPos, pTrans->opername, TSDB_TRANS_OPER_LEN, _OVER); + + pTrans->arbGroupIds = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); + + SDB_GET_INT32(pRaw, dataPos, &arbgroupIdNum, _OVER) + for (int32_t i = 0; i < arbgroupIdNum; ++i) { + int32_t arbGroupId = 0; + SDB_GET_INT32(pRaw, dataPos, &arbGroupId, _OVER) + taosHashPut(pTrans->arbGroupIds, &arbGroupId, sizeof(int32_t), NULL, 0); + } + SDB_GET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) terrno = 0; @@ -462,6 +484,9 @@ void mndTransDropData(STrans *pTrans) { mndTransDropActions(pTrans->commitActions); pTrans->commitActions = NULL; } + if (pTrans->arbGroupIds != NULL) { + taosHashCleanup(pTrans->arbGroupIds); + } if (pTrans->pRpcArray != NULL) { taosArrayDestroy(pTrans->pRpcArray); pTrans->pRpcArray = NULL; @@ -581,6 +606,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->arbGroupIds = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64(); taosInitRWLatch(&pTrans->lockRpcArray); @@ -733,7 +759,9 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname) } } -void mndTransSetArbGroupId(STrans *pTrans, int32_t groupId) { pTrans->arbGroupId = groupId; } +void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId) { + taosHashPut(pTrans->arbGroupIds, &groupId, sizeof(int32_t), NULL, 0); +} void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; } @@ -821,7 +849,16 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { if (pNew->conflict == TRN_CONFLICT_ARBGROUP) { if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; if (pTrans->conflict == TRN_CONFLICT_ARBGROUP) { - if (pNew->arbGroupId == pTrans->arbGroupId) conflict = true; + void *pIter = taosHashIterate(pNew->arbGroupIds, NULL); + while (pIter != NULL) { + int32_t groupId = *(int32_t *)pIter; + if (taosHashGet(pTrans->arbGroupIds, &groupId, sizeof(int32_t)) != NULL) { + taosHashCancelIterate(pNew->arbGroupIds, pIter); + conflict = true; + break; + } + pIter = taosHashIterate(pNew->arbGroupIds, pIter); + } } } @@ -1372,7 +1409,7 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr mInfo("trans:%d, execute %d actions serial, current redoAction:%d", pTrans->id, numOfActions, pTrans->actionPos); for (int32_t action = pTrans->actionPos; action < numOfActions; ++action) { - STransAction *pAction = taosArrayGet(pActions, pTrans->actionPos); + STransAction *pAction = taosArrayGet(pActions, action); code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); if (code == 0) { diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 64bca96702..ee60556639 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -49,6 +49,7 @@ TEST_F(MndTestFunc, 01_Show_Func) { } TEST_F(MndTestFunc, 02_Create_Func) { +#ifndef WINDOWS { SCreateFuncReq createReq = {0}; strcpy(createReq.name, ""); @@ -159,9 +160,11 @@ TEST_F(MndTestFunc, 02_Create_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", ""); EXPECT_EQ(test.GetShowRows(), 1); +#endif } TEST_F(MndTestFunc, 03_Retrieve_Func) { +#ifndef WINDOWS { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 1; @@ -376,9 +379,11 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST); } +#endif } TEST_F(MndTestFunc, 04_Drop_Func) { +#ifndef WINDOWS { SDropFuncReq dropReq = {0}; strcpy(dropReq.name, ""); @@ -441,9 +446,11 @@ TEST_F(MndTestFunc, 04_Drop_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", ""); EXPECT_EQ(test.GetShowRows(), 1); +#endif } TEST_F(MndTestFunc, 05_Actual_code) { +#ifndef WINDOWS { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "udf1"); @@ -507,4 +514,5 @@ TEST_F(MndTestFunc, 05_Actual_code) { } tFreeSRetrieveFuncRsp(&retrieveRsp); } -} \ No newline at end of file +#endif +} diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c85316f810..84e1996f2c 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -355,6 +355,8 @@ typedef struct { int flush_count; } SCacheFlushState; +typedef struct SCompMonitor SCompMonitor; + struct STsdb { char * path; SVnode * pVnode; @@ -375,8 +377,11 @@ struct STsdb { TdThreadMutex pgMutex; struct STFileSystem *pFS; // new SRocksCache rCache; - // compact monitor - struct SCompMonitor *pCompMonitor; + SCompMonitor *pCompMonitor; + struct { + SVHashTable *ht; + SArray *arr; + } *commitInfo; }; struct TSDBKEY { diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 304716744c..7bdc8e1c33 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -49,31 +49,21 @@ int32_t vnodeEncodeConfig(const void* pObj, SJson* pJson); int32_t vnodeDecodeConfig(const SJson* pJson, void* pObj); // vnodeAsync.c -typedef struct SVAsync SVAsync; - typedef enum { EVA_PRIORITY_HIGH = 0, EVA_PRIORITY_NORMAL, EVA_PRIORITY_LOW, } EVAPriority; -#define VNODE_ASYNC_VALID_CHANNEL_ID(channelId) ((channelId) > 0) -#define VNODE_ASYNC_VALID_TASK_ID(taskId) ((taskId) > 0) - -int32_t vnodeAsyncInit(SVAsync** async, char* label); -int32_t vnodeAsyncDestroy(SVAsync** async); -int32_t vnodeAChannelInit(SVAsync* async, int64_t* channelId); -int32_t vnodeAChannelDestroy(SVAsync* async, int64_t channelId, bool waitRunning); -int32_t vnodeAsync(SVAsync* async, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*), void* arg, - int64_t* taskId); -int32_t vnodeAsyncC(SVAsync* async, int64_t channelId, EVAPriority priority, int32_t (*execute)(void*), - void (*complete)(void*), void* arg, int64_t* taskId); -int32_t vnodeAWait(SVAsync* async, int64_t taskId); -int32_t vnodeACancel(SVAsync* async, int64_t taskId); -int32_t vnodeAsyncSetWorkers(SVAsync* async, int32_t numWorkers); - -// vnodeModule.c -extern SVAsync* vnodeAsyncHandle[2]; +int32_t vnodeAsyncOpen(int32_t numOfThreads); +int32_t vnodeAsyncClose(); +int32_t vnodeAChannelInit(int64_t async, SVAChannelID* channelID); +int32_t vnodeAChannelDestroy(SVAChannelID* channelID, bool waitRunning); +int32_t vnodeAsync(SVAChannelID* channelID, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*), + void* arg, SVATaskID* taskID); +int32_t vnodeAWait(SVATaskID* taskID); +int32_t vnodeACancel(SVATaskID* taskID); +int32_t vnodeAsyncSetWorkers(int64_t async, int32_t numWorkers); // vnodeBufPool.c typedef struct SVBufPoolNode SVBufPoolNode; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 419ebd1a6c..06b58213f7 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -237,9 +237,6 @@ int32_t tsdbCacheNewSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, int8_t int32_t tsdbCacheDropSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, bool hasPrimayKey); int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t col_type); int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey); -int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); -int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync); -int32_t tsdbS3Migrate(STsdb* tsdb, int64_t now, int32_t sync); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); @@ -470,6 +467,16 @@ typedef struct SVMonitorObj { taos_counter_t* insertCounter; } SVMonitorObj; +typedef struct { + int64_t async; + int64_t id; +} SVAChannelID; + +typedef struct { + int64_t async; + int64_t id; +} SVATaskID; + struct SVnode { char* path; SVnodeCfg config; @@ -491,8 +498,8 @@ struct SVnode { SVBufPool* onRecycle; // commit variables - int64_t commitChannel; - int64_t commitTask; + SVAChannelID commitChannel; + SVATaskID commitTask; SMeta* pMeta; SSma* pSma; @@ -598,6 +605,24 @@ struct SCompactInfo { void initStorageAPI(SStorageAPI* pAPI); +// a simple hash table impl +typedef struct SVHashTable SVHashTable; + +struct SVHashTable { + uint32_t (*hash)(const void*); + int32_t (*compare)(const void*, const void*); + int32_t numEntries; + uint32_t numBuckets; + struct SVHashEntry** buckets; +}; + +#define vHashNumEntries(ht) ((ht)->numEntries) +int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*)); +int32_t vHashDestroy(SVHashTable** ht); +int32_t vHashPut(SVHashTable* ht, void* obj); +int32_t vHashGet(SVHashTable* ht, const void* obj, void** retObj); +int32_t vHashDrop(SVHashTable* ht, const void* obj); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 5441d0c4c1..7d6eda9cf1 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -622,8 +622,8 @@ int32_t smaRetention(SSma *pSma, int64_t now) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { - code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1); - if (code) goto _end; + // code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1); + // if (code) goto _end; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index f602ee5fe3..42df7f111e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -588,6 +588,13 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud); } + for (uint8_t i = 0; i < pLastCol->rowKey.numOfPKs; ++i) { + SValue *pValue = &pLastCol->rowKey.pks[i]; + if (IS_VAR_DATA_TYPE(pValue->type)) { + taosMemoryFree(pValue->pData); + } + } + if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type) /* && pLastCol->colVal.value.nData > 0*/) { taosMemoryFree(pLastCol->colVal.value.pData); } @@ -1072,6 +1079,8 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray SLastCol *PToFree = pLastCol; if (IS_LAST_KEY(idxKey->key) && !COL_VAL_IS_VALUE(pColVal)) { + taosMemoryFreeClear(PToFree); + rocksdb_free(values_list[i]); continue; } @@ -1238,6 +1247,18 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo ctxArray = taosArrayInit(pBlockData->nColData, sizeof(SLastUpdateCtx)); // 1. prepare last + STsdbRowKey tsdbRowKey = {0}; + tsdbRowGetKey(&lRow, &tsdbRowKey); + + { + SLastUpdateCtx updateCtx = { + .lflag = LFLAG_LAST, + .tsdbRowKey = tsdbRowKey, + .colVal = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, ((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP, + .val = lRow.pBlockData->aTSKEY[lRow.iRow]}))}; + taosArrayPush(ctxArray, &updateCtx); + } + TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0); for (int32_t iColData = 0; iColData < pBlockData->nColData; ++iColData) { @@ -1263,9 +1284,6 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo } // 2. prepare last row - STsdbRowKey tsdbRowKey = {0}; - tsdbRowGetKey(&lRow, &tsdbRowKey); - STSDBRowIter iter = {0}; tsdbRowIterOpen(&iter, &lRow, pTSchema); for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 96c0f2a4ba..ffe5c2c1e0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -17,13 +17,12 @@ // extern dependencies typedef struct { - STsdb *tsdb; - TFileSetArray *fsetArr; - TFileOpArray fopArray[1]; - - // SSkmInfo skmTb[1]; - // SSkmInfo skmRow[1]; + int32_t fid; + STFileSet *fset; +} SFileSetCommitInfo; +typedef struct { + STsdb *tsdb; int32_t minutes; int8_t precision; int32_t minRow; @@ -32,34 +31,34 @@ typedef struct { int32_t sttTrigger; int32_t szPage; int64_t compactVersion; + int64_t cid; + int64_t now; struct { - int64_t cid; - int64_t now; - TSKEY nextKey; - int32_t fid; - int32_t expLevel; - SDiskID did; - TSKEY minKey; - TSKEY maxKey; - STFileSet *fset; - TABLEID tbid[1]; - bool hasTSData; - bool skipTsRow; - SHashObj *pColCmprObj; + SFileSetCommitInfo *info; + + int32_t expLevel; + SDiskID did; + TSKEY minKey; + TSKEY maxKey; + TABLEID tbid[1]; + bool hasTSData; + + bool skipTsRow; + SHashObj *pColCmprObj; } ctx[1]; // reader TSttFileReaderArray sttReaderArray[1]; - // iter TTsdbIterArray dataIterArray[1]; SIterMerger *dataIterMerger; TTsdbIterArray tombIterArray[1]; SIterMerger *tombIterMerger; - // writer SFSetWriter *writer; + + TFileOpArray fopArray[1]; } SCommitter2; static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { @@ -74,8 +73,8 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { .maxRow = committer->maxRow, .szPage = committer->szPage, .cmprAlg = committer->cmprAlg, - .fid = committer->ctx->fid, - .cid = committer->ctx->cid, + .fid = committer->ctx->info->fid, + .cid = committer->cid, .did = committer->ctx->did, .level = 0, }; @@ -83,11 +82,11 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { if (committer->sttTrigger == 1) { config.toSttOnly = false; - if (committer->ctx->fset) { + if (committer->ctx->info->fset) { for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ftype++) { - if (committer->ctx->fset->farr[ftype] != NULL) { + if (committer->ctx->info->fset->farr[ftype] != NULL) { config.files[ftype].exist = true; - config.files[ftype].file = committer->ctx->fset->farr[ftype]->f[0]; + config.files[ftype].file = committer->ctx->info->fset->farr[ftype]->f[0]; } } } @@ -117,7 +116,6 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) { committer->ctx->tbid->suid = 0; committer->ctx->tbid->uid = 0; - for (SRowInfo *row; (row = tsdbIterMergerGetData(committer->dataIterMerger)) != NULL;) { if (row->uid != committer->ctx->tbid->uid) { committer->ctx->tbid->suid = row->suid; @@ -132,7 +130,6 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) { int64_t ts = TSDBROW_TS(&row->row); if (ts > committer->ctx->maxKey) { - committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); TSDB_CHECK_CODE(code, lino, _exit); continue; @@ -152,7 +149,8 @@ _exit: if (code) { TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); } else { - tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->fid, numOfRow); + tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->info->fid, + numOfRow); } return code; } @@ -168,7 +166,7 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { } // do not need to write tomb data if there is no ts data - bool skip = (committer->ctx->fset == NULL && !committer->ctx->hasTSData); + bool skip = (committer->ctx->info->fset == NULL && !committer->ctx->hasTSData); committer->ctx->tbid->suid = 0; committer->ctx->tbid->uid = 0; @@ -187,12 +185,8 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { if (record->ekey < committer->ctx->minKey) { // do nothing } else if (record->skey > committer->ctx->maxKey) { - committer->ctx->nextKey = TMIN(record->skey, committer->ctx->nextKey); + // committer->ctx->nextKey = TMIN(record->skey, committer->ctx->nextKey); } else { - if (record->ekey > committer->ctx->maxKey) { - committer->ctx->nextKey = TMIN(committer->ctx->nextKey, committer->ctx->maxKey + 1); - } - record->skey = TMAX(record->skey, committer->ctx->minKey); record->ekey = TMIN(record->ekey, committer->ctx->maxKey); @@ -211,8 +205,8 @@ _exit: if (code) { TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); } else { - tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode), committer->ctx->fid, - numRecord); + tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode), + committer->ctx->info->fid, numRecord); } return code; } @@ -223,15 +217,15 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) { ASSERT(TARRAY2_SIZE(committer->sttReaderArray) == 0); - if (committer->ctx->fset == NULL // - || committer->sttTrigger > 1 // - || TARRAY2_SIZE(committer->ctx->fset->lvlArr) == 0 // + if (committer->ctx->info->fset == NULL // + || committer->sttTrigger > 1 // + || TARRAY2_SIZE(committer->ctx->info->fset->lvlArr) == 0 // ) { return 0; } SSttLvl *lvl; - TARRAY2_FOREACH(committer->ctx->fset->lvlArr, lvl) { + TARRAY2_FOREACH(committer->ctx->info->fset->lvlArr, lvl) { STFileObj *fobj = NULL; TARRAY2_FOREACH(lvl->fobjArr, fobj) { SSttFileReader *sttReader; @@ -289,7 +283,7 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) { config.from->version = VERSION_MIN; config.from->key = (SRowKey){ .ts = committer->ctx->minKey, - .numOfPKs = 0, // TODO: support multiple primary keys + .numOfPKs = 0, }; code = tsdbIterOpen(&config, &iter); @@ -359,22 +353,15 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { int32_t lino = 0; STsdb *tsdb = committer->tsdb; - int32_t fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision); - // check if can commit - tsdbFSCheckCommit(tsdb, fid); + tsdbFSCheckCommit(tsdb, committer->ctx->info->fid); - committer->ctx->fid = fid; - committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now); - tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey, + committer->ctx->expLevel = tsdbFidLevel(committer->ctx->info->fid, &tsdb->keepCfg, committer->now); + tsdbFidKeyRange(committer->ctx->info->fid, committer->minutes, committer->precision, &committer->ctx->minKey, &committer->ctx->maxKey); code = tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did); TSDB_CHECK_CODE(code, lino, _exit); tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did); - STFileSet fset = {.fid = committer->ctx->fid}; - committer->ctx->fset = &fset; - STFileSet **fsetPtr = TARRAY2_SEARCH(committer->fsetArr, &committer->ctx->fset, tsdbTFileSetCmprFn, TD_EQ); - committer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr; committer->ctx->tbid->suid = 0; committer->ctx->tbid->uid = 0; @@ -391,15 +378,13 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { code = tsdbCommitOpenWriter(committer); TSDB_CHECK_CODE(code, lino, _exit); - // reset nextKey - committer->ctx->nextKey = TSKEY_MAX; - _exit: if (code) { TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); } else { tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode), - __func__, committer->ctx->fid, committer->ctx->minKey, committer->ctx->maxKey, committer->ctx->expLevel); + __func__, committer->ctx->info->fid, committer->ctx->minKey, committer->ctx->maxKey, + committer->ctx->expLevel); } return code; } @@ -421,7 +406,7 @@ _exit: if (code) { TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); } else { - tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid); + tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid); } return code; } @@ -449,7 +434,220 @@ _exit: if (code) { TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); } else { - tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid); + tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid); + } + return code; +} + +static int32_t tFileSetCommitInfoCompare(const void *arg1, const void *arg2) { + SFileSetCommitInfo *info1 = (SFileSetCommitInfo *)arg1; + SFileSetCommitInfo *info2 = (SFileSetCommitInfo *)arg2; + + if (info1->fid < info2->fid) { + return -1; + } else if (info1->fid > info2->fid) { + return 1; + } else { + return 0; + } +} + +static int32_t tFileSetCommitInfoPCompare(const void *arg1, const void *arg2) { + return tFileSetCommitInfoCompare(*(SFileSetCommitInfo **)arg1, *(SFileSetCommitInfo **)arg2); +} + +static uint32_t tFileSetCommitInfoHash(const void *arg) { + SFileSetCommitInfo *info = (SFileSetCommitInfo *)arg; + return MurmurHash3_32((const char *)&info->fid, sizeof(info->fid)); +} + +static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) { + int32_t code = 0; + int32_t lino = 0; + + if (pTsdb->commitInfo) { + for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) { + SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i); + vHashDrop(pTsdb->commitInfo->ht, info); + tsdbTFileSetClear(&info->fset); + taosMemoryFree(info); + } + + vHashDestroy(&pTsdb->commitInfo->ht); + taosArrayDestroy(pTsdb->commitInfo->arr); + pTsdb->commitInfo->arr = NULL; + taosMemoryFreeClear(pTsdb->commitInfo); + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + +static int32_t tsdbCommitInfoInit(STsdb *pTsdb) { + int32_t code = 0; + int32_t lino = 0; + + pTsdb->commitInfo = taosMemoryCalloc(1, sizeof(*pTsdb->commitInfo)); + if (pTsdb->commitInfo == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } + + code = vHashInit(&pTsdb->commitInfo->ht, tFileSetCommitInfoHash, tFileSetCommitInfoCompare); + TSDB_CHECK_CODE(code, lino, _exit); + + pTsdb->commitInfo->arr = taosArrayInit(0, sizeof(SFileSetCommitInfo *)); + if (pTsdb->commitInfo->arr == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } + +_exit: + if (code) { + tsdbCommitInfoDestroy(pTsdb); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + +static int32_t tsdbCommitInfoAdd(STsdb *tsdb, int32_t fid) { + int32_t code = 0; + int32_t lino = 0; + + SFileSetCommitInfo *tinfo; + + if ((tinfo = taosMemoryMalloc(sizeof(*tinfo))) == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } + tinfo->fid = fid; + tinfo->fset = NULL; + + code = vHashPut(tsdb->commitInfo->ht, tinfo); + TSDB_CHECK_CODE(code, lino, _exit); + + if ((taosArrayPush(tsdb->commitInfo->arr, &tinfo)) == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } + taosArraySort(tsdb->commitInfo->arr, tFileSetCommitInfoPCompare); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + +static int32_t tsdbCommitInfoBuild(STsdb *tsdb) { + int32_t code = 0; + int32_t lino = 0; + + STFileSet *fset = NULL; + SRBTreeIter iter; + + code = tsdbCommitInfoInit(tsdb); + TSDB_CHECK_CODE(code, lino, _exit); + + // scan time-series data + iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1); + for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) { + STbData *pTbData = TCONTAINER_OF(node, STbData, rbtn); + + // scan time-series data + STsdbRowKey from = { + .key.ts = INT64_MIN, + .key.numOfPKs = 0, + .version = INT64_MIN, + }; + for (;;) { + int64_t minKey, maxKey; + STbDataIter tbDataIter = {0}; + TSDBROW *row; + int32_t fid; + + tsdbTbDataIterOpen(pTbData, &from, 0, &tbDataIter); + if ((row = tsdbTbDataIterGet(&tbDataIter)) == NULL) { + break; + } + + fid = tsdbKeyFid(TSDBROW_TS(row), tsdb->keepCfg.days, tsdb->keepCfg.precision); + tsdbFidKeyRange(fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey); + + SFileSetCommitInfo *info; + SFileSetCommitInfo tinfo = { + .fid = fid, + }; + vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info); + if (info == NULL) { + code = tsdbCommitInfoAdd(tsdb, fid); + TSDB_CHECK_CODE(code, lino, _exit); + } + + from.key.ts = maxKey + 1; + } + } + + taosThreadMutexLock(&tsdb->mutex); + + // scan tomb data + if (tsdb->imem->nDel > 0) { + TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { + if (tsdbTFileSetIsEmpty(fset)) { + continue; + } + + SFileSetCommitInfo *info; + SFileSetCommitInfo tinfo = { + .fid = fset->fid, + }; + + // check if the file set already on the commit list + vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info); + if (info != NULL) { + continue; + } + + int64_t minKey, maxKey; + bool hasDataToCommit = false; + tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey); + iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1); + for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) { + STbData *pTbData = TCONTAINER_OF(node, STbData, rbtn); + for (SDelData *pDelData = pTbData->pHead; pDelData; pDelData = pDelData->pNext) { + if (pDelData->sKey > maxKey || pDelData->eKey < minKey) { + continue; + } else { + hasDataToCommit = true; + if ((code = tsdbCommitInfoAdd(tsdb, fset->fid))) { + taosThreadMutexUnlock(&tsdb->mutex); + TSDB_CHECK_CODE(code, lino, _exit); + } + break; + } + } + + if (hasDataToCommit) { + break; + } + } + } + } + + // begin tasks on file set + for (int i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) { + SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); + tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset); + if (fset) { + tsdbTFileSetInitCopy(tsdb, fset, &info->fset); + } + } + + taosThreadMutexUnlock(&tsdb->mutex); + +_exit: + if (code) { + tsdbCommitInfoDestroy(tsdb); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -458,11 +656,7 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co int32_t code = 0; int32_t lino = 0; - memset(committer, 0, sizeof(committer[0])); - committer->tsdb = tsdb; - code = tsdbFSCreateCopySnapshot(tsdb->pFS, &committer->fsetArr); - TSDB_CHECK_CODE(code, lino, _exit); committer->minutes = tsdb->keepCfg.days; committer->precision = tsdb->keepCfg.precision; committer->minRow = info->info.config.tsdbCfg.minRows; @@ -471,21 +665,11 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co committer->sttTrigger = info->info.config.sttTrigger; committer->szPage = info->info.config.tsdbPageSize; committer->compactVersion = INT64_MAX; - committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS); - committer->ctx->now = taosGetTimestampSec(); + committer->cid = tsdbFSAllocEid(tsdb->pFS); + committer->now = taosGetTimestampSec(); - committer->ctx->nextKey = tsdb->imem->minKey; - if (tsdb->imem->nDel > 0) { - SRBTreeIter iter[1] = {tRBTreeIterCreate(tsdb->imem->tbDataTree, 1)}; - - for (SRBTreeNode *node = tRBTreeIterNext(iter); node; node = tRBTreeIterNext(iter)) { - STbData *tbData = TCONTAINER_OF(node, STbData, rbtn); - - for (SDelData *delData = tbData->pHead; delData; delData = delData->pNext) { - committer->ctx->nextKey = TMIN(committer->ctx->nextKey, delData->sKey); - } - } - } + code = tsdbCommitInfoBuild(tsdb); + TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { @@ -516,14 +700,13 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) { TARRAY2_DESTROY(committer->sttReaderArray, NULL); TARRAY2_DESTROY(committer->fopArray, NULL); TARRAY2_DESTROY(committer->sttReaderArray, NULL); - tsdbFSDestroyCopySnapshot(&committer->fsetArr); _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, lino, - tstrerror(code), committer->ctx->cid); + tstrerror(code), committer->cid); } else { - tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->cid); + tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->cid); } return code; } @@ -553,17 +736,18 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) { taosThreadMutexUnlock(&tsdb->mutex); tsdbUnrefMemTable(imem, NULL, true); } else { - SCommitter2 committer[1]; + SCommitter2 committer = {0}; - code = tsdbOpenCommitter(tsdb, info, committer); + code = tsdbOpenCommitter(tsdb, info, &committer); TSDB_CHECK_CODE(code, lino, _exit); - while (committer->ctx->nextKey != TSKEY_MAX) { - code = tsdbCommitFileSet(committer); + for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) { + committer.ctx->info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); + code = tsdbCommitFileSet(&committer); TSDB_CHECK_CODE(code, lino, _exit); } - code = tsdbCloseCommitter(committer, code); + code = tsdbCloseCommitter(&committer, code); TSDB_CHECK_CODE(code, lino, _exit); } @@ -580,22 +764,33 @@ int32_t tsdbCommitCommit(STsdb *tsdb) { int32_t code = 0; int32_t lino = 0; - if (tsdb->imem == NULL) goto _exit; + if (tsdb->imem) { + SMemTable *pMemTable = tsdb->imem; + + taosThreadMutexLock(&tsdb->mutex); + + if ((code = tsdbFSEditCommit(tsdb->pFS))) { + taosThreadMutexUnlock(&tsdb->mutex); + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdb->imem = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) { + SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); + if (info->fset) { + tsdbFinishTaskOnFileSet(tsdb, info->fid); + } + } - SMemTable *pMemTable = tsdb->imem; - taosThreadMutexLock(&tsdb->mutex); - code = tsdbFSEditCommit(tsdb->pFS); - if (code) { taosThreadMutexUnlock(&tsdb->mutex); - TSDB_CHECK_CODE(code, lino, _exit); + + tsdbCommitInfoDestroy(tsdb); + tsdbUnrefMemTable(pMemTable, NULL, true); } - tsdb->imem = NULL; - taosThreadMutexUnlock(&tsdb->mutex); - tsdbUnrefMemTable(pMemTable, NULL, true); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); } @@ -611,6 +806,16 @@ int32_t tsdbCommitAbort(STsdb *pTsdb) { code = tsdbFSEditAbort(pTsdb->pFS); TSDB_CHECK_CODE(code, lino, _exit); + taosThreadMutexLock(&pTsdb->mutex); + for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) { + SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i); + if (info->fset) { + tsdbFinishTaskOnFileSet(pTsdb, info->fid); + } + } + taosThreadMutexUnlock(&pTsdb->mutex); + tsdbCommitInfoDestroy(pTsdb); + _exit: if (code) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); @@ -618,4 +823,4 @@ _exit: tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); } return code; -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 3ca26621a1..2345cc599b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -22,9 +22,6 @@ extern void remove_file(const char *fname); -#define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT -#define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1) - typedef struct STFileHashEntry { struct STFileHashEntry *next; char fname[TSDB_FILENAME_LEN]; @@ -290,10 +287,8 @@ static int32_t commit_edit(STFileSystem *fs) { current_fname(fs->tsdb, current, TSDB_FCURRENT); if (fs->etype == TSDB_FEDIT_COMMIT) { current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C); - } else if (fs->etype == TSDB_FEDIT_MERGE) { - current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); } else { - ASSERT(0); + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); } int32_t code; @@ -324,11 +319,8 @@ static int32_t abort_edit(STFileSystem *fs) { if (fs->etype == TSDB_FEDIT_COMMIT) { current_fname(fs->tsdb, fname, TSDB_FCURRENT_C); - } else if (fs->etype == TSDB_FEDIT_MERGE) { - current_fname(fs->tsdb, fname, TSDB_FCURRENT_M); } else { - tsdbError("vgId:%d %s failed since invalid etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); - ASSERT(0); + current_fname(fs->tsdb, fname, TSDB_FCURRENT_M); } int32_t code; @@ -767,9 +759,12 @@ extern int32_t tsdbStopAllCompTask(STsdb *tsdb); int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) { STFileSystem *fs = pTsdb->pFS; - TARRAY2(int64_t) channelArr = {0}; + SArray *channelArray = taosArrayInit(0, sizeof(SVAChannelID)); + if (channelArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - taosThreadMutexLock(&fs->tsdb->mutex); + taosThreadMutexLock(&pTsdb->mutex); // disable pTsdb->bgTaskDisabled = true; @@ -777,20 +772,23 @@ int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) { // collect channel STFileSet *fset; TARRAY2_FOREACH(fs->fSetArr, fset) { - if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) { - TARRAY2_APPEND(&channelArr, fset->bgTaskChannel); - fset->bgTaskChannel = 0; + if (fset->channelOpened) { + taosArrayPush(channelArray, &fset->channel); + fset->channel = (SVAChannelID){0}; + fset->mergeScheduled = false; + tsdbFSSetBlockCommit(fset, false); + fset->channelOpened = false; } - fset->mergeScheduled = false; - tsdbFSSetBlockCommit(fset, false); } - taosThreadMutexUnlock(&fs->tsdb->mutex); + taosThreadMutexUnlock(&pTsdb->mutex); // destroy all channels - int64_t channel; - TARRAY2_FOREACH(&channelArr, channel) { vnodeAChannelDestroy(vnodeAsyncHandle[1], channel, true); } - TARRAY2_DESTROY(&channelArr, NULL); + for (int32_t i = 0; i < taosArrayGetSize(channelArray); i++) { + SVAChannelID *channel = taosArrayGet(channelArray, i); + vnodeAChannelDestroy(channel, true); + } + taosArrayDestroy(channelArray); #ifdef TD_ENTERPRISE tsdbStopAllCompTask(pTsdb); @@ -832,15 +830,10 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e int32_t lino; char current_t[TSDB_FILENAME_LEN]; - switch (etype) { - case TSDB_FEDIT_COMMIT: - current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C); - break; - case TSDB_FEDIT_MERGE: - current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); - break; - default: - ASSERT(0); + if (etype == TSDB_FEDIT_COMMIT) { + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C); + } else { + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); } tsem_wait(&fs->canEdit); @@ -932,8 +925,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { arg->tsdb = fs->tsdb; arg->fid = fset->fid; - code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg, - NULL); + code = vnodeAsync(&fset->channel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg, NULL); TSDB_CHECK_CODE(code, lino, _exit); fset->mergeScheduled = true; } @@ -946,33 +938,6 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { } } - // clear empty level and fset - int32_t i = 0; - while (i < TARRAY2_SIZE(fs->fSetArr)) { - STFileSet *fset = TARRAY2_GET(fs->fSetArr, i); - - int32_t j = 0; - while (j < TARRAY2_SIZE(fset->lvlArr)) { - SSttLvl *lvl = TARRAY2_GET(fset->lvlArr, j); - - if (TARRAY2_SIZE(lvl->fobjArr) == 0) { - TARRAY2_REMOVE(fset->lvlArr, j, tsdbSttLvlClear); - } else { - j++; - } - } - - if (tsdbTFileSetIsEmpty(fset)) { - if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) { - vnodeAChannelDestroy(vnodeAsyncHandle[1], fset->bgTaskChannel, false); - fset->bgTaskChannel = 0; - } - TARRAY2_REMOVE(fs->fSetArr, i, tsdbTFileSetClear); - } else { - i++; - } - } - _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(fs->tsdb->pVnode), __func__, lino, tstrerror(code)); @@ -1211,3 +1176,47 @@ _out: } int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr) { return tsdbTFileSetRangeArrayDestroy(fsrArr); } + +int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset) { + int16_t sttTrigger = tsdb->pVnode->config.sttTrigger; + + tsdbFSGetFSet(tsdb->pFS, fid, fset); + if (sttTrigger == 1 && (*fset)) { + for (;;) { + if ((*fset)->taskRunning) { + (*fset)->numWaitTask++; + + taosThreadCondWait(&(*fset)->beginTask, &tsdb->mutex); + + tsdbFSGetFSet(tsdb->pFS, fid, fset); + ASSERT(fset != NULL); + + (*fset)->numWaitTask--; + ASSERT((*fset)->numWaitTask >= 0); + } else { + (*fset)->taskRunning = true; + break; + } + } + tsdbInfo("vgId:%d begin task on file set:%d", TD_VID(tsdb->pVnode), fid); + } + + return 0; +} + +int32_t tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid) { + int16_t sttTrigger = tsdb->pVnode->config.sttTrigger; + if (sttTrigger == 1) { + STFileSet *fset = NULL; + tsdbFSGetFSet(tsdb->pFS, fid, &fset); + if (fset != NULL && fset->taskRunning) { + fset->taskRunning = false; + if (fset->numWaitTask > 0) { + taosThreadCondSignal(&fset->beginTask); + } + tsdbInfo("vgId:%d finish task on file set:%d", TD_VID(tsdb->pVnode), fid); + } + } + + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 7099ce8b26..6938efde3c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -24,7 +24,9 @@ extern "C" { typedef enum { TSDB_FEDIT_COMMIT = 1, // - TSDB_FEDIT_MERGE + TSDB_FEDIT_MERGE, + TSDB_FEDIT_COMPACT, + TSDB_FEDIT_RETENTION, } EFEditT; typedef enum { @@ -59,6 +61,8 @@ int32_t tsdbFSEditAbort(STFileSystem *fs); // other int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset); int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid); +int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset); +int32_t tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid); // utils int32_t save_fs(const TFileSetArray *arr, const char *fname); int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype); @@ -72,10 +76,6 @@ struct STFileSystem { EFEditT etype; TFileSetArray fSetArr[1]; TFileSetArray fSetArrTmp[1]; - - // background task queue - bool stop; - int64_t taskid; }; #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 0820f53117..598ca0aec3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -456,13 +456,14 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) { TARRAY2_INIT(fset[0]->lvlArr); // background task queue - fset[0]->bgTaskChannel = 0; - fset[0]->mergeScheduled = false; + taosThreadCondInit(&(*fset)->beginTask, NULL); + (*fset)->taskRunning = false; + (*fset)->numWaitTask = 0; // block commit variables taosThreadCondInit(&fset[0]->canCommit, NULL); - fset[0]->numWaitCommit = 0; - fset[0]->blockCommit = false; + (*fset)->numWaitCommit = 0; + (*fset)->blockCommit = false; return 0; } @@ -598,21 +599,19 @@ int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr) { return 0; } -int32_t tsdbTFileSetClear(STFileSet **fset) { - if (!fset[0]) return 0; +void tsdbTFileSetClear(STFileSet **fset) { + if (fset && *fset) { + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if ((*fset)->farr[ftype] == NULL) continue; + tsdbTFileObjUnref((*fset)->farr[ftype]); + } - for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { - if (fset[0]->farr[ftype] == NULL) continue; - tsdbTFileObjUnref(fset[0]->farr[ftype]); + TARRAY2_DESTROY((*fset)->lvlArr, tsdbSttLvlClear); + + taosThreadCondDestroy(&(*fset)->beginTask); + taosThreadCondDestroy(&(*fset)->canCommit); + taosMemoryFreeClear(*fset); } - - TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlClear); - - taosThreadCondDestroy(&fset[0]->canCommit); - taosMemoryFree(fset[0]); - fset[0] = NULL; - - return 0; } int32_t tsdbTFileSetRemove(STFileSet *fset) { @@ -665,6 +664,12 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) { } int32_t tsdbTFileSetOpenChannel(STFileSet *fset) { - if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) return 0; - return vnodeAChannelInit(vnodeAsyncHandle[1], &fset->bgTaskChannel); + int32_t code; + if (!fset->channelOpened) { + if ((code = vnodeAChannelInit(2, &fset->channel))) { + return code; + } + fset->channelOpened = true; + } + return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 86db3b01c7..4b99a46dc7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -42,7 +42,7 @@ typedef enum { int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset); int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset); int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset); -int32_t tsdbTFileSetClear(STFileSet **fset); +void tsdbTFileSetClear(STFileSet **fset); int32_t tsdbTFileSetRemove(STFileSet *fset); int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset, @@ -90,9 +90,15 @@ struct STFileSet { STFileObj *farr[TSDB_FTYPE_MAX]; // file array TSttLvlArray lvlArr[1]; // level array - // background task channel - int64_t bgTaskChannel; - bool mergeScheduled; + // background task + bool channelOpened; + SVAChannelID channel; + bool mergeScheduled; + + // sttTrigger = 1 + TdThreadCond beginTask; + bool taskRunning; + int32_t numWaitTask; // block commit variables TdThreadCond canCommit; diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 971020e7d6..022698b0eb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -593,5 +593,6 @@ _exit: exit(EXIT_FAILURE); } tsdbTFileSetClear(&merger->fset); + taosMemoryFree(arg); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index f4344296b4..3d53d1ada3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -24,8 +24,8 @@ typedef struct { int64_t now; int64_t cid; - TFileSetArray *fsetArr; - TFileOpArray fopArr[1]; + STFileSet *fset; + TFileOpArray fopArr; } SRTNer; static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) { @@ -35,7 +35,7 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) { .of = fobj->f[0], }; - return TARRAY2_APPEND(rtner->fopArr, op); + return TARRAY2_APPEND(&rtner->fopArr, op); } static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) { @@ -71,7 +71,7 @@ static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFi _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); if (fdFrom) taosCloseFile(&fdFrom); if (fdTo) taosCloseFile(&fdTo); } @@ -108,7 +108,7 @@ static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); taosCloseFile(&fdFrom); taosCloseFile(&fdTo); } @@ -128,7 +128,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const .of = fobj->f[0], }; - code = TARRAY2_APPEND(rtner->fopArr, op); + code = TARRAY2_APPEND(&rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); // create new @@ -152,7 +152,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const }, }; - code = TARRAY2_APPEND(rtner->fopArr, op); + code = TARRAY2_APPEND(&rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); // do copy the file @@ -167,7 +167,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); } return code; } @@ -176,80 +176,51 @@ typedef struct { STsdb *tsdb; int64_t now; int32_t fid; + bool s3Migrate; } SRtnArg; -static int32_t tsdbDoRetentionBegin(SRtnArg *arg, SRTNer *rtner) { - int32_t code = 0; - int32_t lino = 0; - - STsdb *tsdb = arg->tsdb; - - rtner->tsdb = tsdb; - rtner->szPage = tsdb->pVnode->config.tsdbPageSize; - rtner->now = arg->now; - rtner->cid = tsdbFSAllocEid(tsdb->pFS); - - code = tsdbFSCreateCopySnapshot(tsdb->pFS, &rtner->fsetArr); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); - } else { - tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); - } - return code; -} - static int32_t tsdbDoRetentionEnd(SRTNer *rtner) { int32_t code = 0; int32_t lino = 0; - if (TARRAY2_SIZE(rtner->fopArr) == 0) goto _exit; - - code = tsdbFSEditBegin(rtner->tsdb->pFS, rtner->fopArr, TSDB_FEDIT_MERGE); - TSDB_CHECK_CODE(code, lino, _exit); - - taosThreadMutexLock(&rtner->tsdb->mutex); - - code = tsdbFSEditCommit(rtner->tsdb->pFS); - if (code) { - taosThreadMutexUnlock(&rtner->tsdb->mutex); + if (TARRAY2_SIZE(&rtner->fopArr) > 0) { + code = tsdbFSEditBegin(rtner->tsdb->pFS, &rtner->fopArr, TSDB_FEDIT_RETENTION); TSDB_CHECK_CODE(code, lino, _exit); + + taosThreadMutexLock(&rtner->tsdb->mutex); + + code = tsdbFSEditCommit(rtner->tsdb->pFS); + if (code) { + taosThreadMutexUnlock(&rtner->tsdb->mutex); + TSDB_CHECK_CODE(code, lino, _exit); + } + + taosThreadMutexUnlock(&rtner->tsdb->mutex); + + TARRAY2_DESTROY(&rtner->fopArr, NULL); } - taosThreadMutexUnlock(&rtner->tsdb->mutex); - - TARRAY2_DESTROY(rtner->fopArr, NULL); - _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); } else { tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); } - tsdbFSDestroyCopySnapshot(&rtner->fsetArr); return code; } -static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { +static int32_t tsdbDoRetention(SRTNer *rtner) { int32_t code = 0; int32_t lino = 0; STFileObj *fobj = NULL; + STFileSet *fset = rtner->fset; int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now); if (expLevel < 0) { // remove the fileset for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) { if (fobj == NULL) continue; - /* - int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs); - if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) { - code = tsdbRemoveFileObjectS3(rtner, fobj); - TSDB_CHECK_CODE(code, lino, _exit); - } else {*/ code = tsdbDoRemoveFileObject(rtner, fobj); TSDB_CHECK_CODE(code, lino, _exit); - //} } SSttLvl *lvl; @@ -275,10 +246,6 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { if (fobj == NULL) continue; if (fobj->f->did.level == did.level) { - /* - code = tsdbCheckMigrateS3(rtner, fobj, ftype, &did); - TSDB_CHECK_CODE(code, lino, _exit); - */ continue; } @@ -306,91 +273,110 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); } return code; } -static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); } +static void tsdbRetentionCancel(void *arg) { taosMemoryFree(arg); } -static int32_t tsdbDoRetentionAsync(void *arg) { +static int32_t tsdbDoS3Migrate(SRTNer *rtner); + +static int32_t tsdbRetention(void *arg) { int32_t code = 0; int32_t lino = 0; - SRTNer rtner[1] = {0}; - code = tsdbDoRetentionBegin(arg, rtner); - TSDB_CHECK_CODE(code, lino, _exit); + SRtnArg *rtnArg = (SRtnArg *)arg; + STsdb *pTsdb = rtnArg->tsdb; + SVnode *pVnode = pTsdb->pVnode; + STFileSet *fset = NULL; + SRTNer rtner = { + .tsdb = pTsdb, + .szPage = pVnode->config.tsdbPageSize, + .now = rtnArg->now, + .cid = tsdbFSAllocEid(pTsdb->pFS), + }; - STFileSet *fset; - TARRAY2_FOREACH(rtner->fsetArr, fset) { - if (fset->fid != ((SRtnArg *)arg)->fid) continue; + // begin task + taosThreadMutexLock(&pTsdb->mutex); + tsdbBeginTaskOnFileSet(pTsdb, rtnArg->fid, &fset); + if (fset && (code = tsdbTFileSetInitCopy(pTsdb, fset, &rtner.fset))) { + taosThreadMutexUnlock(&pTsdb->mutex); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosThreadMutexUnlock(&pTsdb->mutex); - code = tsdbDoRetentionOnFileSet(rtner, fset); + // do retention + if (rtner.fset) { + if (rtnArg->s3Migrate) { + code = tsdbDoS3Migrate(&rtner); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDoRetention(&rtner); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDoRetentionEnd(&rtner); TSDB_CHECK_CODE(code, lino, _exit); } - code = tsdbDoRetentionEnd(rtner); - TSDB_CHECK_CODE(code, lino, _exit); - _exit: - if (code) { - if (TARRAY2_DATA(rtner->fopArr)) { - TARRAY2_DESTROY(rtner->fopArr, NULL); - } - TFileSetArray **fsetArr = &rtner->fsetArr; - if (fsetArr[0]) { - tsdbFSDestroyCopySnapshot(&rtner->fsetArr); - } + if (rtner.fset) { + taosThreadMutexLock(&pTsdb->mutex); + tsdbFinishTaskOnFileSet(pTsdb, rtnArg->fid); + taosThreadMutexUnlock(&pTsdb->mutex); + } - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + // clear resources + tsdbTFileSetClear(&rtner.fset); + TARRAY2_DESTROY(&rtner.fopArr, NULL); + taosMemoryFree(arg); + if (code) { + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(((SRtnArg *)arg)->tsdb->pVnode), __func__, code, lino); } return code; } -int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) { +static int32_t tsdbAsyncRetentionImpl(STsdb *tsdb, int64_t now, bool s3Migrate) { int32_t code = 0; - - taosThreadMutexLock(&tsdb->mutex); - - if (tsdb->bgTaskDisabled) { - taosThreadMutexUnlock(&tsdb->mutex); - return 0; - } + int32_t lino = 0; STFileSet *fset; - TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { - code = tsdbTFileSetOpenChannel(fset); - if (code) { - taosThreadMutexUnlock(&tsdb->mutex); - return code; - } - SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); - if (arg == NULL) { - taosThreadMutexUnlock(&tsdb->mutex); - return TSDB_CODE_OUT_OF_MEMORY; - } + if (!tsdb->bgTaskDisabled) { + TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { + code = tsdbTFileSetOpenChannel(fset); + TSDB_CHECK_CODE(code, lino, _exit); - arg->tsdb = tsdb; - arg->now = now; - arg->fid = fset->fid; + SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); + if (arg == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } - if (sync) { - code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoRetentionAsync, - tsdbFreeRtnArg, arg, NULL); - } else { - code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoRetentionAsync, - tsdbFreeRtnArg, arg, NULL); - } - if (code) { - tsdbFreeRtnArg(arg); - taosThreadMutexUnlock(&tsdb->mutex); - return code; + arg->tsdb = tsdb; + arg->now = now; + arg->fid = fset->fid; + arg->s3Migrate = s3Migrate; + + if ((code = vnodeAsync(&fset->channel, EVA_PRIORITY_LOW, tsdbRetention, tsdbRetentionCancel, arg, NULL))) { + taosMemoryFree(arg); + TSDB_CHECK_CODE(code, lino, _exit); + } } } - taosThreadMutexUnlock(&tsdb->mutex); +_exit: + if (code) { + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(tsdb->pVnode), __func__, code, lino); + } + return code; +} +int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now) { + int32_t code = 0; + taosThreadMutexLock(&tsdb->mutex); + code = tsdbAsyncRetentionImpl(tsdb, now, false); + taosThreadMutexUnlock(&tsdb->mutex); return code; } @@ -462,7 +448,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int .of = fobj->f[0], }; - code = TARRAY2_APPEND(rtner->fopArr, op); + code = TARRAY2_APPEND(&rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); // create new @@ -486,7 +472,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int }, }; - code = TARRAY2_APPEND(rtner->fopArr, op); + code = TARRAY2_APPEND(&rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); char fname[TSDB_FILENAME_LEN]; @@ -566,7 +552,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64 .of = fobj->f[0], }; - code = TARRAY2_APPEND(rtner->fopArr, op); + code = TARRAY2_APPEND(&rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); // create new @@ -590,7 +576,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64 }, }; - code = TARRAY2_APPEND(rtner->fopArr, op); + code = TARRAY2_APPEND(&rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); char fname[TSDB_FILENAME_LEN]; @@ -653,10 +639,11 @@ _exit: return code; } -static int32_t tsdbDoS3MigrateOnFileSet(SRTNer *rtner, STFileSet *fset) { +static int32_t tsdbDoS3Migrate(SRTNer *rtner) { int32_t code = 0; int32_t lino = 0; + STFileSet *fset = rtner->fset; STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA]; if (!fobj) return code; @@ -720,41 +707,7 @@ _exit: return code; } -static int32_t tsdbDoS3MigrateAsync(void *arg) { - int32_t code = 0; - int32_t lino = 0; - SRTNer rtner[1] = {0}; - - code = tsdbDoRetentionBegin(arg, rtner); - TSDB_CHECK_CODE(code, lino, _exit); - - STFileSet *fset; - TARRAY2_FOREACH(rtner->fsetArr, fset) { - if (fset->fid != ((SRtnArg *)arg)->fid) continue; - - code = tsdbDoS3MigrateOnFileSet(rtner, fset); - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tsdbDoRetentionEnd(rtner); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - if (TARRAY2_DATA(rtner->fopArr)) { - TARRAY2_DESTROY(rtner->fopArr, NULL); - } - TFileSetArray **fsetArr = &rtner->fsetArr; - if (fsetArr[0]) { - tsdbFSDestroyCopySnapshot(&rtner->fsetArr); - } - - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); - } - return code; -} - -int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) { +int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now) { int32_t code = 0; extern int8_t tsS3EnabledCfg; @@ -772,45 +725,7 @@ int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) { } taosThreadMutexLock(&tsdb->mutex); - - if (tsdb->bgTaskDisabled) { - taosThreadMutexUnlock(&tsdb->mutex); - return 0; - } - - STFileSet *fset; - TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { - code = tsdbTFileSetOpenChannel(fset); - if (code) { - taosThreadMutexUnlock(&tsdb->mutex); - return code; - } - - SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); - if (arg == NULL) { - taosThreadMutexUnlock(&tsdb->mutex); - return TSDB_CODE_OUT_OF_MEMORY; - } - - arg->tsdb = tsdb; - arg->now = now; - arg->fid = fset->fid; - - if (sync) { - code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync, - tsdbFreeRtnArg, arg, NULL); - } else { - code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync, - tsdbFreeRtnArg, arg, NULL); - } - if (code) { - tsdbFreeRtnArg(arg); - taosThreadMutexUnlock(&tsdb->mutex); - return code; - } - } - + code = tsdbAsyncRetentionImpl(tsdb, now, true); taosThreadMutexUnlock(&tsdb->mutex); - return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c index 313603b19f..6d6533463b 100644 --- a/source/dnode/vnode/src/vnd/vnodeAsync.c +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -16,6 +16,7 @@ #include "vnd.h" #include "vnodeHash.h" +typedef struct SVAsync SVAsync; typedef struct SVATask SVATask; typedef struct SVAChannel SVAChannel; @@ -54,7 +55,7 @@ struct SVATask { int32_t priorScore; SVAChannel *channel; int32_t (*execute)(void *); - void (*complete)(void *); + void (*cancel)(void *); void *arg; EVATaskState state; @@ -67,6 +68,11 @@ struct SVATask { struct SVATask *next; }; +typedef struct { + void (*cancel)(void *); + void *arg; +} SVATaskCancelInfo; + #define VATASK_PIORITY(task_) ((task_)->priority - ((task_)->priorScore / 4)) // async channel @@ -112,6 +118,10 @@ struct SVAsync { SVHashTable *taskTable; }; +SVAsync *vnodeAsyncs[3]; +#define MIN_ASYNC_ID 1 +#define MAX_ASYNC_ID (sizeof(vnodeAsyncs) / sizeof(vnodeAsyncs[0]) - 1) + static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) { int32_t ret; @@ -160,11 +170,6 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) { } async->numTasks--; - // call complete callback - if (task->complete) { - task->complete(task->arg); - } - if (task->numWait == 0) { taosThreadCondDestroy(&task->waitCond); taosMemoryFree(task); @@ -176,7 +181,7 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) { return 0; } -static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) { +static int32_t vnodeAsyncCancelAllTasks(SVAsync *async, SArray *cancelArray) { while (async->queue[0].next != &async->queue[0] || async->queue[1].next != &async->queue[1] || async->queue[2].next != &async->queue[2]) { for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) { @@ -184,6 +189,12 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) { SVATask *task = async->queue[i].next; task->prev->next = task->next; task->next->prev = task->prev; + if (task->cancel) { + taosArrayPush(cancelArray, &(SVATaskCancelInfo){ + .cancel = task->cancel, + .arg = task->arg, + }); + } vnodeAsyncTaskDone(async, task); } } @@ -194,6 +205,7 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) { static void *vnodeAsyncLoop(void *arg) { SVWorker *worker = (SVWorker *)arg; SVAsync *async = worker->async; + SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo)); setThreadName(async->label); @@ -209,12 +221,12 @@ static void *vnodeAsyncLoop(void *arg) { for (;;) { if (async->stop || worker->workerId >= async->numWorkers) { if (async->stop) { // cancel all tasks - vnodeAsyncCancelAllTasks(async); + vnodeAsyncCancelAllTasks(async, cancelArray); } worker->state = EVA_WORKER_STATE_STOP; async->numLaunchWorkers--; taosThreadMutexUnlock(&async->mutex); - return NULL; + goto _exit; } for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) { @@ -259,6 +271,12 @@ static void *vnodeAsyncLoop(void *arg) { worker->runningTask->execute(worker->runningTask->arg); } +_exit: + for (int32_t i = 0; i < taosArrayGetSize(cancelArray); i++) { + SVATaskCancelInfo *cancel = (SVATaskCancelInfo *)taosArrayGet(cancelArray, i); + cancel->cancel(cancel->arg); + } + taosArrayDestroy(cancelArray); return NULL; } @@ -294,7 +312,7 @@ static int32_t vnodeAsyncChannelCompare(const void *obj1, const void *obj2) { return 0; } -int32_t vnodeAsyncInit(SVAsync **async, char *label) { +static int32_t vnodeAsyncInit(SVAsync **async, const char *label) { int32_t ret; if (async == NULL) { @@ -360,7 +378,7 @@ int32_t vnodeAsyncInit(SVAsync **async, char *label) { return 0; } -int32_t vnodeAsyncDestroy(SVAsync **async) { +static int32_t vnodeAsyncDestroy(SVAsync **async) { if ((*async) == NULL) { return TSDB_CODE_INVALID_PARA; } @@ -433,20 +451,39 @@ static int32_t vnodeAsyncLaunchWorker(SVAsync *async) { return 0; } -#ifdef BUILD_NO_CALL -int32_t vnodeAsync(SVAsync *async, EVAPriority priority, int32_t (*execute)(void *), void (*complete)(void *), - void *arg, int64_t *taskId) { - return vnodeAsyncC(async, 0, priority, execute, complete, arg, taskId); -} -#endif +int32_t vnodeAsyncOpen(int32_t numOfThreads) { + int32_t code = 0; + int32_t lino = 0; -int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int32_t (*execute)(void *), - void (*complete)(void *), void *arg, int64_t *taskId) { - if (async == NULL || execute == NULL || channelId < 0) { + // vnode-commit + code = vnodeAsyncInit(&vnodeAsyncs[1], "vnode-commit"); + TSDB_CHECK_CODE(code, lino, _exit); + vnodeAsyncSetWorkers(1, numOfThreads); + + // vnode-merge + code = vnodeAsyncInit(&vnodeAsyncs[2], "vnode-merge"); + TSDB_CHECK_CODE(code, lino, _exit); + vnodeAsyncSetWorkers(2, numOfThreads); + +_exit: + return 0; +} + +int32_t vnodeAsyncClose() { + vnodeAsyncDestroy(&vnodeAsyncs[1]); + vnodeAsyncDestroy(&vnodeAsyncs[2]); + return 0; +} + +int32_t vnodeAsync(SVAChannelID *channelID, EVAPriority priority, int32_t (*execute)(void *), void (*cancel)(void *), + void *arg, SVATaskID *taskID) { + if (channelID == NULL || channelID->async < MIN_ASYNC_ID || channelID->async > MAX_ASYNC_ID || execute == NULL || + channelID->id < 0) { return TSDB_CODE_INVALID_PARA; } - int64_t id; + int64_t id; + SVAsync *async = vnodeAsyncs[channelID->async]; // create task object SVATask *task = (SVATask *)taosMemoryCalloc(1, sizeof(SVATask)); @@ -457,7 +494,7 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int task->priority = priority; task->priorScore = 0; task->execute = execute; - task->complete = complete; + task->cancel = cancel; task->arg = arg; task->state = EVA_TASK_STATE_WAITTING; task->numWait = 0; @@ -466,10 +503,12 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int // schedule task taosThreadMutexLock(&async->mutex); - if (channelId == 0) { + if (channelID->id == 0) { task->channel = NULL; } else { - SVAChannel channel = {.channelId = channelId}; + SVAChannel channel = { + .channelId = channelID->id, + }; vHashGet(async->channelTable, &channel, (void **)&task->channel); if (task->channel == NULL) { taosThreadMutexUnlock(&async->mutex); @@ -540,20 +579,24 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int taosThreadMutexUnlock(&async->mutex); - if (taskId != NULL) { - *taskId = id; + if (taskID != NULL) { + taskID->async = channelID->async; + taskID->id = id; } return 0; } -int32_t vnodeAWait(SVAsync *async, int64_t taskId) { - if (async == NULL || taskId <= 0) { +int32_t vnodeAWait(SVATaskID *taskID) { + if (taskID == NULL || taskID->async < MIN_ASYNC_ID || taskID->async > MAX_ASYNC_ID || taskID->id <= 0) { return TSDB_CODE_INVALID_PARA; } + SVAsync *async = vnodeAsyncs[taskID->async]; SVATask *task = NULL; - SVATask task2 = {.taskId = taskId}; + SVATask task2 = { + .taskId = taskID->id, + }; taosThreadMutexLock(&async->mutex); @@ -574,21 +617,27 @@ int32_t vnodeAWait(SVAsync *async, int64_t taskId) { return 0; } -int32_t vnodeACancel(SVAsync *async, int64_t taskId) { - if (async == NULL) { +int32_t vnodeACancel(SVATaskID *taskID) { + if (taskID == NULL || taskID->async < MIN_ASYNC_ID || taskID->async > MAX_ASYNC_ID || taskID->id <= 0) { return TSDB_CODE_INVALID_PARA; } int32_t ret = 0; + SVAsync *async = vnodeAsyncs[taskID->async]; SVATask *task = NULL; - SVATask task2 = {.taskId = taskId}; + SVATask task2 = { + .taskId = taskID->id, + }; + void (*cancel)(void *) = NULL; + void *arg = NULL; taosThreadMutexLock(&async->mutex); vHashGet(async->taskTable, &task2, (void **)&task); if (task) { if (task->state == EVA_TASK_STATE_WAITTING) { - // remove from queue + cancel = task->cancel; + arg = task->arg; task->next->prev = task->prev; task->prev->next = task->next; vnodeAsyncTaskDone(async, task); @@ -599,14 +648,18 @@ int32_t vnodeACancel(SVAsync *async, int64_t taskId) { taosThreadMutexUnlock(&async->mutex); + if (cancel) { + cancel(arg); + } + return ret; } -int32_t vnodeAsyncSetWorkers(SVAsync *async, int32_t numWorkers) { - if (async == NULL || numWorkers <= 0 || numWorkers > VNODE_ASYNC_MAX_WORKERS) { +int32_t vnodeAsyncSetWorkers(int64_t asyncID, int32_t numWorkers) { + if (asyncID < MIN_ASYNC_ID || asyncID > MAX_ASYNC_ID || numWorkers <= 0 || numWorkers > VNODE_ASYNC_MAX_WORKERS) { return TSDB_CODE_INVALID_PARA; } - + SVAsync *async = vnodeAsyncs[asyncID]; taosThreadMutexLock(&async->mutex); async->numWorkers = numWorkers; if (async->numIdleWorkers > 0) { @@ -617,11 +670,13 @@ int32_t vnodeAsyncSetWorkers(SVAsync *async, int32_t numWorkers) { return 0; } -int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) { - if (async == NULL || channelId == NULL) { +int32_t vnodeAChannelInit(int64_t asyncID, SVAChannelID *channelID) { + if (channelID == NULL || asyncID < MIN_ASYNC_ID || asyncID > MAX_ASYNC_ID) { return TSDB_CODE_INVALID_PARA; } + SVAsync *async = vnodeAsyncs[asyncID]; + // create channel object SVAChannel *channel = (SVAChannel *)taosMemoryMalloc(sizeof(SVAChannel)); if (channel == NULL) { @@ -637,7 +692,7 @@ int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) { // register channel taosThreadMutexLock(&async->mutex); - channel->channelId = *channelId = ++async->nextChannelId; + channel->channelId = channelID->id = ++async->nextChannelId; // add to hash table int32_t ret = vHashPut(async->channelTable, channel); @@ -657,16 +712,24 @@ int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) { taosThreadMutexUnlock(&async->mutex); + channelID->async = asyncID; return 0; } -int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning) { - if (async == NULL || channelId <= 0) { +int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) { + if (channelID == NULL || channelID->async < MIN_ASYNC_ID || channelID->async > MAX_ASYNC_ID || channelID->id <= 0) { return TSDB_CODE_INVALID_PARA; } + SVAsync *async = vnodeAsyncs[channelID->async]; SVAChannel *channel = NULL; - SVAChannel channel2 = {.channelId = channelId}; + SVAChannel channel2 = { + .channelId = channelID->id, + }; + SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo)); + if (cancelArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } taosThreadMutexLock(&async->mutex); @@ -684,6 +747,12 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning SVATask *task = channel->queue[i].next; task->prev->next = task->next; task->next->prev = task->prev; + if (task->cancel) { + taosArrayPush(cancelArray, &(SVATaskCancelInfo){ + .cancel = task->cancel, + .arg = task->arg, + }); + } vnodeAsyncTaskDone(async, task); } } @@ -693,6 +762,12 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning if (channel->scheduled) { channel->scheduled->prev->next = channel->scheduled->next; channel->scheduled->next->prev = channel->scheduled->prev; + if (channel->scheduled->cancel) { + taosArrayPush(cancelArray, &(SVATaskCancelInfo){ + .cancel = channel->scheduled->cancel, + .arg = channel->scheduled->arg, + }); + } vnodeAsyncTaskDone(async, channel->scheduled); } taosMemoryFree(channel); @@ -713,12 +788,16 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning channel->state = EVA_CHANNEL_STATE_CLOSE; } } - } else { - taosThreadMutexUnlock(&async->mutex); - return TSDB_CODE_INVALID_PARA; } taosThreadMutexUnlock(&async->mutex); + for (int32_t i = 0; i < taosArrayGetSize(cancelArray); i++) { + SVATaskCancelInfo *cancel = (SVATaskCancelInfo *)taosArrayGet(cancelArray, i); + cancel->cancel(cancel->arg); + } + taosArrayDestroy(cancelArray); + channelID->async = 0; + channelID->id = 0; return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 645f2620dc..f071775990 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -289,7 +289,7 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { int64_t lastCommitted = pInfo->info.state.committed; // wait last commit task - vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask); + vnodeAWait(&pVnode->commitTask); if (syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg) != 0) goto _exit; @@ -361,15 +361,14 @@ static void vnodeReturnBufPool(SVnode *pVnode) { taosThreadMutexUnlock(&pVnode->mutex); } -static int32_t vnodeCommitTask(void *arg) { +static int32_t vnodeCommit(void *arg) { int32_t code = 0; SCommitInfo *pInfo = (SCommitInfo *)arg; SVnode *pVnode = pInfo->pVnode; // commit - code = vnodeCommitImpl(pInfo); - if (code) { + if ((code = vnodeCommitImpl(pInfo))) { vFatal("vgId:%d, failed to commit vnode since %s", TD_VID(pVnode), terrstr()); taosMsleep(100); exit(EXIT_FAILURE); @@ -379,37 +378,34 @@ static int32_t vnodeCommitTask(void *arg) { vnodeReturnBufPool(pVnode); _exit: + taosMemoryFree(arg); return code; } -static void vnodeCompleteCommit(void *arg) { taosMemoryFree(arg); } +static void vnodeCommitCancel(void *arg) { taosMemoryFree(arg); } int vnodeAsyncCommit(SVnode *pVnode) { int32_t code = 0; + int32_t lino = 0; SCommitInfo *pInfo = (SCommitInfo *)taosMemoryCalloc(1, sizeof(*pInfo)); if (NULL == pInfo) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } // prepare to commit code = vnodePrepareCommit(pVnode, pInfo); - if (TSDB_CODE_SUCCESS != code) { - goto _exit; - } + TSDB_CHECK_CODE(code, lino, _exit); // schedule the task - code = vnodeAsyncC(vnodeAsyncHandle[0], pVnode->commitChannel, EVA_PRIORITY_HIGH, vnodeCommitTask, - vnodeCompleteCommit, pInfo, &pVnode->commitTask); + code = + vnodeAsync(&pVnode->commitChannel, EVA_PRIORITY_HIGH, vnodeCommit, vnodeCommitCancel, pInfo, &pVnode->commitTask); + TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - if (NULL != pInfo) { - taosMemoryFree(pInfo); - } - vError("vgId:%d, %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), - pVnode->state.commitID); + taosMemoryFree(pInfo); + vError("vgId:%d %s failed at line %d since %s" PRId64, TD_VID(pVnode), __func__, lino, tstrerror(code)); } else { vInfo("vgId:%d, vnode async commit done, commitId:%" PRId64 " term:%" PRId64 " applied:%" PRId64, TD_VID(pVnode), pVnode->state.commitID, pVnode->state.applyTerm, pVnode->state.applied); @@ -419,7 +415,7 @@ _exit: int vnodeSyncCommit(SVnode *pVnode) { vnodeAsyncCommit(pVnode); - vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask); + vnodeAWait(&pVnode->commitTask); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeHash.h b/source/dnode/vnode/src/vnd/vnodeHash.h index 0181ca748d..00b3488930 100644 --- a/source/dnode/vnode/src/vnd/vnodeHash.h +++ b/source/dnode/vnode/src/vnd/vnodeHash.h @@ -22,23 +22,6 @@ extern "C" { #endif -typedef struct SVHashTable SVHashTable; - -struct SVHashTable { - uint32_t (*hash)(const void*); - int32_t (*compare)(const void*, const void*); - int32_t numEntries; - uint32_t numBuckets; - struct SVHashEntry** buckets; -}; - -#define vHashNumEntries(ht) ((ht)->numEntries) -int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*)); -int32_t vHashDestroy(SVHashTable** ht); -int32_t vHashPut(SVHashTable* ht, void* obj); -int32_t vHashGet(SVHashTable* ht, const void* obj, void** retObj); -int32_t vHashDrop(SVHashTable* ht, const void* obj); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 44fcbefba7..3a454c53ef 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -18,8 +18,6 @@ static volatile int32_t VINIT = 0; -SVAsync* vnodeAsyncHandle[2]; - int vnodeInit(int nthreads) { int32_t init; @@ -28,13 +26,9 @@ int vnodeInit(int nthreads) { return 0; } - // vnode-commit - vnodeAsyncInit(&vnodeAsyncHandle[0], "vnode-commit"); - vnodeAsyncSetWorkers(vnodeAsyncHandle[0], nthreads); - - // vnode-merge - vnodeAsyncInit(&vnodeAsyncHandle[1], "vnode-merge"); - vnodeAsyncSetWorkers(vnodeAsyncHandle[1], nthreads); + if (vnodeAsyncOpen(nthreads) != 0) { + return -1; + } if (walInit() < 0) { return -1; @@ -48,8 +42,7 @@ void vnodeCleanup() { if (init == 0) return; // set stop - vnodeAsyncDestroy(&vnodeAsyncHandle[0]); - vnodeAsyncDestroy(&vnodeAsyncHandle[1]); + vnodeAsyncClose(); walCleanUp(); smaCleanUp(); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 425cff7ce9..da8c3a6cad 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -399,7 +399,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC taosThreadMutexInit(&pVnode->mutex, NULL); taosThreadCondInit(&pVnode->poolNotEmpty, NULL); - if (vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel) != 0) { + if (vnodeAChannelInit(1, &pVnode->commitChannel) != 0) { vError("vgId:%d, failed to init commit channel", TD_VID(pVnode)); goto _err; } @@ -527,8 +527,8 @@ void vnodePostClose(SVnode *pVnode) { vnodeSyncPostClose(pVnode); } void vnodeClose(SVnode *pVnode) { if (pVnode) { - vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask); - vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true); + vnodeAWait(&pVnode->commitTask); + vnodeAChannelDestroy(&pVnode->commitChannel, true); vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); tqClose(pVnode->pTq); diff --git a/source/dnode/vnode/src/vnd/vnodeRetention.c b/source/dnode/vnode/src/vnd/vnodeRetention.c index 5db20b8fc7..6dca7a9a60 100644 --- a/source/dnode/vnode/src/vnd/vnodeRetention.c +++ b/source/dnode/vnode/src/vnd/vnodeRetention.c @@ -15,20 +15,15 @@ #include "vnd.h" -int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) { - int32_t code = TSDB_CODE_SUCCESS; +extern int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now); +extern int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now); - code = tsdbRetention(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1); - - if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now); - - return code; +int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now) { + // async retention + return tsdbAsyncRetention(pVnode->pTsdb, now); } -int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now) { - int32_t code = TSDB_CODE_SUCCESS; - - code = tsdbS3Migrate(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1); - - return code; +int32_t vnodeAsyncS3Migrate(SVnode *pVnode, int64_t now) { + // async migration + return tsdbAsyncS3Migrate(pVnode->pTsdb, now); } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index ea742108aa..4bb5fbc822 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -622,13 +622,13 @@ extern int32_t tsdbEnableBgTask(STsdb *pTsdb); static int32_t vnodeCancelAndDisableAllBgTask(SVnode *pVnode) { tsdbDisableAndCancelAllBgTask(pVnode->pTsdb); vnodeSyncCommit(pVnode); - vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true); + vnodeAChannelDestroy(&pVnode->commitChannel, true); return 0; } static int32_t vnodeEnableBgTask(SVnode *pVnode) { tsdbEnableBgTask(pVnode->pTsdb); - vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel); + vnodeAChannelInit(1, &pVnode->commitChannel); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index cb71495e9f..cbac6a13de 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -50,7 +50,7 @@ static int32_t vnodeProcessArbCheckSyncReq(SVnode *pVnode, void *pReq, int32_t l static int32_t vnodePreCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token); static int32_t vnodeCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token); -static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp); +static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); extern int32_t vnodeProcessKillCompactReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); extern int32_t vnodeQueryCompactProgress(SVnode *pVnode, SRpcMsg *pMsg); @@ -866,7 +866,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { pMetaRsp->precision = pVnode->config.tsdbCfg.precision; } -extern int32_t vnodeDoRetention(SVnode *pVnode, int64_t now); +extern int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; @@ -880,13 +880,13 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int3 vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp); - code = vnodeDoRetention(pVnode, trimReq.timestamp); + code = vnodeAsyncRetention(pVnode, trimReq.timestamp); _exit: return code; } -extern int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now); +extern int32_t vnodeAsyncS3Migrate(SVnode *pVnode, int64_t now); static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; @@ -900,7 +900,7 @@ static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, vInfo("vgId:%d, s3migrate vnode request will be processed, time:%d", pVnode->config.vgId, s3migrateReq.timestamp); - code = vnodeDoS3Migrate(pVnode, s3migrateReq.timestamp); + code = vnodeAsyncS3Migrate(pVnode, s3migrateReq.timestamp); _exit: return code; @@ -931,13 +931,13 @@ end: return ret; } -static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp) { +static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = -1; SMetaReader mr = {0}; SVDropTtlTableReq ttlReq = {0}; SVFetchTtlExpiredTbsRsp rsp = {0}; SEncoder encoder = {0}; - SArray* pNames = NULL; + SArray *pNames = NULL; pRsp->msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP; pRsp->code = TSDB_CODE_SUCCESS; pRsp->pCont = NULL; @@ -950,8 +950,8 @@ static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* ASSERT(ttlReq.nUids == taosArrayGetSize(ttlReq.pTbUids)); - tb_uid_t suid; - char ctbName[TSDB_TABLE_NAME_LEN]; + tb_uid_t suid; + char ctbName[TSDB_TABLE_NAME_LEN]; SVDropTbReq expiredTb = {.igNotExists = true}; metaReaderDoInit(&mr, pVnode->pMeta, 0); rsp.vgId = TD_VID(pVnode); @@ -965,12 +965,12 @@ static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* } char buf[TSDB_TABLE_NAME_LEN]; for (int32_t i = 0; i < ttlReq.nUids; ++i) { - tb_uid_t* uid = taosArrayGet(ttlReq.pTbUids, i); + tb_uid_t *uid = taosArrayGet(ttlReq.pTbUids, i); expiredTb.suid = *uid; terrno = metaReaderGetTableEntryByUid(&mr, *uid); if (terrno < 0) goto _end; strncpy(buf, mr.me.name, TSDB_TABLE_NAME_LEN); - void* p = taosArrayPush(pNames, buf); + void *p = taosArrayPush(pNames, buf); expiredTb.name = p; if (mr.me.type == TSDB_CHILD_TABLE) { expiredTb.suid = mr.me.ctbEntry.suid; @@ -1144,7 +1144,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, if (i < tbNames->size - 1) { taosStringBuilderAppendChar(&sb, ','); } - //taosMemoryFreeClear(*key); + // taosMemoryFreeClear(*key); } size_t len = 0; @@ -2241,14 +2241,14 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, return TSDB_CODE_SUCCESS; } -extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +extern int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { if (!pVnode->restored) { vInfo("vgId:%d, ignore compact req during restoring. ver:%" PRId64, TD_VID(pVnode), ver); return 0; } - return vnodeProcessCompactVnodeReqImpl(pVnode, ver, pReq, len, pRsp); + return vnodeAsyncCompact(pVnode, ver, pReq, len, pRsp); } static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { @@ -2355,8 +2355,6 @@ _OVER: } #ifndef TD_ENTERPRISE -int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { - return 0; -} +int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; } int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync) { return 0; } #endif diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 9a84127be6..9f421d6de3 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -511,7 +511,6 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB } pInfo->isPrevRowSet = false; - return TSDB_CODE_SUCCESS; } @@ -825,8 +824,12 @@ static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperato SSDataBlock* pResBlock = pSliceInfo->pRes; SInterval* pInterval = &pSliceInfo->interval; - while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && - pSliceInfo->fillType != TSDB_FILL_LINEAR) { + if (pSliceInfo->fillType == TSDB_FILL_NEXT || pSliceInfo->fillType == TSDB_FILL_LINEAR || + pSliceInfo->pPrevGroupKey == NULL) { + return; + } + + while (pSliceInfo->current <= pSliceInfo->win.ekey) { genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, index, false); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -1068,6 +1071,8 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); +// int32_t code = initKeeperInfo(pSliceInfo, pBlock, &pOperator->exprSupp); + code = appendDownstream(pOperator, &downstream, 1); return pOperator; diff --git a/source/libs/monitorfw/inc/taos_linked_list_i.h b/source/libs/monitorfw/inc/taos_linked_list_i.h index ed11a76427..015f8a57ad 100644 --- a/source/libs/monitorfw/inc/taos_linked_list_i.h +++ b/source/libs/monitorfw/inc/taos_linked_list_i.h @@ -47,7 +47,7 @@ int taos_linked_list_push(taos_linked_list_t *self, void *item); /** * @brief API PRIVATE Pop the first item off of the list */ -void *taos_linked_list_pop(taos_linked_list_t *self); +//void *taos_linked_list_pop(taos_linked_list_t *self); /** * @brief API PRIVATE Returns the item at the head of the list or NULL if not present diff --git a/source/libs/monitorfw/src/taos_collector.c b/source/libs/monitorfw/src/taos_collector.c index 17d324462c..414c02121d 100644 --- a/source/libs/monitorfw/src/taos_collector.c +++ b/source/libs/monitorfw/src/taos_collector.c @@ -33,6 +33,8 @@ taos_map_t *taos_collector_default_collect(taos_collector_t *self) { return self taos_collector_t *taos_collector_new(const char *name) { int r = 0; taos_collector_t *self = (taos_collector_t *)taos_malloc(sizeof(taos_collector_t)); + if (self == NULL) return NULL; + memset(self, 0, sizeof(taos_collector_t)); self->name = taos_strdup(name); self->metrics = taos_map_new(); if (self->metrics == NULL) { @@ -66,9 +68,11 @@ int taos_collector_destroy(taos_collector_t *self) { if (r) ret = r; self->metrics = NULL; - r = taos_string_builder_destroy(self->string_builder); - if (r) ret = r; - self->string_builder = NULL; + if(self->string_builder != NULL){ + r = taos_string_builder_destroy(self->string_builder); + if (r) ret = r; + self->string_builder = NULL; + } taos_free((char *)self->name); self->name = NULL; diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index c3ed0112c5..d4838ef301 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -59,6 +59,7 @@ taos_collector_registry_t *taos_collector_registry_new(const char *name) { r = pthread_rwlock_init(self->lock, NULL); if (r) { TAOS_LOG("failed to initialize rwlock"); + taos_free(self); return NULL; } return self; @@ -301,6 +302,9 @@ const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, _OVER: tjsonDelete(pJson); + if(tmp_builder != NULL){ + taos_string_builder_destroy(tmp_builder); + } return NULL; } diff --git a/source/libs/monitorfw/src/taos_linked_list.c b/source/libs/monitorfw/src/taos_linked_list.c index 675400a6fa..2becb08a07 100644 --- a/source/libs/monitorfw/src/taos_linked_list.c +++ b/source/libs/monitorfw/src/taos_linked_list.c @@ -117,6 +117,7 @@ int taos_linked_list_push(taos_linked_list_t *self, void *item) { return 0; } +/* void *taos_linked_list_pop(taos_linked_list_t *self) { TAOS_ASSERT(self != NULL); if (self == NULL) return NULL; @@ -141,6 +142,7 @@ void *taos_linked_list_pop(taos_linked_list_t *self) { } return item; } +*/ int taos_linked_list_remove(taos_linked_list_t *self, void *item) { TAOS_ASSERT(self != NULL); diff --git a/source/libs/monitorfw/src/taos_map.c b/source/libs/monitorfw/src/taos_map.c index 8f0b83884e..ffb7d000fc 100644 --- a/source/libs/monitorfw/src/taos_map.c +++ b/source/libs/monitorfw/src/taos_map.c @@ -90,7 +90,7 @@ taos_map_t *taos_map_new() { return NULL; } - self->addrs = taos_malloc(sizeof(taos_linked_list_t) * self->max_size); + self->addrs = taos_malloc(sizeof(taos_linked_list_t*) * self->max_size); self->free_value_fn = destroy_map_node_value_no_op; for (int i = 0; i < self->max_size; i++) { @@ -273,7 +273,7 @@ int taos_map_ensure_space(taos_map_t *self) { if (r) return r; // Create a new array of addrs - taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t) * new_max); + taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t*) * new_max); // Initialize the new array for (int i = 0; i < new_max; i++) { diff --git a/source/libs/monitorfw/src/taos_metric.c b/source/libs/monitorfw/src/taos_metric.c index 4e9af35f34..5cecbc927f 100644 --- a/source/libs/monitorfw/src/taos_metric.c +++ b/source/libs/monitorfw/src/taos_metric.c @@ -33,6 +33,8 @@ taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name, size_t label_key_count, const char **label_keys) { int r = 0; taos_metric_t *self = (taos_metric_t *)taos_malloc(sizeof(taos_metric_t)); + if (self == NULL) return NULL; + memset(self, 0, sizeof(taos_metric_t)); self->type = metric_type; int len = strlen(name) + 1; self->name = taos_malloc(len); @@ -79,6 +81,7 @@ taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name, r = pthread_rwlock_init(self->rwlock, NULL); if (r) { TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR); + taos_free(self); return NULL; } return self; @@ -91,9 +94,11 @@ int taos_metric_destroy(taos_metric_t *self) { int r = 0; int ret = 0; - r = taos_map_destroy(self->samples); - self->samples = NULL; - if (r) ret = r; + if(self->samples != NULL){ + r = taos_map_destroy(self->samples); + self->samples = NULL; + if (r) ret = r; + } r = taos_metric_formatter_destroy(self->formatter); self->formatter = NULL; @@ -152,8 +157,7 @@ taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const return NULL; // Get l_value - r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys, - label_values); + r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys, label_values); if (r) { TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); } @@ -170,10 +174,12 @@ taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const sample = taos_metric_sample_new(self->type, l_value, 0.0); r = taos_map_set(self->samples, l_value, sample); if (r) { + taos_free((void *)l_value); TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); } } - pthread_rwlock_unlock(self->rwlock); + r = pthread_rwlock_unlock(self->rwlock); + if (r) TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); taos_free((void *)l_value); return sample; } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 1a51620856..84d3f734fe 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -845,6 +845,7 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) { COPY_SCALAR_FIELD(precision); COPY_SCALAR_FIELD(isEmptyResult); COPY_SCALAR_FIELD(timeLineResMode); + COPY_SCALAR_FIELD(timeLineFromOrderBy); COPY_SCALAR_FIELD(timeLineCurMode); COPY_SCALAR_FIELD(hasAggFuncs); COPY_SCALAR_FIELD(hasRepeatScanFuncs); @@ -862,6 +863,8 @@ static int32_t setOperatorCopy(const SSetOperator* pSrc, SSetOperator* pDst) { COPY_CHAR_ARRAY_FIELD(stmtName); COPY_SCALAR_FIELD(precision); COPY_SCALAR_FIELD(timeLineResMode); + COPY_SCALAR_FIELD(timeLineFromOrderBy); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 003255a728..6795fefc14 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -959,7 +959,8 @@ static bool isTimeLineQuery(SNode* pStmt) { return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineCurMode) || (TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineCurMode); } else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) { - return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode; + return (TIME_LINE_MULTI == ((SSetOperator*)pStmt)->timeLineResMode) || + (TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode); } else { return false; } @@ -1000,18 +1001,64 @@ static bool isBlockTimeLineAlignedQuery(SNode* pStmt) { return false; } +SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesNum) { + SNodeList* pPartitionList = NULL; + SNode* pNode = NULL; + if (pOrderList->length <= nodesNum) { + return NULL; + } + + pNode = nodesListGetNode(pOrderList, nodesNum); + SOrderByExprNode* pOrder = (SOrderByExprNode*)pNode; + if (!isPrimaryKeyImpl(pOrder->pExpr)) { + return NULL; + } + + for (int32_t i = 0; i < nodesNum; ++i) { + pNode = nodesListGetNode(pOrderList, i); + pOrder = (SOrderByExprNode*)pNode; + nodesListMakeStrictAppend(&pPartitionList, nodesCloneNode(pOrder->pExpr)); + } + + return pPartitionList; +} + + static bool isTimeLineAlignedQuery(SNode* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt; if (!isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { return false; } - if (QUERY_NODE_SELECT_STMT != nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { - return false; + if (QUERY_NODE_SELECT_STMT == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery; + if (pSelect->pPartitionByList) { + if (!pSub->timeLineFromOrderBy && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) { + return true; + } + if (pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) { + SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length); + bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList); + nodesDestroyList(pPartitionList); + + if (match) { + return true; + } + } + } } - SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery; - if (pSelect->pPartitionByList && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) { - return true; + if (QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + SSetOperator* pSub = (SSetOperator*)((STempTableNode*)pSelect->pFromTable)->pSubquery; + if (pSelect->pPartitionByList && pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) { + SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length); + bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList); + nodesDestroyList(pPartitionList); + + if (match) { + return true; + } + } } + return false; } @@ -6025,9 +6072,19 @@ static void resetResultTimeline(SSelectStmt* pSelect) { if ((QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder)) || (QUERY_NODE_TEMP_TABLE != nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder))) { pSelect->timeLineResMode = TIME_LINE_GLOBAL; - } else { - pSelect->timeLineResMode = TIME_LINE_NONE; + return; + } else if (pSelect->pOrderByList->length > 1) { + for (int32_t i = 1; i < pSelect->pOrderByList->length; ++i) { + pOrder = ((SOrderByExprNode*)nodesListGetNode(pSelect->pOrderByList, i))->pExpr; + if (isPrimaryKeyImpl(pOrder)) { + pSelect->timeLineResMode = TIME_LINE_MULTI; + pSelect->timeLineFromOrderBy = true; + return; + } + } } + + pSelect->timeLineResMode = TIME_LINE_NONE; } static int32_t replaceOrderByAliasForSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { @@ -6180,16 +6237,13 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS } snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName); SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft); - if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight)) { - SColumnNode* pLCol = (SColumnNode*)pLeft; - SColumnNode* pRCol = (SColumnNode*)pRight; + bool isLeftPrimTs = isPrimaryKeyImpl(pLeft); + bool isRightPrimTs = isPrimaryKeyImpl(pRight); + + if (isLeftPrimTs && isRightPrimTs) { SColumnNode* pFCol = (SColumnNode*)pProj; - if (pLCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID && pRCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pFCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - if (pLCol->isPrimTs && pRCol->isPrimTs) { - pFCol->isPrimTs = true; - } - } + pFCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + pFCol->isPrimTs = true; } if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) { return TSDB_CODE_OUT_OF_MEMORY; @@ -6225,9 +6279,19 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS SNode* pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, 0))->pExpr; if (isPrimaryKeyImpl(pOrder)) { pSetOperator->timeLineResMode = TIME_LINE_GLOBAL; - } else { - pSetOperator->timeLineResMode = TIME_LINE_NONE; + return code; + } else if (pSetOperator->pOrderByList->length > 1) { + for (int32_t i = 1; i < pSetOperator->pOrderByList->length; ++i) { + pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, i))->pExpr; + if (isPrimaryKeyImpl(pOrder)) { + pSetOperator->timeLineResMode = TIME_LINE_MULTI; + pSetOperator->timeLineFromOrderBy = true; + return code; + } + } } + + pSetOperator->timeLineResMode = TIME_LINE_NONE; } return code; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 23b8baf031..acc29997a2 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1587,6 +1587,7 @@ static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator TSWAP(pProject->node.pLimit, pSetOperator->pLimit); } pProject->ignoreGroupId = true; + pProject->isSetOpProj = true; int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index a2a0343316..7d10c02529 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3284,18 +3284,34 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* SNodeList* pNewChildTargets = nodesMakeList(); if (NULL == pProjectNode->node.pParent) { - SNode* pProjection = NULL; - FOREACH(pProjection, pProjectNode->pProjections) { - SNode* pChildTarget = NULL; - FOREACH(pChildTarget, pChild->pTargets) { - if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) { - nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget)); + SNode *pProjection = NULL, *pChildTarget = NULL; + bool needOrderMatch = QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChild) && ((SProjectLogicNode*)pChild)->isSetOpProj; + bool orderMatch = true; + if (needOrderMatch) { + // For sql: select ... from (select ... union all select ...); + // When eliminating the outer proj (the outer select), we have to make sure that the outer proj projections and + // union all project targets have same columns in the same order. See detail in TD-30188 + FORBOTH(pProjection, pProjectNode->pProjections, pChildTarget, pChild->pTargets) { + if (!pProjection) break; + if (0 != strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) { + orderMatch = false; break; } + nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget)); + } + } else { + FOREACH(pProjection, pProjectNode->pProjections) { + FOREACH(pChildTarget, pChild->pTargets) { + if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) { + nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget)); + break; + } + } } } - if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets)) { + if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets) && + (!needOrderMatch || (needOrderMatch && orderMatch))) { nodesDestroyList(pChild->pTargets); pChild->pTargets = pNewChildTargets; } else { diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index faa90dcbf8..11e9350e14 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -370,12 +370,12 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) { QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask); tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + QW_ERR_RET(TSDB_CODE_GRANT_VIEW_EXPIRED); } if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) { QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask); tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + QW_ERR_RET(TSDB_CODE_GRANT_AUDIT_EXPIRED); } } } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index c52a8599a0..6d2215264e 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -798,6 +798,7 @@ TEST(queryTest, normalCase) { schedulerFreeJob(&job, 0); + taosThreadJoin(thread1, NULL); } TEST(queryTest, readyFirstCase) { @@ -907,6 +908,8 @@ TEST(queryTest, readyFirstCase) { schedulerDestroy(); schedulerFreeJob(&job, 0); + + taosThreadJoin(thread1, NULL); } TEST(queryTest, flowCtrlCase) { @@ -1001,6 +1004,8 @@ TEST(queryTest, flowCtrlCase) { schedulerDestroy(); schedulerFreeJob(&job, 0); + + taosThreadJoin(thread1, NULL); } TEST(insertTest, normalCase) { @@ -1061,6 +1066,8 @@ TEST(insertTest, normalCase) { schedulerFreeJob(&insertJobRefId, 0); schedulerDestroy(); + + taosThreadJoin(thread1, NULL); } TEST(multiThread, forceFree) { diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 2e65c9ef16..b157597e60 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1155,10 +1155,13 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { taskDbAddRef(pTaskDb); int64_t chkpId = pTaskDb->chkpId; + taskDbRefChkp(pTaskDb, chkpId); code = taskDbDoCheckpoint(pTaskDb, chkpId); - taskDbRemoveRef(pTaskDb); + if (code != 0) { + taskDbUnRefChkp(pTaskDb, chkpId); + } - taskDbRefChkp(pTaskDb, pTaskDb->chkpId); + taskDbRemoveRef(pTaskDb); SStreamTask* pTask = pTaskDb->pTask; SStreamTaskSnap snap = {.streamId = pTask->id.streamId, @@ -1182,14 +1185,15 @@ int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) { for (int i = 0; i < taosArrayGetSize(pSnapInfo); i++) { SStreamTaskSnap* pSnap = taosArrayGet(pSnapInfo, i); sprintf(buf, "0x%" PRIx64 "-0x%x", pSnap->streamId, (int32_t)pSnap->taskId); - STaskDbWrapper* pTaskDb = taosHashGet(pMeta->pTaskDbUnique, buf, strlen(buf)); - if (pTaskDb == NULL) { + STaskDbWrapper** pTaskDb = taosHashGet(pMeta->pTaskDbUnique, buf, strlen(buf)); + if (pTaskDb == NULL || *pTaskDb == NULL) { stWarn("stream backend:%p failed to find task db, streamId:% " PRId64 "", pMeta, pSnap->streamId); + memset(buf, 0, sizeof(buf)); continue; } memset(buf, 0, sizeof(buf)); - taskDbUnRefChkp(pTaskDb, pSnap->chkpId); + taskDbUnRefChkp(*pTaskDb, pSnap->chkpId); } taosThreadMutexUnlock(&pMeta->backendMutex); return 0; @@ -1989,7 +1993,8 @@ void taskDbRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) { void taskDbUnRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) { taosThreadRwlockWrlock(&pTaskDb->chkpDirLock); - for (int i = 0; i < taosArrayGetSize(pTaskDb->chkpInUse); i++) { + int32_t size = taosArrayGetSize(pTaskDb->chkpInUse); + for (int i = 0; i < size; i++) { int64_t* p = taosArrayGet(pTaskDb->chkpInUse, i); if (*p == chkp) { taosArrayRemove(pTaskDb->chkpInUse, i); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 047b169ec9..934ff898a9 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -184,16 +184,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i return code; } -static int32_t handleResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) { +static int32_t handleSanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) { int32_t code = TSDB_CODE_SUCCESS; if (taosArrayGetSize(pRes) > 0) { SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(NULL, pTask, size, pRes); code = doOutputResultBlockImpl(pTask, pStreamBlocks); - if (code != TSDB_CODE_SUCCESS) { - stDebug("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code)); + if (code != TSDB_CODE_SUCCESS) { // should not have error code + stError("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code)); } } else { - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); } return code; } @@ -268,6 +268,17 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0); } + // output queue is full, idle for 5 sec. + if (streamQueueIsFull(pTask->outputq.queue)) { + stWarn("s-task:%s outputQ is full, idle for 1sec and retry", id); + return buildScanhistoryExecRet(TASK_SCANHISTORY_REXEC, STREAM_SCAN_HISTORY_TIMESLICE); + } + + if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { + stWarn("s-task:%s downstream task inputQ blocked, idle for 5sec and retry", id); + return buildScanhistoryExecRet(TASK_SCANHISTORY_REXEC, FILL_HISTORY_TASK_EXEC_INTERVAL); + } + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -284,19 +295,13 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { } // dispatch the generated results - /*int32_t code = */handleResultBlocks(pTask, pRes, size); - - int64_t el = taosGetTimestampMs() - st; - - // downstream task input queue is full, try in 5sec - if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED && (pTask->info.fillHistory == 1)) { - return buildScanhistoryExecRet(TASK_SCANHISTORY_REXEC, FILL_HISTORY_TASK_EXEC_INTERVAL); - } + /*int32_t code = */handleSanhistoryResultBlocks(pTask, pRes, size); if (finished) { return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0); } + int64_t el = taosGetTimestampMs() - st; if (el >= STREAM_SCAN_HISTORY_TIMESLICE && (pTask->info.fillHistory == 1)) { stDebug("s-task:%s fill-history:%d time slice exhausted, elapsed time:%.2fs, retry in 100ms", id, pTask->info.fillHistory, el / 1000.0); @@ -558,7 +563,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { if (streamQueueIsFull(pTask->outputq.queue)) { stWarn("s-task:%s outputQ is full, idle for 500ms and retry", id); - streamTaskSetIdleInfo(pTask, 500); + streamTaskSetIdleInfo(pTask, 1000); return 0; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 0aace1cb5b..59d49a8231 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -562,13 +562,16 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { tEncoderClear(&encoder); int64_t id[2] = {pTask->id.streamId, pTask->id.taskId}; - if (tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn) < 0) { - stError("s-task:%s save to disk failed, code:%s", pTask->id.idStr, tstrerror(terrno)); - return -1; + + code = tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s task meta save to disk failed, code:%s", pTask->id.idStr, tstrerror(terrno)); + } else { + stDebug("s-task:%s task meta save to disk", pTask->id.idStr); } taosMemoryFree(buf); - return 0; + return code; } int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pTaskId) { diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 5596eb3dee..247baea16f 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -381,7 +381,7 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc } } - return TSDB_CODE_SUCCESS; + return code; } int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index eb67e61c4c..25015c4d33 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -144,7 +144,7 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { int32_t streamCreateTaskDbSnapInfo(void* arg, char* path, SArray* pSnap) { return taskDbBuildSnap(arg, pSnap); } -int32_t streamDestroyTasdDbSnapInfo(void* arg, SArray* snap) { return taskDbDestroySnap(arg, snap); } +int32_t streamDestroyTaskDbSnapInfo(void* arg, SArray* snap) { return taskDbDestroySnap(arg, snap); } void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { if (qDebugFlag & DEBUG_DEBUG) { @@ -333,7 +333,7 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) { } taosArrayDestroy(handle->pDbSnapSet); } - streamDestroyTasdDbSnapInfo(handle->pMeta, handle->pSnapInfoSet); + streamDestroyTaskDbSnapInfo(handle->pMeta, handle->pSnapInfoSet); if (handle->pSnapInfoSet) { for (int32_t i = 0; i < taosArrayGetSize(handle->pSnapInfoSet); i++) { SStreamTaskSnap* pSnap = taosArrayGet(handle->pSnapInfoSet, i); diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 83de642e51..fb745f86cb 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -759,7 +759,7 @@ void* getRowStateBuff(SStreamFileState* pFileState) { return pFileState->rowStat void* getStateFileStore(SStreamFileState* pFileState) { return pFileState->pFileStore; } bool isDeteled(SStreamFileState* pFileState, TSKEY ts) { - return pFileState->deleteMark > 0 && ts < (pFileState->maxTs - pFileState->deleteMark); + return pFileState->deleteMark != INT64_MAX && pFileState->maxTs > 0 && ts < (pFileState->maxTs - pFileState->deleteMark); } bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap) { return ts <= (pFileState->flushMark + gap); } diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index f2b45c5b84..e7c4573c14 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -177,11 +177,10 @@ int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) { continue; } - return 0; + return (terrno = 0); } - terrno = TSDB_CODE_FS_NO_VALID_DISK; - return -1; + return (terrno = TSDB_CODE_FS_NO_VALID_DISK); } const char *tfsGetPrimaryPath(STfs *pTfs) { return TFS_PRIMARY_DISK(pTfs)->path; } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 5a3dc867e6..4635ec340d 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -2741,7 +2741,7 @@ int32_t tsDecompressTimestamp2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t tsCompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf, int32_t nBuf) { DEFINE_VAR(cmprAlg) - if (lvl != 0 && lossyFloat) { + if (l2 == L2_TSZ && lvl != 0 && lossyFloat) { return tsCompressFloatLossyImp(pIn, nEle, pOut); } FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_FLOAT, 1); @@ -2760,7 +2760,7 @@ int32_t tsDecompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int int32_t tsCompressDouble2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf, int32_t nBuf) { DEFINE_VAR(cmprAlg) - if (lvl != 0 && lossyDouble) { + if (l2 == L2_TSZ && lvl != 0 && lossyDouble) { // lossy mode return tsCompressDoubleLossyImp(pIn, nEle, pOut); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ad811cc891..0f594af0e9 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -259,7 +259,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING, "Database in creating TAOS_DEFINE_ERROR(TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE, "Encryption is not allowed to be changed after database is created") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY, "Inconsistent encryption key") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ENCRYPT_KEY, "The cluster has not been set properly for database encryption") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED, "The database encryption function grant expired") // mnode-node TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") @@ -499,9 +498,20 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_LACK_OF_BASIC, "Lack of basic functio TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OBJ_NOT_EXIST, "Grant object not exist") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_LAST_ACTIVE_NOT_FOUND, "The historial active code does not match") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MACHINES_MISMATCH, "Cluster machines mismatch with active code") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE, "Expire time of optional grant item is too large") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE, "Expiration time of optional grant item is too large") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUPLICATED_ACTIVE, "The active code can't be activated repeatedly") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED, "Number of view has reached the licensed upper limit") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED, "Number of views has reached the licensed upper limit") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_BASIC_EXPIRED, "License expired for basic functions") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_EXPIRED, "License expired for stream function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SUBSCRIPTION_EXPIRED, "License expired for subscription function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_EXPIRED, "License expired for view function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_AUDIT_EXPIRED, "License expired for audit function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CSV_EXPIRED, "License expired for CSV function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED, "License expired for multi-tier storage function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED, "License expired for object storage function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED,"License expired for dual-replica HA function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED, "License expired for database encryption function") + // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") diff --git a/tests/army/community/insert/test_column_tag_boundary.py b/tests/army/community/insert/test_column_tag_boundary.py index 4c04fd3f9b..4f1245d728 100644 --- a/tests/army/community/insert/test_column_tag_boundary.py +++ b/tests/army/community/insert/test_column_tag_boundary.py @@ -22,6 +22,7 @@ class TDTestCase(TBase): self.child_table_num = 1 self.insert_round_num = 700 self.row_num_per_round = 15 + self.start_ts = 1704082431000 def prepare_data(self): # database @@ -39,7 +40,7 @@ class TDTestCase(TBase): for j in range(self.insert_round_num): sql = "insert into ct_binary%s values" % (i+1) for k in range(self.row_num_per_round): - sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), 'a' * self.max_column_length) + sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), 'a' * self.max_column_length) tdSql.execute(sql) tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_binary{i+1} {j+1} times successfully") tdSql.execute("flush database db;") @@ -63,7 +64,7 @@ class TDTestCase(TBase): for j in range(self.insert_round_num): sql = "insert into ct_varchar%s values" % (i+1) for k in range(self.row_num_per_round): - sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), 'b' * self.max_column_length) + sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), 'b' * self.max_column_length) tdSql.execute(sql) tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_varchar{i+1} {j+1} times successfully") tdSql.execute("flush database db;") @@ -98,7 +99,7 @@ class TDTestCase(TBase): for j in range(self.insert_round_num): sql = "insert into ct_nchar%s values" % (i+1) for k in range(self.row_num_per_round): - sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), column) + sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), column) tdSql.execute(sql) tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_nchar{i+1} {j+1} times successfully") tdSql.execute("flush database db;") @@ -124,7 +125,7 @@ class TDTestCase(TBase): for j in range(self.insert_round_num): sql = "insert into ct_varbinary%s values" % (i+1) for k in range(row_num_per_round): - sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), '\\x' + column) + sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), '\\x' + column) tdSql.execute(sql) tdLog.info(f"Insert {row_num_per_round} rows data into ct_varbinary{i+1} {j+1} times successfully") tdSql.execute("flush database db;") @@ -153,7 +154,7 @@ class TDTestCase(TBase): for j in range(self.insert_round_num): sql = "insert into ct_json_tag%s values" % (i+1) for k in range(row_num_per_round): - sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), '\\x' + column) + sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), '\\x' + column) tdSql.execute(sql) tdLog.info(f"Insert {row_num_per_round} rows data into ct_json_tag{i+1} {j+1} times successfully") tdSql.execute("flush database db;") diff --git a/tests/army/community/query/accuracy/test_query_accuracy.py b/tests/army/community/query/accuracy/test_query_accuracy.py new file mode 100644 index 0000000000..f7b9c00a28 --- /dev/null +++ b/tests/army/community/query/accuracy/test_query_accuracy.py @@ -0,0 +1,77 @@ +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.eos import * +import random +import string + + +class TDTestCase(TBase): + """Add test case to verify the complicated query accuracy + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def prepare_data(self): + # database for case TS-4806 + tdSql.execute("create database db_ts4806;") + tdSql.execute("use db_ts4806;") + # super table + tdSql.execute("create table st (ts timestamp, adl float, bdl float, cdl float, ady float, bdy float, cdy float) \ + tags(pt_radio float, ct_ratio float, rated_cap float, ta_id varchar(128), id varchar(128), area_code \ + varchar(128), zdy_flag int, elec_cust_name bigint,bureau_code bigint, fl_name varchar(32), classify_id \ + varchar(128));") + # child table + tdSql.execute("create table ct_1 using st tags(1.2, 1.3, 3.4, '271000276', '30000001', '10001', 1, 10001, 2000001, 'beijing', '13169');") + tdSql.execute("create table ct_2 using st tags(2.1, 1.2, 3.3, '271000277', '30000002', '10002', 1, 10002, 2000002, 'shanghai', '13141');") + tdSql.execute("create table ct_3 using st tags(3.1, 4.2, 5.3, '271000278', '30000003', '10003', 0, 10003, 2000003, 'guangzhou', '13151');") + # insert data for ts4806 + start_ts = 1705783972000 + data = [ + (1.1, 2.2, 3.3, 1.1, 2.2, 3.3), + (1.2, 2.3, 3.4, 1.2, 2.3, 3.4), + (1.3, 2.4, 3.5, 1.3, 2.4, 3.5), + (1.4, 2.5, 3.6, 1.4, 2.5, 3.6), + (1.5, 2.6, 3.7, 1.5, 2.6, 3.7), + (1.6, 2.7, 3.8, 1.6, 2.7, 3.8), + (1.7, 2.8, 3.9, 1.7, 2.8, 3.9), + (1.8, 2.9, 4.0, 1.8, 2.9, 4.0), + (1.9, 4.2, 4.1, 1.9, 3.0, 4.1), + (1.2, 3.1, 4.2, 2.0, 3.1, 4.2) + ] + index = [1, 2, 5, 0, 7, 3, 8, 4, 6, 9] + for ct in ['ct_1', 'ct_2']: + for i in range(10): + sql = f"insert into {ct} values" + for j in range(1000): + sql += f"({start_ts + i * 1000 * 1000 + j * 1000}, {','.join([str(item) for item in data[index[i]]])})," + sql += ";" + tdSql.execute(sql) + + def test_ts4806(self): + tdSql.execute("use db_ts4806;") + tdSql.query("select _wstart, cj.id, count(*) from st cj where cj.ts >= '2024-01-21 04:52:52.000' and cj.ts <= ' 2024-01-21 07:39:31.000' \ + and cj.zdy_flag = 1 and cj.id in ('30000001', '30000002') partition by cj.id event_window start with \ + (CASE WHEN cj.adl >= cj.bdl AND cj.adl >= cj.cdl THEN cj.adl WHEN cj.bdl >= cj.adl AND cj.bdl >= cj.cdl \ + THEN cj.bdl ELSE cj.cdl END) * cj.ct_ratio * 0.4 * 1.732 / cj.rated_cap > 1 end with (CASE WHEN cj.adl >= \ + cj.bdl AND cj.adl >= cj.cdl THEN cj.adl WHEN cj.bdl >= cj.adl AND cj.bdl >= cj.cdl THEN cj.bdl ELSE cj.cdl \ + END) * cj.ct_ratio * 0.4 * 1.732 / cj.rated_cap <= 1 HAVING count(*) >= 4 order by _wstart, cj.id;") + tdSql.checkRows(5) + tdSql.checkData(4, 1, '30000002') + tdSql.checkData(4, 2, 1001) + + def run(self): + self.prepare_data() + self.test_ts4806() + + def stop(self): + tdSql.execute("drop database db_ts4806;") + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/army/enterprise/alter/alterConfig.py b/tests/army/enterprise/alter/alterConfig.py new file mode 100644 index 0000000000..7413e5e5a6 --- /dev/null +++ b/tests/army/enterprise/alter/alterConfig.py @@ -0,0 +1,55 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + def alterSupportVnodes(self): + tdLog.info(f"test function of altering supportVnodes") + + tdSql.execute("alter dnode 1 'supportVnodes' '128'") + time.sleep(1) + tdSql.query('show dnodes') + tdSql.checkData(0, 3, "128") + + tdSql.execute("alter dnode 1 'supportVnodes' '64'") + time.sleep(1) + tdSql.query('show dnodes') + tdSql.checkData(0, 3, "64") + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # TS-4721 + self.alterSupportVnodes() + + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/enterprise/db-encrypt/basic.py b/tests/army/enterprise/db-encrypt/basic.py new file mode 100644 index 0000000000..8d30bbcfe2 --- /dev/null +++ b/tests/army/enterprise/db-encrypt/basic.py @@ -0,0 +1,63 @@ +import taos +import sys +import os +import subprocess +import glob +import shutil +import time + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.srvCtl import * +from frame.caseBase import * +from frame import * +from frame.autogen import * +# from frame.server.dnodes import * +# from frame.server.cluster import * + + +class TDTestCase(TBase): + + def init(self, conn, logSql, replicaVar=1): + super(TDTestCase, self).init(conn, logSql, replicaVar=1, checkColName="c1") + self.valgrind = 0 + self.db = "test" + self.stb = "meters" + self.childtable_count = 10 + tdSql.init(conn.cursor(), logSql) + + def create_encrypt_db(self): + + tdSql.execute("create encrypt_key '1234567890'") + autoGen = AutoGen() + autoGen.create_db(self.db, 2, 1, "ENCRYPT_ALGORITHM 'sm4'") + tdSql.execute(f"use {self.db}") + autoGen.create_stable(self.stb, 2, 3, 8, 8) + autoGen.create_child(self.stb, "d", self.childtable_count) + autoGen.insert_data(1000) + + tdSql.query(f"select * from {self.db}.{self.stb}") + tdSql.checkRows(1000 * self.childtable_count) + + self.timestamp_step = 1000 + self.insert_rows = 1000 + + self.checkInsertCorrect() + + def create_encrypt_db_error(self): + tdSql.error("create encrypt_key '123'") + tdSql.error("create encrypt_key '12345678abcdefghi'") + tdSql.error("create database test ENCRYPT_ALGORITHM 'sm4'") + + def run(self): + self.create_encrypt_db_error() + self.create_encrypt_db() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 342389562a..61687eeccd 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -11,6 +11,7 @@ # army-test # ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f enterprise/db-encrypt/basic.py ,,n,army,python3 ./test.py -f enterprise/s3/s3Basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f community/query/function/test_func_elapsed.py @@ -20,10 +21,12 @@ ,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f community/query/accuracy/test_query_accuracy.py ,,y,army,./pytest.sh python3 ./test.py -f community/insert/insert_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py ,,n,army,python3 ./test.py -f community/query/show.py -N 3 +,,n,army,python3 ./test.py -f enterprise/alter/alterConfig.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/compressBasic.py -N 3 @@ -994,6 +997,7 @@ ,,n,system-test,python3 ./test.py -f eco-system/meta/database/keep_time_offset.py #tsim test +,,y,script,./test.sh -f tsim/query/timeline.sim ,,y,script,./test.sh -f tsim/join/join.sim ,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim ,,y,script,./test.sh -f tsim/parser/where.sim diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index ef9413a9d5..e58f890ccd 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -315,19 +315,22 @@ function run_thread() { fi if [ -n "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" - local build_dir=$log_dir/build_${hosts[index]} - local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build" - # if [ $ent -ne 0 ]; then - # remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" - # fi - mkdir "$build_dir" 2>/dev/null - if [ $? -eq 0 ]; then - # scp build binary - cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/" - echo "$cmd" - $cmd >/dev/null - fi fi + # scp build binary and unit test log + local build_dir=$log_dir/build_${hosts[index]} + local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build" + local remote_unit_test_log_dir="${workdirs[index]}/${DEBUGPATH}/Testing/Temporary/" + + mkdir "$build_dir" 2>/dev/null + if [ $? -eq 0 ]; then + cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/" + echo "$cmd" + $cmd >/dev/null + cmd="$scpcmd:${remote_unit_test_log_dir}/* ${build_dir}/" + echo "$cmd" + $cmd >/dev/null + fi + # get remote sim dir local remote_sim_dir="${workdirs[index]}/tmp/thread_volume/$thread_no" local tarcmd="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index df50e8031c..7bb2f42495 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -201,6 +201,9 @@ class TDCom: self.cast_tag_stb_filter_des_select_elm = "ts, t1, t2, t3, t4, cast(t1 as TINYINT UNSIGNED), t6, t7, t8, t9, t10, cast(t2 as varchar(256)), t12, cast(t3 as bool)" self.tag_count = len(self.tag_filter_des_select_elm.split(",")) self.state_window_range = list() + + self.custom_col_val = 0 + self.part_val_list = [1, 2] # def init(self, conn, logSql): # # tdSql.init(conn.cursor(), logSql) @@ -1259,7 +1262,7 @@ class TDCom: default_ctbname_index_start_num += 1 tdSql.execute(create_stable_sql) - def sgen_column_value_list(self, column_elm_list, need_null, ts_value=None): + def sgen_column_value_list(self, column_elm_list, need_null, ts_value=None, additional_ts=None, custom_col_index=None, col_value_type=None, force_pk_val=None): """_summary_ Args: @@ -1269,6 +1272,8 @@ class TDCom: """ self.column_value_list = list() self.ts_value = self.genTs()[0] + if additional_ts is not None: + self.additional_ts = self.genTs(additional_ts=additional_ts)[2] if ts_value is not None: self.ts_value = ts_value @@ -1292,7 +1297,22 @@ class TDCom: for i in range(int(len(self.column_value_list)/2)): index_num = random.randint(0, len(self.column_value_list)-1) self.column_value_list[index_num] = None - self.column_value_list = [self.ts_value] + self.column_value_list + + if custom_col_index is not None: + if col_value_type == "Random": + pass + elif col_value_type == "Incremental": + self.column_value_list[custom_col_index] = self.custom_col_val + self.custom_col_val += 1 + elif col_value_type == "Part_equal": + self.column_value_list[custom_col_index] = random.choice(self.part_val_list) + + self.column_value_list = [self.ts_value] + [self.additional_ts] + self.column_value_list if additional_ts is not None else [self.ts_value] + self.column_value_list + if col_value_type == "Incremental" and custom_col_index==1: + self.column_value_list[custom_col_index] = self.custom_col_val if force_pk_val is None else force_pk_val + if col_value_type == "Part_equal" and custom_col_index==1: + self.column_value_list[custom_col_index] = random.randint(0, self.custom_col_val) if force_pk_val is None else force_pk_val + def screate_table(self, dbname=None, tbname="tb", use_name="table", column_elm_list=None, count=1, default_tbname_prefix="tb", default_tbname_index_start_num=1, @@ -1333,7 +1353,7 @@ class TDCom: default_tbname_index_start_num += 1 tdSql.execute(create_table_sql) - def sinsert_rows(self, dbname=None, tbname=None, column_ele_list=None, ts_value=None, count=1, need_null=False): + def sinsert_rows(self, dbname=None, tbname=None, column_ele_list=None, ts_value=None, count=1, need_null=False, custom_col_index=None, col_value_type="random"): """insert rows Args: @@ -1353,7 +1373,7 @@ class TDCom: if tbname is not None: self.tbname = tbname - self.sgen_column_value_list(column_ele_list, need_null, ts_value) + self.sgen_column_value_list(column_ele_list, need_null, ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) # column_value_str = ", ".join(str(v) for v in self.column_value_list) column_value_str = "" for column_value in self.column_value_list: @@ -1370,7 +1390,7 @@ class TDCom: else: for num in range(count): ts_value = self.genTs()[0] - self.sgen_column_value_list(column_ele_list, need_null, f'{ts_value}+{num}s') + self.sgen_column_value_list(column_ele_list, need_null, f'{ts_value}+{num}s', custom_col_index=custom_col_index, col_value_type=col_value_type) column_value_str = "" for column_value in self.column_value_list: if column_value is None: @@ -1777,7 +1797,7 @@ class TDCom: self.sdelete_rows(tbname=self.ctb_name, start_ts=self.time_cast(self.record_history_ts, "-")) self.sdelete_rows(tbname=self.tb_name, start_ts=self.time_cast(self.record_history_ts, "-")) - def prepare_data(self, interval=None, watermark=None, session=None, state_window=None, state_window_max=127, interation=3, range_count=None, precision="ms", fill_history_value=0, ext_stb=None): + def prepare_data(self, interval=None, watermark=None, session=None, state_window=None, state_window_max=127, interation=3, range_count=None, precision="ms", fill_history_value=0, ext_stb=None, custom_col_index=None, col_value_type="random"): """prepare stream data Args: @@ -1840,8 +1860,8 @@ class TDCom: if fill_history_value == 1: for i in range(self.range_count): ts_value = str(self.date_time)+f'-{self.default_interval*(i+1)}s' - self.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value) - self.sinsert_rows(tbname=self.tb_name, ts_value=ts_value) + self.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) + self.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) if i == 1: self.record_history_ts = ts_value @@ -1862,6 +1882,18 @@ class TDCom: time.sleep(1) return tbname + def get_group_id_from_stb(self, stbname): + tdSql.query(f'select distinct group_id from {stbname}') + cnt = 0 + while len(tdSql.queryResult) == 0: + tdSql.query(f'select distinct group_id from {stbname}') + if cnt < self.default_interval: + cnt += 1 + time.sleep(1) + else: + return False + return tdSql.queryResult[0][0] + def update_json_file_replica(self, json_file_path, new_replica_value, output_file_path=None): """ Read a JSON file, update the 'replica' value, and write the result back to a file. diff --git a/tests/script/tsim/query/timeline.sim b/tests/script/tsim/query/timeline.sim new file mode 100644 index 0000000000..743a6f1a9f --- /dev/null +++ b/tests/script/tsim/query/timeline.sim @@ -0,0 +1,51 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database test; +sql use test; + +sql CREATE STABLE `demo` (`_ts` TIMESTAMP, `faev` DOUBLE) TAGS (`deviceid` VARCHAR(256)); +sql CREATE TABLE demo_1 USING demo (deviceid) TAGS ('1'); +sql CREATE TABLE demo_2 USING demo (deviceid) TAGS ('2'); +sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-11-30 00:00:00.000', 1.0); +sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-04 01:00:00.001', 2.0); +sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-04 02:00:00.002', 3.0); +sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-05 03:00:00.003', 4.0); +sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-11-30 00:00:00.000', 5.0); +sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-28 01:00:00.001', 6.0); +sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-28 02:00:00.002', 7.0); +sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-29 03:00:00.003', 8.0); + +sql_error select diff(faev) from ((select ts, faev from demo union all select ts, faev from demo)); +sql_error select diff(faev) from (select _ts, faev from demo union all select _ts, faev from demo order by faev, _ts); +sql_error select diff(faev) from (select _ts, faev from demo union all select _ts, faev from demo order by faev, _ts) partition by faev; +sql select diff(faev) from (select _ts, faev from demo union all select _ts + 1s, faev from demo order by faev, _ts) partition by faev; +sql_error select diff(faev) from (select _ts, faev, deviceid from demo union all select _ts + 1s, faev, deviceid from demo order by deviceid, _ts) partition by faev; +sql select diff(faev) from (select _ts, faev, deviceid from demo union all select _ts + 1s, faev, deviceid from demo order by faev, _ts, deviceid) partition by faev; + +sql_error select diff(faev) from (select _ts, faev from demo); +sql_error select diff(faev) from (select _ts, faev from demo order by faev, _ts); +sql select diff(faev) from (select _ts, faev from demo order by faev, _ts) partition by faev; +sql_error select diff(faev) from (select _ts, faev, deviceid from demo order by faev, _ts) partition by deviceid; +sql_error select diff(faev) from (select _ts, faev, deviceid from demo order by deviceid, _ts) partition by faev; +sql select diff(faev) from (select _ts, faev, deviceid from demo order by faev, _ts, deviceid) partition by faev; + +sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n))) ORDER BY deviceid, ts) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n))) ORDER BY ts, deviceid) PARTITION by deviceid; + + +sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM (SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY deviceid, ts) PARTITION by deviceid; +sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM (SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY ts, deviceid) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY deviceid, ts) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY ts, deviceid) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0) ORDER BY deviceid, ts) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0) ORDER BY ts, deviceid) PARTITION by deviceid; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 944b2fbb1e..35f5e39214 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -222,7 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(254, 255)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(255, 256)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(54, len(tdSql.queryResult)) @@ -336,7 +336,7 @@ class TDTestCase: tdSql.checkEqual(True, result[i][1] in key_status_list[1]) index += 1 tdSql.checkEqual(True, index > 0) - + tdSql.query(f'show encryptions') result = tdSql.queryResult index = 0 @@ -344,7 +344,7 @@ class TDTestCase: tdSql.checkEqual(True, result[i][1] in key_status_list[1]) index += 1 tdSql.checkEqual(True, index > 0) - + # loaded/sm4 tdSql.execute('drop database if exists db2') tdSql.execute('create encrypt_key \'12345678\'') @@ -357,7 +357,7 @@ class TDTestCase: tdSql.checkEqual(True, result[i][1] in key_status_list[3]) index += 1 tdSql.checkEqual(True, index > 0) - + tdSql.query(f'show encryptions') result = tdSql.queryResult index = 0 diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 81c5b66185..1cb95b59c5 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -15,6 +15,30 @@ class TDTestCase: #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file + def interp_on_empty_table(self): + dbname = "db" + tbname = "t" + + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + + tdSql.execute(f'''create table if not exists {dbname}.{tbname} (ts timestamp, k int)''') + + tdLog.printNoPrefix("==========step2:interp query on empty table") + + tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(prev)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(next)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(value, 2)") + tdSql.checkRows(0) + def run(self): dbname = "db" tbname = "tb" @@ -5658,6 +5682,7 @@ class TDTestCase: tdSql.checkData(0, 0, '2023-08-06 23:59:00') tdSql.checkData(0, 1, None) + self.interp_on_empty_table() def stop(self): diff --git a/tests/system-test/2-query/union1.py b/tests/system-test/2-query/union1.py index 8db5ce01f3..853dfe9582 100644 --- a/tests/system-test/2-query/union1.py +++ b/tests/system-test/2-query/union1.py @@ -240,9 +240,22 @@ class TDTestCase: tdSql.error( " '' union all select c1 from ct1 " ) # tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ") + def test_select_from_union_all(self): + tdSql.query('select c8, ts from ((select ts, c8,c1 from stb1 order by c1) union all select ts, c8, c1 from stb1 limit 15)') + tdSql.checkRows(15) + tdSql.query('select c8, ts from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)') + tdSql.checkRows(15) + tdSql.query('select ts, c1 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)') + tdSql.checkRows(15) + tdSql.query('select ts, c1, c8 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)') + tdSql.checkRows(15) + tdSql.query('select ts, c8, c1, 123 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)') + tdSql.checkRows(15) + def all_test(self): self.__test_error() self.union_check() + self.test_select_from_union_all() def __create_tb(self): diff --git a/tests/system-test/8-stream/at_once_interval.py b/tests/system-test/8-stream/at_once_interval.py index 763b88bc69..eb581e84c4 100644 --- a/tests/system-test/8-stream/at_once_interval.py +++ b/tests/system-test/8-stream/at_once_interval.py @@ -15,9 +15,12 @@ class TDTestCase: def at_once_interval(self, interval, partition="tbname", delete=False, fill_value=None, fill_history_value=None, case_when=None): tdLog.info(f"*** testing stream at_once+interval: interval: {interval}, partition: {partition}, fill_history: {fill_history_value}, fill: {fill_value}, delete: {delete}, case_when: {case_when} ***") + col_value_type = "Incremental" if partition=="c1" else "random" + custom_col_index = 1 if partition=="c1" else None + self.tdCom.custom_col_val = 0 self.delete = delete self.tdCom.case_name = sys._getframe().f_code.co_name - self.tdCom.prepare_data(interval=interval, fill_history_value=fill_history_value) + self.tdCom.prepare_data(interval=interval, fill_history_value=fill_history_value, custom_col_index=custom_col_index, col_value_type=col_value_type) self.stb_name = self.tdCom.stb_name.replace(f"{self.tdCom.dbname}.", "") self.ctb_name = self.tdCom.ctb_name.replace(f"{self.tdCom.dbname}.", "") self.tb_name = self.tdCom.tb_name.replace(f"{self.tdCom.dbname}.", "") @@ -76,15 +79,15 @@ class TDTestCase: for i in range(self.tdCom.range_count): ts_value = str(self.tdCom.date_time+self.tdCom.dataDict["interval"])+f'+{i*10}s' ts_cast_delete_value = self.tdCom.time_cast(ts_value) - self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value) + self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) if i%2 == 0: - self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value) + self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) if self.delete and i%2 != 0: self.tdCom.sdelete_rows(tbname=self.tdCom.ctb_name, start_ts=ts_cast_delete_value) self.tdCom.date_time += 1 - self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value) + self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) if i%2 == 0: - self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value) + self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) if self.delete and i%2 != 0: self.tdCom.sdelete_rows(tbname=self.tdCom.tb_name, start_ts=ts_cast_delete_value) self.tdCom.date_time += 1 @@ -102,6 +105,7 @@ class TDTestCase: if self.tdCom.subtable: for tname in [self.stb_name, self.ctb_name]: + group_id = self.tdCom.get_group_id_from_stb(f'{tname}_output') tdSql.query(f'select * from {self.ctb_name}') ptn_counter = 0 for c1_value in tdSql.queryResult: @@ -116,11 +120,11 @@ class TDTestCase: tbname = self.tdCom.get_subtable_wait(f'{tname}_{self.tdCom.subtable_prefix}{abs_c1_value}{self.tdCom.subtable_suffix}') tdSql.query(f'select count(*) from `{tbname}`') elif partition == "tbname" and ptn_counter == 0: - tbname = self.tdCom.get_subtable_wait(f'{tname}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}') + tbname = self.tdCom.get_subtable_wait(f'{tname}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}_{tname}_output_{group_id}') tdSql.query(f'select count(*) from `{tbname}`') ptn_counter += 1 tdSql.checkEqual(tdSql.queryResult[0][0] > 0, True) - + group_id = self.tdCom.get_group_id_from_stb(f'{self.tb_name}_output') tdSql.query(f'select * from {self.tb_name}') ptn_counter = 0 for c1_value in tdSql.queryResult: @@ -135,7 +139,7 @@ class TDTestCase: tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{abs_c1_value}{self.tdCom.subtable_suffix}') tdSql.query(f'select count(*) from `{tbname}`') elif partition == "tbname" and ptn_counter == 0: - tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}') + tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}_{self.tb_name}_output_{group_id}') tdSql.query(f'select count(*) from `{tbname}`') ptn_counter += 1 diff --git a/tests/system-test/8-stream/at_once_session.py b/tests/system-test/8-stream/at_once_session.py index 6f25e5ad97..cdded9388c 100644 --- a/tests/system-test/8-stream/at_once_session.py +++ b/tests/system-test/8-stream/at_once_session.py @@ -15,9 +15,12 @@ class TDTestCase: def at_once_session(self, session, ignore_expired=None, ignore_update=None, partition="tbname", delete=False, fill_history_value=None, case_when=None, subtable=True): tdLog.info(f"*** testing stream at_once+interval: session: {session}, ignore_expired: {ignore_expired}, ignore_update: {ignore_update}, partition: {partition}, delete: {delete}, fill_history: {fill_history_value}, case_when: {case_when}, subtable: {subtable} ***") + col_value_type = "Incremental" if partition=="c1" else "random" + custom_col_index = 1 if partition=="c1" else None + self.tdCom.custom_col_val = 0 self.delete = delete self.tdCom.case_name = sys._getframe().f_code.co_name - self.tdCom.prepare_data(session=session, fill_history_value=fill_history_value) + self.tdCom.prepare_data(session=session, fill_history_value=fill_history_value, custom_col_index=custom_col_index, col_value_type=col_value_type) self.stb_name = self.tdCom.stb_name.replace(f"{self.tdCom.dbname}.", "") self.ctb_name = self.tdCom.ctb_name.replace(f"{self.tdCom.dbname}.", "") self.tb_name = self.tdCom.tb_name.replace(f"{self.tdCom.dbname}.", "") @@ -79,11 +82,11 @@ class TDTestCase: if i == 0: record_window_close_ts = window_close_ts for ts_value in [self.tdCom.date_time, window_close_ts]: - self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True) - self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True) + self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type) + self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type) if self.tdCom.update and i%2 == 0: - self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True) - self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True) + self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type) + self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type) if self.delete and i%2 != 0: dt = f'cast({self.tdCom.date_time-1} as timestamp)' self.tdCom.sdelete_rows(tbname=self.ctb_name, start_ts=dt) @@ -166,6 +169,7 @@ class TDTestCase: self.tdCom.sdelete_rows(tbname=self.tb_name, start_ts=self.tdCom.time_cast(self.tdCom.record_history_ts, "-")) if self.tdCom.subtable: + group_id = self.tdCom.get_group_id_from_stb(f'{self.ctb_name}_output') tdSql.query(f'select * from {self.ctb_name}') ptn_counter = 0 for c1_value in tdSql.queryResult: @@ -182,11 +186,11 @@ class TDTestCase: tbname = self.tdCom.get_subtable_wait(f'{self.ctb_name}_{self.tdCom.subtable_prefix}{partition_elm_alias}{self.tdCom.subtable_suffix}') tdSql.query(f'select count(*) from `{tbname}`') elif partition == "tbname" and ptn_counter == 0: - tbname = self.tdCom.get_subtable_wait(f'{self.ctb_name}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}') + tbname = self.tdCom.get_subtable_wait(f'{self.ctb_name}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}_{self.ctb_name}_output_{group_id}') tdSql.query(f'select count(*) from `{tbname}`') ptn_counter += 1 tdSql.checkEqual(tdSql.queryResult[0][0] > 0, True) if subtable is not None else tdSql.checkEqual(tdSql.queryResult[0][0] >= 0, True) - + group_id = self.tdCom.get_group_id_from_stb(f'{self.tb_name}_output') tdSql.query(f'select * from {self.tb_name}') ptn_counter = 0 for c1_value in tdSql.queryResult: @@ -203,7 +207,7 @@ class TDTestCase: tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{partition_elm_alias}{self.tdCom.subtable_suffix}') tdSql.query(f'select count(*) from `{tbname}`') elif partition == "tbname" and ptn_counter == 0: - tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}') + tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}_{self.tb_name}_output_{group_id}') tdSql.query(f'select count(*) from `{tbname}`') ptn_counter += 1 diff --git a/tests/system-test/8-stream/stream_multi_agg.py b/tests/system-test/8-stream/stream_multi_agg.py index 3532825493..acb80f528b 100644 --- a/tests/system-test/8-stream/stream_multi_agg.py +++ b/tests/system-test/8-stream/stream_multi_agg.py @@ -39,7 +39,7 @@ class TDTestCase: os.system("nohup taosBenchmark -y -B 1 -t 40 -S 1000 -n 10 -i 1000 -v 5 > /dev/null 2>&1 &") time.sleep(10) - tdSql.query("use test") + tdSql.execute("use test", queryTimes=100) tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)") tdLog.debug("========create stream and insert data ok========") time.sleep(15) @@ -66,7 +66,7 @@ class TDTestCase: os.system("taosBenchmark -d db -t 20 -v 6 -n 1000 -y > /dev/null 2>&1") # create stream - tdSql.execute("use db") + tdSql.execute("use db", queryTimes=100) tdSql.execute("create stream stream1 fill_history 1 into sta as select count(*) as cnt from meters interval(10a);",show=True) time.sleep(5) diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index 2837c817be..060717c20e 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -89,8 +89,8 @@ else export LD_PRELOAD="$(realpath "$(gcc -print-file-name=libasan.so)") $(realpath "$(gcc -print-file-name=libstdc++.so)")" echo "Preload AsanSo:" $? - $* -a 2>$AsanFile - + $* -a 2> $AsanFile + cat $AsanFile unset LD_PRELOAD for ((i = 1; i <= 20; i++)); do AsanFileLen=$(cat $AsanFile | wc -l) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 188abb4b58..a16a03d30a 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -140,8 +140,8 @@ ELSE () BUILD_COMMAND COMMAND set CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client COMMAND set CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib - COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'" - COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'" + COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" + COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" INSTALL_COMMAND COMMAND cmake -E echo "Comparessing taosadapter.exe" @@ -167,8 +167,8 @@ ELSE () PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'" - COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" INSTALL_COMMAND COMMAND cmake -E echo "Copy taosadapter" COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin @@ -192,19 +192,19 @@ ELSE () PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'" - COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" +# COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" INSTALL_COMMAND - COMMAND cmake -E echo "Comparessing taosadapter.exe" - COMMAND upx taosadapter || : +# COMMAND cmake -E echo "Comparessing taosadapter.exe" +# COMMAND upx taosadapter || : COMMAND cmake -E echo "Copy taosadapter" COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E echo "Copy taosadapter.toml" COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ - COMMAND cmake -E echo "Copy taosadapter-debug" - COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin +# COMMAND cmake -E echo "Copy taosadapter-debug" +# COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin ) ENDIF () ENDIF () diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c index 162ecd229c..0f123fb560 100644 --- a/utils/test/c/write_raw_block_test.c +++ b/utils/test/c/write_raw_block_test.c @@ -78,7 +78,7 @@ int buildStable(TAOS* pConn) { } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into ntba values(now,'hello')"); + pRes = taos_query(pConn, "insert into ntba values(now + 1s,'hello')"); if (taos_errno(pRes) != 0) { printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes)); return -1;