From 1547b7ba8819c651709f7f589f70ae28ebf1671e Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 17 Jul 2024 17:57:14 +0800 Subject: [PATCH 01/52] enh: return with code --- include/util/tutil.h | 12 ++++ source/dnode/mnode/impl/inc/mndUser.h | 8 +-- source/dnode/mnode/impl/src/mndDb.c | 11 ++-- source/dnode/mnode/impl/src/mndProfile.c | 4 +- source/dnode/mnode/impl/src/mndStb.c | 3 +- source/dnode/mnode/impl/src/mndUser.c | 83 ++++++++++++------------ 6 files changed, 65 insertions(+), 56 deletions(-) diff --git a/include/util/tutil.h b/include/util/tutil.h index 31ce34f667..0a4c6fd73c 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -172,6 +172,18 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } \ } while (0) +#define TAOS_CHECK_EXEC_GOTO(CMD, LINO, EXEC, LABEL) \ + do { \ + code = (CMD); \ + if (code != TSDB_CODE_SUCCESS) { \ + if (LINO) { \ + *((int32_t *)(LINO)) = __LINE__; \ + } \ + (EXEC); \ + goto LABEL; \ + } \ + } while (0) + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index eda39dd29f..a443673a35 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -29,14 +29,14 @@ enum { }; int32_t mndInitUser(SMnode *pMnode); void mndCleanupUser(SMnode *pMnode); -SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName); +int32_t mndAcquireUser(SMnode *pMnode, const char *userName, SUserObj **ppUser); void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); -SHashObj *mndDupDbHash(SHashObj *pOld); -SHashObj *mndDupTableHash(SHashObj *pOld); -SHashObj *mndDupTopicHash(SHashObj *pOld); +int32_t mndDupDbHash(SHashObj *pOld, SHashObj **ppNew); +int32_t mndDupTableHash(SHashObj *pOld, SHashObj **ppNew); +int32_t mndDupTopicHash(SHashObj *pOld, SHashObj **ppNew); int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen, int64_t ipWhiteListVer); int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 8432855c6f..10adf6546c 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -871,6 +871,7 @@ _exit: static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; + int32_t lino = 0; SDbObj *pDb = NULL; SUserObj *pUser = NULL; SCreateDbReq createReq = {0}; @@ -931,10 +932,7 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndAcquireUser(pMnode, pReq->info.conn.user, &pUser), &lino, _OVER); code = mndCreateDb(pMnode, pReq, &createReq, pUser); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -946,7 +944,7 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("db:%s, failed to create since %s", createReq.db, terrstr()); + mError("db:%s, failed to create at line %d since %s", createReq.db, lino, terrstr()); } mndReleaseDb(pMnode, pDb); @@ -2397,9 +2395,10 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SDbObj *pDb = NULL; + SUserObj *pUser = NULL; ESdbStatus objStatus = 0; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); + (void)mndAcquireUser(pMnode, pReq->info.conn.user, &pUser); if (pUser == NULL) return 0; bool sysinfo = pUser->sysInfo; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 4224d79391..3376ecf130 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -251,9 +251,9 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->info.conn.user); + code = mndAcquireUser(pMnode, pReq->info.conn.user, &pUser); if (pUser == NULL) { - mGError("user:%s, failed to login from %s while acquire user since %s", pReq->info.conn.user, ip, terrstr()); + mGError("user:%s, failed to login from %s while acquire user since %s", pReq->info.conn.user, ip, tstrerror(code)); goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 73542bbb1e..3fac879ca9 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2851,8 +2851,9 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { int32_t code = -1; STableInfoReq infoReq = {0}; STableMetaRsp metaRsp = {0}; + SUserObj *pUser = NULL; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); + code = mndAcquireUser(pMnode, pReq->info.conn.user, &pUser); if (pUser == NULL) return 0; bool sysinfo = pUser->sysInfo; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index fd5267e471..333761fc00 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1305,12 +1305,13 @@ static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser) { return 0; } -SHashObj *mndDupTableHash(SHashObj *pOld) { - SHashObj *pNew = +int32_t mndDupTableHash(SHashObj *pOld, SHashObj **ppNew) { + *ppNew = taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (pNew == NULL) { + if (*ppNew == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; + TAOS_CHECK_RETURN(NULL); } char *tb = taosHashIterate(pOld, NULL); @@ -1443,17 +1444,18 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { return 0; } -SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName) { +int32_t mndAcquireUser(SMnode *pMnode, const char *userName, SUserObj **ppUser) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); if (pUser == NULL) { - if (terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { - terrno = TSDB_CODE_MND_USER_NOT_EXIST; + if (code == TSDB_CODE_SDB_OBJ_NOT_THERE) { + code = TSDB_CODE_MND_USER_NOT_EXIST; } else { - terrno = TSDB_CODE_MND_USER_NOT_AVAILABLE; + code = TSDB_CODE_MND_USER_NOT_AVAILABLE; } } - return pUser; + TAOS_RETURN(code); } void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { @@ -1552,21 +1554,21 @@ _OVER: static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; - int32_t code = -1; + int32_t code = 0; + int32_t lino = 0; SUserObj *pUser = NULL; SUserObj *pOperUser = NULL; SCreateUserReq createReq = {0}; if (tDeserializeSCreateUserReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_INVALID_MSG, &lino, _OVER); } mInfo("user:%s, start to create, createdb:%d, is_import:%d", createReq.user, createReq.isImport, createReq.createDb); #ifndef TD_ENTERPRISE if (createReq.isImport == 1) { - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_OPS_NOT_SUPPORT, &lino, _OVER); // enterprise feature } #endif @@ -1599,15 +1601,15 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { } } - pUser = mndAcquireUser(pMnode, createReq.user); + code = mndAcquireUser(pMnode, createReq.user, &pUser); if (pUser != NULL) { terrno = TSDB_CODE_MND_USER_ALREADY_EXIST; goto _OVER; } - pOperUser = mndAcquireUser(pMnode, pReq->info.conn.user); + code = mndAcquireUser(pMnode, pReq->info.conn.user, &pOperUser); if (pOperUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + code = TSDB_CODE_MND_NO_USER_FROM_CONN; goto _OVER; } @@ -1615,6 +1617,7 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { code = terrno; goto _OVER; } + TAOS_CHECK_GOTO(grantCheck(TSDB_GRANT_USER), &lino, _OVER); code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -1640,7 +1643,7 @@ _OVER: mndReleaseUser(pMnode, pOperUser); tFreeSCreateUserReq(&createReq); - return code; + TAOS_RETURN(code); } int32_t mndProcessGetUserWhiteListReq(SRpcMsg *pReq) { @@ -1681,7 +1684,7 @@ int32_t mndProcessGetUserWhiteListReq(SRpcMsg *pReq) { _OVER: mndReleaseUser(pMnode, pUser); tFreeSGetUserWhiteListRsp(&wlRsp); - return code; + TAOS_RETURN(code); } int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq) { @@ -1736,32 +1739,34 @@ static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpc return 0; } -SHashObj *mndDupObjHash(SHashObj *pOld, int32_t dataLen) { - SHashObj *pNew = +static int32_t mndDupObjHash(SHashObj *pOld, int32_t dataLen, SHashObj **ppNew) { + int32_t code = 0; + + *ppNew = taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (pNew == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + if (*ppNew == NULL) { + code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } char *db = taosHashIterate(pOld, NULL); while (db != NULL) { int32_t len = strlen(db) + 1; - if (taosHashPut(pNew, db, len, db, dataLen) != 0) { + if ((code = taosHashPut(*ppNew, db, len, db, dataLen)) != 0) { + if (terrno != 0) code = terrno; // TODO: remove this line after terrno is removed taosHashCancelIterate(pOld, db); - taosHashCleanup(pNew); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + taosHashCleanup(*ppNew); + TAOS_RETURN(code); } db = taosHashIterate(pOld, db); } - return pNew; + TAOS_RETURN(code); } -SHashObj *mndDupDbHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_DB_FNAME_LEN); } +int32_t mndDupDbHash(SHashObj *pOld, SHashObj **ppNew) { return mndDupObjHash(pOld, TSDB_DB_FNAME_LEN, ppNew); } -SHashObj *mndDupTopicHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_TOPIC_FNAME_LEN); } +int32_t mndDupTopicHash(SHashObj *pOld, SHashObj **ppNew) { return mndDupObjHash(pOld, TSDB_TOPIC_FNAME_LEN, ppNew); } static int32_t mndTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj *useDbHash, SAlterUserReq *alterReq, SSdb *pSdb) { @@ -2300,7 +2305,7 @@ _OVER: static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; - int32_t code = -1; + int32_t code = 0; SUserObj *pUser = NULL; SGetUserAuthReq authReq = {0}; SGetUserAuthRsp authRsp = {0}; @@ -2312,25 +2317,17 @@ static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq) { mTrace("user:%s, start to get auth", authReq.user); - pUser = mndAcquireUser(pMnode, authReq.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_USER_NOT_EXIST; - goto _OVER; - } + TAOS_CHECK_GOTO(mndAcquireUser(pMnode, authReq.user, &pUser), NULL, _OVER); - code = mndSetUserAuthRsp(pMnode, pUser, &authRsp); - if (code) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndSetUserAuthRsp(pMnode, pUser, &authRsp), NULL, _OVER); int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &authRsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); } - tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp); + TAOS_CHECK_GOTO(tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp), NULL, _OVER); pReq->info.rsp = pRsp; pReq->info.rspLen = contLen; @@ -2341,7 +2338,7 @@ _OVER: mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); - return code; + TAOS_RETURN(code); } static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { From b2c60945d7581c9b9a8cd689309ed90055bba0b9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 19 Jul 2024 18:10:43 +0800 Subject: [PATCH 02/52] enh: return error code --- include/util/tutil.h | 12 -- source/dnode/mnode/impl/inc/mndUser.h | 28 ++--- source/dnode/mnode/impl/src/mndUser.c | 169 ++++++++++++++------------ source/libs/tfs/inc/tfsInt.h | 14 +-- source/libs/tfs/src/tfs.c | 46 +++---- source/libs/tfs/src/tfsDisk.c | 40 +++--- source/libs/tfs/src/tfsTier.c | 38 +++--- 7 files changed, 184 insertions(+), 163 deletions(-) diff --git a/include/util/tutil.h b/include/util/tutil.h index b97d2ff39d..2aa28ac1df 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -172,18 +172,6 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } \ } while (0) -#define TAOS_CHECK_EXEC_GOTO(CMD, LINO, EXEC, LABEL) \ - do { \ - code = (CMD); \ - if (code != TSDB_CODE_SUCCESS) { \ - if (LINO) { \ - *((int32_t *)(LINO)) = __LINE__; \ - } \ - (EXEC); \ - goto LABEL; \ - } \ - } while (0) - #define TAOS_UNUSED(expr) (void)(expr) #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index a443673a35..5043f6da3d 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -27,22 +27,22 @@ enum { IP_WHITE_ADD, IP_WHITE_DROP, }; -int32_t mndInitUser(SMnode *pMnode); -void mndCleanupUser(SMnode *pMnode); -int32_t mndAcquireUser(SMnode *pMnode, const char *userName, SUserObj **ppUser); -void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); +int32_t mndInitUser(SMnode *pMnode); +void mndCleanupUser(SMnode *pMnode); +int32_t mndAcquireUser(SMnode *pMnode, const char *userName, SUserObj **ppUser); +void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test -SSdbRaw *mndUserActionEncode(SUserObj *pUser); -int32_t mndDupDbHash(SHashObj *pOld, SHashObj **ppNew); -int32_t mndDupTableHash(SHashObj *pOld, SHashObj **ppNew); -int32_t mndDupTopicHash(SHashObj *pOld, SHashObj **ppNew); -int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, - int32_t *pRspLen, int64_t ipWhiteListVer); -int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); -int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb); -int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view); -int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic); +SSdbRaw *mndUserActionEncode(SUserObj *pUser); +int32_t mndDupDbHash(SHashObj *pOld, SHashObj **ppNew); +int32_t mndDupTableHash(SHashObj *pOld, SHashObj **ppNew); +int32_t mndDupTopicHash(SHashObj *pOld, SHashObj **ppNew); +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen, int64_t ipWhiteListVer); +int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); +int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb); +int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view); +int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic); int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew); void mndUserFreeObj(SUserObj *pUser); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index c47e2e03a7..a019533133 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -95,11 +95,11 @@ #define ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(_type, _priv) \ (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_SUBSCRIBE_PRIV(_priv)) -static SIpWhiteList *createDefaultIpWhiteList(); -SIpWhiteList *createIpWhiteList(void *buf, int32_t len); -static bool updateIpWhiteList(SIpWhiteList *pOld, SIpWhiteList *pNew); -static bool isIpWhiteListEqual(SIpWhiteList *a, SIpWhiteList *b); -static bool isIpRangeEqual(SIpV4Range *a, SIpV4Range *b); +static int32_t createDefaultIpWhiteList(SIpWhiteList **ppWhiteList); +SIpWhiteList *createIpWhiteList(void *buf, int32_t len); +static bool updateIpWhiteList(SIpWhiteList *pOld, SIpWhiteList *pNew); +static bool isIpWhiteListEqual(SIpWhiteList *a, SIpWhiteList *b); +static bool isIpRangeEqual(SIpV4Range *a, SIpV4Range *b); void destroyIpWhiteTab(SHashObj *pIpWhiteTab); @@ -645,20 +645,24 @@ SIpWhiteList *createIpWhiteList(void *buf, int32_t len) { return p; } -static SIpWhiteList *createDefaultIpWhiteList() { - SIpWhiteList *pWhiteList = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range) * 1); - pWhiteList->num = 1; - SIpV4Range *range = &(pWhiteList->pIpRange[0]); +static int32_t createDefaultIpWhiteList(SIpWhiteList **ppWhiteList) { + *ppWhiteList = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range) * 1); + if (*ppWhiteList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (*ppWhiteList)->num = 1; + SIpV4Range *range = &((*ppWhiteList)->pIpRange[0]); struct in_addr addr; if (uv_inet_pton(AF_INET, "127.0.0.1", &addr) == 0) { range->ip = addr.s_addr; range->mask = 32; } - return pWhiteList; + return 0; } static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) { + int32_t code = 0; SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)pass, strlen(pass), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); @@ -668,7 +672,7 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char userObj.sysInfo = 1; userObj.enable = 1; userObj.ipWhiteListVer = taosGetTimestampMs(); - userObj.pIpWhiteList = createDefaultIpWhiteList(); + TAOS_CHECK_RETURN(createDefaultIpWhiteList(&userObj.pIpWhiteList)); if (strcmp(user, TSDB_DEFAULT_USER) == 0) { userObj.superUser = 1; userObj.createdb = 1; @@ -830,7 +834,10 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { } SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); - if (pRaw == NULL) goto _OVER; + if (pRaw == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } int32_t dataPos = 0; SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER) @@ -973,6 +980,10 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t num = pUser->pIpWhiteList->num; int32_t tlen = sizeof(SIpWhiteList) + num * sizeof(SIpV4Range) + 4; char *buf = taosMemoryCalloc(1, tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } int32_t len = tSerializeIpWhiteList(buf, tlen, pUser->pIpWhiteList); SDB_SET_INT32(pRaw, dataPos, len, _OVER); @@ -1306,12 +1317,11 @@ static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser) { } int32_t mndDupTableHash(SHashObj *pOld, SHashObj **ppNew) { + int32_t code = 0; *ppNew = taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (*ppNew == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - TAOS_CHECK_RETURN(NULL); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } char *tb = taosHashIterate(pOld, NULL); @@ -1320,24 +1330,24 @@ int32_t mndDupTableHash(SHashObj *pOld, SHashObj **ppNew) { char *key = taosHashGetKey(tb, &keyLen); int32_t valueLen = strlen(tb) + 1; - if (taosHashPut(pNew, key, keyLen, tb, valueLen) != 0) { + if ((code = taosHashPut(*ppNew, key, keyLen, tb, valueLen)) != 0) { taosHashCancelIterate(pOld, tb); - taosHashCleanup(pNew); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + taosHashCleanup(*ppNew); + code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; // TODO: remove this line after terrno is removed + TAOS_RETURN(code); } tb = taosHashIterate(pOld, tb); } - return pNew; + TAOS_RETURN(code); } -SHashObj *mndDupUseDbHash(SHashObj *pOld) { - SHashObj *pNew = +int32_t mndDupUseDbHash(SHashObj *pOld, SHashObj **ppNew) { + int32_t code = 0; + *ppNew = taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (pNew == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + if (*ppNew == NULL) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } int32_t *db = taosHashIterate(pOld, NULL); @@ -1345,42 +1355,43 @@ SHashObj *mndDupUseDbHash(SHashObj *pOld) { size_t keyLen = 0; char *key = taosHashGetKey(db, &keyLen); - if (taosHashPut(pNew, key, keyLen, db, sizeof(*db)) != 0) { + if ((code = taosHashPut(*ppNew, key, keyLen, db, sizeof(*db))) != 0) { taosHashCancelIterate(pOld, db); - taosHashCleanup(pNew); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + taosHashCleanup(*ppNew); + code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; // TODO: remove this line after terrno is removed + TAOS_RETURN(code); } db = taosHashIterate(pOld, db); } - return pNew; + TAOS_RETURN(code); } int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { + int32_t code = 0; memcpy(pNew, pUser, sizeof(SUserObj)); pNew->authVersion++; pNew->updateTime = taosGetTimestampMs(); taosRLockLatch(&pUser->lock); - pNew->readDbs = mndDupDbHash(pUser->readDbs); - pNew->writeDbs = mndDupDbHash(pUser->writeDbs); - pNew->readTbs = mndDupTableHash(pUser->readTbs); - pNew->writeTbs = mndDupTableHash(pUser->writeTbs); - pNew->alterTbs = mndDupTableHash(pUser->alterTbs); - pNew->readViews = mndDupTableHash(pUser->readViews); - pNew->writeViews = mndDupTableHash(pUser->writeViews); - pNew->alterViews = mndDupTableHash(pUser->alterViews); - pNew->topics = mndDupTopicHash(pUser->topics); - pNew->useDbs = mndDupUseDbHash(pUser->useDbs); + TAOS_CHECK_GOTO(mndDupDbHash(pUser->readDbs, &pNew->readDbs), NULL, _OVER); + TAOS_CHECK_GOTO(mndDupDbHash(pUser->writeDbs, &pNew->writeDbs), NULL, _OVER); + TAOS_CHECK_GOTO(mndDupTableHash(pUser->readTbs, &pNew->readTbs), NULL, _OVER); + TAOS_CHECK_GOTO(mndDupTableHash(pUser->writeTbs, &pNew->writeTbs), NULL, _OVER); + TAOS_CHECK_GOTO(mndDupTableHash(pUser->alterTbs, &pNew->alterTbs), NULL, _OVER); + TAOS_CHECK_GOTO(mndDupTableHash(pUser->readViews, &pNew->readViews), NULL, _OVER); + TAOS_CHECK_GOTO(mndDupTableHash(pUser->writeViews, &pNew->writeViews), NULL, _OVER); + TAOS_CHECK_GOTO(mndDupTableHash(pUser->alterViews, &pNew->alterViews), NULL, _OVER); + TAOS_CHECK_GOTO(mndDupTopicHash(pUser->topics, &pNew->topics), NULL, _OVER); + TAOS_CHECK_GOTO(mndDupUseDbHash(pUser->useDbs, &pNew->useDbs), NULL, _OVER); pNew->pIpWhiteList = cloneIpWhiteList(pUser->pIpWhiteList); - - taosRUnLockLatch(&pUser->lock); - - if (pNew->readDbs == NULL || pNew->writeDbs == NULL || pNew->topics == NULL) { - return -1; + if (pNew->pIpWhiteList == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; } - return 0; + +_OVER: + taosRUnLockLatch(&pUser->lock); + TAOS_RETURN(code); } void mndUserFreeObj(SUserObj *pUser) { @@ -1436,6 +1447,10 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { int32_t sz = sizeof(SIpWhiteList) + pNew->pIpWhiteList->num * sizeof(SIpV4Range); pOld->pIpWhiteList = taosMemoryRealloc(pOld->pIpWhiteList, sz); + if (pOld->pIpWhiteList == NULL) { + taosWUnLockLatch(&pOld->lock); + return TSDB_CODE_OUT_OF_MEMORY; + } memcpy(pOld->pIpWhiteList, pNew->pIpWhiteList, sz); pOld->ipWhiteListVer = pNew->ipWhiteListVer; @@ -1464,6 +1479,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { } static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SRpcMsg *pReq) { + int32_t code = 0; SUserObj userObj = {0}; if (pCreate->isImport != 1) { taosEncryptPass_c((uint8_t *)pCreate->pass, strlen(pCreate->pass), userObj.pass); @@ -1481,14 +1497,16 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate userObj.createdb = pCreate->createDb; if (pCreate->numIpRanges == 0) { - userObj.pIpWhiteList = createDefaultIpWhiteList(); - + TAOS_CHECK_RETURN(createDefaultIpWhiteList(&userObj.pIpWhiteList)); } else { SHashObj *pUniqueTab = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); + if(pUniqueTab == NULL){ + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } int32_t dummpy = 0; for (int i = 0; i < pCreate->numIpRanges; i++) { SIpV4Range range = {.ip = pCreate->pIpRanges[i].ip, .mask = pCreate->pIpRanges[i].mask}; - taosHashPut(pUniqueTab, &range, sizeof(range), &dummpy, sizeof(dummpy)); + if(code = taosHashPut(pUniqueTab, &range, sizeof(range), &dummpy, sizeof(dummpy)); } taosHashPut(pUniqueTab, &defaultIpRange, sizeof(defaultIpRange), &dummpy, sizeof(dummpy)); @@ -2575,6 +2593,7 @@ static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int3 } static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + int32_t code = 0; SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -2806,15 +2825,15 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ SUserAuthBatchRsp batchRsp = {0}; batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp)); if (batchRsp.pArray == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } int32_t code = 0; for (int32_t i = 0; i < numOfUses; ++i) { - SUserObj *pUser = mndAcquireUser(pMnode, pUsers[i].user); + SUserObj *pUser = NULL; + code = mndAcquireUser(pMnode, pUsers[i].user, &pUser); if (pUser == NULL) { - if (TSDB_CODE_MND_USER_NOT_EXIST == terrno) { + if (TSDB_CODE_MND_USER_NOT_EXIST == code) { SGetUserAuthRsp rsp = {.dropped = 1}; memcpy(rsp.user, pUsers[i].user, TSDB_USER_LEN); taosArrayPush(batchRsp.pArray, &rsp); @@ -2837,7 +2856,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ goto _OVER; } - taosArrayPush(batchRsp.pArray, &rsp); + (void)taosArrayPush(batchRsp.pArray, &rsp); mndReleaseUser(pMnode, pUser); } @@ -2852,9 +2871,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ int32_t rspLen = tSerializeSUserAuthBatchRsp(NULL, 0, &batchRsp); void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tFreeSUserAuthBatchRsp(&batchRsp); - return -1; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); } tSerializeSUserAuthBatchRsp(pRsp, rspLen, &batchRsp); @@ -2870,7 +2887,7 @@ _OVER: *pRspLen = 0; tFreeSUserAuthBatchRsp(&batchRsp); - return code; + TAOS_RETURN(code); } int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { @@ -2885,8 +2902,7 @@ int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pUser); if (pIter == NULL) break; - code = -1; - if (mndUserDupObj(pUser, &newUser) != 0) { + if ((code = mndUserDupObj(pUser, &newUser)) != 0) { break; } @@ -2897,7 +2913,8 @@ int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { (void)taosHashRemove(newUser.writeDbs, db, len); SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + if (pCommitRaw == NULL || (code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) { + code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; // TODO: remove this line when terrno is removed break; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -2905,13 +2922,12 @@ int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { mndUserFreeObj(&newUser); sdbRelease(pSdb, pUser); - code = 0; } if (pUser != NULL) sdbRelease(pSdb, pUser); if (pIter != NULL) sdbCancelFetch(pSdb, pIter); mndUserFreeObj(&newUser); - return code; + TAOS_RETURN(code); } int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb) { @@ -2926,8 +2942,7 @@ int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb) { pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pUser); if (pIter == NULL) break; - code = -1; - if (mndUserDupObj(pUser, &newUser) != 0) { + if ((code = mndUserDupObj(pUser, &newUser)) != 0) { break; } @@ -2940,7 +2955,8 @@ int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb) { (void)taosHashRemove(newUser.alterTbs, stb, len); SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + if (pCommitRaw == NULL || (code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) { + code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; // TODO: remove this line when terrno is removed break; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -2948,13 +2964,12 @@ int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb) { mndUserFreeObj(&newUser); sdbRelease(pSdb, pUser); - code = 0; } if (pUser != NULL) sdbRelease(pSdb, pUser); if (pIter != NULL) sdbCancelFetch(pSdb, pIter); mndUserFreeObj(&newUser); - return code; + TAOS_RETURN(code); } int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view) { @@ -2969,8 +2984,7 @@ int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view) { pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pUser); if (pIter == NULL) break; - code = -1; - if (mndUserDupObj(pUser, &newUser) != 0) { + if ((code = mndUserDupObj(pUser, &newUser)) != 0) { break; } @@ -2983,7 +2997,8 @@ int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view) { (void)taosHashRemove(newUser.alterViews, view, len); SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + if (pCommitRaw == NULL || (code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) { + code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; // TODO: remove this line when terrno is removed break; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -2991,13 +3006,12 @@ int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view) { mndUserFreeObj(&newUser); sdbRelease(pSdb, pUser); - code = 0; } if (pUser != NULL) sdbRelease(pSdb, pUser); if (pIter != NULL) sdbCancelFetch(pSdb, pIter); mndUserFreeObj(&newUser); - return code; + TAOS_RETURN(code); } int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { @@ -3014,8 +3028,7 @@ int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { break; } - code = -1; - if (mndUserDupObj(pUser, &newUser) != 0) { + if ((code = mndUserDupObj(pUser, &newUser)) != 0) { break; } @@ -3023,7 +3036,8 @@ int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { if (inTopic) { (void)taosHashRemove(newUser.topics, topic, len); SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + if (pCommitRaw == NULL || (code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) { + code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; // TODO: remove this line when terrno is removed break; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -3031,13 +3045,12 @@ int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { mndUserFreeObj(&newUser); sdbRelease(pSdb, pUser); - code = 0; } if (pUser != NULL) sdbRelease(pSdb, pUser); if (pIter != NULL) sdbCancelFetch(pSdb, pIter); mndUserFreeObj(&newUser); - return code; + TAOS_RETURN(code); } int64_t mndGetUserIpWhiteListVer(SMnode *pMnode, SUserObj *pUser) { diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index b3bde3abba..3c2b67da01 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -74,16 +74,16 @@ typedef struct STfs { SHashObj *hash; // name to did map } STfs; -STfsDisk *tfsNewDisk(int32_t level, int32_t id, int8_t disable, const char *dir); +int32_t tfsNewDisk(int32_t level, int32_t id, int8_t disable, const char *dir, STfsDisk **ppDisk); STfsDisk *tfsFreeDisk(STfsDisk *pDisk); int32_t tfsUpdateDiskSize(STfsDisk *pDisk); -int32_t tfsInitTier(STfsTier *pTier, int32_t level); -void tfsDestroyTier(STfsTier *pTier); -STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg); -void tfsUpdateTierSize(STfsTier *pTier); -int32_t tfsAllocDiskOnTier(STfsTier *pTier); -void tfsPosNextId(STfsTier *pTier); +int32_t tfsInitTier(STfsTier *pTier, int32_t level); +void tfsDestroyTier(STfsTier *pTier); +int32_t tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg, STfsDisk **ppDisk); +void tfsUpdateTierSize(STfsTier *pTier); +int32_t tfsAllocDiskOnTier(STfsTier *pTier); +void tfsPosNextId(STfsTier *pTier); #define tfsLockTier(pTier) taosThreadSpinLock(&(pTier)->lock) #define tfsUnLockTier(pTier) taosThreadSpinUnlock(&(pTier)->lock) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 1c64885bf5..3432b2a069 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -495,7 +495,9 @@ static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg) { } SDiskID did = {.level = pCfg->level}; - STfsDisk *pDisk = tfsMountDiskToTier(TFS_TIER_AT(pTfs, did.level), pCfg); + STfsDisk *pDisk = NULL; + + tfsMountDiskToTier(TFS_TIER_AT(pTfs, did.level), pCfg, &pDisk); if (pDisk == NULL) { fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, terrstr()); return -1; @@ -515,33 +517,28 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { if (pCfg->level < 0 || pCfg->level >= TFS_MAX_TIERS) { fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level); - terrno = TSDB_CODE_FS_INVLD_CFG; - return -1; + TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); } if (pCfg->primary < 0 || pCfg->primary > 1) { fError("failed to mount %s to FS since invalid primary %d", pCfg->dir, pCfg->primary); - terrno = TSDB_CODE_FS_INVLD_CFG; - return -1; + TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); } if (pCfg->disable < 0 || pCfg->disable > 1) { fError("failed to mount %s to FS since invalid disable %" PRIi8, pCfg->dir, pCfg->disable); - terrno = TSDB_CODE_FS_INVLD_CFG; - return -1; + TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); } if (pCfg->primary) { if (pCfg->level != 0) { fError("failed to mount %s to FS since disk is primary but level %d not 0", pCfg->dir, pCfg->level); - terrno = TSDB_CODE_FS_INVLD_CFG; - return -1; + TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); } if (TFS_PRIMARY_DISK(pTfs) != NULL) { fError("failed to mount %s to FS since duplicate primary mount", pCfg->dir); - terrno = TSDB_CODE_FS_DUP_PRIMARY; - return -1; + TAOS_RETURN(TSDB_CODE_FS_DUP_PRIMARY); } } @@ -578,15 +575,14 @@ static int32_t tfsFormatDir(char *idir, char *odir) { int32_t code = wordexp(idir, &wep, 0); if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; + TAOS_RETURN(TAOS_SYSTEM_ERROR(code)); } char tmp[PATH_MAX] = {0}; if (taosRealPath(wep.we_wordv[0], tmp, PATH_MAX) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(errno); wordfree(&wep); - return -1; + TAOS_RETURN(code); } strcpy(odir, tmp); @@ -597,23 +593,20 @@ static int32_t tfsFormatDir(char *idir, char *odir) { static int32_t tfsCheck(STfs *pTfs) { if (TFS_PRIMARY_DISK(pTfs) == NULL) { fError("no primary disk is set"); - terrno = TSDB_CODE_FS_NO_PRIMARY_DISK; - return -1; + TAOS_RETURN(TSDB_CODE_FS_NO_PRIMARY_DISK); } for (int32_t level = 0; level < pTfs->nlevel; level++) { if (TFS_TIER_AT(pTfs, level)->ndisk == 0) { fError("no disk at level %d", level); - terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER; - return -1; + TAOS_RETURN(TSDB_CODE_FS_NO_MOUNT_AT_TIER); } if (level == 0) { tfsUpdateTierSize(TFS_TIER_AT(pTfs, level)); if (TFS_TIER_AT(pTfs, level)->nAvailDisks == 0) { fError("no disk to create new file at level %d", level); - terrno = TSDB_CODE_FS_NO_VALID_DISK; - return -1; + TAOS_RETURN(TSDB_CODE_FS_NO_VALID_DISK); } } } @@ -684,7 +677,9 @@ static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter) { int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo) { pInfo->datadirs = taosArrayInit(32, sizeof(SMonDiskDesc)); - if (pInfo->datadirs == NULL) return -1; + if (pInfo->datadirs == NULL) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } tfsUpdateSize(pTfs); @@ -697,7 +692,12 @@ int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo) { dinfo.size = pDisk->size; dinfo.level = pDisk->level; tstrncpy(dinfo.name, pDisk->path, sizeof(dinfo.name)); - taosArrayPush(pInfo->datadirs, &dinfo); + if (taosArrayPush(pInfo->datadirs, &dinfo) == NULL) { + tfsUnLock(pTfs); + taosArrayDestroy(pInfo->datadirs); + pInfo->datadirs = NULL; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } } tfsUnLock(pTfs); diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index 06eb1cd42e..424eb886ba 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -16,25 +16,34 @@ #define _DEFAULT_SOURCE #include "tfsInt.h" -STfsDisk *tfsNewDisk(int32_t level, int32_t id, int8_t disable, const char *path) { - STfsDisk *pDisk = taosMemoryCalloc(1, sizeof(STfsDisk)); - if (pDisk == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; +int32_t tfsNewDisk(int32_t level, int32_t id, int8_t disable, const char *path, STfsDisk **ppDisk) { + int32_t code = 0; + int32_t lino = 0; + STfsDisk *pDisk = NULL; + + if ((pDisk = taosMemoryCalloc(1, sizeof(STfsDisk))) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - pDisk->path = taosStrdup(path); - if (pDisk->path == NULL) { - taosMemoryFree(pDisk); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + if ((pDisk->path = taosStrdup(path)) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } pDisk->level = level; pDisk->id = id; pDisk->disable = disable; - taosGetDiskSize(pDisk->path, &pDisk->size); - return pDisk; + if (taosGetDiskSize(pDisk->path, &pDisk->size) < 0) { + code = TAOS_SYSTEM_ERROR(errno); // TODO: refactor this line + TAOS_CHECK_GOTO(code, &lino, _OVER); + } +_OVER: + if (code != 0) { + pDisk = tfsFreeDisk(pDisk); + fError("%s failed at line %d since %s, disk:%s level:%d id:%d ", __func__, lino, tstrerror(code), path, level, id); + } + *ppDisk = pDisk; + + TAOS_RETURN(code); } STfsDisk *tfsFreeDisk(STfsDisk *pDisk) { @@ -48,9 +57,10 @@ STfsDisk *tfsFreeDisk(STfsDisk *pDisk) { int32_t tfsUpdateDiskSize(STfsDisk *pDisk) { if (taosGetDiskSize(pDisk->path, &pDisk->size) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - fError("failed to get disk:%s size, level:%d id:%d since %s", pDisk->path, pDisk->level, pDisk->id, terrstr()); - return -1; + int32_t code = TAOS_SYSTEM_ERROR(errno); // TODO: refactor this line + fError("failed to get disk:%s size, level:%d id:%d since %s", pDisk->path, pDisk->level, pDisk->id, + tstrerror(code)); + TAOS_RETURN(code); } return 0; diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index cb79c334bf..a0fece3691 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -22,8 +22,7 @@ int32_t tfsInitTier(STfsTier *pTier, int32_t level) { memset(pTier, 0, sizeof(STfsTier)); if (taosThreadSpinInit(&pTier->lock, 0) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } pTier->level = level; @@ -39,10 +38,13 @@ void tfsDestroyTier(STfsTier *pTier) { taosThreadSpinDestroy(&pTier->lock); } -STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg) { +int32_t tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg, STfsDisk **ppDisk) { + int32_t code = 0; + int32_t lino = 0; + STfsDisk *pDisk = NULL; + if (pTier->ndisk >= TFS_MAX_DISKS_PER_TIER) { - terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; - return NULL; + TAOS_CHECK_GOTO(TSDB_CODE_FS_TOO_MANY_MOUNT, &lino, _OVER); } int32_t id = 0; @@ -61,18 +63,25 @@ STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg) { } if (id >= TFS_MAX_DISKS_PER_TIER) { - terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; - return NULL; + TAOS_CHECK_GOTO(TSDB_CODE_FS_TOO_MANY_MOUNT, &lino, _OVER); } - STfsDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->disable, pCfg->dir); - if (pDisk == NULL) return NULL; + TAOS_CHECK_GOTO(tfsNewDisk(pCfg->level, id, pCfg->disable, pCfg->dir, &pDisk), &lino, _OVER); pTier->disks[id] = pDisk; pTier->ndisk++; - fDebug("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id); - return pTier->disks[id]; +_OVER: + if (code != 0) { + pDisk = tfsFreeDisk(pDisk); + fError("%s failed at line %d since %s, disk:%s level:%d id:%d", __func__, lino, tstrerror(code), pCfg->dir, + pCfg->level, id); + } else { + *ppDisk = pTier->disks[id]; + fDebug("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id); + } + + TAOS_RETURN(code); } void tfsUpdateTierSize(STfsTier *pTier) { @@ -100,13 +109,11 @@ void tfsUpdateTierSize(STfsTier *pTier) { // Round-Robin to allocate disk on a tier int32_t tfsAllocDiskOnTier(STfsTier *pTier) { - terrno = TSDB_CODE_FS_NO_VALID_DISK; - tfsLockTier(pTier); if (pTier->ndisk <= 0 || pTier->nAvailDisks <= 0) { tfsUnLockTier(pTier); - return -1; + TAOS_RETURN(TSDB_CODE_FS_NO_VALID_DISK); } int32_t retId = -1; @@ -149,6 +156,9 @@ int32_t tfsAllocDiskOnTier(STfsTier *pTier) { } tfsUnLockTier(pTier); + if (retId < 0) { + TAOS_RETURN(TSDB_CODE_FS_NO_VALID_DISK); + } return retId; } From cd8a54cbf7dfd9adb80091323950eec814c8540f Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 21 Jul 2024 11:43:39 +0800 Subject: [PATCH 03/52] enh: return error code --- include/libs/tfs/tfs.h | 6 +- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 7 +- source/dnode/mnode/impl/src/mndUser.c | 16 +- source/dnode/vnode/src/sma/smaCommit.c | 13 +- source/dnode/vnode/src/tsdb/tsdbFS2.c | 8 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 3 +- source/dnode/vnode/test/tsdbSmaTest.cpp | 2 +- source/libs/tfs/src/tfs.c | 218 ++++++++++++------------ source/libs/tfs/src/tfsDisk.c | 8 +- source/libs/tfs/src/tfsTier.c | 8 +- source/libs/tfs/test/tfsTest.cpp | 33 ++-- 11 files changed, 167 insertions(+), 155 deletions(-) diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 2b90e3226c..446c1d6fd7 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -44,9 +44,9 @@ typedef struct { * * @param pCfg Config of the fs. * @param ndisk Length of the config. - * @return STfs* The fs object. + * @param ppTfs The fs object. */ -STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk); +int32_t tfsOpen(SDiskCfg *pCfg, int32_t ndisk, STfs **ppTfs); /** * @brief Close a fs. @@ -275,7 +275,7 @@ int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2); * @param rname The rel name of file. * @return STfsDir* The dir object. */ -STfsDir *tfsOpendir(STfs *pTfs, const char *rname); +int32_t tfsOpendir(STfs *pTfs, const char *rname, STfsDir **ppDir); /** * @brief Get a file from dir and move to next pos. diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index f636629b3a..fa73b608e5 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -110,6 +110,7 @@ static bool dmCheckDiskSpace() { } int32_t dmDiskInit() { + int32_t code = 0; SDnode *pDnode = dmInstance(); SDiskCfg dCfg = {.level = 0, .primary = 1, .disable = 0}; tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN); @@ -120,10 +121,10 @@ int32_t dmDiskInit() { numOfDisks = 1; } - pDnode->pTfs = tfsOpen(pDisks, numOfDisks); + code = tfsOpen(pDisks, numOfDisks, &pDnode->pTfs); if (pDnode->pTfs == NULL) { - dError("failed to init tfs since %s", terrstr()); - return -1; + dError("failed to init tfs since %s", tstrerror(code)); + TAOS_RETURN(code); } return 0; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index a019533133..12c365ce46 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -648,7 +648,7 @@ SIpWhiteList *createIpWhiteList(void *buf, int32_t len) { static int32_t createDefaultIpWhiteList(SIpWhiteList **ppWhiteList) { *ppWhiteList = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range) * 1); if (*ppWhiteList == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } (*ppWhiteList)->num = 1; SIpV4Range *range = &((*ppWhiteList)->pIpRange[0]); @@ -722,7 +722,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { } SSdbRaw *mndUserActionEncode(SUserObj *pUser) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = TSDB_CODE_OUT_OF_MEMORY; int32_t ipWhiteReserve = pUser->pIpWhiteList ? (sizeof(SIpV4Range) * pUser->pIpWhiteList->num + sizeof(SIpWhiteList) + 4) : 16; @@ -835,8 +835,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); if (pRaw == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); } int32_t dataPos = 0; @@ -981,8 +980,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t tlen = sizeof(SIpWhiteList) + num * sizeof(SIpV4Range) + 4; char *buf = taosMemoryCalloc(1, tlen); if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); } int32_t len = tSerializeIpWhiteList(buf, tlen, pUser->pIpWhiteList); @@ -995,11 +993,11 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) - terrno = 0; + code = 0; _OVER: - if (terrno != 0) { - mError("user:%s, failed to encode to raw:%p since %s", pUser->user, pRaw, terrstr()); + if (code != 0) { + mError("user:%s, failed to encode to raw:%p since %s", pUser->user, pRaw, tstrerror(code)); sdbFreeRaw(pRaw); return NULL; } diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 3512f1476f..e707f2150d 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -99,8 +99,7 @@ int32_t smaBegin(SSma *pSma) { } } _exit: - terrno = code; - return code; + TAOS_RETURN(code); } extern int32_t tsdbCommitCommit(STsdb *tsdb); @@ -119,7 +118,7 @@ _exit: if (code) { smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } - return code; + TAOS_RETURN(code); } /** @@ -202,7 +201,7 @@ _exit: if (code) { smaError("vgId:%d, %s failed at line %d since %s(%d)", SMA_VID(pSma), __func__, lino, tstrerror(code), isCommit); } - return code; + TAOS_RETURN(code); } /** @@ -229,7 +228,7 @@ _exit: if (code) { smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } - return code; + TAOS_RETURN(code); } /** @@ -241,7 +240,7 @@ _exit: static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); if (!pEnv) { - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); @@ -276,5 +275,5 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 9b8a979d01..7742e55db7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -523,12 +523,8 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { } { // clear unreferenced files - STfsDir *dir = tfsOpendir(fs->tsdb->pVnode->pTfs, fs->tsdb->path); - if (dir == NULL) { - code = TAOS_SYSTEM_ERROR(terrno); - lino = __LINE__; - goto _exit; - } + STfsDir *dir = NULL; + TAOS_CHECK_GOTO(tfsOpendir(fs->tsdb->pVnode->pTfs, fs->tsdb->path, &dir), &lino, _exit); STFileHash fobjHash = {0}; code = tsdbFSCreateFileObjHash(fs, &fobjHash); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index ea9209c6b4..a2b39e5284 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -175,7 +175,8 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr snprintf(tsdbFilePrefix, TSDB_FILENAME_LEN, "tsdb%sv", TD_DIRSEP); int32_t prefixLen = strlen(tsdbFilePrefix); - STfsDir *tsdbDir = tfsOpendir(pTfs, tsdbPath); + STfsDir *tsdbDir = NULL; + (void)tfsOpendir(pTfs, tsdbPath, &tsdbDir); if (tsdbDir == NULL) return 0; while (1) { diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 8b19c0dc95..15e8d666c1 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -371,7 +371,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { pDisks.disable = 0; strncpy(pDisks.dir, TD_DATA_DIR_PATH, TSDB_FILENAME_LEN); int32_t numOfDisks = 1; - pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks); + (void)tfsOpen(&pDisks, numOfDisks, &pTsdb->pTfs); EXPECT_NE(pTsdb->pTfs, nullptr); // generate SSubmitReq msg and update expire window diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 3432b2a069..07c1fab54f 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -21,61 +21,56 @@ static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg); static int32_t tfsCheck(STfs *pTfs); static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg); static int32_t tfsFormatDir(char *idir, char *odir); -static STfsDisk *tfsGetDiskByName(STfs *pTfs, const char *dir); +static int32_t tfsGetDiskByName(STfs *pTfs, const char *dir, STfsDisk **ppDisk); static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir); static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter); -STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk) { +int32_t tfsOpen(SDiskCfg *pCfg, int32_t ndisk, STfs **ppTfs) { + int32_t code = 0; + int32_t lino = 0; + STfs *pTfs = NULL; + if (ndisk <= 0 || ndisk > TFS_MAX_DISKS) { - terrno = TSDB_CODE_INVALID_PARA; - return NULL; + TAOS_CHECK_GOTO(TSDB_CODE_INVALID_PARA, &lino, _exit); } - STfs *pTfs = taosMemoryCalloc(1, sizeof(STfs)); + pTfs = taosMemoryCalloc(1, sizeof(STfs)); if (pTfs == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (taosThreadSpinInit(&pTfs->lock, 0) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - tfsClose(pTfs); - return NULL; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } for (int32_t level = 0; level < TFS_MAX_TIERS; level++) { STfsTier *pTier = &pTfs->tiers[level]; - if (tfsInitTier(pTier, level) < 0) { - tfsClose(pTfs); - return NULL; - } + TAOS_CHECK_GOTO(tfsInitTier(pTier, level), &lino, _exit); } pTfs->hash = taosHashInit(TFS_MAX_DISKS * 2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (pTfs->hash == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tfsClose(pTfs); - return NULL; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } for (int32_t idisk = 0; idisk < ndisk; idisk++) { - if (tfsMount(pTfs, &pCfg[idisk]) < 0) { - tfsClose(pTfs); - return NULL; - } + TAOS_CHECK_GOTO(tfsMount(pTfs, &pCfg[idisk]), &lino, _exit); } - if (tfsCheck(pTfs) < 0) { - tfsClose(pTfs); - return NULL; - } + TAOS_CHECK_GOTO(tfsCheck(pTfs) < 0, &lino, _exit); tfsUpdateSize(pTfs); for (int32_t level = 0; level < pTfs->nlevel; level++) { tfsPosNextId(&pTfs->tiers[level]); } - return pTfs; +_exit: + if (code != 0) { + tfsClose(pTfs); + pTfs = NULL; + } + *ppTfs = pTfs; + TAOS_RETURN(code); } void tfsClose(STfs *pTfs) { @@ -149,7 +144,7 @@ bool tfsDiskSpaceSufficient(STfs *pTfs, int32_t level, int32_t disk) { int32_t tfsGetDisksAtLevel(STfs *pTfs, int32_t level) { if (level < 0 || level >= pTfs->nlevel) { - return 0; + TAOS_RETURN(0); } STfsTier *pTier = TFS_TIER_AT(pTfs, level); @@ -177,10 +172,10 @@ int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) { continue; } - return (terrno = 0); + TAOS_RETURN(0); } - return (terrno = TSDB_CODE_FS_NO_VALID_DISK); + TAOS_RETURN(TSDB_CODE_FS_NO_VALID_DISK); } const char *tfsGetPrimaryPath(STfs *pTfs) { return TFS_PRIMARY_DISK(pTfs)->path; } @@ -270,22 +265,27 @@ int32_t tfsMkdirAt(STfs *pTfs, const char *rname, SDiskID diskId) { char aname[TMPNAME_LEN]; if (pDisk == NULL) { - return -1; + TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); } snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname); if (taosMkDir(aname) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } - return 0; + TAOS_RETURN(0); } int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { - if (tfsMkdirAt(pTfs, rname, diskId) < 0) { + int32_t code = 0; + int32_t lino = 0; + char *s = NULL; + char *dir = NULL; + if ((code = tfsMkdirAt(pTfs, rname, diskId)) < 0) { if (errno == ENOENT) { // Try to create upper - char *s = taosStrdup(rname); + if ((s = taosStrdup(rname)) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } // Make a copy of dirname(s) because the implementation of 'dirname' differs on different platforms. // Some platform may modify the contents of the string passed into dirname(). Others may return a pointer to @@ -293,25 +293,30 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { // the pointer directly in this recursion. // See // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html - char *dir = taosStrdup(taosDirName(s)); - if (strlen(dir) >= strlen(rname) || tfsMkdirRecurAt(pTfs, dir, diskId) < 0) { - taosMemoryFree(s); - taosMemoryFree(dir); - return -1; + if ((dir = taosStrdup(taosDirName(s))) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } - taosMemoryFree(s); - taosMemoryFree(dir); - if (tfsMkdirAt(pTfs, rname, diskId) < 0) { - return -1; + if (strlen(dir) >= strlen(rname)) { // TODO: check if it is necessary for equal length + TAOS_CHECK_GOTO(TSDB_CODE_APP_ERROR, &lino, _exit); } + TAOS_CHECK_GOTO(tfsMkdirRecurAt(pTfs, dir, diskId), &lino, _exit); + TAOS_CHECK_GOTO(tfsMkdirAt(pTfs, rname, diskId), &lino, _exit); } else { - return -1; + TAOS_CHECK_GOTO(code, &lino, _exit); } + } else { + TAOS_RETURN(code); } - return 0; +_exit: + if (code != 0) { + fError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + taosMemoryFree(s); + taosMemoryFree(dir); + TAOS_RETURN(code); } int32_t tfsMkdirRecur(STfs *pTfs, const char *rname) { @@ -319,13 +324,11 @@ int32_t tfsMkdirRecur(STfs *pTfs, const char *rname) { STfsTier *pTier = TFS_TIER_AT(pTfs, level); for (int32_t id = 0; id < pTier->ndisk; id++) { SDiskID did = {.id = id, .level = level}; - if (tfsMkdirRecurAt(pTfs, rname, did) < 0) { - return -1; - } + TAOS_CHECK_RETURN(tfsMkdirRecurAt(pTfs, rname, did)); } } - return 0; + TAOS_RETURN(0); } int32_t tfsMkdir(STfs *pTfs, const char *rname) { @@ -333,13 +336,11 @@ int32_t tfsMkdir(STfs *pTfs, const char *rname) { STfsTier *pTier = TFS_TIER_AT(pTfs, level); for (int32_t id = 0; id < pTier->ndisk; id++) { SDiskID did = {.id = id, .level = level}; - if (tfsMkdirAt(pTfs, rname, did) < 0) { - return -1; - } + TAOS_CHECK_RETURN(tfsMkdirAt(pTfs, rname, did)); } } - return 0; + TAOS_RETURN(0); } bool tfsDirExistAt(STfs *pTfs, const char *rname, SDiskID diskId) { @@ -352,7 +353,7 @@ bool tfsDirExistAt(STfs *pTfs, const char *rname, SDiskID diskId) { int32_t tfsRmdir(STfs *pTfs, const char *rname) { if (rname[0] == 0) { - return 0; + TAOS_RETURN(0); } char aname[TMPNAME_LEN] = "\0"; @@ -367,7 +368,7 @@ int32_t tfsRmdir(STfs *pTfs, const char *rname) { } } - return 0; + TAOS_RETURN(0); } static int32_t tfsRenameAt(STfs *pTfs, SDiskID diskId, const char *orname, const char *nrname) { @@ -382,12 +383,12 @@ static int32_t tfsRenameAt(STfs *pTfs, SDiskID diskId, const char *orname, const snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname); if (taosRenameFile(oaname, naname) != 0 && errno != ENOENT) { - terrno = TAOS_SYSTEM_ERROR(errno); - fError("failed to rename %s to %s since %s", oaname, naname, terrstr()); - return -1; + int32_t code = TAOS_SYSTEM_ERROR(errno); + fError("%s failed to rename %s to %s since %s", __func__, oaname, naname, tstrerror(code)); + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(0); } int32_t tfsRename(STfs *pTfs, int32_t diskPrimary, const char *orname, const char *nrname) { @@ -399,14 +400,12 @@ int32_t tfsRename(STfs *pTfs, int32_t diskPrimary, const char *orname, const cha } SDiskID diskId = {.level = level, .id = id}; - if (tfsRenameAt(pTfs, diskId, orname, nrname)) { - return -1; - } + TAOS_CHECK_RETURN(tfsRenameAt(pTfs, diskId, orname, nrname)); } } SDiskID diskId = {.level = 0, .id = diskPrimary}; - return tfsRenameAt(pTfs, diskId, orname, nrname); + TAOS_RETURN(tfsRenameAt(pTfs, diskId, orname, nrname)); } int32_t tfsSearch(STfs *pTfs, int32_t level, const char *fname) { @@ -426,11 +425,11 @@ int32_t tfsSearch(STfs *pTfs, int32_t level, const char *fname) { return -1; } -STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { +int32_t tfsOpendir(STfs *pTfs, const char *rname, STfsDir **ppDir) { + int32_t code = 0; STfsDir *pDir = taosMemoryCalloc(1, sizeof(STfsDir)); if (pDir == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exit); } SDiskID diskId = {.id = 0, .level = 0}; @@ -438,12 +437,11 @@ STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { pDir->pTfs = pTfs; tstrncpy(pDir->dirName, rname, TSDB_FILENAME_LEN); - if (tfsOpendirImpl(pTfs, pDir) < 0) { - taosMemoryFree(pDir); - return NULL; - } + TAOS_CHECK_GOTO(tfsOpendirImpl(pTfs, pDir), NULL, _exit); - return pDir; +_exit: + taosMemoryFree(pDir); + TAOS_RETURN(code); } const STfsFile *tfsReaddir(STfsDir *pTfsDir) { @@ -490,30 +488,35 @@ void tfsClosedir(STfsDir *pTfsDir) { } static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg) { - if (tfsCheckAndFormatCfg(pTfs, pCfg) < 0) { - return -1; - } + int32_t code = 0; + int32_t lino = 0; + + TAOS_CHECK_GOTO(tfsCheckAndFormatCfg(pTfs, pCfg), &lino, _exit); SDiskID did = {.level = pCfg->level}; STfsDisk *pDisk = NULL; - - tfsMountDiskToTier(TFS_TIER_AT(pTfs, did.level), pCfg, &pDisk); - if (pDisk == NULL) { - fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, terrstr()); - return -1; - } + + TAOS_CHECK_GOTO(tfsMountDiskToTier(TFS_TIER_AT(pTfs, did.level), pCfg, &pDisk), &lino, _exit); did.id = pDisk->id; - taosHashPut(pTfs->hash, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did)); + if (taosHashPut(pTfs->hash, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), + sizeof(did)) != 0) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } if (pTfs->nlevel < pCfg->level + 1) { pTfs->nlevel = pCfg->level + 1; } - return 0; +_exit: + if (code != 0) { + fError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + TAOS_RETURN(code); } static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { - char dirName[TSDB_FILENAME_LEN] = "\0"; + int32_t code = 0; + char dirName[TSDB_FILENAME_LEN] = "\0"; if (pCfg->level < 0 || pCfg->level >= TFS_MAX_TIERS) { fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level); @@ -542,32 +545,34 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { } } - if (tfsFormatDir(pCfg->dir, dirName) < 0) { - fError("failed to mount %s to FS since %s", pCfg->dir, terrstr()); - return -1; + if ((code = tfsFormatDir(pCfg->dir, dirName)) < 0) { + fError("failed to mount %s to FS since %s", pCfg->dir, tstrerror(code)); + TAOS_RETURN(code); } - if (tfsGetDiskByName(pTfs, dirName) != NULL) { + STfsDisk *pDisk = NULL; + if ((code = tfsGetDiskByName(pTfs, dirName, NULL)) != 0) { + fError("failed to mount %s to FS since %s", pCfg->dir, tstrerror(code)); + TAOS_RETURN(code); + } + if (pDisk != NULL) { fError("failed to mount %s to FS since duplicate mount", pCfg->dir); - terrno = TSDB_CODE_FS_INVLD_CFG; - return -1; + TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); } if (!taosCheckAccessFile(dirName, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) { fError("failed to mount %s to FS since no R/W access rights", pCfg->dir); - terrno = TSDB_CODE_FS_INVLD_CFG; - return -1; + TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); } if (!taosIsDir(dirName)) { fError("failed to mount %s to FS since not a directory", pCfg->dir); - terrno = TSDB_CODE_FS_INVLD_CFG; - return -1; + TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); } strncpy(pCfg->dir, dirName, TSDB_FILENAME_LEN); - return 0; + TAOS_RETURN(0); } static int32_t tfsFormatDir(char *idir, char *odir) { @@ -587,7 +592,7 @@ static int32_t tfsFormatDir(char *idir, char *odir) { strcpy(odir, tmp); wordfree(&wep); - return 0; + TAOS_RETURN(0); } static int32_t tfsCheck(STfs *pTfs) { @@ -611,17 +616,19 @@ static int32_t tfsCheck(STfs *pTfs) { } } - return 0; + TAOS_RETURN(0); } -static STfsDisk *tfsGetDiskByName(STfs *pTfs, const char *dir) { +static int32_t tfsGetDiskByName(STfs *pTfs, const char *dir, STfsDisk **ppDisk) { void *pr = taosHashGet(pTfs->hash, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN)); - if (pr == NULL) return NULL; + if (pr == NULL) { + TAOS_RETURN(terrno); + } - SDiskID did = *(SDiskID *)pr; - STfsDisk *pDisk = TFS_DISK_AT(pTfs, did); + SDiskID did = *(SDiskID *)pr; + *ppDisk = TFS_DISK_AT(pTfs, did); - return pDisk; + TAOS_RETURN(0); } static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pTfsDir) { @@ -635,7 +642,7 @@ static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pTfsDir) { while (true) { pDisk = tfsNextDisk(pTfs, &pTfsDir->iter); - if (pDisk == NULL) return 0; + if (pDisk == NULL) TAOS_RETURN(0); pTfsDir->did.level = pDisk->level; pTfsDir->did.id = pDisk->id; @@ -647,9 +654,10 @@ static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pTfsDir) { } pTfsDir->pDir = taosOpenDir(adir); if (pTfsDir->pDir != NULL) break; + fWarn("%s failed to open dir %s since %s", __func__, adir, tstrerror(TAOS_SYSTEM_ERROR(errno))); } - return 0; + TAOS_RETURN(0); } static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter) { @@ -702,5 +710,5 @@ int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo) { } tfsUnLock(pTfs); - return 0; + TAOS_RETURN(0); } diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index 424eb886ba..07fa8d79b9 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -22,11 +22,11 @@ int32_t tfsNewDisk(int32_t level, int32_t id, int8_t disable, const char *path, STfsDisk *pDisk = NULL; if ((pDisk = taosMemoryCalloc(1, sizeof(STfsDisk))) == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if ((pDisk->path = taosStrdup(path)) == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } pDisk->level = level; @@ -34,9 +34,9 @@ int32_t tfsNewDisk(int32_t level, int32_t id, int8_t disable, const char *path, pDisk->disable = disable; if (taosGetDiskSize(pDisk->path, &pDisk->size) < 0) { code = TAOS_SYSTEM_ERROR(errno); // TODO: refactor this line - TAOS_CHECK_GOTO(code, &lino, _OVER); + TAOS_CHECK_GOTO(code, &lino, _exit); } -_OVER: +_exit: if (code != 0) { pDisk = tfsFreeDisk(pDisk); fError("%s failed at line %d since %s, disk:%s level:%d id:%d ", __func__, lino, tstrerror(code), path, level, id); diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index a0fece3691..685f95c517 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -44,7 +44,7 @@ int32_t tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg, STfsDisk **ppDisk) { STfsDisk *pDisk = NULL; if (pTier->ndisk >= TFS_MAX_DISKS_PER_TIER) { - TAOS_CHECK_GOTO(TSDB_CODE_FS_TOO_MANY_MOUNT, &lino, _OVER); + TAOS_CHECK_GOTO(TSDB_CODE_FS_TOO_MANY_MOUNT, &lino, _exit); } int32_t id = 0; @@ -63,15 +63,15 @@ int32_t tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg, STfsDisk **ppDisk) { } if (id >= TFS_MAX_DISKS_PER_TIER) { - TAOS_CHECK_GOTO(TSDB_CODE_FS_TOO_MANY_MOUNT, &lino, _OVER); + TAOS_CHECK_GOTO(TSDB_CODE_FS_TOO_MANY_MOUNT, &lino, _exit); } - TAOS_CHECK_GOTO(tfsNewDisk(pCfg->level, id, pCfg->disable, pCfg->dir, &pDisk), &lino, _OVER); + TAOS_CHECK_GOTO(tfsNewDisk(pCfg->level, id, pCfg->disable, pCfg->dir, &pDisk), &lino, _exit); pTier->disks[id] = pDisk; pTier->ndisk++; -_OVER: +_exit: if (code != 0) { pDisk = tfsFreeDisk(pDisk); fError("%s failed at line %d since %s, disk:%s level:%d id:%d", __func__, lino, tstrerror(code), pCfg->dir, diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index d8c117818e..bb89fbe69f 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -40,11 +40,12 @@ TEST_F(TfsTest, 01_Open_Close) { dCfg.disable = 0; taosRemoveDir(root); - STfs *pTfs = tfsOpen(&dCfg, 1); + STfs *pTfs = NULL; + (void)tfsOpen(&dCfg, 1, &pTfs); ASSERT_EQ(pTfs, nullptr); taosMulMkDir(root); - pTfs = tfsOpen(&dCfg, 1); + (void)tfsOpen(&dCfg, 1, &pTfs); ASSERT_NE(pTfs, nullptr); tfsUpdateSize(pTfs); @@ -68,7 +69,8 @@ TEST_F(TfsTest, 02_AllocDisk) { taosRemoveDir(root); taosMkDir(root); - STfs *pTfs = tfsOpen(&dCfg, 1); + STfs *pTfs = NULL; + (void)tfsOpen(&dCfg, 1, &pTfs); ASSERT_NE(pTfs, nullptr); SDiskID did; @@ -120,7 +122,8 @@ TEST_F(TfsTest, 03_Dir) { taosRemoveDir(root); taosMkDir(root); - STfs *pTfs = tfsOpen(&dCfg, 1); + STfs *pTfs = NULL; + (void)tfsOpen(&dCfg, 1, &pTfs); ASSERT_NE(pTfs, nullptr); char p1[] = "p1"; @@ -175,7 +178,8 @@ TEST_F(TfsTest, 04_File) { taosRemoveDir(root); taosMkDir(root); - STfs *pTfs = tfsOpen(&dCfg, 1); + STfs *pTfs = NULL; + (void)tfsOpen(&dCfg, 1, &pTfs); ASSERT_NE(pTfs, nullptr); STfsFile file0; @@ -264,7 +268,8 @@ TEST_F(TfsTest, 04_File) { EXPECT_NE(taosDirExist(af2), 1); { - STfsDir *pDir = tfsOpendir(pTfs, "t3"); + STfsDir *pDir = NULL; + tfsOpendir(pTfs, "t3", &pDir); const STfsFile *pf1 = tfsReaddir(pDir); EXPECT_NE(pf1, nullptr); @@ -281,7 +286,8 @@ TEST_F(TfsTest, 04_File) { EXPECT_GT(tfsCopyFile(&f1, &f2), 0); { - STfsDir *pDir = tfsOpendir(pTfs, "t3"); + STfsDir *pDir = NULL; + tfsOpendir(pTfs, "t3", &pDir); const STfsFile *pf1 = tfsReaddir(pDir); EXPECT_NE(pf1, nullptr); @@ -386,17 +392,18 @@ TEST_F(TfsTest, 05_MultiDisk) { taosMkDir(root22); taosMkDir(root23); - STfs *pTfs = tfsOpen(dCfg, 9); + STfs *pTfs = NULL; + (void)tfsOpen(dCfg, 9, &pTfs); ASSERT_EQ(pTfs, nullptr); dCfg[0].primary = 1; dCfg[1].primary = 1; - pTfs = tfsOpen(dCfg, 9); + (void)tfsOpen(dCfg, 9, &pTfs); ASSERT_EQ(pTfs, nullptr); dCfg[0].primary = 0; dCfg[1].primary = 1; - pTfs = tfsOpen(dCfg, 9); + (void)tfsOpen(dCfg, 9, &pTfs); ASSERT_NE(pTfs, nullptr); tfsUpdateSize(pTfs); @@ -693,7 +700,8 @@ TEST_F(TfsTest, 05_MultiDisk) { tfsRemoveFile(&f2); { - STfsDir *pDir = tfsOpendir(pTfs, "t3"); + STfsDir *pDir = NULL; + tfsOpendir(pTfs, "t3", &pDir); const STfsFile *pf1 = tfsReaddir(pDir); EXPECT_NE(pf1, nullptr); @@ -711,7 +719,8 @@ TEST_F(TfsTest, 05_MultiDisk) { EXPECT_GT(tfsCopyFile(&f1, &f2), 0); { - STfsDir *pDir = tfsOpendir(pTfs, "t3"); + STfsDir *pDir = NULL; + tfsOpendir(pTfs, "t3", &pDir); const STfsFile *pf1 = tfsReaddir(pDir); EXPECT_NE(pf1, nullptr); From 98a1145091cc0ad9d50e5fcdef841e53be3a7a12 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 21 Jul 2024 23:45:59 +0800 Subject: [PATCH 04/52] enh: return error code --- source/common/src/trow.c | 288 ++++++----------------- source/dnode/mnode/impl/inc/mndShow.h | 7 +- source/dnode/mnode/impl/src/mndCluster.c | 19 +- source/dnode/mnode/impl/src/mndUser.c | 34 ++- source/dnode/mnode/sdb/src/sdb.c | 5 +- source/dnode/vnode/src/sma/smaEnv.c | 109 +++++---- source/dnode/vnode/src/sma/smaSnapshot.c | 43 ++-- source/dnode/vnode/src/sma/smaUtil.c | 7 +- 8 files changed, 191 insertions(+), 321 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 3c38c89e7f..075f29454f 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -17,8 +17,8 @@ #include "trow.h" #include "tlog.h" -static bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); -static bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal); +static bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); +static bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal); void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) { pIter->pSchema = pSchema; @@ -110,8 +110,7 @@ bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; @@ -131,55 +130,11 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa *pValType = ((*pDestByte) & 0x03); break; default: - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } -#if 0 -int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { - if (!pBitmap || colIdx < 0) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; - int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pValType = (((*pDestByte) & 0x80) >> 7); - break; - case 1: - *pValType = (((*pDestByte) & 0x40) >> 6); - break; - case 2: - *pValType = (((*pDestByte) & 0x20) >> 5); - break; - case 3: - *pValType = (((*pDestByte) & 0x10) >> 4); - break; - case 4: - *pValType = (((*pDestByte) & 0x08) >> 3); - break; - case 5: - *pValType = (((*pDestByte) & 0x04) >> 2); - break; - case 6: - *pValType = (((*pDestByte) & 0x02) >> 1); - break; - case 7: - *pValType = ((*pDestByte) & 0x01); - break; - default: - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} -#endif - int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { switch (bitmapMode) { case 0: @@ -192,10 +147,9 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT break; #endif default: - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal) { @@ -416,7 +370,6 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl return true; } - bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { if (pIter->colIdx >= pIter->pSchema->numOfCols) { return false; @@ -452,6 +405,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r int32_t maxVarDataLen = 0; int32_t iColVal = 0; int32_t nBound = 0; + int32_t code = 0; void *varBuf = NULL; bool isAlloc = false; @@ -481,7 +435,8 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r } } else { varDataLen += sizeof(VarDataLenT); - if (pTColumn->type == TSDB_DATA_TYPE_VARCHAR || pTColumn->type == TSDB_DATA_TYPE_VARBINARY || pTColumn->type == TSDB_DATA_TYPE_GEOMETRY) { + if (pTColumn->type == TSDB_DATA_TYPE_VARCHAR || pTColumn->type == TSDB_DATA_TYPE_VARBINARY || + pTColumn->type == TSDB_DATA_TYPE_GEOMETRY) { varDataLen += CHAR_BYTES; if (maxVarDataLen < CHAR_BYTES + sizeof(VarDataLenT)) { maxVarDataLen = CHAR_BYTES + sizeof(VarDataLenT); @@ -494,7 +449,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r } } } else { - if(pColVal && COL_VAL_IS_VALUE(pColVal)) { + if (pColVal && COL_VAL_IS_VALUE(pColVal)) { nonVarDataLen += TYPE_BYTES[pTColumn->type]; } } @@ -516,8 +471,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r } if (!(*ppRow)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } if (maxVarDataLen > 0) { @@ -526,8 +480,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r if (isAlloc) { taosMemoryFreeClear(*ppRow); } - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } } @@ -567,18 +520,28 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r } if (TD_IS_TP_ROW(rb.pBuf)) { - tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal); + TAOS_CHECK_GOTO( + tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal), + NULL, _exit); } else { - tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, rb.offset, iBound - 1); + TAOS_CHECK_GOTO( + tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, rb.offset, iBound - 1), NULL, + _exit); } ++iColVal; } tdSRowEnd(&rb); +_exit: taosMemoryFreeClear(varBuf); + if (code != 0) { + if (isAlloc) { + taosMemoryFreeClear(*ppRow); + } + } - return 0; + TAOS_RETURN(code); } static FORCE_INLINE int32_t tdCompareColId(const void *arg1, const void *arg2) { @@ -632,83 +595,31 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell return true; } -#if 0 -int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { - if (!pBitmap || colIdx < 0) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; - int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7); - // set the value and clear other partitions for offset 0 - // *pDestByte |= (valType << 7); - break; - case 1: - *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6); - // *pDestByte |= (valType << 6); - break; - case 2: - *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5); - // *pDestByte |= (valType << 5); - break; - case 3: - *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4); - // *pDestByte |= (valType << 4); - break; - case 4: - *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3); - // *pDestByte |= (valType << 3); - break; - case 5: - *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2); - // *pDestByte |= (valType << 2); - break; - case 6: - *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1); - // *pDestByte |= (valType << 1); - break; - case 7: - *pDestByte = ((*pDestByte) & 0xFE) | valType; - // *pDestByte |= (valType); - break; - default: - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} -#endif int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { #ifdef TD_SUPPORT_BITMAP ASSERT(colIdx < tdRowGetNCols(pRow) - 1); - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { + int32_t code = 0; + if ((code = tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0)) != TSDB_CODE_SUCCESS) { output->valType = TD_VTYPE_NONE; - return terrno; + TAOS_RETURN(code); } if (tdValTypeIsNorm(output->valType)) { if (offset < 0) { - terrno = TSDB_CODE_INVALID_PARA; output->valType = TD_VTYPE_NONE; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } output->val = POINTER_SHIFT(pRow, offset); } #else if (offset < 0) { - terrno = TSDB_CODE_INVALID_PARA; output->valType = TD_VTYPE_NONE; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } output->val = POINTER_SHIFT(pRow, offset); output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; #endif - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, @@ -720,12 +631,13 @@ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t } else { output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { + int32_t code = 0; + if ((code = tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0)) != TSDB_CODE_SUCCESS) { output->valType = TD_VTYPE_NONE; - return terrno; + TAOS_RETURN(code); } if (output->valType == TD_VTYPE_NORM) { @@ -736,7 +648,7 @@ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t } } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, TDRowValT valType, const void *val, @@ -745,23 +657,21 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp if (!val) { #ifdef TD_SUPPORT_BITMAP if (valType == TD_VTYPE_NORM) { - terrno = TSDB_CODE_INVALID_PTR; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } #else - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + d #endif } // TS KEY is stored in STSRow.ts and not included in STSRow.data field. if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { if (!val) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } TD_ROW_KEY(pRow) = *(TSKEY *)val; // The primary TS key is Norm all the time, thus its valType is not stored in bitmap. - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. @@ -773,10 +683,9 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp break; case TD_VTYPE_NONE: if (!pBuilder->hasNone) pBuilder->hasNone = true; - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); default: - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } if (TD_IS_TP_ROW(pRow)) { @@ -784,22 +693,19 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp } else { tdAppendColValToKvRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset, colId); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) { if (colIdx < 1) { - terrno = TSDB_CODE_INVALID_PARA; ASSERTS(0, "colIdx is %" PRIi64, colIdx); - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } --colIdx; #ifdef TD_SUPPORT_BITMAP - if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { - return terrno; - } + TAOS_CHECK_RETURN(tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0)); #endif STSRow *row = pBuilder->pBuf; @@ -821,21 +727,18 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo } } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset) { if (colIdx < 1) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } --colIdx; #ifdef TD_SUPPORT_BITMAP - if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { - return terrno; - } + TAOS_CHECK_RETURN(tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0)); #endif STSRow *row = pBuilder->pBuf; @@ -857,51 +760,13 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo } } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } -#if 0 -int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, - int32_t allNullLen, int32_t boundNullLen) { - if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) { - uint32_t tpLen = allNullLen; - uint32_t kvLen = sizeof(col_id_t) + sizeof(SKvRowIdx) * nBoundCols + boundNullLen; - if (isSelectKVRow(kvLen, tpLen)) { - pBuilder->rowType = TD_ROW_KV; - } else { - pBuilder->rowType = TD_ROW_TP; - } - - } else { - pBuilder->rowType = TD_ROW_TP; - } - pBuilder->flen = flen; - pBuilder->nCols = nCols; - pBuilder->nBoundCols = nBoundCols; - if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } -#ifdef TD_SUPPORT_BITMAP - // the primary TS key is stored separatedly - pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); - if (nBoundCols > 0) { - pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); - } else { - pBuilder->nBoundBitmaps = 0; - } -#else - pBuilder->nBitmaps = 0; - pBuilder->nBoundBitmaps = 0; -#endif - return TSDB_CODE_SUCCESS; -} -#endif int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } if (pBuilder->hasNone) pBuilder->hasNone = false; @@ -937,18 +802,16 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->offset = 0; break; default: - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } ASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); @@ -966,10 +829,9 @@ int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { #endif break; default: - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } void tdSRowReset(SRowBuilder *pBuilder) { @@ -985,8 +847,7 @@ int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) { pBuilder->flen = flen; pBuilder->nCols = nCols; if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } #ifdef TD_SUPPORT_BITMAP // the primary TS key is stored separatedly @@ -995,7 +856,7 @@ int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) { pBuilder->nBitmaps = 0; pBuilder->nBoundBitmaps = 0; #endif - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen) { @@ -1003,8 +864,7 @@ int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, pBuilder->nCols = nCols; pBuilder->nBoundCols = nBoundCols; if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } #ifdef TD_SUPPORT_BITMAP // the primary TS key is stored separatedly @@ -1018,24 +878,12 @@ int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, pBuilder->nBitmaps = 0; pBuilder->nBoundBitmaps = 0; #endif - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } -#if 0 -bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) { - TDRowValT valType = 0; - tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode); - if (tdValTypeIsNorm(valType)) { - return true; - } - return false; -} -#endif - int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { if (!pBitmap || colIdx < 0) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; @@ -1060,10 +908,9 @@ int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { // *pDestByte |= (valType); break; default: - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode) { @@ -1078,14 +925,11 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int break; #endif default: - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } - - void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal) { STColumn *pTColumn = &pTSchema->columns[iCol]; SCellVal cv = {0}; diff --git a/source/dnode/mnode/impl/inc/mndShow.h b/source/dnode/mnode/impl/inc/mndShow.h index de7068e9ad..c47b5d1e7b 100644 --- a/source/dnode/mnode/impl/inc/mndShow.h +++ b/source/dnode/mnode/impl/inc/mndShow.h @@ -23,11 +23,14 @@ extern "C" { #endif -#define COL_DATA_SET_VAL_RET(pData, isNull, pObj) \ +#define COL_DATA_SET_VAL_GOTO(pData, isNull, pObj, LINO) \ do { \ if ((code = colDataSetVal(pColInfo, numOfRows, (pData), (isNull))) != 0) { \ if (pObj) sdbRelease(pSdb, (pObj)); \ - return code; \ + if (LINO) { \ + *((int32_t *)(LINO)) = __LINE__; \ + } \ + goto _OVER; \ } \ } while (0) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 74e0940cba..26d2bc2c44 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -281,6 +281,7 @@ static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock * SMnode *pMnode = pMsg->info.node; SSdb *pSdb = pMnode->pSdb; int32_t code = 0; + int32_t lino = 0; int32_t numOfRows = 0; int32_t cols = 0; SClusterObj *pCluster = NULL; @@ -291,31 +292,31 @@ static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock * cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - COL_DATA_SET_VAL_RET((const char *)&pCluster->id, false, pCluster); + COL_DATA_SET_VAL_GOTO((const char *)&pCluster->id, false, pCluster, &lino); char buf[tListLen(pCluster->name) + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(buf, pCluster->name, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - COL_DATA_SET_VAL_RET(buf, false, pCluster); + COL_DATA_SET_VAL_GOTO(buf, false, pCluster, &lino); int32_t upTime = mndGetClusterUpTimeImp(pCluster); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - COL_DATA_SET_VAL_RET((const char *)&upTime, false, pCluster); + COL_DATA_SET_VAL_GOTO((const char *)&upTime, false, pCluster, &lino); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - COL_DATA_SET_VAL_RET((const char *)&pCluster->createdTime, false, pCluster); + COL_DATA_SET_VAL_GOTO((const char *)&pCluster->createdTime, false, pCluster, &lino); char ver[12] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(ver, tsVersionName, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - COL_DATA_SET_VAL_RET((const char *)ver, false, pCluster); + COL_DATA_SET_VAL_GOTO((const char *)ver, false, pCluster, &lino); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (tsExpireTime <= 0) { colDataSetNULL(pColInfo, numOfRows); } else { - COL_DATA_SET_VAL_RET((const char *)&tsExpireTime, false, pCluster); + COL_DATA_SET_VAL_GOTO((const char *)&tsExpireTime, false, pCluster, &lino); } sdbRelease(pSdb, pCluster); @@ -323,6 +324,12 @@ static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock * } pShow->numOfRows += numOfRows; + +_OVER: + if (code != 0) { + mError("failed to retrieve cluster info at line %d since %s", lino, tstrerror(code)); + TAOS_RETURN(code); + } return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 12c365ce46..2335bf7bf8 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -123,7 +123,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBloc static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter); SHashObj *mndFetchAllIpWhite(SMnode *pMnode); static int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq); -bool mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type); +static int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type, int8_t *pUpdate); void ipWhiteMgtUpdateAll(SMnode *pMnode); typedef struct { @@ -136,8 +136,11 @@ static SIpWhiteMgt ipWhiteMgt; const static SIpV4Range defaultIpRange = {.ip = 16777343, .mask = 32}; -void ipWhiteMgtInit() { +static int32_t ipWhiteMgtInit() { ipWhiteMgt.pIpWhiteTab = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), 1, HASH_ENTRY_LOCK); + if (ipWhiteMgt.pIpWhiteTab == NULL) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } ipWhiteMgt.ver = 0; taosThreadRwlockInit(&ipWhiteMgt.rw, NULL); } @@ -147,13 +150,21 @@ void ipWhiteMgtCleanup() { } int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { - bool update = true; + int32_t code = 0; + int32_t lino = 0; + bool update = true; taosThreadRwlockWrlock(&ipWhiteMgt.rw); SIpWhiteList **ppList = taosHashGet(ipWhiteMgt.pIpWhiteTab, user, strlen(user)); if (ppList == NULL || *ppList == NULL) { SIpWhiteList *p = cloneIpWhiteList(pNew); - taosHashPut(ipWhiteMgt.pIpWhiteTab, user, strlen(user), &p, sizeof(void *)); + if (p == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } + if ((code = taosHashPut(ipWhiteMgt.pIpWhiteTab, user, strlen(user), &p, sizeof(void *))) != 0) { + taosMemoryFree(p); + TAOS_CHECK_GOTO(code, &lino, _OVER); + } } else { SIpWhiteList *pOld = *ppList; if (isIpWhiteListEqual(pOld, pNew)) { @@ -161,7 +172,13 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { } else { taosMemoryFree(pOld); SIpWhiteList *p = cloneIpWhiteList(pNew); - taosHashPut(ipWhiteMgt.pIpWhiteTab, user, strlen(user), &p, sizeof(void *)); + if (p == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } + if ((code = taosHashPut(ipWhiteMgt.pIpWhiteTab, user, strlen(user), &p, sizeof(void *))) != 0) { + taosMemoryFree(p); + TAOS_CHECK_GOTO(code, &lino, _OVER); + } } } SArray *fqdns = mndGetAllDnodeFqdns(pMnode); @@ -185,6 +202,7 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { if (update) ipWhiteMgt.ver++; +_OVER: taosThreadRwlockUnlock(&ipWhiteMgt.rw); return 0; } @@ -212,6 +230,7 @@ bool isRangeInWhiteList(SIpWhiteList *pList, SIpV4Range *range) { } return false; } +#if 0 int32_t ipWhiteUpdateForAllUser(SIpWhiteList *pList) { taosThreadRwlockWrlock(&ipWhiteMgt.rw); @@ -256,6 +275,7 @@ int32_t ipWhiteUpdateForAllUser(SIpWhiteList *pList) { taosThreadRwlockUnlock(&ipWhiteMgt.rw); return 0; } +#endif void ipWhiteMgtUpdateAll(SMnode *pMnode) { ipWhiteMgt.ver++; @@ -292,7 +312,7 @@ int64_t mndGetIpWhiteVer(SMnode *pMnode) { return ver; } -bool mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type) { +int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type, bool *pUpdate) { bool update = false; SIpV4Range range = {.ip = taosGetIpv4FromFqdn(fqdn), .mask = 32}; mDebug("ip-white-list may update for user: %s, fqdn: %s", user, fqdn); @@ -500,7 +520,7 @@ SHashObj *mndFetchAllIpWhite(SMnode *pMnode) { } int32_t mndInitUser(SMnode *pMnode) { - ipWhiteMgtInit(); + TAOS_CHECK_REUTNR(ipWhiteMgtInit()); SSdbTable table = { .sdbType = SDB_USER, diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index c4b32fe87c..2e4790dac8 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -134,15 +134,14 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { SHashObj *hash = taosHashInit(64, taosGetDefaultHashFunction(hashType), true, HASH_ENTRY_LOCK); if (hash == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } pSdb->maxId[sdbType] = 0; pSdb->hashObjs[sdbType] = hash; mInfo("sdb table:%s is initialized", sdbTableName(sdbType)); - return 0; + TAOS_RETURN(0); } static int32_t sdbCreateDir(SSdb *pSdb) { diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index dd12f2bca2..b3e072f823 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -39,6 +39,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat); */ // implementation int32_t smaInit() { + int32_t code = 0; int8_t old; int32_t nLoops = 0; while (1) { @@ -56,8 +57,9 @@ int32_t smaInit() { if (smaMgmt.rsetId < 0) { atomic_store_8(&smaMgmt.inited, 0); - smaError("failed to init sma rset since %s", terrstr()); - return TSDB_CODE_FAILED; + code = terrno; + smaError("failed to init sma rset since %s", tstrerror(code)); + TAOS_RETURN(code); } int32_t type = (8 == POINTER_BYTES) ? TSDB_DATA_TYPE_UBIGINT : TSDB_DATA_TYPE_UINT; @@ -66,21 +68,22 @@ int32_t smaInit() { smaMgmt.tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); if (!smaMgmt.refHash || !smaMgmt.tmrHandle) { + code = terrno; taosCloseRef(smaMgmt.rsetId); if (smaMgmt.refHash) { taosHashCleanup(smaMgmt.refHash); smaMgmt.refHash = NULL; } atomic_store_8(&smaMgmt.inited, 0); - smaError("failed to init sma tmr handle since %s", terrstr()); - return TSDB_CODE_FAILED; + smaError("failed to init sma tmr handle since %s", tstrerror(code)); + TAOS_RETURN(code); } atomic_store_8(&smaMgmt.inited, 1); smaInfo("sma mgmt env is initialized, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(code); } /** @@ -110,13 +113,13 @@ void smaCleanUp() { } static int32_t tdNewSmaEnv(SSma *pSma, int8_t smaType, SSmaEnv **ppEnv) { + int32_t code = 0; SSmaEnv *pEnv = NULL; pEnv = (SSmaEnv *)taosMemoryCalloc(1, sizeof(SSmaEnv)); *ppEnv = pEnv; if (!pEnv) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } SMA_ENV_TYPE(pEnv) = smaType; @@ -126,26 +129,23 @@ static int32_t tdNewSmaEnv(SSma *pSma, int8_t smaType, SSmaEnv **ppEnv) { (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), *ppEnv) : atomic_store_ptr(&SMA_RSMA_ENV(pSma), *ppEnv); - if ((terrno = tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma)) != TSDB_CODE_SUCCESS) { + if ((code = tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma)) != TSDB_CODE_SUCCESS) { tdFreeSmaEnv(pEnv); *ppEnv = NULL; (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), NULL) : atomic_store_ptr(&SMA_RSMA_ENV(pSma), NULL); - return TSDB_CODE_FAILED; + TAOS_RETURN(code); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(code); } static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, SSmaEnv **ppEnv) { - if (!(*ppEnv)) { - if (tdNewSmaEnv(pSma, smaType, ppEnv) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_FAILED; - } + TAOS_CHECK_RETURN(tdNewSmaEnv(pSma, smaType, ppEnv)); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } /** @@ -187,9 +187,8 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS int32_t code = 0; int32_t lino = 0; - - if (*pSmaStat) { // no lock - return code; // success, return directly + if (*pSmaStat) { // no lock + TAOS_RETURN(code); // success, return directly } /** @@ -201,7 +200,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat) + sizeof(TdThread) * tsNumOfVnodeRsmaThreads); if (!(*pSmaStat)) { code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(code, &lino, _exit); } if (smaType == TSDB_SMA_TYPE_ROLLUP) { @@ -211,20 +210,20 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS tsem_init(&pRSmaStat->notEmpty, 0, 0); if (!(pRSmaStat->blocks = taosArrayInit(1, sizeof(SSDataBlock)))) { code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(code, &lino, _exit); } SSDataBlock datablock = {.info.type = STREAM_CHECKPOINT}; - taosArrayPush(pRSmaStat->blocks, &datablock); + (void)taosArrayPush(pRSmaStat->blocks, &datablock); // init smaMgmt smaInit(); int64_t refId = taosAddRef(smaMgmt.rsetId, pRSmaStat); if (refId < 0) { - smaError("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d failed since:%s", SMA_VID(pSma), - refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM, tstrerror(terrno)); code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); + smaError("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d failed since:%s", SMA_VID(pSma), + refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM, tstrerror(code)); + TAOS_CHECK_GOTO(code, &lino, _exit); } else { smaDebug("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d succeed", SMA_VID(pSma), refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM); @@ -236,14 +235,11 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (!RSMA_INFO_HASH(pRSmaStat)) { code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(code, &lino, _exit); } taosHashSetFreeFp(RSMA_INFO_HASH(pRSmaStat), tRSmaInfoHashFreeNode); - if (tdRsmaStartExecutor(pSma) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } + TAOS_CHECK_GOTO(tdRsmaStartExecutor(pSma), &lino, _exit); taosInitRWLatch(RSMA_FS_LOCK(pRSmaStat)); } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { @@ -256,7 +252,7 @@ _exit: } else { smaDebug("vgId:%d, %s succeed, type:%" PRIi8, SMA_VID(pSma), __func__, smaType); } - return code; + TAOS_RETURN(code); } static void tdDestroyTSmaStat(STSmaStat *pStat) { @@ -340,45 +336,45 @@ static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { } int32_t tdLockSma(SSma *pSma) { - int code = taosThreadMutexLock(&pSma->mutex); - if (code != 0) { - smaError("vgId:%d, failed to lock since %s", SMA_VID(pSma), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; + int errCode = taosThreadMutexLock(&pSma->mutex); + if (errCode != 0) { + int32_t code = TAOS_SYSTEM_ERROR(errCode); + smaError("vgId:%d, failed to lock since %s", SMA_VID(pSma), tstrerror(code)); + TAOS_RETURN(code); } pSma->locked = true; - return 0; + TAOS_RETURN(0); } int32_t tdUnLockSma(SSma *pSma) { - pSma->locked = false; - int code = taosThreadMutexUnlock(&pSma->mutex); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - smaError("vgId:%d, failed to unlock since %s", SMA_VID(pSma), strerror(errno)); - return -1; + int errCode = taosThreadMutexUnlock(&pSma->mutex); + if (errCode != 0) { + int32_t code = TAOS_SYSTEM_ERROR(errCode); + smaError("vgId:%d, failed to unlock since %s", SMA_VID(pSma), tstrerror(code)); + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { + int32_t code = 0; SSmaEnv *pEnv = NULL; switch (smaType) { case TSDB_SMA_TYPE_TIME_RANGE: if ((pEnv = (SSmaEnv *)atomic_load_ptr(&SMA_TSMA_ENV(pSma)))) { - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } break; case TSDB_SMA_TYPE_ROLLUP: if ((pEnv = (SSmaEnv *)atomic_load_ptr(&SMA_RSMA_ENV(pSma)))) { - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } break; default: smaError("vgId:%d, undefined smaType:%" PRIi8, SMA_VID(pSma), smaType); - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } // init sma env @@ -386,15 +382,15 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&SMA_TSMA_ENV(pSma)) : atomic_load_ptr(&SMA_RSMA_ENV(pSma)); if (!pEnv) { - if (tdInitSmaEnv(pSma, smaType, &pEnv) < 0) { + if ((code = tdInitSmaEnv(pSma, smaType, &pEnv)) < 0) { tdUnLockSma(pSma); - return TSDB_CODE_FAILED; + TAOS_RETURN(code); } } tdUnLockSma(pSma); - return TSDB_CODE_SUCCESS; -}; + TAOS_RETURN(TSDB_CODE_SUCCESS); +} void *tdRSmaExecutorFunc(void *param) { setThreadName("vnode-rsma"); @@ -404,6 +400,7 @@ void *tdRSmaExecutorFunc(void *param) { } static int32_t tdRsmaStartExecutor(const SSma *pSma) { + int32_t code = 0; TdThreadAttr thAttr = {0}; taosThreadAttrInit(&thAttr); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); @@ -414,15 +411,15 @@ static int32_t tdRsmaStartExecutor(const SSma *pSma) { for (int32_t i = 0; i < tsNumOfVnodeRsmaThreads; ++i) { if (taosThreadCreate(&pthread[i], &thAttr, tdRSmaExecutorFunc, (void *)pSma) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - smaError("vgId:%d, failed to create pthread for rsma since %s", SMA_VID(pSma), terrstr()); - return -1; + code = TAOS_SYSTEM_ERROR(errno); + smaError("vgId:%d, failed to create pthread for rsma since %s", SMA_VID(pSma), tstrerror(code)); + TAOS_RETURN(code); } smaDebug("vgId:%d, success to create pthread for rsma", SMA_VID(pSma)); } taosThreadAttrDestroy(&thAttr); - return 0; + TAOS_RETURN(code); } static int32_t tdRsmaStopExecutor(const SSma *pSma) { @@ -433,7 +430,7 @@ static int32_t tdRsmaStopExecutor(const SSma *pSma) { TdThread *pthread = NULL; if (!(pEnv = SMA_RSMA_ENV(pSma)) || !(pStat = SMA_ENV_STAT(pEnv))) { - return 0; + TAOS_RETURN(0); } pEnv->flag |= SMA_ENV_FLG_CLOSE; @@ -453,5 +450,5 @@ static int32_t tdRsmaStopExecutor(const SSma *pSma) { smaInfo("vgId:%d, rsma executor stopped, number:%d", SMA_VID(pSma), tsNumOfVnodeRsmaThreads); } - return 0; + TAOS_RETURN(0); } diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index c93d9a7de6..cd8a334f4a 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -38,8 +38,7 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead // alloc pReader = (SRSmaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } pReader->pSma = pSma; pReader->sver = sver; @@ -50,7 +49,7 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead if (pSma->pRSmaTsdb[i]) { code = tsdbSnapReaderOpen(pSma->pRSmaTsdb[i], sver, ever, (i == 0 ? SNAP_DATA_RSMA1 : SNAP_DATA_RSMA2), NULL, &pReader->pDataReader[i]); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(code, &lino, _exit); } } @@ -61,7 +60,7 @@ _exit: *ppReader = NULL; smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } - return code; + TAOS_RETURN(code); } int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { @@ -80,7 +79,7 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { if (!pReader->rsmaDataDone[i]) { smaInfo("vgId:%d, vnode snapshot rsma read level %d not done", SMA_VID(pReader->pSma), i); code = tsdbSnapRead(pTsdbSnapReader, ppData); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(code, &lino, _exit); if (*ppData) { goto _exit; } else { @@ -93,12 +92,12 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { _exit: if (code) { - smaError("vgId:%d, vnode snapshot rsma read failed since %s", SMA_VID(pReader->pSma), tstrerror(code)); + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pReader->pSma), __func__, lino, tstrerror(code)); rsmaSnapReaderClose(&pReader); } else { smaInfo("vgId:%d, vnode snapshot rsma read succeed", SMA_VID(pReader->pSma)); } - return code; + TAOS_RETURN(code); } int32_t rsmaSnapReaderClose(SRSmaSnapReader** ppReader) { @@ -114,7 +113,7 @@ int32_t rsmaSnapReaderClose(SRSmaSnapReader** ppReader) { smaInfo("vgId:%d, vnode snapshot rsma reader closed", SMA_VID(pReader->pSma)); taosMemoryFreeClear(*ppReader); - return code; + TAOS_RETURN(code); } // SRSmaSnapWriter ======================================== @@ -137,8 +136,7 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, void** ppRang // alloc pWriter = (SRSmaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); if (!pWriter) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } pWriter->pSma = pSma; pWriter->sver = sver; @@ -148,7 +146,7 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, void** ppRang for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { code = tsdbSnapWriterOpen(pSma->pRSmaTsdb[i], sver, ever, ((void**)ppRanges)[i], &pWriter->pDataWriter[i]); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(code, &lino, _exit); } } @@ -162,7 +160,7 @@ _exit: } else { smaInfo("vgId:%d, rsma snapshot writer open succeed", TD_VID(pSma->pVnode)); } - return code; + TAOS_RETURN(code); } int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter) { @@ -171,13 +169,13 @@ int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter) { if (pWriter->pDataWriter[i]) { code = tsdbSnapWriterPrepareClose(pWriter->pDataWriter[i]); if (code) { - smaError("vgId:%d, failed to prepare close tsdbSnapWriter since %s. i: %d", SMA_VID(pWriter->pSma), terrstr(), - i); - return -1; + smaError("vgId:%d, failed to prepare close tsdbSnapWriter since %s. i: %d", SMA_VID(pWriter->pSma), + tstrerror(code), i); + TAOS_RETURN(code); } } } - return code; + TAOS_RETURN(code); } int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { @@ -206,13 +204,13 @@ int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pWriter->pDataWriter[i]) { code = tsdbSnapWriterClose(&pWriter->pDataWriter[i], rollback); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(code, &lino, _exit); } } // rsma restore code = tdRSmaRestore(pWriter->pSma, RSMA_RESTORE_SYNC, pWriter->ever, rollback); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(code, &lino, _exit); smaInfo("vgId:%d, vnode snapshot rsma writer restore from sync succeed", SMA_VID(pSma)); _exit: @@ -221,12 +219,13 @@ _exit: if (code) { if (pOutFD) taosCloseFile(&pOutFD); if (pInFD) taosCloseFile(&pInFD); - smaError("vgId:%d, vnode snapshot rsma writer close failed since %s", SMA_VID(pSma), tstrerror(code)); + smaError("vgId:%d, vnode snapshot rsma writer close failed at line %d since %s", SMA_VID(pSma), lino, + tstrerror(code)); } else { smaInfo("vgId:%d, vnode snapshot rsma writer close succeed", pSma ? SMA_VID(pSma) : 0); } - return code; + TAOS_RETURN(code); } int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { @@ -244,7 +243,7 @@ int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) } else { code = TSDB_CODE_RSMA_FS_SYNC; } - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(code, &lino, _exit); _exit: if (code) { @@ -253,5 +252,5 @@ _exit: } else { smaInfo("vgId:%d, rsma snapshot write for data type %" PRIi8 " succeed", SMA_VID(pWriter->pSma), pHdr->type); } - return code; + TAOS_RETURN(code); } diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index e45cbac329..d94e58f954 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -39,9 +39,10 @@ void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { return taosAcquireRef(rse int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId) { if (taosReleaseRef(rsetId, refId) < 0) { - smaWarn("rsma release ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr()); - return TSDB_CODE_FAILED; + int32_t code = terrno; + smaWarn("rsma release ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, tstrerror(code)); + TAOS_RETURN(code); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } From b337fa628b58cd7d4ec6602c957b64071b163ca7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 22 Jul 2024 15:29:37 +0800 Subject: [PATCH 05/52] enh: return error code --- include/util/tbase58.h | 4 +- include/util/tbase64.h | 4 +- source/dnode/mgmt/node_util/src/dmFile.c | 5 +- source/dnode/mnode/impl/inc/mndDnode.h | 2 +- source/dnode/mnode/impl/inc/mndUser.h | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 18 +- source/dnode/mnode/impl/src/mndUser.c | 404 ++++++++++++++++++----- source/util/src/tbase58.c | 56 ++-- source/util/src/tbase64.c | 30 +- source/util/test/tbaseCodecTest.cpp | 12 +- 10 files changed, 397 insertions(+), 140 deletions(-) diff --git a/include/util/tbase58.h b/include/util/tbase58.h index ab62e1926e..8210cbf4ba 100644 --- a/include/util/tbase58.h +++ b/include/util/tbase58.h @@ -25,8 +25,8 @@ extern "C" { #define TBASE_MAX_ILEN 4096 #define TBASE_MAX_OLEN 5653 -uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen); -char *base58_encode(const uint8_t *value, int32_t vlen); +int32_t base58_decode(const char *value, size_t inlen, int32_t *outlen, uint8_t **result); +int32_t base58_encode(const uint8_t *value, int32_t vlen, char **result); #ifdef __cplusplus } diff --git a/include/util/tbase64.h b/include/util/tbase64.h index 22b8f95c5a..e342e6a0fa 100644 --- a/include/util/tbase64.h +++ b/include/util/tbase64.h @@ -22,8 +22,8 @@ extern "C" { #endif -uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen); -char *base64_encode(const uint8_t *value, int32_t vlen); +int32_t base64_decode(const char *value, int32_t inlen, int32_t *outlen, uint8_t **result); +int32_t base64_encode(const uint8_t *value, int32_t vlen, char **result); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 752abb83a2..a062b96a3d 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -308,6 +308,7 @@ _OVER: int32_t dmUpdateEncryptKey(char *key, bool toLogFile) { #ifdef TD_ENTERPRISE int32_t code = -1; + int32_t lino = 0; char *machineId = NULL; char *encryptCode = NULL; @@ -344,9 +345,7 @@ int32_t dmUpdateEncryptKey(char *key, bool toLogFile) { goto _OVER; } - if (generateEncryptCode(key, machineId, &encryptCode) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(generateEncryptCode(key, machineId, &encryptCode), &lino, _OVER); if(dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode, toLogFile) != 0){ goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h index ab104e316e..7708b954b7 100644 --- a/source/dnode/mnode/impl/inc/mndDnode.h +++ b/source/dnode/mnode/impl/inc/mndDnode.h @@ -30,7 +30,7 @@ SEpSet mndGetDnodeEpset(SDnodeObj *pDnode); SEpSet mndGetDnodeEpsetById(SMnode *pMnode, int32_t dnodeId); int32_t mndGetDnodeSize(SMnode *pMnode); bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs); -void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo); +int32_t mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index 5043f6da3d..0048a0badd 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -49,7 +49,7 @@ void mndUserFreeObj(SUserObj *pUser); int64_t mndGetIpWhiteVer(SMnode *pMnode); -void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock); +int32_t mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock); int32_t mndRefreshUserIpWhiteList(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 73addea6fe..879f533635 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -175,7 +175,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; - mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); + mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); // TODO: check the return value _OVER: mndTransDrop(pTrans); @@ -417,8 +417,9 @@ static void mndGetDnodeEps(SMnode *pMnode, SArray *pDnodeEps) { } } -void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { - SSdb *pSdb = pMnode->pSdb; +int32_t mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { + SSdb *pSdb = pMnode->pSdb; + int32_t code = 0; int32_t numOfEps = 0; void *pIter = NULL; @@ -440,8 +441,13 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { dInfo.isMnode = 0; } - taosArrayPush(pDnodeInfo, &dInfo); + if(taosArrayPush(pDnodeInfo, &dInfo) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + sdbCancelFetch(pSdb, pIter); + break; + } } + TAOS_RETURN(code); } #define CHECK_MONITOR_PARA(para,err) \ @@ -883,7 +889,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; - mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); + mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); // TODO: check the return value _OVER: mndTransDrop(pTrans); sdbFreeRaw(pRaw); @@ -1165,7 +1171,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1); + mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1); // TODO: check the return value code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 2335bf7bf8..bae617013c 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -96,7 +96,7 @@ (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_SUBSCRIBE_PRIV(_priv)) static int32_t createDefaultIpWhiteList(SIpWhiteList **ppWhiteList); -SIpWhiteList *createIpWhiteList(void *buf, int32_t len); +static int32_t createIpWhiteList(void *buf, int32_t len, SIpWhiteList **ppWhiteList); static bool updateIpWhiteList(SIpWhiteList *pOld, SIpWhiteList *pNew); static bool isIpWhiteListEqual(SIpWhiteList *a, SIpWhiteList *b); static bool isIpRangeEqual(SIpV4Range *a, SIpV4Range *b); @@ -121,11 +121,11 @@ static int32_t mndRetrieveUsersFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter); -SHashObj *mndFetchAllIpWhite(SMnode *pMnode); +static int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab); static int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq); -static int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type, int8_t *pUpdate); +static int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type, bool *pUpdate); -void ipWhiteMgtUpdateAll(SMnode *pMnode); +static int32_t ipWhiteMgtUpdateAll(SMnode *pMnode); typedef struct { SHashObj *pIpWhiteTab; int64_t ver; @@ -159,9 +159,11 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { if (ppList == NULL || *ppList == NULL) { SIpWhiteList *p = cloneIpWhiteList(pNew); if (p == NULL) { + update = false; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } if ((code = taosHashPut(ipWhiteMgt.pIpWhiteTab, user, strlen(user), &p, sizeof(void *))) != 0) { + update = false; taosMemoryFree(p); TAOS_CHECK_GOTO(code, &lino, _OVER); } @@ -173,20 +175,26 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { taosMemoryFree(pOld); SIpWhiteList *p = cloneIpWhiteList(pNew); if (p == NULL) { + update = false; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } if ((code = taosHashPut(ipWhiteMgt.pIpWhiteTab, user, strlen(user), &p, sizeof(void *))) != 0) { + update = false; taosMemoryFree(p); TAOS_CHECK_GOTO(code, &lino, _OVER); } } } - SArray *fqdns = mndGetAllDnodeFqdns(pMnode); + SArray *fqdns = mndGetAllDnodeFqdns(pMnode); // TODO: update this line after refactor api for (int i = 0; i < taosArrayGetSize(fqdns); i++) { char *fqdn = taosArrayGetP(fqdns, i); - update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, TSDB_DEFAULT_USER, fqdn, IP_WHITE_ADD); - update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, IP_WHITE_ADD); + bool upd = false; + TAOS_CHECK_GOTO(mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, TSDB_DEFAULT_USER, fqdn, IP_WHITE_ADD, &upd), &lino, + _OVER); + update |= upd; + TAOS_CHECK_GOTO(mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, IP_WHITE_ADD, &upd), &lino, _OVER); + update |= upd; } for (int i = 0; i < taosArrayGetSize(fqdns); i++) { @@ -204,7 +212,10 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { _OVER: taosThreadRwlockUnlock(&ipWhiteMgt.rw); - return 0; + if (code != 0) { + mError("failed to update ip white list for user: %s at line %d since %s", user, lino, tstrerror(code)); + } + TAOS_RETURN(code); } int32_t ipWhiteMgtRemove(char *user) { bool update = true; @@ -277,14 +288,17 @@ int32_t ipWhiteUpdateForAllUser(SIpWhiteList *pList) { } #endif -void ipWhiteMgtUpdateAll(SMnode *pMnode) { - ipWhiteMgt.ver++; - SHashObj *pNew = mndFetchAllIpWhite(pMnode); +static int32_t ipWhiteMgtUpdateAll(SMnode *pMnode) { + SHashObj *pNew = NULL; + TAOS_CHECK_RETURN(mndFetchAllIpWhite(pMnode, &pNew)); + SHashObj *pOld = ipWhiteMgt.pIpWhiteTab; ipWhiteMgt.pIpWhiteTab = pNew; + ipWhiteMgt.ver++; destroyIpWhiteTab(pOld); + TAOS_RETURN(0); } void ipWhiteMgtUpdate2(SMnode *pMnode) { taosThreadRwlockWrlock(&ipWhiteMgt.rw); @@ -313,6 +327,8 @@ int64_t mndGetIpWhiteVer(SMnode *pMnode) { } int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type, bool *pUpdate) { + int32_t code = 0; + int32_t lino = 0; bool update = false; SIpV4Range range = {.ip = taosGetIpv4FromFqdn(fqdn), .mask = 32}; mDebug("ip-white-list may update for user: %s, fqdn: %s", user, fqdn); @@ -325,22 +341,34 @@ int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8 if (type == IP_WHITE_ADD) { if (pList == NULL) { SIpWhiteList *pNewList = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range)); + if(pNewList == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memcpy(pNewList->pIpRange, &range, sizeof(SIpV4Range)); pNewList->num = 1; - taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *)); + if (taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *)) != 0) { + taosMemoryFree(pNewList); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } update = true; } else { if (!isRangeInWhiteList(pList, &range)) { int32_t sz = sizeof(SIpWhiteList) + sizeof(SIpV4Range) * (pList->num + 1); SIpWhiteList *pNewList = taosMemoryCalloc(1, sz); + if (pNewList == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memcpy(pNewList->pIpRange, pList->pIpRange, sizeof(SIpV4Range) * (pList->num)); pNewList->pIpRange[pList->num].ip = range.ip; pNewList->pIpRange[pList->num].mask = range.mask; pNewList->num = pList->num + 1; - taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *)); + if (taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *)) != 0) { + taosMemoryFree(pNewList); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } taosMemoryFree(pList); update = true; } @@ -355,6 +383,9 @@ int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8 int32_t idx = 0; int32_t sz = sizeof(SIpWhiteList) + sizeof(SIpV4Range) * (pList->num - 1); SIpWhiteList *pNewList = taosMemoryCalloc(1, sz); + if(pNewList == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } for (int i = 0; i < pList->num; i++) { SIpV4Range *e = &pList->pIpRange[i]; if (!isIpRangeEqual(e, &range)) { @@ -364,7 +395,10 @@ int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8 } } pNewList->num = idx; - taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *)); + if (taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *) != 0)) { + taosMemoryFree(pNewList); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } taosMemoryFree(pList); } update = true; @@ -375,29 +409,44 @@ int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8 mDebug("ip-white-list update for user: %s, fqdn: %s", user, fqdn); } - return update; +_OVER: + if (pUpdate) *pUpdate = update; + if (code != 0) { + mError("failed to update ip-white-list for user: %s, fqdn: %s at line %d since %s", user, fqdn, lino, + tstrerror(code)); + } + TAOS_RETURN(code); } int32_t mndRefreshUserIpWhiteList(SMnode *pMnode) { + int32_t code = 0; taosThreadRwlockWrlock(&ipWhiteMgt.rw); - ipWhiteMgtUpdateAll(pMnode); + if ((code = ipWhiteMgtUpdateAll(pMnode)) != 0) { + taosThreadRwlockUnlock(&ipWhiteMgt.rw); + TAOS_RETURN(code); + } ipWhiteMgt.ver = taosGetTimestampMs(); taosThreadRwlockUnlock(&ipWhiteMgt.rw); - return 0; + TAOS_RETURN(code); } -void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock) { + +int32_t mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock) { + int32_t code = 0; + int32_t lino = 0; + bool update = false; + if (lock) { taosThreadRwlockWrlock(&ipWhiteMgt.rw); if (ipWhiteMgt.ver == 0) { - ipWhiteMgtUpdateAll(pMnode); + TAOS_CHECK_GOTO(ipWhiteMgtUpdateAll(pMnode), &lino, _OVER); ipWhiteMgt.ver = taosGetTimestampMs(); - mInfo("ip-white-list, user: %" PRId64 "", ipWhiteMgt.ver); + mInfo("update ip-white-list, user: %s, ver: %" PRId64, user, ipWhiteMgt.ver); } } - bool update = mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, type); + TAOS_CHECK_GOTO(mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, type, &update), &lino, _OVER); void *pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, NULL); while (pIter) { @@ -405,24 +454,43 @@ void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t t char *key = taosHashGetKey(pIter, &klen); char *keyDup = taosMemoryCalloc(1, klen + 1); + if (keyDup == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memcpy(keyDup, key, klen); - update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, keyDup, fqdn, type); + code = mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, keyDup, fqdn, type, &update); + if (code != 0) { + update = false; + taosMemoryFree(keyDup); + TAOS_CHECK_GOTO(code, &lino, _OVER); + } taosMemoryFree(keyDup); pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, pIter); } +_OVER: if (update) ipWhiteMgt.ver++; - if (lock) taosThreadRwlockUnlock(&ipWhiteMgt.rw); + if (code != 0) { + mError("failed to update ip-white-list for user: %s, fqdn: %s at line %d since %s", user, fqdn, lino, + tstrerror(code)); + } + + TAOS_RETURN(code); } -int64_t ipWhiteMgtFillMsg(SUpdateIpWhite *pUpdate) { + +static int64_t ipWhiteMgtFillMsg(SUpdateIpWhite *pUpdate) { int64_t ver = 0; taosThreadRwlockWrlock(&ipWhiteMgt.rw); ver = ipWhiteMgt.ver; int32_t num = taosHashGetSize(ipWhiteMgt.pIpWhiteTab); pUpdate->pUserIpWhite = taosMemoryCalloc(1, num * sizeof(SUpdateUserIpWhite)); + if (pUpdate->pUserIpWhite == NULL) { + taosThreadRwlockUnlock(&ipWhiteMgt.rw); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } void *pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, NULL); int32_t i = 0; @@ -437,6 +505,11 @@ int64_t ipWhiteMgtFillMsg(SUpdateIpWhite *pUpdate) { memcpy(pUser->user, key, klen); pUser->numOfRange = list->num; pUser->pIpRanges = taosMemoryCalloc(1, list->num * sizeof(SIpV4Range)); + if (pUser->pIpRanges == NULL) { + taosThreadRwlockUnlock(&ipWhiteMgt.rw); + tFreeSUpdateIpWhiteReq(pUpdate); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } memcpy(pUser->pIpRanges, list->pIpRange, list->num * sizeof(SIpV4Range)); i++; } @@ -446,7 +519,7 @@ int64_t ipWhiteMgtFillMsg(SUpdateIpWhite *pUpdate) { pUpdate->ver = ver; taosThreadRwlockUnlock(&ipWhiteMgt.rw); - return 0; + TAOS_RETURN(0); } void destroyIpWhiteTab(SHashObj *pIpWhiteTab) { @@ -461,22 +534,54 @@ void destroyIpWhiteTab(SHashObj *pIpWhiteTab) { taosHashCleanup(pIpWhiteTab); } -SHashObj *mndFetchAllIpWhite(SMnode *pMnode) { +int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { + int32_t code = 0; + int32_t lino = 0; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; - SHashObj *pIpWhiteTab = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), 1, HASH_ENTRY_LOCK); + SHashObj *pIpWhiteTab = NULL; + SArray *pUserNames = NULL; + SArray *fqdns = NULL; + + pIpWhiteTab = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), 1, HASH_ENTRY_LOCK); + if (pIpWhiteTab == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } + pUserNames = taosArrayInit(8, sizeof(void *)); + if(pUserNames == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } - SArray *pUserNames = taosArrayInit(8, sizeof(void *)); while (1) { SUserObj *pUser = NULL; pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pUser); if (pIter == NULL) break; SIpWhiteList *pWhiteList = cloneIpWhiteList(pUser->pIpWhiteList); - taosHashPut(pIpWhiteTab, pUser->user, strlen(pUser->user), &pWhiteList, sizeof(void *)); + if (pWhiteList == NULL) { + sdbRelease(pSdb, pUser); + sdbCancelFetch(pSdb, pIter); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } + if(taosHashPut(pIpWhiteTab, pUser->user, strlen(pUser->user), &pWhiteList, sizeof(void *)) != 0) { + taosMemoryFree(pWhiteList); + sdbRelease(pSdb, pUser); + sdbCancelFetch(pSdb, pIter); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } char *name = taosStrdup(pUser->user); - taosArrayPush(pUserNames, &name); + if(name == NULL) { + sdbRelease(pSdb, pUser); + sdbCancelFetch(pSdb, pIter); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } + if(taosArrayPush(pUserNames, &name) == NULL){ + taosMemoryFree(name); + sdbRelease(pSdb, pUser); + sdbCancelFetch(pSdb, pIter); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } sdbRelease(pSdb, pUser); } @@ -491,20 +596,27 @@ SHashObj *mndFetchAllIpWhite(SMnode *pMnode) { } if (found == false) { char *name = taosStrdup(TSDB_DEFAULT_USER); - taosArrayPush(pUserNames, &name); + if (name == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } + if (taosArrayPush(pUserNames, &name) == NULL) { + taosMemoryFree(name); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } } - SArray *fqdns = mndGetAllDnodeFqdns(pMnode); + fqdns = mndGetAllDnodeFqdns(pMnode); // TODO: refactor this line after refactor api for (int i = 0; i < taosArrayGetSize(fqdns); i++) { char *fqdn = taosArrayGetP(fqdns, i); for (int j = 0; j < taosArrayGetSize(pUserNames); j++) { char *name = taosArrayGetP(pUserNames, j); - mndUpdateIpWhiteImpl(pIpWhiteTab, name, fqdn, IP_WHITE_ADD); + TAOS_CHECK_GOTO(mndUpdateIpWhiteImpl(pIpWhiteTab, name, fqdn, IP_WHITE_ADD, NULL), &lino, _OVER); } } +_OVER: for (int i = 0; i < taosArrayGetSize(fqdns); i++) { char *fqdn = taosArrayGetP(fqdns, i); taosMemoryFree(fqdn); @@ -516,11 +628,17 @@ SHashObj *mndFetchAllIpWhite(SMnode *pMnode) { } taosArrayDestroy(pUserNames); - return pIpWhiteTab; + if (code != 0) { + mError("failed to fetch all ip white list at line %d since %s", lino, tstrerror(code)); + destroyIpWhiteTab(pIpWhiteTab); + pIpWhiteTab = NULL; + } + *ppIpWhiteTab = pIpWhiteTab; + TAOS_RETURN(code); } int32_t mndInitUser(SMnode *pMnode) { - TAOS_CHECK_REUTNR(ipWhiteMgtInit()); + TAOS_CHECK_RETURN(ipWhiteMgtInit()); SSdbTable table = { .sdbType = SDB_USER, @@ -634,34 +752,50 @@ int32_t tSerializeIpWhiteList(void *buf, int32_t len, SIpWhiteList *pList) { } int32_t tDerializeIpWhileList(void *buf, int32_t len, SIpWhiteList *pList) { + int32_t code = 0; + int32_t lino = 0; SDecoder decoder = {0}; tDecoderInit(&decoder, buf, len); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pList->num) < 0) return -1; + TAOS_CHECK_GOTO(tStartDecode(&decoder), &lino, _OVER); + TAOS_CHECK_GOTO(tDecodeI32(&decoder, &pList->num), &lino, _OVER); for (int i = 0; i < pList->num; i++) { SIpV4Range *pRange = &(pList->pIpRange[i]); - if (tDecodeU32(&decoder, &pRange->ip) < 0) return -1; - if (tDecodeU32(&decoder, &pRange->mask) < 0) return -1; + TAOS_CHECK_GOTO(tDecodeU32(&decoder, &pRange->ip), &lino, _OVER); + TAOS_CHECK_GOTO(tDecodeU32(&decoder, &pRange->mask), &lino, _OVER); } + +_OVER: tEndDecode(&decoder); tDecoderClear(&decoder); + if(code != 0) { + mError("failed to deserialize ip white list at line %d since %s", lino, tstrerror(code)); + } - return 0; + TAOS_RETURN(code); } -SIpWhiteList *createIpWhiteList(void *buf, int32_t len) { - int32_t num = 0; - SDecoder decoder = {0}; + +static int32_t createIpWhiteList(void *buf, int32_t len, SIpWhiteList **ppList) { + int32_t code = 0; + int32_t lino = 0; + int32_t num = 0; + SIpWhiteList *p = NULL; + SDecoder decoder = {0}; tDecoderInit(&decoder, buf, len); - if (tStartDecode(&decoder) < 0) return NULL; - if (tDecodeI32(&decoder, &num) < 0) return NULL; + TAOS_CHECK_GOTO(tStartDecode(&decoder), &lino, _OVER), &lino, _OVER); + TAOS_CHECK_GOTO(tDecodeI32(&decoder, &num), &lino, _OVER), &lino, _OVER); + + p = taosMemoryCalloc(1, sizeof(SIpWhiteList) + num * sizeof(SIpV4Range)); + if(p == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } + TAOS_CHECK_GOTO(tDerializeIpWhileList(buf, len, p), &lino, _OVER); + +_OVER: tEndDecode(&decoder); tDecoderClear(&decoder); - - SIpWhiteList *p = taosMemoryCalloc(1, sizeof(SIpWhiteList) + num * sizeof(SIpV4Range)); - tDerializeIpWhileList(buf, len, p); return p; } @@ -1027,23 +1161,29 @@ _OVER: } static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = 0; + int32_t lino = 0; SSdbRow *pRow = NULL; SUserObj *pUser = NULL; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) { + TAOS_CHECK_GOTO(TSDB_CODE_INVALID_PTR, &lino, _OVER); + } if (sver < 1 || sver > USER_VER_NUMBER) { - terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_SDB_INVALID_DATA_VER, &lino, _OVER); } pRow = sdbAllocRow(sizeof(SUserObj)); - if (pRow == NULL) goto _OVER; + if (pRow == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } pUser = sdbGetRowObj(pRow); - if (pUser == NULL) goto _OVER; + if (pUser == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } int32_t dataPos = 0; SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER) @@ -1074,20 +1214,23 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { pUser->writeDbs = taosHashInit(numOfWriteDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pUser->topics = taosHashInit(numOfTopics, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (pUser->readDbs == NULL || pUser->writeDbs == NULL || pUser->topics == NULL) goto _OVER; + if (pUser->readDbs == NULL || pUser->writeDbs == NULL || pUser->topics == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + goto _OVER; + } for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER) int32_t len = strlen(db) + 1; - taosHashPut(pUser->readDbs, db, len, db, TSDB_DB_FNAME_LEN); + TAOS_CHECK_GOTO(taosHashPut(pUser->readDbs, db, len, db, TSDB_DB_FNAME_LEN), &lino, _OVER); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER) int32_t len = strlen(db) + 1; - taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN); + TAOS_CHECK_GOTO(taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN), &lino, _OVER); } if (sver >= 2) { @@ -1095,7 +1238,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { char topic[TSDB_TOPIC_FNAME_LEN] = {0}; SDB_GET_BINARY(pRaw, dataPos, topic, TSDB_TOPIC_FNAME_LEN, _OVER) int32_t len = strlen(topic) + 1; - taosHashPut(pUser->topics, topic, len, topic, TSDB_TOPIC_FNAME_LEN); + TAOS_CHECK_GOTO(taosHashPut(pUser->topics, topic, len, topic, TSDB_TOPIC_FNAME_LEN), &lino, _OVER); } } @@ -1133,21 +1276,38 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { pUser->useDbs = taosHashInit(numOfUseDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (pUser->readTbs == NULL || pUser->writeTbs == NULL || pUser->alterTbs == NULL || pUser->readViews == NULL || + pUser->writeViews == NULL || pUser->alterViews == NULL || pUser->useDbs == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + goto _OVER; + } + for (int32_t i = 0; i < numOfReadTbs; ++i) { int32_t keyLen = 0; SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); char *key = taosMemoryCalloc(keyLen, sizeof(char)); + if (key == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); char *value = taosMemoryCalloc(valuelen, sizeof(char)); + if (value == NULL) { + taosMemoryFree(key); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) - taosHashPut(pUser->readTbs, key, keyLen, value, valuelen); + if (taosHashPut(pUser->readTbs, key, keyLen, value, valuelen) != 0) { + taosMemoryFree(key); + taosMemoryFree(value); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } taosMemoryFree(key); taosMemoryFree(value); @@ -1158,16 +1318,27 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); char *key = taosMemoryCalloc(keyLen, sizeof(char)); + if (key == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); char *value = taosMemoryCalloc(valuelen, sizeof(char)); + if (value == NULL) { + taosMemoryFree(key); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) - taosHashPut(pUser->writeTbs, key, keyLen, value, valuelen); + if (taosHashPut(pUser->writeTbs, key, keyLen, value, valuelen) != 0) { + taosMemoryFree(key); + taosMemoryFree(value); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } taosMemoryFree(key); taosMemoryFree(value); @@ -1179,16 +1350,27 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); char *key = taosMemoryCalloc(keyLen, sizeof(char)); + if (key == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); char *value = taosMemoryCalloc(valuelen, sizeof(char)); + if (value == NULL) { + taosMemoryFree(key); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) - taosHashPut(pUser->alterTbs, key, keyLen, value, valuelen); + if (taosHashPut(pUser->alterTbs, key, keyLen, value, valuelen) != 0) { + taosMemoryFree(key); + taosMemoryFree(value); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } taosMemoryFree(key); taosMemoryFree(value); @@ -1199,16 +1381,27 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); char *key = taosMemoryCalloc(keyLen, sizeof(char)); + if (key == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); char *value = taosMemoryCalloc(valuelen, sizeof(char)); + if (value == NULL) { + taosMemoryFree(key); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) - taosHashPut(pUser->readViews, key, keyLen, value, valuelen); + if (taosHashPut(pUser->readViews, key, keyLen, value, valuelen) != 0) { + taosMemoryFree(key); + taosMemoryFree(value); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } taosMemoryFree(key); taosMemoryFree(value); @@ -1219,16 +1412,27 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); char *key = taosMemoryCalloc(keyLen, sizeof(char)); + if (key == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); char *value = taosMemoryCalloc(valuelen, sizeof(char)); + if (value == NULL) { + taosMemoryFree(key); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) - taosHashPut(pUser->writeViews, key, keyLen, value, valuelen); + if (taosHashPut(pUser->writeViews, key, keyLen, value, valuelen) != 0) { + taosMemoryFree(key); + taosMemoryFree(value); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } taosMemoryFree(key); taosMemoryFree(value); @@ -1239,16 +1443,27 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); char *key = taosMemoryCalloc(keyLen, sizeof(char)); + if (key == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); char *value = taosMemoryCalloc(valuelen, sizeof(char)); + if (value == NULL) { + taosMemoryFree(key); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) - taosHashPut(pUser->alterViews, key, keyLen, value, valuelen); + if (taosHashPut(pUser->alterViews, key, keyLen, value, valuelen) != 0) { + taosMemoryFree(key); + taosMemoryFree(value); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } taosMemoryFree(key); taosMemoryFree(value); @@ -1260,13 +1475,19 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); char *key = taosMemoryCalloc(keyLen, sizeof(char)); + if (key == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); int32_t ref = 0; SDB_GET_INT32(pRaw, dataPos, &ref, _OVER); - taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref)); + if (taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref)) != 0) { + taosMemoryFree(key); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } taosMemoryFree(key); } } @@ -1276,17 +1497,19 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &len, _OVER); char *buf = taosMemoryMalloc(len); - if (buf == NULL) goto _OVER; + if(buf == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } SDB_GET_BINARY(pRaw, dataPos, buf, len, _OVER); - pUser->pIpWhiteList = createIpWhiteList(buf, len); + code = createIpWhiteList(buf, len, &pUser->pIpWhiteList); taosMemoryFree(buf); SDB_GET_INT64(pRaw, dataPos, &pUser->ipWhiteListVer, _OVER); } if (pUser->pIpWhiteList == NULL) { - pUser->pIpWhiteList = createDefaultIpWhiteList(); + createDefaultIpWhiteList(&pUser->pIpWhiteList); pUser->ipWhiteListVer = taosGetTimestampMs(); } @@ -1724,29 +1947,50 @@ _OVER: } int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq) { + int32_t code = 0; + int32_t lino = 0; + int32_t len = 0; + void *pRsp = NULL; + SUpdateIpWhite ipWhite = {0}; + // impl later SRetrieveIpWhiteReq req = {0}; if (tDeserializeRetrieveIpWhite(pReq->pCont, pReq->contLen, &req) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; + code = TSDB_CODE_INVALID_MSG; + TAOS_CHECK_GOTO(code, &lino, _OVER); } - SUpdateIpWhite ipWhite = {0}; - ipWhiteMgtFillMsg(&ipWhite); + TAOS_CHECK_GOTO(ipWhiteMgtFillMsg(&ipWhite), &lino, _OVER); - int32_t len = tSerializeSUpdateIpWhite(NULL, 0, &ipWhite); + len = tSerializeSUpdateIpWhite(NULL, 0, &ipWhite); + if (len < 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_CHECK_GOTO(code, &lino, _OVER); + } - void *pRsp = rpcMallocCont(len); - tSerializeSUpdateIpWhite(pRsp, len, &ipWhite); + pRsp = rpcMallocCont(len); + if (!pRsp) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } + len = tSerializeSUpdateIpWhite(pRsp, len, &ipWhite); + if (len < 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_CHECK_GOTO(code, &lino, _OVER); + } +_OVER: + if (code != 0) { + mError("failed to process retrieve ip white request at line %d since %s", lino, tstrerror(code)); + rpcFreeCont(pRsp); + pRsp = NULL; + len = 0; + } + pReq->code = code; pReq->info.rsp = pRsp; pReq->info.rspLen = len; - //} - tFreeSUpdateIpWhiteReq(&ipWhite); - return 0; -_OVER: - return -1; + tFreeSUpdateIpWhiteReq(&ipWhite); + TAOS_RETURN(code); } static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpcMsg *pReq) { diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index 5eb72879c3..274249badf 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -21,7 +21,7 @@ #define TBASE_BUF_SIZE 256 static const char *basis_58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; -char *base58_encode(const uint8_t *value, int32_t vlen) { +int32_t base58_encode(const uint8_t *value, int32_t vlen, char **result) { const uint8_t *pb = value; const uint8_t *pe = pb + vlen; uint8_t buf[TBASE_BUF_SIZE] = {0}; @@ -29,9 +29,10 @@ char *base58_encode(const uint8_t *value, int32_t vlen) { bool bfree = false; int32_t nz = 0, size = 0, len = 0; + *result = NULL; + if (vlen > TBASE_MAX_ILEN) { - terrno = TSDB_CODE_INVALID_PARA; - return NULL; + return TSDB_CODE_INVALID_PARA; } while (pb != pe && *pb == 0) { @@ -42,8 +43,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) { size = (pe - pb) * 69 / 50 + 1; if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } bfree = true; } @@ -62,17 +62,17 @@ char *base58_encode(const uint8_t *value, int32_t vlen) { const uint8_t *pi = pbuf + (size - len); while (pi != pbuf + size && *pi == 0) ++pi; - uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - pi) + 1); - if (!result) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + uint8_t *pResult = taosMemoryCalloc(1, nz + (pbuf + size - pi) + 1); + if (!pResult) { if (bfree) taosMemoryFree(pbuf); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } - memset(result, '1', nz); - while (pi != pbuf + size) result[nz++] = basis_58[*pi++]; + memset(pResult, '1', nz); + while (pi != pbuf + size) pResult[nz++] = basis_58[*pi++]; if (bfree) taosMemoryFree(pbuf); - return result; + *result = pResult; + return 0; } static const signed char index_58[256] = { @@ -86,7 +86,7 @@ static const signed char index_58[256] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; -uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { +int32_t base58_decode(const char *value, size_t inlen, int32_t *outlen, uint8_t **result) { const char *pb = value; const char *pe = value + inlen; uint8_t buf[TBASE_BUF_SIZE] = {0}; @@ -94,15 +94,15 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { bool bfree = false; int32_t nz = 0, size = 0, len = 0; + *result = NULL; + if (inlen > TBASE_MAX_OLEN) { - terrno = TSDB_CODE_INVALID_PARA; - return NULL; + return TSDB_CODE_INVALID_PARA; } while (pb != pe) { if (*pb == 0) { - terrno = TSDB_CODE_INVALID_PARA; - return NULL; + return TSDB_CODE_INVALID_PARA; } ++pb; } @@ -117,8 +117,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { size = (int32_t)(pe - pb) * 733 / 1000 + 1; if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } bfree = true; } @@ -126,9 +125,8 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { while (pb != pe && *pb && !isspace(*pb)) { int32_t num = index_58[(uint8_t)*pb]; if (num == -1) { - terrno = TSDB_CODE_INVALID_PARA; if (bfree) taosMemoryFree(pbuf); - return NULL; + return TSDB_CODE_INVALID_PARA; } int32_t i = 0; for (int32_t j = size - 1; (num != 0 || i < len) && (j >= 0); --j, ++i) { @@ -143,23 +141,23 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { while (pb != pe && isspace(*pb)) ++pb; if (*pb != 0) { if (bfree) taosMemoryFree(pbuf); - return NULL; + return TSDB_CODE_INVALID_DATA_FMT; } const uint8_t *it = pbuf + (size - len); while (it != pbuf + size && *it == 0) ++it; - uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - it) + 1); - if (!result) { + uint8_t *pResult = taosMemoryCalloc(1, nz + (pbuf + size - it) + 1); + if (!pResult) { if (bfree) taosMemoryFree(pbuf); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } - memset(result, 0, nz); - while (it != pbuf + size) result[nz++] = *it++; + memset(pResult, 0, nz); + while (it != pbuf + size) pResult[nz++] = *it++; if (outlen) *outlen = nz; if (bfree) taosMemoryFree(pbuf); - return result; + *result = pResult; + return 0; } \ No newline at end of file diff --git a/source/util/src/tbase64.c b/source/util/src/tbase64.c index a2f4ddbc51..86b069b853 100644 --- a/source/util/src/tbase64.c +++ b/source/util/src/tbase64.c @@ -18,10 +18,13 @@ static char basis_64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -char *base64_encode(const uint8_t *value, int32_t vlen) { +int32_t base64_encode(const uint8_t *value, int32_t vlen, char **result) { uint8_t oval = 0; - char *result = (char *)taosMemoryMalloc((size_t)(vlen * 4) / 3 + 10); - char *out = result; + *result = (char *)taosMemoryMalloc((size_t)(vlen * 4) / 3 + 10); + if (*result == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + char *out = *result; while (vlen >= 3) { *out++ = basis_64[value[0] >> 2]; *out++ = basis_64[((value[0] << 4) & 0x30) | (value[1] >> 4)]; @@ -39,7 +42,7 @@ char *base64_encode(const uint8_t *value, int32_t vlen) { *out++ = '='; } *out = '\0'; - return result; + return 0; } #define CHAR64(c) (((c) < 0 || (c) > 127) ? -1 : index_64[(c)]) @@ -51,17 +54,20 @@ static signed char index_64[128] = { 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1}; -uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen) { - int32_t c1, c2, c3, c4; - uint8_t *result = (uint8_t *)taosMemoryMalloc((size_t)(inlen * 3) / 4 + 1); - uint8_t *out = result; +int32_t base64_decode(const char *value, int32_t inlen, int32_t *outlen, uint8_t **result) { + int32_t c1, c2, c3, c4; + *result = (uint8_t *)taosMemoryMalloc((size_t)(inlen * 3) / 4 + 1); + if (*result == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + uint8_t *out = *result; *outlen = 0; while (1) { if (value[0] == 0) { *out = '\0'; - return result; + return 0; } // skip \r\n @@ -93,9 +99,9 @@ uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen) { } base64_decode_error: - taosMemoryFree(result); - result = 0; + taosMemoryFree(*result); + *result = 0; *outlen = 0; - return result; + return TSDB_CODE_INVALID_DATA_FMT; } diff --git a/source/util/test/tbaseCodecTest.cpp b/source/util/test/tbaseCodecTest.cpp index 63bbfcaa68..da6135ea66 100644 --- a/source/util/test/tbaseCodecTest.cpp +++ b/source/util/test/tbaseCodecTest.cpp @@ -19,14 +19,16 @@ using namespace std; static void checkBase58Codec(uint8_t *pRaw, int32_t rawLen, int32_t index) { int64_t start = taosGetTimestampUs(); - char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen); + char *pEnc = NULL; + (void)base58_encode((const uint8_t *)pRaw, rawLen, &pEnc); ASSERT_NE(nullptr, pEnc); int32_t encLen = strlen(pEnc); int64_t endOfEnc = taosGetTimestampUs(); std::cout << "index:" << index << ", encLen is " << encLen << ", cost:" << endOfEnc - start << " us" << std::endl; int32_t decLen = 0; - char *pDec = (char *)base58_decode((const char *)pEnc, encLen, &decLen); + char *pDec = NULL; + (void)base58_decode((const char *)pEnc, encLen, &decLen, (uint8_t**)&pDec); std::cout << "index:" << index << ", decLen is " << decLen << ", cost:" << taosGetTimestampUs() - endOfEnc << " us" << std::endl; ASSERT_NE(nullptr, pDec); @@ -68,9 +70,11 @@ TEST(TD_BASE_CODEC_TEST, tbase58_test) { // 2. overflow case char tmp[1]; - char *pEnc = base58_encode((const uint8_t *)tmp, TBASE_MAX_ILEN + 1); + char *pEnc = NULL; + (void)base58_encode((const uint8_t *)tmp, TBASE_MAX_ILEN + 1, &pEnc); ASSERT_EQ(nullptr, pEnc); - char *pDec = (char *)base58_decode((const char *)tmp, TBASE_MAX_OLEN + 1, NULL); + char *pDec = NULL; + (void)base58_decode((const char *)tmp, TBASE_MAX_OLEN + 1, NULL, (uint8_t**)&pDec); ASSERT_EQ(nullptr, pDec); taosMemoryFreeClear(pRaw); From 280c5d4b3db4acebba17b5a7428c3e13ffdab6f6 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 22 Jul 2024 16:33:42 +0800 Subject: [PATCH 06/52] enh: return error code --- source/dnode/mnode/impl/src/mndAcct.c | 4 ++ source/dnode/mnode/impl/src/mndArbGroup.c | 4 ++ source/dnode/mnode/impl/src/mndCluster.c | 4 ++ source/dnode/mnode/impl/src/mndCompact.c | 4 ++ .../dnode/mnode/impl/src/mndCompactDetail.c | 10 +++-- source/dnode/mnode/impl/src/mndConsumer.c | 4 ++ source/dnode/mnode/impl/src/mndDb.c | 4 ++ source/dnode/mnode/impl/src/mndDnode.c | 4 ++ source/dnode/mnode/impl/src/mndFunc.c | 4 ++ source/dnode/mnode/impl/src/mndIndex.c | 4 ++ source/dnode/mnode/impl/src/mndMnode.c | 4 ++ source/dnode/mnode/impl/src/mndQnode.c | 4 ++ source/dnode/mnode/impl/src/mndSma.c | 4 ++ source/dnode/mnode/impl/src/mndSnode.c | 4 ++ source/dnode/mnode/impl/src/mndStb.c | 4 ++ source/dnode/mnode/impl/src/mndStream.c | 2 + source/dnode/mnode/impl/src/mndStreamTrans.c | 2 + source/dnode/mnode/impl/src/mndSubscribe.c | 4 ++ source/dnode/mnode/impl/src/mndTopic.c | 4 ++ source/dnode/mnode/impl/src/mndTrans.c | 9 +++- source/dnode/mnode/impl/src/mndUser.c | 40 +++++++++++------ source/dnode/mnode/impl/src/mndVgroup.c | 4 ++ source/dnode/mnode/sdb/inc/sdb.h | 43 +++++++++++-------- 23 files changed, 138 insertions(+), 36 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 99f545ba74..425624d717 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -113,6 +113,8 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) { } static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, ACCT_VER_NUMBER, sizeof(SAcctObj) + ACCT_RESERVE_SIZE); @@ -153,6 +155,8 @@ _OVER: } static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SAcctObj *pAcct = NULL; SSdbRow *pRow = NULL; diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index c3c150ba50..2adb420abe 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -114,6 +114,8 @@ void mndArbGroupInitFromVgObj(SVgObj *pVgObj, SArbGroup *outGroup) { } SSdbRaw *mndArbGroupActionEncode(SArbGroup *pGroup) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = sizeof(SArbGroup) + ARBGROUP_RESERVE_SIZE; @@ -152,6 +154,8 @@ _OVER: } SSdbRow *mndArbGroupActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SArbGroup *pGroup = NULL; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 47f289dcfd..366e4543fe 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -144,6 +144,8 @@ int64_t mndGetClusterUpTime(SMnode *pMnode) { } static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, CLUSTER_VER_NUMBE, sizeof(SClusterObj) + CLUSTER_RESERVE_SIZE); @@ -172,6 +174,8 @@ _OVER: } static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SClusterObj *pCluster = NULL; SSdbRow *pRow = NULL; diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 065a9a327d..cdb4680ad6 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -88,6 +88,8 @@ int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj) { } SSdbRaw *mndCompactActionEncode(SCompactObj *pCompact) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_SUCCESS; void *buf = NULL; @@ -136,6 +138,8 @@ OVER: } SSdbRow *mndCompactActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; SSdbRow *pRow = NULL; SCompactObj *pCompact = NULL; void *buf = NULL; diff --git a/source/dnode/mnode/impl/src/mndCompactDetail.c b/source/dnode/mnode/impl/src/mndCompactDetail.c index 7d73cf8dcd..c68fbfac2d 100644 --- a/source/dnode/mnode/impl/src/mndCompactDetail.c +++ b/source/dnode/mnode/impl/src/mndCompactDetail.c @@ -144,6 +144,8 @@ int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailO } SSdbRaw *mndCompactDetailActionEncode(SCompactDetailObj *pCompact) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_SUCCESS; void *buf = NULL; @@ -193,9 +195,11 @@ OVER: } SSdbRow *mndCompactDetailActionDecode(SSdbRaw *pRaw) { - SSdbRow *pRow = NULL; - SCompactDetailObj *pCompact = NULL; - void *buf = NULL; + int32_t code = 0; + int32_t lino = 0; + SSdbRow *pRow = NULL; + SCompactDetailObj *pCompact = NULL; + void *buf = NULL; terrno = TSDB_CODE_SUCCESS; int8_t sver = 0; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 7788bf005a..384bfb6747 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -700,6 +700,8 @@ _over: } SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; void *buf = NULL; @@ -736,6 +738,8 @@ CM_ENCODE_OVER: } SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; SSdbRow *pRow = NULL; SMqConsumerObj *pConsumer = NULL; void *buf = NULL; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 1c9be5cf81..da041bc0dd 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -88,6 +88,8 @@ int32_t mndInitDb(SMnode *pMnode) { void mndCleanupDb(SMnode *pMnode) {} SSdbRaw *mndDbActionEncode(SDbObj *pDb) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = sizeof(SDbObj) + pDb->cfg.numOfRetensions * sizeof(SRetention) + DB_RESERVE_SIZE; @@ -166,6 +168,8 @@ _OVER: } static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SDbObj *pDb = NULL; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 879f533635..c49c295234 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -184,6 +184,8 @@ _OVER: } static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER_NUMBER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE); @@ -215,6 +217,8 @@ _OVER: } static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SDnodeObj *pDnode = NULL; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 333fdf83bb..ffbfc7fef7 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -61,6 +61,8 @@ int32_t mndInitFunc(SMnode *pMnode) { void mndCleanupFunc(SMnode *pMnode) {} static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = pFunc->commentSize + pFunc->codeSize + sizeof(SFuncObj) + SDB_FUNC_RESERVE_SIZE; @@ -101,6 +103,8 @@ _OVER: } static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SFuncObj *pFunc = NULL; diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index bb77f6c69b..acb8a23756 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -207,6 +207,8 @@ void mndCleanupIdx(SMnode *pMnode) { } static SSdbRaw *mndIdxActionEncode(SIdxObj *pIdx) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; // int32_t size = @@ -244,6 +246,8 @@ _OVER: } static SSdbRow *mndIdxActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SIdxObj *pIdx = NULL; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 7b1ca9c625..014d6468ff 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -127,6 +127,8 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { } static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, MNODE_VER_NUMBER, sizeof(SMnodeObj) + MNODE_RESERVE_SIZE); @@ -154,6 +156,8 @@ _OVER: } static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SMnodeObj *pObj = NULL; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 6cf0fbd387..d6647f88f2 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -75,6 +75,8 @@ void mndReleaseQnode(SMnode *pMnode, SQnodeObj *pObj) { } static SSdbRaw *mndQnodeActionEncode(SQnodeObj *pObj) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRaw *pRaw = sdbAllocRaw(SDB_QNODE, QNODE_VER_NUMBER, sizeof(SQnodeObj) + QNODE_RESERVE_SIZE); @@ -100,6 +102,8 @@ _OVER: } static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SQnodeObj *pObj = NULL; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 946df84a0f..001ad83388 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -112,6 +112,8 @@ int32_t mndInitSma(SMnode *pMnode) { void mndCleanupSma(SMnode *pMnode) {} static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = @@ -173,6 +175,8 @@ _OVER: } static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SSmaObj *pSma = NULL; diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 4243ccb77c..3f65e1c58b 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -79,6 +79,8 @@ void mndReleaseSnode(SMnode *pMnode, SSnodeObj *pObj) { } static SSdbRaw *mndSnodeActionEncode(SSnodeObj *pObj) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRaw *pRaw = sdbAllocRaw(SDB_SNODE, SNODE_VER_NUMBER, sizeof(SSnodeObj) + SNODE_RESERVE_SIZE); @@ -104,6 +106,8 @@ _OVER: } static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SSnodeObj *pObj = NULL; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3fac879ca9..c5e6f38238 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -115,6 +115,8 @@ int32_t mndInitStb(SMnode *pMnode) { void mndCleanupStb(SMnode *pMnode) {} SSdbRaw *mndStbActionEncode(SStbObj *pStb) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + pStb->commentLen + @@ -205,6 +207,8 @@ _OVER: } static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SStbObj *pStb = NULL; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index d57dc6e52e..32a76afa5b 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -160,6 +160,8 @@ void mndCleanupStream(SMnode *pMnode) { } SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index f252791618..b39487b79f 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -177,6 +177,8 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnCo } SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; void *buf = NULL; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e2bedc258a..43e291dc6b 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -1096,6 +1096,8 @@ end: void mndCleanupSubscribe(SMnode *pMnode) {} static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; void *buf = NULL; int32_t tlen = tEncodeSubscribeObj(NULL, pSub); @@ -1132,6 +1134,8 @@ SUB_ENCODE_OVER: } static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SMqSubscribeObj *pSub = NULL; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index bcb38a3902..2c7f83729a 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -76,6 +76,8 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]) { } SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; void *swBuf = NULL; @@ -163,6 +165,8 @@ TOPIC_ENCODE_OVER: } SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SMqTopicObj *pTopic = NULL; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 58dab20859..49d1a4ca97 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -105,6 +105,8 @@ static int32_t mndTransGetActionsSize(SArray *pArray) { } static int32_t mndTransEncodeAction(SSdbRaw *pRaw, int32_t *offset, SArray *pActions, int32_t actionsNum) { + int32_t code = 0; + int32_t lino = 0; int32_t dataPos = *offset; int8_t unused = 0; int32_t ret = -1; @@ -142,6 +144,8 @@ _OVER: } SSdbRaw *mndTransEncode(STrans *pTrans) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_INVALID_MSG; int8_t sver = taosArrayGetSize(pTrans->prepareActions) ? TRANS_VER2_NUMBER : TRANS_VER1_NUMBER; @@ -225,6 +229,8 @@ _OVER: } static int32_t mndTransDecodeAction(SSdbRaw *pRaw, int32_t *offset, SArray *pActions, int32_t actionNum) { + int32_t code = 0; + int32_t lino = 0; STransAction action = {0}; int32_t dataPos = *offset; int8_t unused = 0; @@ -279,7 +285,8 @@ _OVER: SSdbRow *mndTransDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_INVALID_MSG; - + int32_t code = 0; + int32_t lino = 0; SSdbRow *pRow = NULL; STrans *pTrans = NULL; char *pData = NULL; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index bae617013c..32b82633fe 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -784,11 +784,11 @@ static int32_t createIpWhiteList(void *buf, int32_t len, SIpWhiteList **ppList) SDecoder decoder = {0}; tDecoderInit(&decoder, buf, len); - TAOS_CHECK_GOTO(tStartDecode(&decoder), &lino, _OVER), &lino, _OVER); - TAOS_CHECK_GOTO(tDecodeI32(&decoder, &num), &lino, _OVER), &lino, _OVER); + TAOS_CHECK_GOTO(tStartDecode(&decoder), &lino, _OVER); + TAOS_CHECK_GOTO(tDecodeI32(&decoder, &num), &lino, _OVER); p = taosMemoryCalloc(1, sizeof(SIpWhiteList) + num * sizeof(SIpV4Range)); - if(p == NULL) { + if (p == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } TAOS_CHECK_GOTO(tDerializeIpWhileList(buf, len, p), &lino, _OVER); @@ -796,7 +796,12 @@ static int32_t createIpWhiteList(void *buf, int32_t len, SIpWhiteList **ppList) _OVER: tEndDecode(&decoder); tDecoderClear(&decoder); - return p; + if (code != 0) { + taosMemoryFreeClear(p); + mError("failed to create ip white list at line %d since %s", lino, tstrerror(code)); + } + *ppList = p; + TAOS_RETURN(code); } static int32_t createDefaultIpWhiteList(SIpWhiteList **ppWhiteList) { @@ -876,7 +881,8 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { } SSdbRaw *mndUserActionEncode(SUserObj *pUser) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = 0; + int32_t lino = 0; int32_t ipWhiteReserve = pUser->pIpWhiteList ? (sizeof(SIpV4Range) * pUser->pIpWhiteList->num + sizeof(SIpWhiteList) + 4) : 16; @@ -989,7 +995,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); if (pRaw == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } int32_t dataPos = 0; @@ -1744,21 +1750,30 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate if(pUniqueTab == NULL){ TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - int32_t dummpy = 0; + int32_t dummpy = 0; for (int i = 0; i < pCreate->numIpRanges; i++) { SIpV4Range range = {.ip = pCreate->pIpRanges[i].ip, .mask = pCreate->pIpRanges[i].mask}; - if(code = taosHashPut(pUniqueTab, &range, sizeof(range), &dummpy, sizeof(dummpy)); + if ((code = taosHashPut(pUniqueTab, &range, sizeof(range), &dummpy, sizeof(dummpy))) != 0) { + taosHashCleanup(pUniqueTab); + TAOS_RETURN(code); + } + } + if ((code = taosHashPut(pUniqueTab, &defaultIpRange, sizeof(defaultIpRange), &dummpy, sizeof(dummpy))) != 0) { + taosHashCleanup(pUniqueTab); + TAOS_RETURN(code); } - taosHashPut(pUniqueTab, &defaultIpRange, sizeof(defaultIpRange), &dummpy, sizeof(dummpy)); if (taosHashGetSize(pUniqueTab) > MND_MAX_USE_HOST) { - terrno = TSDB_CODE_MND_TOO_MANY_USER_HOST; taosHashCleanup(pUniqueTab); - return terrno; + TAOS_RETURN(TSDB_CODE_MND_TOO_MANY_USER_HOST); } int32_t numOfRanges = taosHashGetSize(pUniqueTab); SIpWhiteList *p = taosMemoryCalloc(1, sizeof(SIpWhiteList) + numOfRanges * sizeof(SIpV4Range)); + if (p == NULL) { + taosHashCleanup(pUniqueTab); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } void *pIter = taosHashIterate(pUniqueTab, NULL); int32_t i = 0; while (pIter) { @@ -1781,9 +1796,8 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-user"); if (pTrans == NULL) { mError("user:%s, failed to create since %s", pCreate->user, terrstr()); - taosMemoryFree(userObj.pIpWhiteList); - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } mInfo("trans:%d, used to create user:%s", pTrans->id, pCreate->user); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0422bfabff..160d16667f 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -89,6 +89,8 @@ int32_t mndInitVgroup(SMnode *pMnode) { void mndCleanupVgroup(SMnode *pMnode) {} SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, VGROUP_VER_NUMBER, sizeof(SVgObj) + VGROUP_RESERVE_SIZE); @@ -127,6 +129,8 @@ _OVER: } SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SVgObj *pVgroup = NULL; diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index fc9b89a141..5bb8ae2b5e 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -39,18 +39,20 @@ extern "C" { #define SDB_GET_VAL(pData, dataPos, val, pos, func, type) \ { \ - if (func(pRaw, dataPos, val) != 0) { \ + if ((code = func(pRaw, dataPos, val)) != 0) { \ + lino = __LINE__; \ goto pos; \ } \ dataPos += sizeof(type); \ } -#define SDB_GET_BINARY(pRaw, dataPos, val, valLen, pos) \ - { \ - if (sdbGetRawBinary(pRaw, dataPos, val, valLen) != 0) { \ - goto pos; \ - } \ - dataPos += valLen; \ +#define SDB_GET_BINARY(pRaw, dataPos, val, valLen, pos) \ + { \ + if ((code = sdbGetRawBinary(pRaw, dataPos, val, valLen)) != 0) { \ + lino = __LINE__; \ + goto pos; \ + } \ + dataPos += valLen; \ } #define SDB_GET_INT64(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt64, int64_t) @@ -67,7 +69,8 @@ extern "C" { #define SDB_SET_VAL(pRaw, dataPos, val, pos, func, type) \ { \ - if (func(pRaw, dataPos, val) != 0) { \ + if ((code = func(pRaw, dataPos, val)) != 0) { \ + lino = __LINE__; \ goto pos; \ } \ dataPos += sizeof(type); \ @@ -79,12 +82,13 @@ extern "C" { #define SDB_SET_INT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt8, int8_t) #define SDB_SET_UINT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawUInt8, uint8_t) -#define SDB_SET_BINARY(pRaw, dataPos, val, valLen, pos) \ - { \ - if (sdbSetRawBinary(pRaw, dataPos, val, valLen) != 0) { \ - goto pos; \ - } \ - dataPos += valLen; \ +#define SDB_SET_BINARY(pRaw, dataPos, val, valLen, pos) \ + { \ + if ((code = sdbSetRawBinary(pRaw, dataPos, val, valLen)) != 0) { \ + lino = __LINE__; \ + goto pos; \ + } \ + dataPos += valLen; \ } #define SDB_SET_RESERVE(pRaw, dataPos, valLen, pos) \ @@ -93,11 +97,12 @@ extern "C" { SDB_SET_BINARY(pRaw, dataPos, val, valLen, pos) \ } -#define SDB_SET_DATALEN(pRaw, dataLen, pos) \ - { \ - if (sdbSetRawDataLen(pRaw, dataLen) != 0) { \ - goto pos; \ - } \ +#define SDB_SET_DATALEN(pRaw, dataLen, pos) \ + { \ + if ((code = sdbSetRawDataLen(pRaw, dataLen)) != 0) { \ + lino = __LINE__; \ + goto pos; \ + } \ } typedef struct SMnode SMnode; From 67aeb8d27110f728212073da0f0bddb12af5671c Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 22 Jul 2024 19:24:00 +0800 Subject: [PATCH 07/52] enh: return error code --- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 9 +- source/dnode/mnode/impl/src/mndUser.c | 319 +++++++++++++----------- 2 files changed, 169 insertions(+), 159 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index fcb21267e8..e5f601abfe 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -109,13 +109,6 @@ static int32_t dmCheckDiskSpace() { return code; } -int32_t tfsOpenWrapper(SDiskCfg *pCfg, int32_t ndisk, STfs **tfs) { - *tfs = tfsOpen(pCfg, ndisk); - if (*tfs == NULL) { - return terrno; - } - return 0; -} int32_t dmDiskInit() { SDnode *pDnode = dmInstance(); SDiskCfg dCfg = {.level = 0, .primary = 1, .disable = 0}; @@ -127,7 +120,7 @@ int32_t dmDiskInit() { numOfDisks = 1; } - int32_t code = tfsOpenWrapper(pDisks, numOfDisks, &pDnode->pTfs); + int32_t code = tfsOpen(pDisks, numOfDisks, &pDnode->pTfs); if (code != 0) { dError("failed to init tfs since %s", tstrerror(code)); TAOS_RETURN(code); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 32b82633fe..8e7ad30332 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -143,6 +143,7 @@ static int32_t ipWhiteMgtInit() { } ipWhiteMgt.ver = 0; taosThreadRwlockInit(&ipWhiteMgt.rw, NULL); + TAOS_RETURN(0); } void ipWhiteMgtCleanup() { destroyIpWhiteTab(ipWhiteMgt.pIpWhiteTab); @@ -731,24 +732,32 @@ int32_t convertIpWhiteListToStr(SIpWhiteList *pList, char **buf) { } return strlen(*buf); } -int32_t tSerializeIpWhiteList(void *buf, int32_t len, SIpWhiteList *pList) { +int32_t tSerializeIpWhiteList(void *buf, int32_t len, SIpWhiteList *pList, uint32_t *pLen) { + int32_t code = 0; + int32_t lino = 0; + int32_t tlen = 0; SEncoder encoder = {0}; tEncoderInit(&encoder, buf, len); - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, pList->num) < 0) return -1; + TAOS_CHECK_GOTO(tStartEncode(&encoder), &lino, _OVER); + TAOS_CHECK_GOTO(tEncodeI32(&encoder, pList->num), &lino, _OVER); for (int i = 0; i < pList->num; i++) { SIpV4Range *pRange = &(pList->pIpRange[i]); - if (tEncodeU32(&encoder, pRange->ip) < 0) return -1; - if (tEncodeU32(&encoder, pRange->mask) < 0) return -1; + TAOS_CHECK_GOTO(tEncodeU32(&encoder, pRange->ip), &lino, _OVER); + TAOS_CHECK_GOTO(tEncodeU32(&encoder, pRange->mask), &lino, _OVER); } tEndEncode(&encoder); - int32_t tlen = encoder.pos; + tlen = encoder.pos; +_OVER: tEncoderClear(&encoder); - return tlen; + if (code != 0) { + mError("failed to serialize ip white list at line %d since %s", lino, tstrerror(code)); + } + if (pLen) *pLen = tlen; + TAOS_RETURN(code); } int32_t tDerializeIpWhileList(void *buf, int32_t len, SIpWhiteList *pList) { @@ -869,7 +878,7 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char return 0; _ERROR: taosMemoryFree(userObj.pIpWhiteList); - return -1; + TAOS_RETURN(terrno ? terrno : -1); } static int32_t mndCreateDefaultUsers(SMnode *pMnode) { @@ -883,7 +892,6 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t code = 0; int32_t lino = 0; - int32_t ipWhiteReserve = pUser->pIpWhiteList ? (sizeof(SIpV4Range) * pUser->pIpWhiteList->num + sizeof(SIpWhiteList) + 4) : 16; int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); @@ -898,6 +906,8 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t numOfUseDbs = taosHashGetSize(pUser->useDbs); int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN + numOfTopics * TSDB_TOPIC_FNAME_LEN + ipWhiteReserve; + char *buf = NULL; + SSdbRaw *pRaw = NULL; char *stb = taosHashIterate(pUser->readTbs, NULL); while (stb != NULL) { @@ -993,7 +1003,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { useDb = taosHashIterate(pUser->useDbs, useDb); } - SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); + pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); if (pRaw == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } @@ -1138,31 +1148,32 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { // save white list int32_t num = pUser->pIpWhiteList->num; int32_t tlen = sizeof(SIpWhiteList) + num * sizeof(SIpV4Range) + 4; - char *buf = taosMemoryCalloc(1, tlen); - if (buf == NULL) { + if ((buf = taosMemoryCalloc(1, tlen)) == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); } - int32_t len = tSerializeIpWhiteList(buf, tlen, pUser->pIpWhiteList); + int32_t len = 0; + TAOS_CHECK_GOTO(tSerializeIpWhiteList(buf, tlen, pUser->pIpWhiteList, &len), &lino, _OVER); SDB_SET_INT32(pRaw, dataPos, len, _OVER); SDB_SET_BINARY(pRaw, dataPos, buf, len, _OVER); - taosMemoryFree(buf); + SDB_SET_INT64(pRaw, dataPos, pUser->ipWhiteListVer, _OVER); SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) - code = 0; - _OVER: + taosMemoryFree(buf); if (code != 0) { - mError("user:%s, failed to encode to raw:%p since %s", pUser->user, pRaw, tstrerror(code)); + mError("user:%s, failed to encode user action to raw:%p at line %d since %s", pUser->user, pRaw, lino, + tstrerror(code)); sdbFreeRaw(pRaw); - return NULL; + pRaw = NULL; + terrno = code; } - mTrace("user:%s, encode to raw:%p, row:%p", pUser->user, pRaw, pUser); + mTrace("user:%s, encode user action to raw:%p, row:%p", pUser->user, pRaw, pUser); return pRaw; } @@ -1921,42 +1932,53 @@ _OVER: int32_t mndProcessGetUserWhiteListReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; - int32_t code = -1; + int32_t code = 0; + int32_t lino = 0; + int32_t contLen = 0; + void *pRsp = NULL; SUserObj *pUser = NULL; SGetUserWhiteListReq wlReq = {0}; SGetUserWhiteListRsp wlRsp = {0}; if (tDeserializeSGetUserWhiteListReq(pReq->pCont, pReq->contLen, &wlReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_INVALID_MSG, &lino, _OVER); } mTrace("user: %s, start to get whitelist", wlReq.user); - pUser = mndAcquireUser(pMnode, wlReq.user); + code = mndAcquireUser(pMnode, wlReq.user, &pUser); if (pUser == NULL) { - terrno = TSDB_CODE_MND_USER_NOT_EXIST; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_USER_NOT_EXIST, &lino, _OVER); } - code = mndSetUserWhiteListRsp(pMnode, pUser, &wlRsp); - if (code) { - goto _OVER; + TAOS_CHECK_GOTO(mndSetUserWhiteListRsp(pMnode, pUser, &wlRsp), &lino, _OVER); + + contLen = tSerializeSGetUserWhiteListRsp(NULL, 0, &wlRsp); + if (contLen < 0) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - int32_t contLen = tSerializeSGetUserWhiteListRsp(NULL, 0, &wlRsp); - void *pRsp = rpcMallocCont(contLen); + pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - tSerializeSGetUserWhiteListRsp(pRsp, contLen, &wlRsp); + contLen = tSerializeSGetUserWhiteListRsp(pRsp, contLen, &wlRsp); + if(contLen < 0) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } - pReq->info.rsp = pRsp; - pReq->info.rspLen = contLen; - code = 0; _OVER: mndReleaseUser(pMnode, pUser); tFreeSGetUserWhiteListRsp(&wlRsp); + if (code != 0) { + mError("user:%s, failed to get whitelist at line %d since %s", wlReq.user, lino, tstrerror(code)); + rpcFreeCont(pRsp); + pRsp = NULL; + contLen = 0; + } + pReq->code = code; + pReq->info.rsp = pRsp; + pReq->info.rspLen = contLen; + TAOS_RETURN(code); } @@ -2073,18 +2095,13 @@ static int32_t mndTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj *useD if (alterReq->tagCond != NULL && alterReq->tagCondLen != 0) { char *value = taosHashGet(hash, tbFName, len); if (value != NULL) { - terrno = TSDB_CODE_MND_PRIVILEDGE_EXIST; - return -1; + TAOS_RETURN(TSDB_CODE_MND_PRIVILEDGE_EXIST); } int32_t condLen = alterReq->tagCondLen; - if (taosHashPut(hash, tbFName, len, alterReq->tagCond, condLen) != 0) { - return -1; - } + TAOS_CHECK_RETURN(taosHashPut(hash, tbFName, len, alterReq->tagCond, condLen)); } else { - if (taosHashPut(hash, tbFName, len, alterReq->isView ? "v" : "t", 2) != 0) { - return -1; - } + TAOS_CHECK_RETURN(taosHashPut(hash, tbFName, len, alterReq->isView ? "v" : "t", 2)); } int32_t dbKeyLen = strlen(alterReq->objname) + 1; @@ -2093,11 +2110,9 @@ static int32_t mndTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj *useD if (NULL != currRef) { ref = (*currRef) + 1; } - if (taosHashPut(useDbHash, alterReq->objname, dbKeyLen, &ref, sizeof(ref)) != 0) { - return -1; - } + TAOS_CHECK_RETURN(taosHashPut(useDbHash, alterReq->objname, dbKeyLen, &ref, sizeof(ref))); - return 0; + TAOS_RETURN(0); } static int32_t mndRemoveTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj *useDbHash, SAlterUserReq *alterReq, @@ -2108,7 +2123,7 @@ static int32_t mndRemoveTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj int32_t len = strlen(tbFName) + 1; if (taosHashRemove(hash, tbFName, len) != 0) { - return -1; + TAOS_RETURN(TSDB_CODE_NOT_FOUND); } int32_t dbKeyLen = strlen(alterReq->objname) + 1; @@ -2119,14 +2134,12 @@ static int32_t mndRemoveTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj if (1 == *currRef) { if (taosHashRemove(useDbHash, alterReq->objname, dbKeyLen) != 0) { - return -1; + TAOS_RETURN(TSDB_CODE_NOT_FOUND); } return 0; } int32_t ref = (*currRef) - 1; - if (taosHashPut(useDbHash, alterReq->objname, dbKeyLen, &ref, sizeof(ref)) != 0) { - return -1; - } + TAOS_CHECK_RETURN(taosHashPut(useDbHash, alterReq->objname, dbKeyLen, &ref, sizeof(ref))); return 0; } @@ -2151,8 +2164,10 @@ static char *mndUserAuditTypeStr(int32_t type) { } static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode *pMnode, SUserObj *pNewUser) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + int32_t code = 0; + int32_t lino = 0; if (ALTER_USER_ADD_READ_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) { @@ -2161,11 +2176,12 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname); if (pDb == NULL) { mndReleaseDb(pMnode, pDb); - return -1; + TAOS_RETURN(terrno); } - if (taosHashPut(pNewUser->readDbs, pAlterReq->objname, len, pAlterReq->objname, TSDB_DB_FNAME_LEN) != 0) { + if ((code = taosHashPut(pNewUser->readDbs, pAlterReq->objname, len, pAlterReq->objname, TSDB_DB_FNAME_LEN)) != + 0) { mndReleaseDb(pMnode, pDb); - return -1; + TAOS_CHECK_GOTO(code, &lino, _OVER); } mndReleaseDb(pMnode, pDb); } else { @@ -2174,7 +2190,11 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb); if (pIter == NULL) break; int32_t len = strlen(pDb->name) + 1; - taosHashPut(pNewUser->readDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN); + if ((code = taosHashPut(pNewUser->readDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN)) != 0) { + sdbRelease(pSdb, pDb); + sdbCancelFetch(pSdb, pIter); + TAOS_CHECK_GOTO(code, &lino, _OVER); + } sdbRelease(pSdb, pDb); } } @@ -2187,11 +2207,12 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname); if (pDb == NULL) { mndReleaseDb(pMnode, pDb); - return -1; + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } - if (taosHashPut(pNewUser->writeDbs, pAlterReq->objname, len, pAlterReq->objname, TSDB_DB_FNAME_LEN) != 0) { + if ((code = taosHashPut(pNewUser->writeDbs, pAlterReq->objname, len, pAlterReq->objname, TSDB_DB_FNAME_LEN)) != + 0) { mndReleaseDb(pMnode, pDb); - return -1; + TAOS_CHECK_GOTO(code, &lino, _OVER); } mndReleaseDb(pMnode, pDb); } else { @@ -2200,7 +2221,11 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb); if (pIter == NULL) break; int32_t len = strlen(pDb->name) + 1; - taosHashPut(pNewUser->writeDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN); + if ((code = taosHashPut(pNewUser->writeDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN)) != 0) { + sdbRelease(pSdb, pDb); + sdbCancelFetch(pSdb, pIter); + TAOS_CHECK_GOTO(code, &lino, _OVER); + } sdbRelease(pSdb, pDb); } } @@ -2213,7 +2238,7 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname); if (pDb == NULL) { mndReleaseDb(pMnode, pDb); - return -1; + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } taosHashRemove(pNewUser->readDbs, pAlterReq->objname, len); mndReleaseDb(pMnode, pDb); @@ -2229,7 +2254,7 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname); if (pDb == NULL) { mndReleaseDb(pMnode, pDb); - return -1; + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } taosHashRemove(pNewUser->writeDbs, pAlterReq->objname, len); mndReleaseDb(pMnode, pDb); @@ -2252,32 +2277,32 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode if (ALTER_USER_ADD_READ_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) { - if (mndTablePriviledge(pMnode, pReadTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1; + TAOS_CHECK_GOTO(mndTablePriviledge(pMnode, pReadTbs, pNewUser->useDbs, pAlterReq, pSdb), &lino, _OVER); } if (ALTER_USER_ADD_WRITE_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) { - if (mndTablePriviledge(pMnode, pWriteTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1; + TAOS_CHECK_GOTO(mndTablePriviledge(pMnode, pWriteTbs, pNewUser->useDbs, pAlterReq, pSdb), &lino, _OVER); } if (ALTER_USER_ADD_ALTER_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) { - if (mndTablePriviledge(pMnode, pAlterTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1; + TAOS_CHECK_GOTO(mndTablePriviledge(pMnode, pAlterTbs, pNewUser->useDbs, pAlterReq, pSdb), &lino, _OVER); } if (ALTER_USER_DEL_READ_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_DEL_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) { - if (mndRemoveTablePriviledge(pMnode, pReadTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1; + TAOS_CHECK_GOTO(mndRemoveTablePriviledge(pMnode, pReadTbs, pNewUser->useDbs, pAlterReq, pSdb), &lino, _OVER); } if (ALTER_USER_DEL_WRITE_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_DEL_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) { - if (mndRemoveTablePriviledge(pMnode, pWriteTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1; + TAOS_CHECK_GOTO(mndRemoveTablePriviledge(pMnode, pWriteTbs, pNewUser->useDbs, pAlterReq, pSdb), &lino, _OVER); } if (ALTER_USER_DEL_ALTER_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_DEL_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) { - if (mndRemoveTablePriviledge(pMnode, pAlterTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1; + TAOS_CHECK_GOTO(mndRemoveTablePriviledge(pMnode, pAlterTbs, pNewUser->useDbs, pAlterReq, pSdb), &lino, _OVER); } if (ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) { @@ -2285,9 +2310,12 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pAlterReq->objname); if (pTopic == NULL) { mndReleaseTopic(pMnode, pTopic); - return -1; + TAOS_CHECK_GOTO(terrno, &lino, _OVER); + } + if ((code = taosHashPut(pNewUser->topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN)) != 0) { + mndReleaseTopic(pMnode, pTopic); + TAOS_CHECK_GOTO(code, &lino, _OVER); } - taosHashPut(pNewUser->topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN); mndReleaseTopic(pMnode, pTopic); } @@ -2296,60 +2324,53 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pAlterReq->objname); if (pTopic == NULL) { mndReleaseTopic(pMnode, pTopic); - return -1; + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } taosHashRemove(pNewUser->topics, pAlterReq->objname, len); mndReleaseTopic(pMnode, pTopic); } - return TSDB_CODE_SUCCESS; +_OVER: + if (code != 0) { + mError("user:%s, failed to alter user privileges at line %d since %s", pAlterReq->user, lino, terrstr()); + } + TAOS_RETURN(code); } static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; - int32_t code = -1; + int32_t code = 0; + int32_t lino = 0; SUserObj *pUser = NULL; SUserObj *pOperUser = NULL; SUserObj newUser = {0}; SAlterUserReq alterReq = {0}; - if (tDeserializeSAlterUserReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSAlterUserReq(pReq->pCont, pReq->contLen, &alterReq), &lino, _OVER); mInfo("user:%s, start to alter", alterReq.user); if (alterReq.user[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_USER_FORMAT, &lino, _OVER); } if (TSDB_ALTER_USER_PASSWD == alterReq.alterType && (alterReq.pass[0] == 0 || strlen(alterReq.pass) >= TSDB_PASSWORD_LEN)) { - terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_PASS_FORMAT, &lino, _OVER); } - pUser = mndAcquireUser(pMnode, alterReq.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_USER_NOT_EXIST; - goto _OVER; - } + TAOS_CHECK_GOTO(mndAcquireUser(pMnode, alterReq.user, &pUser), &lino, _OVER); - pOperUser = mndAcquireUser(pMnode, pReq->info.conn.user); + mndAcquireUser(pMnode, pReq->info.conn.user, &pOperUser); if (pOperUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_NO_USER_FROM_CONN, &lino, _OVER); } - if (mndCheckAlterUserPrivilege(pOperUser, pUser, &alterReq) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckAlterUserPrivilege(pOperUser, pUser, &alterReq), &lino, _OVER); - if (mndUserDupObj(pUser, &newUser) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndUserDupObj(pUser, &newUser), &lino, _OVER); if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) { char pass[TSDB_PASSWORD_LEN + 1] = {0}; @@ -2377,16 +2398,19 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { } if (ALTER_USER_ADD_PRIVS(alterReq.alterType) || ALTER_USER_DEL_PRIVS(alterReq.alterType)) { - if (0 != mndProcessAlterUserPrivilegesReq(&alterReq, pMnode, &newUser)) goto _OVER; + TAOS_CHECK_GOTO(mndProcessAlterUserPrivilegesReq(&alterReq, pMnode, &newUser), &lino, _OVER); } if (alterReq.alterType == TSDB_ALTER_USER_ADD_WHITE_LIST) { taosMemoryFreeClear(newUser.pIpWhiteList); - int32_t num = pUser->pIpWhiteList->num + alterReq.numIpRanges; - - SIpWhiteList *pNew = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range) * num); + int32_t num = pUser->pIpWhiteList->num + alterReq.numIpRanges; int32_t idx = pUser->pIpWhiteList->num; + SIpWhiteList *pNew = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range) * num); + + if (pNew == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } bool exist = false; memcpy(pNew->pIpRange, pUser->pIpWhiteList->pIpRange, sizeof(SIpV4Range) * idx); @@ -2403,27 +2427,27 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { } if (exist) { taosMemoryFree(pNew); - terrno = TSDB_CODE_MND_USER_HOST_EXIST; - code = terrno; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_USER_HOST_EXIST, &lino, _OVER); } pNew->num = idx; newUser.pIpWhiteList = pNew; newUser.ipWhiteListVer = pUser->ipWhiteListVer + 1; if (pNew->num > MND_MAX_USE_HOST) { - terrno = TSDB_CODE_MND_TOO_MANY_USER_HOST; - code = terrno; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_TOO_MANY_USER_HOST, &lino, _OVER); } } if (alterReq.alterType == TSDB_ALTER_USER_DROP_WHITE_LIST) { taosMemoryFreeClear(newUser.pIpWhiteList); int32_t num = pUser->pIpWhiteList->num; - SIpWhiteList *pNew = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range) * num); bool noexist = true; bool localHost = false; + SIpWhiteList *pNew = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range) * num); + + if(pNew == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } if (pUser->pIpWhiteList->num > 0) { int idx = 0; @@ -2461,14 +2485,10 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { } if (localHost) { - terrno = TSDB_CODE_MND_USER_LOCAL_HOST_NOT_DROP; - code = terrno; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_USER_LOCAL_HOST_NOT_DROP, &lino, _OVER); } if (noexist) { - terrno = TSDB_CODE_MND_USER_HOST_NOT_EXIST; - code = terrno; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_USER_HOST_NOT_EXIST, &lino, _OVER); } } @@ -2525,14 +2545,14 @@ _OVER: mndReleaseUser(pMnode, pUser); mndUserFreeObj(&newUser); - return code; + TAOS_RETURN(code); } static int32_t mndDropUser(SMnode *pMnode, SRpcMsg *pReq, SUserObj *pUser) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-user"); if (pTrans == NULL) { mError("user:%s, failed to drop since %s", pUser->user, terrstr()); - return -1; + TAOS_RETURN(terrno); } mInfo("trans:%d, used to drop user:%s", pTrans->id, pUser->user); @@ -2540,97 +2560,94 @@ static int32_t mndDropUser(SMnode *pMnode, SRpcMsg *pReq, SUserObj *pUser) { if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); - return -1; + TAOS_RETURN(terrno); } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); - return -1; + TAOS_RETURN(terrno); } ipWhiteMgtRemove(pUser->user); mndTransDrop(pTrans); - return 0; + TAOS_RETURN(0); } static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; - int32_t code = -1; + int32_t code = 0; + int32_t lino = 0; SUserObj *pUser = NULL; SDropUserReq dropReq = {0}; - if (tDeserializeSDropUserReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSDropUserReq(pReq->pCont, pReq->contLen, &dropReq), &lino, _OVER); mInfo("user:%s, start to drop", dropReq.user); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER), &lino, _OVER); if (dropReq.user[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_USER_FORMAT, &lino, _OVER); } - pUser = mndAcquireUser(pMnode, dropReq.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_USER_NOT_EXIST; - goto _OVER; - } + TAOS_CHECK_GOTO(mndAcquireUser(pMnode, dropReq.user, &pUser), &lino, _OVER); - code = mndDropUser(pMnode, pReq, pUser); + TAOS_CHECK_GOTO(mndDropUser(pMnode, pReq, pUser), &lino, _OVER); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; auditRecord(pReq, pMnode->clusterId, "dropUser", "", dropReq.user, dropReq.sql, dropReq.sqlLen); _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("user:%s, failed to drop since %s", dropReq.user, terrstr()); + mError("user:%s, failed to drop at line %d since %s", dropReq.user, lino, tstrerror(code)); } mndReleaseUser(pMnode, pUser); tFreeSDropUserReq(&dropReq); - return code; + TAOS_RETURN(code); } static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = 0; + int32_t lino = 0; + int32_t contLen = 0; + void *pRsp = NULL; SUserObj *pUser = NULL; SGetUserAuthReq authReq = {0}; SGetUserAuthRsp authRsp = {0}; - if (tDeserializeSGetUserAuthReq(pReq->pCont, pReq->contLen, &authReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - + TAOS_CHECK_GOTO(tDeserializeSGetUserAuthReq(pReq->pCont, pReq->contLen, &authReq), &lino, _OVER); mTrace("user:%s, start to get auth", authReq.user); - TAOS_CHECK_GOTO(mndAcquireUser(pMnode, authReq.user, &pUser), NULL, _OVER); + TAOS_CHECK_GOTO(mndAcquireUser(pMnode, authReq.user, &pUser), &lino, _OVER); - TAOS_CHECK_GOTO(mndSetUserAuthRsp(pMnode, pUser, &authRsp), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetUserAuthRsp(pMnode, pUser, &authRsp), &lino, _OVER); - int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &authRsp); - void *pRsp = rpcMallocCont(contLen); + contLen = tSerializeSGetUserAuthRsp(NULL, 0, &authRsp); + if (contLen < 0) { + TAOS_CHECK_GOTO(contLen, &lino, _OVER); + } + pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); } - TAOS_CHECK_GOTO(tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp), NULL, _OVER); - - pReq->info.rsp = pRsp; - pReq->info.rspLen = contLen; - code = 0; + TAOS_CHECK_GOTO(tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp), &lino, _OVER); _OVER: - mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); + if (code != 0) { + mError("user:%s, failed to get auth at line %d since %s", authReq.user, lino, tstrerror(code)); + rpcFreeCont(pRsp); + pRsp = NULL; + contLen = 0; + } + pReq->info.rsp = pRsp; + pReq->info.rspLen = contLen; + pReq->code = code; TAOS_RETURN(code); } From 2ddf03fd74c7d6899d815d8de16119d4488eabec Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 23 Jul 2024 00:43:33 +0800 Subject: [PATCH 08/52] enh: return error code --- include/util/tutil.h | 9 +++++++++ source/dnode/mnode/impl/inc/mndShow.h | 8 +++----- source/dnode/mnode/impl/src/mndCluster.c | 12 ++++++------ source/dnode/mnode/impl/src/mndUser.c | 9 +++++---- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/include/util/tutil.h b/include/util/tutil.h index 5af79dfc49..0e79480bfe 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -176,6 +176,15 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } \ } while (0) +#define TAOS_CHECK_EXIT(CMD) \ + do { \ + code = (CMD); \ + if (code < TSDB_CODE_SUCCESS) { \ + lino = __LINE__; \ + goto _exit; \ + } \ + } while (0) + #define TAOS_UNUSED(expr) (void)(expr) #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/inc/mndShow.h b/source/dnode/mnode/impl/inc/mndShow.h index c47b5d1e7b..25268c52bb 100644 --- a/source/dnode/mnode/impl/inc/mndShow.h +++ b/source/dnode/mnode/impl/inc/mndShow.h @@ -23,14 +23,12 @@ extern "C" { #endif -#define COL_DATA_SET_VAL_GOTO(pData, isNull, pObj, LINO) \ +#define COL_DATA_SET_VAL_GOTO(pData, isNull, pObj, LABEL) \ do { \ if ((code = colDataSetVal(pColInfo, numOfRows, (pData), (isNull))) != 0) { \ if (pObj) sdbRelease(pSdb, (pObj)); \ - if (LINO) { \ - *((int32_t *)(LINO)) = __LINE__; \ - } \ - goto _OVER; \ + lino = __LINE__; \ + goto LABEL; \ } \ } while (0) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 366e4543fe..6a6285e68e 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -302,31 +302,31 @@ static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock * cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - COL_DATA_SET_VAL_GOTO((const char *)&pCluster->id, false, pCluster, &lino); + COL_DATA_SET_VAL_GOTO((const char *)&pCluster->id, false, pCluster, _OVER); char buf[tListLen(pCluster->name) + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(buf, pCluster->name, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - COL_DATA_SET_VAL_GOTO(buf, false, pCluster, &lino); + COL_DATA_SET_VAL_GOTO(buf, false, pCluster, _OVER); int32_t upTime = mndGetClusterUpTimeImp(pCluster); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - COL_DATA_SET_VAL_GOTO((const char *)&upTime, false, pCluster, &lino); + COL_DATA_SET_VAL_GOTO((const char *)&upTime, false, pCluster, _OVER); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - COL_DATA_SET_VAL_GOTO((const char *)&pCluster->createdTime, false, pCluster, &lino); + COL_DATA_SET_VAL_GOTO((const char *)&pCluster->createdTime, false, pCluster, _OVER); char ver[12] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(ver, tsVersionName, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - COL_DATA_SET_VAL_GOTO((const char *)ver, false, pCluster, &lino); + COL_DATA_SET_VAL_GOTO((const char *)ver, false, pCluster, _OVER); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (tsExpireTime <= 0) { colDataSetNULL(pColInfo, numOfRows); } else { - COL_DATA_SET_VAL_GOTO((const char *)&tsExpireTime, false, pCluster, &lino); + COL_DATA_SET_VAL_GOTO((const char *)&tsExpireTime, false, pCluster, _OVER); } sdbRelease(pSdb, pCluster); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 8e7ad30332..a7ba8a846e 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1718,10 +1718,11 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { } int32_t mndAcquireUser(SMnode *pMnode, const char *userName, SUserObj **ppUser) { - int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); - if (pUser == NULL) { + int32_t code = 0; + SSdb *pSdb = pMnode->pSdb; + + *ppUser = sdbAcquire(pSdb, SDB_USER, userName); + if (*ppUser == NULL) { if (code == TSDB_CODE_SDB_OBJ_NOT_THERE) { code = TSDB_CODE_MND_USER_NOT_EXIST; } else { From ac9b27dcce38c6cd2fa607330d7fc8f1c612b9b5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 23 Jul 2024 14:19:02 +0800 Subject: [PATCH 09/52] fix:[TD-31017]process return value in client for tmq --- source/client/inc/clientInt.h | 3 +- source/client/src/clientMain.c | 19 +- source/client/src/clientTmq.c | 915 ++++++++++++++++++++------------- source/common/src/tmsg.c | 2 +- 4 files changed, 578 insertions(+), 361 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 7c342b85d4..507738acc9 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -332,8 +332,7 @@ static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { return (SReqResultInfo*)&msg->common.resInfo; } -SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4); - +int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pResInfo); static FORCE_INLINE SReqResultInfo* tscGetCurResInfo(TAOS_RES* res) { if (TD_RES_QUERY(res)) return &(((SRequestObj*)res)->body.resInfo); return tmqGetCurResInfo(res); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index f65edc103a..3955610d5c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -424,9 +424,11 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return doAsyncFetchRows(pRequest, true, true); } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SMqRspObj *msg = ((SMqRspObj *)res); - SReqResultInfo *pResultInfo; + SReqResultInfo *pResultInfo = NULL; if (msg->common.resIter == -1) { - pResultInfo = tmqGetNextResInfo(res, true); + if(tmqGetNextResInfo(res, true, &pResultInfo) != 0){ + return NULL; + } } else { pResultInfo = tmqGetCurResInfo(res); } @@ -436,8 +438,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { pResultInfo->current += 1; return pResultInfo->row; } else { - pResultInfo = tmqGetNextResInfo(res, true); - if (pResultInfo == NULL) { + if (tmqGetNextResInfo(res, true, &pResultInfo) != 0){ return NULL; } @@ -752,8 +753,9 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { (*numOfRows) = pResultInfo->numOfRows; return pRequest->code; } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { - SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, true); - if (pResultInfo == NULL) return -1; + SReqResultInfo *pResultInfo = NULL; + int32_t code = tmqGetNextResInfo(res, true, &pResultInfo); + if (code != 0) return code; pResultInfo->current = pResultInfo->numOfRows; (*rows) = pResultInfo->row; @@ -774,8 +776,9 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { } if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { - SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, false); - if (pResultInfo == NULL) { + SReqResultInfo *pResultInfo = NULL; + int32_t code = tmqGetNextResInfo(res, false, &pResultInfo); + if (code != 0) { (*numOfRows) = 0; return 0; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index c363686343..e1b7eab40b 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -29,6 +29,22 @@ #define DEFAULT_HEARTBEAT_INTERVAL 3000 #define DEFAULT_ASKEP_INTERVAL 1000 +#define CLIENT_TMQ_NULL_CHECK(c) \ + do { \ + if (c == NULL) { \ + code = TSDB_CODE_OUT_OF_MEMORY; \ + goto END; \ + } \ + } while (0) + +#define CLIENT_TMQ_RETURN_CHECK(c) \ + do { \ + code = c; \ + if (code != 0) { \ + goto END; \ + } \ + } while (0) + struct SMqMgmt { tmr_h timer; int32_t rsetId; @@ -112,7 +128,7 @@ struct tmq_t { typedef struct SAskEpInfo { int32_t code; - tsem_t sem; + tsem2_t sem; } SAskEpInfo; enum { @@ -125,6 +141,7 @@ enum { TMQ_CONSUMER_STATUS__READY, TMQ_CONSUMER_STATUS__NO_TOPIC, TMQ_CONSUMER_STATUS__RECOVER, + TMQ_CONSUMER_STATUS__CLOSED, }; enum { @@ -181,9 +198,7 @@ typedef struct { } SMqPollRspWrapper; typedef struct { - // int64_t refId; - // int32_t epoch; - tsem_t rspSem; + tsem2_t rspSem; int32_t rspErr; } SMqSubscribeCbParam; @@ -201,7 +216,7 @@ typedef struct { } SMqPollCbParam; typedef struct SMqVgCommon { - tsem_t rsp; + tsem2_t rsp; int32_t numOfRsp; SArray* pList; TdThreadMutex mutex; @@ -211,12 +226,12 @@ typedef struct SMqVgCommon { } SMqVgCommon; typedef struct SMqSeekParam { - tsem_t sem; + tsem2_t sem; int32_t code; } SMqSeekParam; typedef struct SMqCommittedParam { - tsem_t sem; + tsem2_t sem; int32_t code; SMqVgOffset vgOffset; } SMqCommittedParam; @@ -234,36 +249,31 @@ typedef struct { int32_t waitingRspNum; int32_t code; tmq_commit_cb* callbackFn; - /*SArray* successfulOffsets;*/ - /*SArray* failedOffsets;*/ void* userParam; } SMqCommitCbParamSet; typedef struct { SMqCommitCbParamSet* params; - // SMqVgOffset* pOffset; char topicName[TSDB_TOPIC_FNAME_LEN]; int32_t vgId; tmq_t* pTmq; } SMqCommitCbParam; typedef struct SSyncCommitInfo { - tsem_t sem; + tsem2_t sem; int32_t code; } SSyncCommitInfo; static int32_t syncAskEp(tmq_t* tmq); -static int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg); static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet); static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, SMqCommitCbParamSet* pParamSet); -static void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId); +static int32_t commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId); static int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpset); tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); if (conf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return conf; } @@ -445,8 +455,7 @@ static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { taosMemoryFree(pBuf->pData); taosMemoryFree(pBuf->pEpSet); - commitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId); - return 0; + return commitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId); } static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, @@ -457,9 +466,9 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse pOffset.offset.val = *offset; int32_t groupLen = strlen(tmq->groupId); - memcpy(pOffset.offset.subKey, tmq->groupId, groupLen); + (void)memcpy(pOffset.offset.subKey, tmq->groupId, groupLen); pOffset.offset.subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pOffset.offset.subKey + groupLen + 1, pTopicName); + (void)strcpy(pOffset.offset.subKey + groupLen + 1, pTopicName); int32_t len = 0; int32_t code = 0; @@ -479,7 +488,11 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse SEncoder encoder; tEncoderInit(&encoder, abuf, len); - tEncodeMqVgOffset(&encoder, &pOffset); + if(tEncodeMqVgOffset(&encoder, &pOffset) < 0) { + tEncoderClear(&encoder); + taosMemoryFree(buf); + return TSDB_CODE_INVALID_PARA; + } tEncoderClear(&encoder); // build param @@ -490,7 +503,6 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse } pParam->params = pParamSet; - // pParam->pOffset = pOffset; pParam->vgId = vgId; pParam->pTmq = tmq; @@ -514,35 +526,35 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse pMsgSendInfo->msgType = TDMT_VND_TMQ_COMMIT_OFFSET; int64_t transporterId = 0; - atomic_add_fetch_32(&pParamSet->waitingRspNum, 1); + (void)atomic_add_fetch_32(&pParamSet->waitingRspNum, 1); code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, pMsgSendInfo); if (code != 0) { - atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); + (void)atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); return code; } return code; } -static SMqClientTopic* getTopicByName(tmq_t* tmq, const char* pTopicName) { +static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic **topic) { int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); for (int32_t i = 0; i < numOfTopics; ++i) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (strcmp(pTopic->topicName, pTopicName) != 0) { + if (pTopic == NULL || strcmp(pTopic->topicName, pTopicName) != 0) { continue; } - - return pTopic; + *topic = pTopic; + return 0; } tscError("consumer:0x%" PRIx64 ", total:%d, failed to find topic:%s", tmq->consumerId, numOfTopics, pTopicName); - return NULL; + return TSDB_CODE_TMQ_INVALID_TOPIC; } -static SMqCommitCbParamSet* prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, - int32_t rspNum) { +static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, + int32_t rspNum, SMqCommitCbParamSet** ppParamSet) { SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pParamSet->refId = tmq->refId; @@ -550,21 +562,22 @@ static SMqCommitCbParamSet* prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* p pParamSet->callbackFn = pCommitFp; pParamSet->userParam = userParam; pParamSet->waitingRspNum = rspNum; - - return pParamSet; + *ppParamSet = pParamSet; + return 0; } static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClientVg** pVg) { - SMqClientTopic* pTopic = getTopicByName(tmq, pTopicName); - if (pTopic == NULL) { + SMqClientTopic* pTopic = NULL; + int32_t code = getTopicByName(tmq, pTopicName, &pTopic); + if (code != 0) { tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); - return TSDB_CODE_TMQ_INVALID_TOPIC; + return code; } int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); for (int32_t i = 0; i < numOfVgs; ++i) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); - if (pClientVg->vgId == vgId) { + if (pClientVg && pClientVg->vgId == vgId) { *pVg = pClientVg; break; } @@ -591,16 +604,23 @@ static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STq goto end; } char offsetBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(offsetBuf, tListLen(offsetBuf), offsetVal); - - char commitBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); - - SMqCommitCbParamSet* pParamSet = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 0); - if (pParamSet == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = tFormatOffset(offsetBuf, tListLen(offsetBuf), offsetVal); + if (code != 0) { goto end; } + + char commitBuf[TSDB_OFFSET_LEN] = {0}; + code = tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); + if (code != 0) { + goto end; + } + + SMqCommitCbParamSet* pParamSet = NULL; + code = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 0, &pParamSet); + if (code != 0) { + goto end; + } + code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, offsetVal, pTopicName, pParamSet); if (code != TSDB_CODE_SUCCESS) { tscError("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s", @@ -666,32 +686,39 @@ end: static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) { int32_t code = 0; // init as 1 to prevent concurrency issue - SMqCommitCbParamSet* pParamSet = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 1); - if (pParamSet == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + SMqCommitCbParamSet* pParamSet = NULL; + code = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 1, &pParamSet); + if (code != 0) { goto end; } - taosRLockLatch(&tmq->lock); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); for (int32_t i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + if (pTopic != NULL) { + code = TSDB_CODE_TMQ_INVALID_TOPIC; + taosRUnLockLatch(&tmq->lock); + goto end; + } int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); - tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, numOfVgroups); for (int32_t j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - + if (pVg != NULL) { + code = TSDB_CODE_INVALID_PARA; + taosRUnLockLatch(&tmq->lock); + goto end; + } if (pVg->offsetInfo.endOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.endOffset, &pVg->offsetInfo.committedOffset)) { char offsetBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(offsetBuf, tListLen(offsetBuf), &pVg->offsetInfo.endOffset); + (void)tFormatOffset(offsetBuf, tListLen(offsetBuf), &pVg->offsetInfo.endOffset); char commitBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); + (void)tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, &pVg->offsetInfo.endOffset, pTopic->topicName, pParamSet); if (code != TSDB_CODE_SUCCESS) { @@ -720,7 +747,12 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us // request is sent if (pParamSet->waitingRspNum != 1) { // count down since waiting rsp num init as 1 - commitRspCountDown(pParamSet, tmq->consumerId, "", 0); + code = commitRspCountDown(pParamSet, tmq->consumerId, "", 0); + if (code != 0){ + tscError("consumer:0x%" PRIx64 " commit rsp count down failed, code:%s", tmq->consumerId, tstrerror(code)); + pParamSet = NULL; + goto end; + } return; } @@ -733,9 +765,9 @@ end: } static void generateTimedTask(int64_t refId, int32_t type) { - tmq_t* tmq; - int8_t* pTaskType; - int32_t code; + tmq_t* tmq = NULL; + int8_t* pTaskType = NULL; + int32_t code = 0; tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) return; @@ -744,7 +776,7 @@ static void generateTimedTask(int64_t refId, int32_t type) { if (code == TSDB_CODE_SUCCESS) { *pTaskType = type; if (taosWriteQitem(tmq->delayedTask, pTaskType) == 0) { - tsem2_post(&tmq->rspSem); + (void)tsem2_post(&tmq->rspSem); } } @@ -761,7 +793,7 @@ void tmqReplayTask(void* param, void* tmrId) { tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) return; - tsem2_post(&tmq->rspSem); + (void)tsem2_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); } @@ -771,23 +803,29 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) { } int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { - if (pMsg == NULL) { + if (code != 0){ return code; } + if (pMsg == NULL || param == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqHbRsp rsp = {0}; - tDeserializeSMqHbRsp(pMsg->pData, pMsg->len, &rsp); + code = tDeserializeSMqHbRsp(pMsg->pData, pMsg->len, &rsp); + if (code != 0) { + return code; + } - int64_t refId = *(int64_t*)param; + int64_t refId = (int64_t)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq != NULL) { taosWLockLatch(&tmq->lock); for (int32_t i = 0; i < taosArrayGetSize(rsp.topicPrivileges); i++) { STopicPrivilege* privilege = taosArrayGet(rsp.topicPrivileges, i); - if (privilege->noPrivilege == 1) { + if (privilege && privilege->noPrivilege == 1) { int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); for (int32_t j = 0; j < topicNumCur; j++) { SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j); - if (strcmp(pTopicCur->topicName, privilege->topic) == 0) { + if (pTopicCur && strcmp(pTopicCur->topicName, privilege->topic) == 0) { tscInfo("consumer:0x%" PRIx64 ", has no privilege, topic:%s", tmq->consumerId, privilege->topic); pTopicCur->noPrivilege = 1; } @@ -814,23 +852,41 @@ void tmqSendHbReq(void* param, void* tmrId) { SMqHbReq req = {0}; req.consumerId = tmq->consumerId; req.epoch = tmq->epoch; - taosRLockLatch(&tmq->lock); req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows)); + if (req.topics == NULL){ + return; + } + taosRLockLatch(&tmq->lock); for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + if (pTopic == NULL) { + continue; + } int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); TopicOffsetRows* data = taosArrayReserve(req.topics, 1); - strcpy(data->topicName, pTopic->topicName); + if (data == NULL){ + continue; + } + (void)strcpy(data->topicName, pTopic->topicName); data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows)); + if (data->offsetRows == NULL){ + continue; + } for (int j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + if (pVg == NULL){ + continue; + } OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); + if (offRows == NULL){ + continue; + } offRows->vgId = pVg->vgId; offRows->rows = pVg->numOfRows; offRows->offset = pVg->offsetInfo.endOffset; offRows->ever = pVg->offsetInfo.walVerEnd == -1 ? 0 : pVg->offsetInfo.walVerEnd; char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset); + (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset); tscDebug("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%" PRId64 ", rows:%" PRId64, tmq->consumerId, tmq->groupId, offRows->vgId, buf, offRows->ever, offRows->rows); } @@ -865,9 +921,7 @@ void tmqSendHbReq(void* param, void* tmrId) { sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; - sendInfo->paramFreeFp = taosMemoryFree; - sendInfo->param = taosMemoryMalloc(sizeof(int64_t)); - *(int64_t*)sendInfo->param = refId; + sendInfo->param = (void*)refId; sendInfo->fp = tmqHbCb; sendInfo->msgType = TDMT_MND_TMQ_HB; @@ -893,48 +947,53 @@ static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) { } } -int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { - STaosQall* qall; - int32_t code; +void tmqHandleAllDelayedTask(tmq_t* pTmq) { + STaosQall* qall = NULL; + int32_t code = 0; code = taosAllocateQall(&qall); - if (code) return code; + if (code) { + tscError("consumer:0x%" PRIx64 ", failed to allocate qall, code:%s", pTmq->consumerId, tstrerror(code)); + return; + } - taosReadAllQitems(pTmq->delayedTask, qall); + (void)taosReadAllQitems(pTmq->delayedTask, qall); int32_t numOfItems = taosQallItemSize(qall); if (numOfItems == 0) { taosFreeQall(qall); - return TSDB_CODE_SUCCESS; + return; } tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, numOfItems); int8_t* pTaskType = NULL; - taosGetQitem(qall, (void**)&pTaskType); + (void)taosGetQitem(qall, (void**)&pTaskType); while (pTaskType != NULL) { if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) { - askEp(pTmq, NULL, false, false); - + code = askEp(pTmq, NULL, false, false); + if (code != 0) { + tscError("consumer:0x%" PRIx64 " failed to ask ep, code:%s", pTmq->consumerId, tstrerror(code)); + continue; + } tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); - taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->epTimer); + (void)taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn; asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, pTmq->autoCommitInterval / 1000.0); - taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, + (void)taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->commitTimer); } else { tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType); } taosFreeQitem(pTaskType); - taosGetQitem(qall, (void**)&pTaskType); + (void)taosGetQitem(qall, (void**)&pTaskType); } taosFreeQall(qall); - return 0; } static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { @@ -964,7 +1023,7 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { void tmqClearUnhandleMsg(tmq_t* tmq) { SMqRspWrapper* rspWrapper = NULL; while (1) { - taosGetQitem(tmq->qall, (void**)&rspWrapper); + (void)taosGetQitem(tmq->qall, (void**)&rspWrapper); if (rspWrapper) { tmqFreeRspWrapper(rspWrapper); taosFreeQitem(rspWrapper); @@ -974,9 +1033,9 @@ void tmqClearUnhandleMsg(tmq_t* tmq) { } rspWrapper = NULL; - taosReadAllQitems(tmq->mqueue, tmq->qall); + (void)taosReadAllQitems(tmq->mqueue, tmq->qall); while (1) { - taosGetQitem(tmq->qall, (void**)&rspWrapper); + (void)taosGetQitem(tmq->qall, (void**)&rspWrapper); if (rspWrapper) { tmqFreeRspWrapper(rspWrapper); taosFreeQitem(rspWrapper); @@ -997,7 +1056,7 @@ int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { if (pMsg) { taosMemoryFree(pMsg->pEpSet); } - tsem_post(&pParam->rspSem); + (void)tsem2_post(&pParam->rspSem); return 0; } @@ -1005,43 +1064,38 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { if (tmq == NULL) return TSDB_CODE_INVALID_PARA; if (*topics == NULL) { *topics = tmq_list_new(); + if (*topics == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } taosRLockLatch(&tmq->lock); for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i); - tmq_list_append(*topics, strchr(topic->topicName, '.') + 1); + if(topic == NULL) { + tscError("topic is null"); + continue; + } + char* tmp = strchr(topic->topicName, '.'); + if(tmp == NULL) { + tscError("topic name is invalid:%s", topic->topicName); + continue; + } + if(tmq_list_append(*topics, tmp+ 1) != 0) { + tscError("failed to append topic:%s", tmp + 1); + continue; + } } taosRUnLockLatch(&tmq->lock); return 0; } -int32_t tmq_unsubscribe(tmq_t* tmq) { - if (tmq == NULL) return TSDB_CODE_INVALID_PARA; - if (tmq->status != TMQ_CONSUMER_STATUS__READY) { - tscInfo("consumer:0x%" PRIx64 " not in ready state, unsubscribe it directly", tmq->consumerId); - return 0; - } - if (tmq->autoCommit) { - int32_t rsp = tmq_commit_sync(tmq, NULL); - if (rsp != 0) { - return rsp; - } - } - taosSsleep(2); // sleep 2s for hb to send offset and rows to server - - tmq_list_t* lst = tmq_list_new(); - int32_t rsp = tmq_subscribe(tmq, lst); - tmq_list_destroy(lst); - return rsp; -} - static void freeClientVg(void* param) { SMqClientVg* pVg = param; tOffsetDestroy(&pVg->offsetInfo.endOffset); tOffsetDestroy(&pVg->offsetInfo.beginOffset); tOffsetDestroy(&pVg->offsetInfo.committedOffset); } -static void freeClientVgImpl(void* param) { +static void freeClientTopic(void* param) { SMqClientTopic* pTopic = param; taosMemoryFreeClear(pTopic->schema.pSchema); taosArrayDestroyEx(pTopic->vgs, freeClientVg); @@ -1051,7 +1105,6 @@ void tmqFreeImpl(void* handle) { tmq_t* tmq = (tmq_t*)handle; int64_t id = tmq->consumerId; - // TODO stop timer if (tmq->mqueue) { tmqClearUnhandleMsg(tmq); taosCloseQueue(tmq->mqueue); @@ -1062,19 +1115,19 @@ void tmqFreeImpl(void* handle) { } taosFreeQall(tmq->qall); - tsem2_destroy(&tmq->rspSem); + (void)tsem2_destroy(&tmq->rspSem); - taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl); + taosArrayDestroyEx(tmq->clientTopics, freeClientTopic); taos_close_internal(tmq->pTscObj); if (tmq->commitTimer) { - taosTmrStopA(&tmq->commitTimer); + (void)taosTmrStopA(&tmq->commitTimer); } if (tmq->epTimer) { - taosTmrStopA(&tmq->epTimer); + (void)taosTmrStopA(&tmq->epTimer); } if (tmq->hbLiveTimer) { - taosTmrStopA(&tmq->hbLiveTimer); + (void)taosTmrStopA(&tmq->hbLiveTimer); } taosMemoryFree(tmq); @@ -1102,32 +1155,34 @@ void tmqMgmtClose(void) { } if (tmqMgmt.rsetId >= 0) { - taosCloseRef(tmqMgmt.rsetId); + (void)taosCloseRef(tmqMgmt.rsetId); tmqMgmt.rsetId = -1; } } #define SET_ERROR_MSG_TMQ(MSG) \ - if (errstr != NULL) snprintf(errstr, errstrLen, MSG); + if (errstr != NULL) (void)snprintf(errstr, errstrLen, MSG); tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { - int32_t code; + int32_t code = 0; if (conf == NULL) { SET_ERROR_MSG_TMQ("configure is null") return NULL; } - taosThreadOnce(&tmqInit, tmqMgmtInit); + code = taosThreadOnce(&tmqInit, tmqMgmtInit); + if (code != 0) { + SET_ERROR_MSG_TMQ("tmq init error") + return NULL; + } if (tmqInitRes != 0) { - terrno = tmqInitRes; SET_ERROR_MSG_TMQ("tmq timer init error") return NULL; } tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tscError("failed to create consumer, groupId:%s, code:%s", conf->groupId, terrstr()); + tscError("failed to create consumer, groupId:%s", conf->groupId); SET_ERROR_MSG_TMQ("malloc tmq failed") return NULL; } @@ -1136,33 +1191,34 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass; pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); + if (pTmq->clientTopics == NULL) { + tscError("failed to create consumer, groupId:%s", conf->groupId); + SET_ERROR_MSG_TMQ("malloc client topics failed") + goto _failed; + } code = taosOpenQueue(&pTmq->mqueue); if (code) { - terrno = code; - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); SET_ERROR_MSG_TMQ("open queue failed") goto _failed; } code = taosOpenQueue(&pTmq->delayedTask); if (code) { - terrno = code; - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); SET_ERROR_MSG_TMQ("open delayed task queue failed") goto _failed; } code = taosAllocateQall(&pTmq->qall); if (code) { - terrno = code; - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); SET_ERROR_MSG_TMQ("allocate qall failed") goto _failed; } if (conf->groupId[0] == 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty") goto _failed; } @@ -1173,8 +1229,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->epoch = 0; // set conf - strcpy(pTmq->clientId, conf->clientId); - strcpy(pTmq->groupId, conf->groupId); + (void)strcpy(pTmq->clientId, conf->clientId); + (void)strcpy(pTmq->groupId, conf->groupId); pTmq->withTbName = conf->withTbName; pTmq->useSnapshot = conf->snapEnable; pTmq->autoCommit = conf->autoCommit; @@ -1205,7 +1261,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, NULL, conf->port, CONN_TYPE__TMQ); if (pTmq->pTscObj == NULL) { tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); - tsem2_destroy(&pTmq->rspSem); + (void)tsem2_destroy(&pTmq->rspSem); SET_ERROR_MSG_TMQ("init tscObj failed") goto _failed; } @@ -1217,10 +1273,13 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { } pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, DEFAULT_HEARTBEAT_INTERVAL, (void*)pTmq->refId, tmqMgmt.timer); - + if (pTmq->hbLiveTimer == NULL) { + SET_ERROR_MSG_TMQ("start heartbeat timer failed") + goto _failed; + } char buf[TSDB_OFFSET_LEN] = {0}; STqOffsetVal offset = {.type = pTmq->resetOffsetCfg}; - tFormatOffset(buf, tListLen(buf), &offset); + (void)tFormatOffset(buf, tListLen(buf), &offset); tscInfo("consumer:0x%" PRIx64 " is setup, refId:%" PRId64 ", groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s", pTmq->consumerId, pTmq->refId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, @@ -1248,8 +1307,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, 256); tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN); - req.topicNames = taosArrayInit(sz, sizeof(void*)); + req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto FAIL; @@ -1264,22 +1323,38 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); - + if (topic == NULL) { + code = TSDB_CODE_INVALID_PARA; + goto FAIL; + } SName name = {0}; - tNameSetDbName(&name, tmq->pTscObj->acctId, topic, strlen(topic)); + code = tNameSetDbName(&name, tmq->pTscObj->acctId, topic, strlen(topic)); + if (code) { + tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId, code); + goto FAIL; + } char* topicFName = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); if (topicFName == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto FAIL; } - tNameExtractFullName(&name, topicFName); - tscInfo("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName); + code = tNameExtractFullName(&name, topicFName); + if (code) { + tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId, code); + taosMemoryFree(topicFName); + goto FAIL; + } - taosArrayPush(req.topicNames, &topicFName); + if (taosArrayPush(req.topicNames, &topicFName) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(topicFName); + goto FAIL; + } + tscInfo("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName); } int32_t tlen = tSerializeSCMSubscribeReq(NULL, &req); - buf = taosMemoryMalloc(tlen); if (buf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1287,7 +1362,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } void* abuf = buf; - tSerializeSCMSubscribeReq(&abuf, &req); + (void)tSerializeSCMSubscribeReq(&abuf, &req); sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { @@ -1297,7 +1372,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } SMqSubscribeCbParam param = {.rspErr = 0}; - if (tsem_init(¶m.rspSem, 0, 0) != 0) { + if (tsem2_init(¶m.rspSem, 0, 0) != 0) { code = TSDB_CODE_TSC_INTERNAL_ERROR; taosMemoryFree(buf); taosMemoryFree(sendInfo); @@ -1305,7 +1380,6 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } sendInfo->msgInfo = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL}; - sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; sendInfo->param = ¶m; @@ -1320,8 +1394,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } - tsem_wait(¶m.rspSem); - tsem_destroy(¶m.rspSem); + (void)tsem2_wait(¶m.rspSem); + (void)tsem2_destroy(¶m.rspSem); if (param.rspErr != 0) { code = param.rspErr; @@ -1343,11 +1417,12 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { taosMsleep(500); } - // init ep timer tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer); - // init auto commit timer - tmq->commitTimer = - taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer); + tmq->commitTimer =taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer); + if (tmq->epTimer == NULL || tmq->commitTimer == NULL) { + code = TSDB_CODE_TSC_INTERNAL_ERROR; + goto FAIL; + } FAIL: taosArrayDestroyP(req.topicNames, taosMemoryFree); @@ -1361,21 +1436,21 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para conf->commitCbUserParam = param; } -static SMqClientVg* getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId) { +static void getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId, SMqClientVg** pVg) { int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); for (int i = 0; i < topicNumCur; i++) { SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); - if (strcmp(pTopicCur->topicName, topicName) == 0) { + if (pTopicCur && strcmp(pTopicCur->topicName, topicName) == 0) { int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); for (int32_t j = 0; j < vgNumCur; j++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); - if (pVgCur->vgId == vgId) { - return pVgCur; + if (pVgCur && pVgCur->vgId == vgId) { + *pVg = pVgCur; + return; } } } } - return NULL; } static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName) { @@ -1391,7 +1466,8 @@ static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName) { static void setVgIdle(tmq_t* tmq, char* topicName, int32_t vgId) { taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = getVgInfo(tmq, topicName, vgId); + SMqClientVg* pVg = NULL; + getVgInfo(tmq, topicName, vgId, &pVg); if (pVg) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } @@ -1402,7 +1478,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tmq_t* tmq = NULL; SMqPollCbParam* pParam = (SMqPollCbParam*)param; if (pParam == NULL || pMsg == NULL) { - goto FAIL; + goto FAIL2; } int64_t refId = pParam->refId; int32_t vgId = pParam->vgId; @@ -1410,15 +1486,14 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { code = TSDB_CODE_TMQ_CONSUMER_CLOSED; - goto FAIL; + goto FAIL2; } - SMqPollRspWrapper* pRspWrapper; + SMqPollRspWrapper* pRspWrapper = NULL; code = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0, (void**)&pRspWrapper); if (code) { tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d, since out of memory", tmq->consumerId, vgId); - taosReleaseRef(tmqMgmt.rsetId, refId); - goto FAIL; + goto FAIL1; } if (code != 0) { @@ -1438,7 +1513,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); - code = -1; + code = TSDB_CODE_TMQ_CONSUMER_MISMATCH; goto END; } @@ -1451,56 +1526,51 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { pMsg->pEpSet = NULL; if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { - SDecoder decoder; + SDecoder decoder = {0}; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); if (tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp) < 0) { tDecoderClear(&decoder); - taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; goto END; } tDecoderClear(&decoder); - memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); + (void)memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset); + (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset); tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64, tmq->consumerId, vgId, pRspWrapper->dataRsp.common.reqOffset.version, buf, rspType, requestId); } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { - SDecoder decoder; + SDecoder decoder = {0}; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); if (tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp) < 0) { tDecoderClear(&decoder); - taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; goto END; } tDecoderClear(&decoder); - memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); + (void)memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); } else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - SDecoder decoder; + SDecoder decoder = {0}; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); if (tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp) < 0) { tDecoderClear(&decoder); - taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; goto END; } tDecoderClear(&decoder); - memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead)); + (void)memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead)); } else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { - SDecoder decoder; + SDecoder decoder = {0}; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); if (tSemiDecodeMqBatchMetaRsp(&decoder, &pRspWrapper->batchMetaRsp) < 0) { tDecoderClear(&decoder); - taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; + goto END; } tDecoderClear(&decoder); - memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead)); - tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, vgId, - requestId); + (void)memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead)); + tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, vgId,requestId); } else { // invalid rspType tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType); } @@ -1508,16 +1578,21 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { END: pRspWrapper->code = code; pRspWrapper->vgId = vgId; - strcpy(pRspWrapper->topicName, pParam->topicName); - taosWriteQitem(tmq->mqueue, pRspWrapper); + (void)strcpy(pRspWrapper->topicName, pParam->topicName); + code = taosWriteQitem(tmq->mqueue, pRspWrapper); + if(code != 0){ + tscError("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code); + } int32_t total = taosQueueItemSize(tmq->mqueue); tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, reqId:0x%" PRIx64, tmq->consumerId, rspType, vgId, total, requestId); + +FAIL1: taosReleaseRef(tmqMgmt.rsetId, refId); -FAIL: - if (tmq) tsem2_post(&tmq->rspSem); +FAIL2: + if (tmq) (void)tsem2_post(&tmq->rspSem); if (pMsg) taosMemoryFreeClear(pMsg->pData); if (pMsg) taosMemoryFreeClear(pMsg->pEpSet); @@ -1546,11 +1621,16 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic tscInfo("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d", tmq->consumerId, pTopic->topicName, vgNumGet); pTopic->vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); - + if (pTopic->vgs == NULL) { + tscError("consumer:0x%" PRIx64 ", failed to init vgs for topic:%s", tmq->consumerId, pTopic->topicName); + return; + } for (int32_t j = 0; j < vgNumGet; j++) { SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - - makeTopicVgroupKey(vgKey, pTopic->topicName, pVgEp->vgId); + if (pVgEp == NULL){ + continue; + } + (void)sprintf(vgKey, "%s:%d", pTopic->topicName, pVgEp->vgId); SVgroupSaveInfo* pInfo = taosHashGet(pVgOffsetHashMap, vgKey, strlen(vgKey)); STqOffsetVal offsetNew = {0}; @@ -1583,7 +1663,11 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic clientVg.offsetInfo.committedOffset = offsetNew; clientVg.offsetInfo.beginOffset = offsetNew; } - taosArrayPush(pTopic->vgs, &clientVg); + if (taosArrayPush(pTopic->vgs, &clientVg) == NULL){ + tscError("consumer:0x%" PRIx64 ", failed to push vg:%d into topic:%s", tmq->consumerId, pVgEp->vgId, + pTopic->topicName); + freeClientVg(&clientVg); + } } } @@ -1607,7 +1691,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) SHashObj* pVgOffsetHashMap = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); if (pVgOffsetHashMap == NULL) { - taosArrayDestroy(newTopics); + (void)taosArrayDestroy(newTopics); return false; } @@ -1620,15 +1704,18 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) for (int32_t i = 0; i < topicNumCur; i++) { // find old topic SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); - if (pTopicCur->vgs) { + if (pTopicCur && pTopicCur->vgs) { int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); tscInfo("consumer:0x%" PRIx64 ", current vg num: %d", tmq->consumerId, vgNumCur); for (int32_t j = 0; j < vgNumCur; j++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); - makeTopicVgroupKey(vgKey, pTopicCur->topicName, pVgCur->vgId); + if (pVgCur == NULL) { + continue; + } + (void)sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.endOffset); + (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.endOffset); tscInfo("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); @@ -1637,7 +1724,9 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) .commitOffset = pVgCur->offsetInfo.committedOffset, .numOfRows = pVgCur->numOfRows, .vgStatus = pVgCur->vgStatus}; - taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)); + if(taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0){ + tscError("consumer:0x%" PRIx64 ", failed to put vg:%d into hashmap", tmq->consumerId, pVgCur->vgId); + } } } } @@ -1645,15 +1734,21 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) for (int32_t i = 0; i < topicNumGet; i++) { SMqClientTopic topic = {0}; SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); + if (pTopicEp == NULL) { + continue; + } initClientTopicFromRsp(&topic, pTopicEp, pVgOffsetHashMap, tmq); - taosArrayPush(newTopics, &topic); + if(taosArrayPush(newTopics, &topic) == NULL){ + tscError("consumer:0x%" PRIx64 ", failed to push topic:%s into new topics", tmq->consumerId, topic.topicName); + freeClientTopic(&topic); + } } taosHashCleanup(pVgOffsetHashMap); // destroy current buffered existed topics info if (tmq->clientTopics) { - taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl); + taosArrayDestroyEx(tmq->clientTopics, freeClientTopic); } tmq->clientTopics = newTopics; taosWUnLockLatch(&tmq->lock); @@ -1668,9 +1763,9 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { int32_t groupLen = strlen(tmq->groupId); - memcpy(pReq->subKey, tmq->groupId, groupLen); + (void)memcpy(pReq->subKey, tmq->groupId, groupLen); pReq->subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pReq->subKey + groupLen + 1, pTopic->topicName); + (void)strcpy(pReq->subKey + groupLen + 1, pTopic->topicName); pReq->withTbName = tmq->withTbName; pReq->consumerId = tmq->consumerId; @@ -1685,36 +1780,41 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->enableBatchMeta = tmq->enableBatchMeta; } -SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { +int32_t tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqMetaRspObj** ppRspObj) { SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj)); if (pRspObj == NULL) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pRspObj->resType = RES_TYPE__TMQ_META; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); pRspObj->vgId = pWrapper->vgHandle->vgId; - memcpy(&pRspObj->metaRsp, &pWrapper->metaRsp, sizeof(SMqMetaRsp)); - return pRspObj; + (void)memcpy(&pRspObj->metaRsp, &pWrapper->metaRsp, sizeof(SMqMetaRsp)); + *ppRspObj = pRspObj; + return 0; } -SMqBatchMetaRspObj* tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { +int32_t tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqBatchMetaRspObj** ppRspObj) { SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj)); if (pRspObj == NULL) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META; tstrncpy(pRspObj->common.topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->common.db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); pRspObj->common.vgId = pWrapper->vgHandle->vgId; - memcpy(&pRspObj->rsp, &pWrapper->batchMetaRsp, sizeof(SMqBatchMetaRsp)); + (void)memcpy(&pRspObj->rsp, &pWrapper->batchMetaRsp, sizeof(SMqBatchMetaRsp)); tscDebug("build batchmeta Rsp from wrapper"); - return pRspObj; + *ppRspObj = pRspObj; + return 0; } void changeByteEndian(char* pData) { + if (pData == NULL) { + return; + } char* p = pData; // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column @@ -1740,6 +1840,9 @@ void changeByteEndian(char* pData) { } static void tmqGetRawDataRowsPrecisionFromRes(void* pRetrieve, void** rawData, int64_t* rows, int32_t* precision) { + if (pRetrieve == NULL) { + return; + } if (*(int64_t*)pRetrieve == 0) { *rawData = ((SRetrieveTableRsp*)pRetrieve)->data; *rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows); @@ -1771,6 +1874,10 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg if (!pDataRsp->withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable pDataRsp->withSchema = true; pDataRsp->blockSchema = taosArrayInit(pDataRsp->blockNum, sizeof(void*)); + if (pDataRsp->blockSchema == NULL){ + tscError("failed to allocate memory for blockSchema"); + return; + } } // extract the rows in this data packet for (int32_t i = 0; i < pDataRsp->blockNum; ++i) { @@ -1786,33 +1893,38 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg if (needTransformSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); if (schema) { - taosArrayPush(pDataRsp->blockSchema, &schema); + if (taosArrayPush(pDataRsp->blockSchema, &schema) == NULL){ + tscError("failed to push schema into blockSchema"); + continue; + } } } } } -SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { +int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj** ppRspObj) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); if (pRspObj == NULL) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pRspObj->common.resType = RES_TYPE__TMQ; - memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); + (void)memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); - return pRspObj; + *ppRspObj = pRspObj; + return 0; } -SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { +int32_t tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqTaosxRspObj** ppRspObj) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); if (pRspObj == NULL) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pRspObj->common.resType = RES_TYPE__TMQ_METADATA; - memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); + (void)memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); - return pRspObj; + *ppRspObj = pRspObj; + return 0; } static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* pVg, int64_t timeout) { @@ -1849,13 +1961,14 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p } pParam->refId = pTmq->refId; - strcpy(pParam->topicName, pTopic->topicName); + (void)strcpy(pParam->topicName, pTopic->topicName); pParam->vgId = pVg->vgId; pParam->requestId = req.reqId; sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFreeClear(pParam); taosMemoryFreeClear(msg); return code; } @@ -1870,7 +1983,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p int64_t transporterId = 0; char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset); + (void)tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset); code = asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, reqId:0x%" PRIx64, pTmq->consumerId, pTopic->topicName, pVg->vgId, code, pTmq->epoch, offsetFormatBuf, req.reqId); @@ -1898,6 +2011,9 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { for (int i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + if (pTopic == NULL){ + continue; + } int32_t numOfVg = taosArrayGetSize(pTopic->vgs); if (pTopic->noPrivilege) { tscDebug("consumer:0x%" PRIx64 " has no privilegr for topic:%s", tmq->consumerId, pTopic->topicName); @@ -1905,6 +2021,9 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { } for (int j = 0; j < numOfVg; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + if (pVg == NULL) { + continue; + } int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs; if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, @@ -1966,11 +2085,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { while (1) { SMqRspWrapper* pRspWrapper = NULL; - taosGetQitem(tmq->qall, (void**)&pRspWrapper); + (void)taosGetQitem(tmq->qall, (void**)&pRspWrapper); if (pRspWrapper == NULL) { - taosReadAllQitems(tmq->mqueue, tmq->qall); - taosGetQitem(tmq->qall, (void**)&pRspWrapper); + (void)taosReadAllQitems(tmq->mqueue, tmq->qall); + (void)taosGetQitem(tmq->qall, (void**)&pRspWrapper); if (pRspWrapper == NULL) { return NULL; } @@ -1984,17 +2103,19 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); tscDebug("consumer:0x%" PRIx64 " wait for the rebalance, set status to be RECOVER", tmq->consumerId); } else if (pRspWrapper->code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { - terrno = pRspWrapper->code; - tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId, - tstrerror(pRspWrapper->code)); + tscInfo("consumer:0x%" PRIx64 " return null since no committed offset", tmq->consumerId); } else { if (pRspWrapper->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { // for vnode transform - askEp(tmq, NULL, false, true); + int32_t code = askEp(tmq, NULL, false, true); + if (code != 0) { + tscError("consumer:0x%" PRIx64 " failed to ask ep, code:%s", tmq->consumerId, tstrerror(code)); + } } tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, since %s", tmq->consumerId, pollRspWrapper->vgId, tstrerror(pRspWrapper->code)); taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + SMqClientVg* pVg = NULL; + getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); if (pVg) pVg->emptyBlockReceiveTs = taosGetTimestampMs(); taosWUnLockLatch(&tmq->lock); } @@ -2008,7 +2129,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { if (pDataRsp->head.epoch == consumerEpoch) { taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + SMqClientVg* pVg = NULL; + getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); pollRspWrapper->vgHandle = pVg; pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { @@ -2032,7 +2154,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { tmq->consumerId, pDataRsp->blockNum != 0); char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset); + (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset); if (pDataRsp->blockNum == 0) { tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, @@ -2043,14 +2165,18 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { taosWUnLockLatch(&tmq->lock); } else { // build rsp int64_t numOfRows = 0; - SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows); + SMqRspObj* pRsp = NULL; + (void)tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp); tmq->totalRows += numOfRows; pVg->emptyBlockReceiveTs = 0; - if (tmq->replayEnable) { + if (pRsp && tmq->replayEnable) { pVg->blockReceiveTs = taosGetTimestampMs(); pVg->blockSleepForReplay = pRsp->rsp.sleepTime; if (pVg->blockSleepForReplay > 0) { - taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, (void*)(tmq->refId), tmqMgmt.timer); + if (taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, (void*)(tmq->refId), tmqMgmt.timer) == NULL) { + tscError("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%"PRId64, tmq->consumerId, + pVg->vgId, pVg->blockSleepForReplay); + } } } tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 @@ -2076,7 +2202,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + SMqClientVg* pVg = NULL; + getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); pollRspWrapper->vgHandle = pVg; pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { @@ -2091,7 +2218,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, &pollRspWrapper->metaRsp.rspOffset, pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId, true); // build rsp - SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); + SMqMetaRspObj* pRsp = NULL; + (void)tmqBuildMetaRspFromWrapper(pollRspWrapper, &pRsp); taosFreeQitem(pRspWrapper); taosWUnLockLatch(&tmq->lock); return pRsp; @@ -2110,7 +2238,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { if (pollRspWrapper->batchMetaRsp.head.epoch == consumerEpoch) { taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + SMqClientVg* pVg = NULL; + getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); pollRspWrapper->vgHandle = pVg; pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { @@ -2123,11 +2252,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } // build rsp - void* pRsp = NULL; updateVgInfo(pVg, &pollRspWrapper->batchMetaRsp.rspOffset, &pollRspWrapper->batchMetaRsp.rspOffset, pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever, tmq->consumerId, true); - pRsp = tmqBuildBatchMetaRspFromWrapper(pollRspWrapper); + SMqBatchMetaRspObj* pRsp = NULL; + (void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp) ; taosFreeQitem(pRspWrapper); taosWUnLockLatch(&tmq->lock); return pRsp; @@ -2145,7 +2274,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { if (pDataRsp->head.epoch == consumerEpoch) { taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + SMqClientVg* pVg = NULL; + getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); pollRspWrapper->vgHandle = pVg; pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { @@ -2174,11 +2304,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { // build rsp int64_t numOfRows = 0; - void* pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows); + SMqTaosxRspObj* pRsp = NULL; + if (tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp) !=0 ) { + tscError("consumer:0x%" PRIx64 " build taosx rsp failed, vgId:%d", tmq->consumerId, pVg->vgId); + } tmq->totalRows += numOfRows; char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); + (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, @@ -2198,7 +2331,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { tscDebug("consumer:0x%" PRIx64 " ep msg received", tmq->consumerId); SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)pRspWrapper; SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; - doUpdateLocalEp(tmq, pEpRspWrapper->epoch, rspMsg); + (void)doUpdateLocalEp(tmq, pEpRspWrapper->epoch, rspMsg); tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pRspWrapper); } else { @@ -2251,9 +2384,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { if (rspObj) { tscDebug("consumer:0x%" PRIx64 " return rsp %p", tmq->consumerId, rspObj); return (TAOS_RES*)rspObj; - } else if (terrno == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { - tscInfo("consumer:0x%" PRIx64 " return null since no committed offset", tmq->consumerId); - return NULL; } if (timeout >= 0) { @@ -2264,10 +2394,9 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { tmq->consumerId, tmq->epoch, startTime, currentTime); return NULL; } - tsem2_timewait(&tmq->rspSem, (timeout - elapsedTime)); + (void)tsem2_timewait(&tmq->rspSem, (timeout - elapsedTime)); } else { - // use tsem_timewait instead of tsem_wait to avoid unexpected stuck - tsem2_timewait(&tmq->rspSem, 1000); + (void)tsem2_timewait(&tmq->rspSem, 1000); } } } @@ -2281,11 +2410,12 @@ static void displayConsumeStatistics(tmq_t* pTmq) { tscDebug("consumer:0x%" PRIx64 " rows dist begin: ", pTmq->consumerId); for (int32_t i = 0; i < numOfTopics; ++i) { SMqClientTopic* pTopics = taosArrayGet(pTmq->clientTopics, i); - + if (pTopics == NULL) continue; tscDebug("consumer:0x%" PRIx64 " topic:%d", pTmq->consumerId, i); int32_t numOfVgs = taosArrayGetSize(pTopics->vgs); for (int32_t j = 0; j < numOfVgs; ++j) { SMqClientVg* pVg = taosArrayGet(pTopics->vgs, j); + if (pVg == NULL) continue; tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows); } } @@ -2293,34 +2423,54 @@ static void displayConsumeStatistics(tmq_t* pTmq) { tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId); } +static int32_t innerClose(tmq_t* tmq){ + if (tmq->status != TMQ_CONSUMER_STATUS__READY) { + tscInfo("consumer:0x%" PRIx64 " not in ready state, unsubscribe it directly", tmq->consumerId); + return 0; + } + if (tmq->autoCommit) { + int32_t code = tmq_commit_sync(tmq, NULL); + if (code != 0) { + return code; + } + } + tmqSendHbReq((void*)(tmq->refId), NULL); + + tmq_list_t* lst = tmq_list_new(); + if (lst == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t code = tmq_subscribe(tmq, lst); + tmq_list_destroy(lst); + return code; +} +int32_t tmq_unsubscribe(tmq_t* tmq) { + if (tmq == NULL) return TSDB_CODE_INVALID_PARA; + int32_t code = 0; + if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) { + code = innerClose(tmq); + if(code == 0){ + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); + } + } + return code; +} + int32_t tmq_consumer_close(tmq_t* tmq) { if (tmq == NULL) return TSDB_CODE_INVALID_PARA; tscInfo("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status); displayConsumeStatistics(tmq); - - if (tmq->status == TMQ_CONSUMER_STATUS__READY) { - // if auto commit is set, commit before close consumer. Otherwise, do nothing. - if (tmq->autoCommit) { - int32_t code = tmq_commit_sync(tmq, NULL); - if (code != 0) { - return code; - } + int32_t code = 0; + if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) { + code = innerClose(tmq); + if(code == 0){ + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); + taosRemoveRef(tmqMgmt.rsetId, tmq->refId); } - tmqSendHbReq((void*)(tmq->refId), NULL); - - tmq_list_t* lst = tmq_list_new(); - int32_t code = tmq_subscribe(tmq, lst); - tmq_list_destroy(lst); - if (code != 0) { - return code; - } - } else { - tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId); } - taosRemoveRef(tmqMgmt.rsetId, tmq->refId); - return 0; + return code; } const char* tmq_err2str(int32_t err) { @@ -2332,7 +2482,7 @@ const char* tmq_err2str(int32_t err) { if (*(taosGetErrMsg()) == 0) { return tstrerror(err); } else { - snprintf(taosGetErrMsgReturn(), ERR_MSG_LEN, "%s,detail:%s", tstrerror(err), taosGetErrMsg()); + (void)snprintf(taosGetErrMsgReturn(), ERR_MSG_LEN, "%s,detail:%s", tstrerror(err), taosGetErrMsg()); return (const char*)taosGetErrMsgReturn(); } } @@ -2360,10 +2510,17 @@ const char* tmq_get_topic_name(TAOS_RES* res) { return NULL; } if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { - return strchr(((SMqRspObjCommon*)res)->topic, '.') + 1; + char *tmp = strchr(((SMqRspObjCommon*)res)->topic, '.'); + if (tmp == NULL) { + return NULL; + } + return tmp + 1; } else if (TD_RES_TMQ_META(res)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - return strchr(pMetaRspObj->topic, '.') + 1; + char *tmp = strchr(((SMqMetaRspObj*)res)->topic, '.'); + if (tmp == NULL) { + return NULL; + } + return tmp + 1; } else { return NULL; } @@ -2375,10 +2532,17 @@ const char* tmq_get_db_name(TAOS_RES* res) { } if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { - return strchr(((SMqRspObjCommon*)res)->db, '.') + 1; + char *tmp = strchr(((SMqRspObjCommon*)res)->db, '.'); + if (tmp == NULL) { + return NULL; + } + return tmp + 1; } else if (TD_RES_TMQ_META(res)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - return strchr(pMetaRspObj->db, '.') + 1; + char *tmp = strchr(((SMqMetaRspObj*)res)->db, '.'); + if (tmp == NULL) { + return NULL; + } + return tmp + 1; } else { return NULL; } @@ -2391,8 +2555,7 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return ((SMqRspObjCommon*)res)->vgId; } else if (TD_RES_TMQ_META(res)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - return pMetaRspObj->vgId; + return ((SMqMetaRspObj*)res)->vgId; } else { return TSDB_CODE_INVALID_PARA; } @@ -2463,7 +2626,7 @@ void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* static void commitCallBackFn(tmq_t* UNUSED_PARAM(tmq), int32_t code, void* param) { SSyncCommitInfo* pInfo = (SSyncCommitInfo*)param; pInfo->code = code; - tsem_post(&pInfo->sem); + (void)tsem2_post(&pInfo->sem); } int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { @@ -2475,7 +2638,15 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { int32_t code = 0; SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); - tsem_init(&pInfo->sem, 0, 0); + if(pInfo == NULL) { + tscError("failed to allocate memory for sync commit"); + return TSDB_CODE_OUT_OF_MEMORY; + } + if (tsem2_init(&pInfo->sem, 0, 0) != 0) { + tscError("failed to init sem for sync commit"); + taosMemoryFree(pInfo); + return TSDB_CODE_OUT_OF_MEMORY; + } pInfo->code = 0; if (pRes == NULL) { @@ -2484,10 +2655,10 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { asyncCommitFromResult(tmq, pRes, commitCallBackFn, pInfo); } - tsem_wait(&pInfo->sem); + (void)tsem2_wait(&pInfo->sem); code = pInfo->code; - tsem_destroy(&pInfo->sem); + (void)tsem2_destroy(&pInfo->sem); taosMemoryFree(pInfo); tscInfo("consumer:0x%" PRIx64 " sync res commit done, code:%s", tmq->consumerId, tstrerror(code)); @@ -2518,7 +2689,7 @@ int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, int32_t accId = tmq->pTscObj->acctId; char tname[TSDB_TOPIC_FNAME_LEN] = {0}; - sprintf(tname, "%d.%s", accId, pTopicName); + (void)snprintf(tname, TSDB_TOPIC_FNAME_LEN, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); SMqClientVg* pVg = NULL; @@ -2544,17 +2715,20 @@ int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, return TSDB_CODE_OUT_OF_MEMORY; } - tsem_init(&pInfo->sem, 0, 0); + if (tsem2_init(&pInfo->sem, 0, 0) != 0) { + taosMemoryFree(pInfo); + return TSDB_CODE_OUT_OF_MEMORY; + } pInfo->code = 0; code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, commitCallBackFn, pInfo); if (code == 0) { - tsem_wait(&pInfo->sem); + (void)tsem2_wait(&pInfo->sem); code = pInfo->code; } if (code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; - tsem_destroy(&pInfo->sem); + (void)tsem2_destroy(&pInfo->sem); taosMemoryFree(pInfo); tscInfo("consumer:0x%" PRIx64 " sync send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, @@ -2629,7 +2803,7 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { if (pParam->sync) { SMqAskEpRsp rsp = {0}; tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); - doUpdateLocalEp(tmq, head->epoch, &rsp); + (void)doUpdateLocalEp(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } else { SMqAskEpRspWrapper* pWrapper; @@ -2640,7 +2814,7 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; pWrapper->epoch = head->epoch; - memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); + (void)memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg); taosWriteQitem(tmq->mqueue, pWrapper); @@ -2654,7 +2828,7 @@ FAIL: SAskEpInfo* pInfo = pParam->pParam; if (pInfo) { pInfo->code = code; - tsem_post(&pInfo->sem); + (void)tsem2_post(&pInfo->sem); } } @@ -2669,15 +2843,18 @@ FAIL: int32_t syncAskEp(tmq_t* pTmq) { SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); if (pInfo == NULL) return TSDB_CODE_OUT_OF_MEMORY; - tsem_init(&pInfo->sem, 0, 0); + if (tsem2_init(&pInfo->sem, 0, 0) != 0) { + taosMemoryFree(pInfo); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } int32_t code = askEp(pTmq, pInfo, true, false); if (code == 0) { - tsem_wait(&pInfo->sem); + (void)tsem2_wait(&pInfo->sem); code = pInfo->code; } - tsem_destroy(&pInfo->sem); + (void)tsem2_destroy(&pInfo->sem); taosMemoryFree(pInfo); return code; } @@ -2686,7 +2863,7 @@ int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { SMqAskEpReq req = {0}; req.consumerId = pTmq->consumerId; req.epoch = updateEpSet ? -1 : pTmq->epoch; - strcpy(req.cgroup, pTmq->groupId); + (void)strcpy(req.cgroup, pTmq->groupId); int code = 0; SMqAskEpCbParam* pParam = NULL; void* pReq = NULL; @@ -2740,18 +2917,13 @@ int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { return asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); } -int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg) { - return sprintf(dst, "%s:%d", topicName, vg); -} - int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { int64_t refId = pParamSet->refId; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { taosMemoryFree(pParamSet); - terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; - return terrno; + return TSDB_CODE_TMQ_CONSUMER_CLOSED; } // if no more waiting rsp @@ -2760,23 +2932,23 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { } taosMemoryFree(pParamSet); - taosReleaseRef(tmqMgmt.rsetId, refId); - return 0; + return taosReleaseRef(tmqMgmt.rsetId, refId); } -void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { +int32_t commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); if (waitingRspNum == 0) { tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, vgId); - tmqCommitDone(pParamSet); + return tmqCommitDone(pParamSet); } else { tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d commit-rsp received, remain:%d", consumerId, pTopic, vgId, waitingRspNum); } + return 0; } -SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { +int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pResInfo) { SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; pRspObj->resIter++; @@ -2784,7 +2956,9 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { if (common->withSchema) { doFreeReqResultInfo(&pRspObj->resInfo); SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(common->blockSchema, pRspObj->resIter); - setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); + if (pSW){ + setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); + } } void* pRetrieve = taosArrayGetP(common->blockData, pRspObj->resIter); @@ -2798,15 +2972,17 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { pRspObj->resInfo.current = 0; pRspObj->resInfo.precision = precision; - // TODO handle the compressed case pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows; - setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols, pRspObj->resInfo.numOfRows, + int32_t code = setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols, pRspObj->resInfo.numOfRows, convertUcs4); - - return &pRspObj->resInfo; + if (code != 0){ + return code; + } + *pResInfo = &pRspObj->resInfo; + return code; } - return NULL; + return TSDB_CODE_TSC_INTERNAL_ERROR; } static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { @@ -2820,28 +2996,36 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { if (code != TSDB_CODE_SUCCESS) { tscError("consumer:0x%" PRIx64 " failed to get the wal info from vgId:%d for topic:%s", pCommon->consumerId, pParam->vgId, pCommon->pTopicName); - pCommon->code = code; + } else { - SMqDataRsp rsp; - SDecoder decoder; + SMqDataRsp rsp = {0}; + SDecoder decoder = {0}; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - tDecodeMqDataRsp(&decoder, &rsp); + code = tDecodeMqDataRsp(&decoder, &rsp); tDecoderClear(&decoder); + if (code != 0){ + goto END; + } SMqRspHead* pHead = pMsg->pData; - tmq_topic_assignment assignment = {.begin = pHead->walsver, .end = pHead->walever + 1, .currentOffset = rsp.common.rspOffset.version, .vgId = pParam->vgId}; taosThreadMutexLock(&pCommon->mutex); - taosArrayPush(pCommon->pList, &assignment); + if(taosArrayPush(pCommon->pList, &assignment) == NULL){ + tscError("consumer:0x%" PRIx64 " failed to push the wal info from vgId:%d for topic:%s", pCommon->consumerId, + pParam->vgId, pCommon->pTopicName); + code = TSDB_CODE_TSC_INTERNAL_ERROR; + } taosThreadMutexUnlock(&pCommon->mutex); } +END: + pCommon->code = code; if (total == pParam->totalReq) { - tsem_post(&pCommon->rsp); + (void)tsem2_post(&pCommon->rsp); } if (pMsg) { @@ -2849,15 +3033,15 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pEpSet); } - return 0; + return code; } static void destroyCommonInfo(SMqVgCommon* pCommon) { if (pCommon == NULL) { return; } - taosArrayDestroy(pCommon->pList); - tsem_destroy(&pCommon->rsp); + (void)taosArrayDestroy(pCommon->pList); + (void)tsem2_destroy(&pCommon->rsp); taosThreadMutexDestroy(&pCommon->mutex); taosMemoryFree(pCommon->pTopicName); taosMemoryFree(pCommon); @@ -2877,7 +3061,7 @@ static int32_t tmCommittedCb(void* param, SDataBuf* pMsg, int32_t code) { goto end; } if (pMsg) { - SDecoder decoder; + SDecoder decoder = {0}; tDecoderInit(&decoder, (uint8_t*)pMsg->pData, pMsg->len); if (tDecodeMqVgOffset(&decoder, &pParam->vgOffset) < 0) { tOffsetDestroy(&pParam->vgOffset.offset); @@ -2893,8 +3077,8 @@ end: taosMemoryFree(pMsg->pEpSet); } pParam->code = code; - tsem_post(&pParam->sem); - return 0; + (void)tsem2_post(&pParam->sem); + return code; } int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* epSet) { @@ -2904,9 +3088,9 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep pOffset.consumerId = tmq->consumerId; int32_t groupLen = strlen(tmq->groupId); - memcpy(pOffset.offset.subKey, tmq->groupId, groupLen); + (void)memcpy(pOffset.offset.subKey, tmq->groupId, groupLen); pOffset.offset.subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pOffset.offset.subKey + groupLen + 1, tname); + (void)strcpy(pOffset.offset.subKey + groupLen + 1, tname); int32_t len = 0; tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code); @@ -2925,7 +3109,12 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep SEncoder encoder; tEncoderInit(&encoder, abuf, len); - tEncodeMqVgOffset(&encoder, &pOffset); + code = tEncodeMqVgOffset(&encoder, &pOffset); + if (code < 0) { + taosMemoryFree(buf); + tEncoderClear(&encoder); + return code; + } tEncoderClear(&encoder); SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); @@ -2940,7 +3129,12 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep taosMemoryFree(sendInfo); return TSDB_CODE_OUT_OF_MEMORY; } - tsem_init(&pParam->sem, 0, 0); + if (tsem2_init(&pParam->sem, 0, 0) != 0){ + taosMemoryFree(buf); + taosMemoryFree(sendInfo); + taosMemoryFree(pParam); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } sendInfo->msgInfo = (SDataBuf){.pData = buf, .len = sizeof(SMsgHead) + len, .handle = NULL}; sendInfo->requestId = generateRequestId(); @@ -2952,12 +3146,12 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep int64_t transporterId = 0; code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, sendInfo); if (code != 0) { - tsem_destroy(&pParam->sem); + (void)tsem2_destroy(&pParam->sem); taosMemoryFree(pParam); return code; } - tsem_wait(&pParam->sem); + (void)tsem2_wait(&pParam->sem); code = pParam->code; if (code == TSDB_CODE_SUCCESS) { if (pParam->vgOffset.offset.val.type == TMQ_OFFSET__LOG) { @@ -2967,7 +3161,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; } } - tsem_destroy(&pParam->sem); + (void)tsem2_destroy(&pParam->sem); taosMemoryFree(pParam); return code; @@ -2981,7 +3175,7 @@ int64_t tmq_position(tmq_t* tmq, const char* pTopicName, int32_t vgId) { int32_t accId = tmq->pTscObj->acctId; char tname[TSDB_TOPIC_FNAME_LEN] = {0}; - sprintf(tname, "%d.%s", accId, pTopicName); + (void)snprintf(tname, TSDB_TOPIC_FNAME_LEN, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); @@ -3040,7 +3234,7 @@ int64_t tmq_committed(tmq_t* tmq, const char* pTopicName, int32_t vgId) { int32_t accId = tmq->pTscObj->acctId; char tname[TSDB_TOPIC_FNAME_LEN] = {0}; - sprintf(tname, "%d.%s", accId, pTopicName); + (void)snprintf(tname, TSDB_TOPIC_FNAME_LEN, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); @@ -3094,13 +3288,14 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a int32_t accId = tmq->pTscObj->acctId; char tname[TSDB_TOPIC_FNAME_LEN] = {0}; - sprintf(tname, "%d.%s", accId, pTopicName); - int32_t code = TSDB_CODE_SUCCESS; + (void)snprintf(tname, TSDB_TOPIC_FNAME_LEN, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); - SMqClientTopic* pTopic = getTopicByName(tmq, tname); - if (pTopic == NULL) { - code = TSDB_CODE_TMQ_INVALID_TOPIC; + + SMqClientTopic* pTopic = NULL; + int32_t code = getTopicByName(tmq, tname, &pTopic); + if (code != 0) { + tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); goto end; } @@ -3108,6 +3303,9 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a *numOfAssignment = taosArrayGetSize(pTopic->vgs); for (int32_t j = 0; j < (*numOfAssignment); ++j) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); + if (pClientVg == NULL){ + continue; + } int32_t type = pClientVg->offsetInfo.beginOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type); @@ -3128,6 +3326,9 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a for (int32_t j = 0; j < (*numOfAssignment); ++j) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); + if (pClientVg == NULL){ + continue; + } if (pClientVg->offsetInfo.beginOffset.type != TMQ_OFFSET__LOG) { needFetch = true; break; @@ -3145,24 +3346,31 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a if (needFetch) { pCommon = taosMemoryCalloc(1, sizeof(SMqVgCommon)); if (pCommon == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = terrno; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } pCommon->pList = taosArrayInit(4, sizeof(tmq_topic_assignment)); - tsem_init(&pCommon->rsp, 0, 0); + if (pCommon->pList == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + if (tsem2_init(&pCommon->rsp, 0, 0) != 0){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } taosThreadMutexInit(&pCommon->mutex, 0); pCommon->pTopicName = taosStrdup(pTopic->topicName); pCommon->consumerId = tmq->consumerId; - terrno = TSDB_CODE_OUT_OF_MEMORY; for (int32_t i = 0; i < (*numOfAssignment); ++i) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); - + if (pClientVg == NULL){ + continue; + } SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam)); if (pParam == NULL) { - code = terrno; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } @@ -3178,21 +3386,21 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req); if (msgSize < 0) { taosMemoryFree(pParam); - code = terrno; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } char* msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { taosMemoryFree(pParam); - code = terrno; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) { taosMemoryFree(msg); taosMemoryFree(pParam); - code = terrno; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } @@ -3200,7 +3408,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a if (sendInfo == NULL) { taosMemoryFree(pParam); taosMemoryFree(msg); - code = terrno; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } @@ -3214,7 +3422,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a int64_t transporterId = 0; char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.beginOffset); + (void)tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.beginOffset); tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); @@ -3224,10 +3432,9 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } } - tsem_wait(&pCommon->rsp); + (void)tsem2_wait(&pCommon->rsp); code = pCommon->code; - terrno = code; if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -3242,6 +3449,9 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a for (int32_t i = 0; i < taosArrayGetSize(pTopic->vgs); ++i) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); + if (pClientVg == NULL){ + continue; + } if (pClientVg->vgId != p->vgId) { continue; } @@ -3285,7 +3495,7 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { } SMqSeekParam* pParam = param; pParam->code = code; - tsem_post(&pParam->sem); + (void)tsem2_post(&pParam->sem); return 0; } @@ -3299,7 +3509,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ int32_t accId = tmq->pTscObj->acctId; char tname[TSDB_TOPIC_FNAME_LEN] = {0}; - sprintf(tname, "%d.%s", accId, pTopicName); + (void)snprintf(tname, TSDB_TOPIC_FNAME_LEN, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); @@ -3366,7 +3576,12 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ taosMemoryFree(sendInfo); return TSDB_CODE_OUT_OF_MEMORY; } - tsem_init(&pParam->sem, 0, 0); + if (tsem2_init(&pParam->sem, 0, 0) != 0){ + taosMemoryFree(msg); + taosMemoryFree(sendInfo); + taosMemoryFree(pParam); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } sendInfo->msgInfo = (SDataBuf){.pData = msg, .len = msgSize, .handle = NULL}; sendInfo->requestId = generateRequestId(); @@ -3378,14 +3593,14 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ int64_t transporterId = 0; code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); if (code != 0) { - tsem_destroy(&pParam->sem); + (void)tsem2_destroy(&pParam->sem); taosMemoryFree(pParam); return code; } - tsem_wait(&pParam->sem); + (void)tsem2_wait(&pParam->sem); code = pParam->code; - tsem_destroy(&pParam->sem); + (void)tsem2_destroy(&pParam->sem); taosMemoryFree(pParam); tscInfo("consumer:0x%" PRIx64 "send seek to vgId:%d, return code:%s", tmq->consumerId, vgId, tstrerror(code)); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9efff055bf..cf62beefd0 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9248,7 +9248,7 @@ void tOffsetCopy(STqOffsetVal *pLeft, const STqOffsetVal *pRight) { *pLeft = *pRight; if (IS_VAR_DATA_TYPE(pRight->primaryKey.type)) { pLeft->primaryKey.pData = taosMemoryMalloc(pRight->primaryKey.nData); - memcpy(pLeft->primaryKey.pData, pRight->primaryKey.pData, pRight->primaryKey.nData); + (void)memcpy(pLeft->primaryKey.pData, pRight->primaryKey.pData, pRight->primaryKey.nData); } } From 62a781ac6e7ffcb4dc9f49cc30dc018a9e21065e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 23 Jul 2024 14:32:28 +0800 Subject: [PATCH 10/52] fix:[TD-31017]process return value in client for tmq --- include/common/tmsg.h | 4 +++- source/client/src/clientTmq.c | 39 +++++++++++++++++++---------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a5dea8a44e..2befefc732 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -4036,7 +4036,9 @@ static FORCE_INLINE void* tDecodeSMqAskEpRsp(void* buf, SMqAskEpRsp* pRsp) { for (int32_t i = 0; i < sz; i++) { SMqSubTopicEp topicEp; buf = tDecodeMqSubTopicEp(buf, &topicEp); - taosArrayPush(pRsp->topics, &topicEp); + if (taosArrayPush(pRsp->topics, &topicEp) == NULL) { + return NULL; + } } return buf; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index e1b7eab40b..3f578648fa 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -794,7 +794,7 @@ void tmqReplayTask(void* param, void* tmrId) { if (tmq == NULL) return; (void)tsem2_post(&tmq->rspSem); - taosReleaseRef(tmqMgmt.rsetId, refId); + (void)taosReleaseRef(tmqMgmt.rsetId, refId); } void tmqAssignDelayedCommitTask(void* param, void* tmrId) { @@ -833,7 +833,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { } } taosWUnLockLatch(&tmq->lock); - taosReleaseRef(tmqMgmt.rsetId, refId); + (void)taosReleaseRef(tmqMgmt.rsetId, refId); } tDestroySMqHbRsp(&rsp); taosMemoryFree(pMsg->pData); @@ -936,9 +936,9 @@ void tmqSendHbReq(void* param, void* tmrId) { OVER: tDestroySMqHbReq(&req); if (tmrId != NULL) { - taosTmrReset(tmqSendHbReq, DEFAULT_HEARTBEAT_INTERVAL, param, tmqMgmt.timer, &tmq->hbLiveTimer); + (void)taosTmrReset(tmqSendHbReq, DEFAULT_HEARTBEAT_INTERVAL, param, tmqMgmt.timer, &tmq->hbLiveTimer); } - taosReleaseRef(tmqMgmt.rsetId, refId); + (void)taosReleaseRef(tmqMgmt.rsetId, refId); } static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) { @@ -1589,7 +1589,7 @@ END: tmq->consumerId, rspType, vgId, total, requestId); FAIL1: - taosReleaseRef(tmqMgmt.rsetId, refId); + (void)taosReleaseRef(tmqMgmt.rsetId, refId); FAIL2: if (tmq) (void)tsem2_post(&tmq->rspSem); @@ -2748,7 +2748,7 @@ void tmq_commit_offset_async(tmq_t* tmq, const char* pTopicName, int32_t vgId, i int32_t accId = tmq->pTscObj->acctId; char tname[TSDB_TOPIC_FNAME_LEN] = {0}; - sprintf(tname, "%d.%s", accId, pTopicName); + (void)snprintf(tname, TSDB_TOPIC_FNAME_LEN, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); SMqClientVg* pVg = NULL; @@ -2802,11 +2802,12 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); if (pParam->sync) { SMqAskEpRsp rsp = {0}; - tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); - (void)doUpdateLocalEp(tmq, head->epoch, &rsp); + if(tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL){ + (void)doUpdateLocalEp(tmq, head->epoch, &rsp); + } tDeleteSMqAskEpRsp(&rsp); } else { - SMqAskEpRspWrapper* pWrapper; + SMqAskEpRspWrapper* pWrapper = NULL; code = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0, (void**)&pWrapper); if (code) { goto END; @@ -2815,13 +2816,15 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; pWrapper->epoch = head->epoch; (void)memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg); - - taosWriteQitem(tmq->mqueue, pWrapper); + if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) != NULL){ + taosFreeQitem(pWrapper); + }else{ + (void)taosWriteQitem(tmq->mqueue, pWrapper); + } } END: - taosReleaseRef(tmqMgmt.rsetId, pParam->refId); + (void)taosReleaseRef(tmqMgmt.rsetId, pParam->refId); FAIL: if (pParam->sync) { @@ -3013,13 +3016,13 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { .currentOffset = rsp.common.rspOffset.version, .vgId = pParam->vgId}; - taosThreadMutexLock(&pCommon->mutex); + (void)taosThreadMutexLock(&pCommon->mutex); if(taosArrayPush(pCommon->pList, &assignment) == NULL){ tscError("consumer:0x%" PRIx64 " failed to push the wal info from vgId:%d for topic:%s", pCommon->consumerId, pParam->vgId, pCommon->pTopicName); code = TSDB_CODE_TSC_INTERNAL_ERROR; } - taosThreadMutexUnlock(&pCommon->mutex); + (void)taosThreadMutexUnlock(&pCommon->mutex); } END: @@ -3042,7 +3045,7 @@ static void destroyCommonInfo(SMqVgCommon* pCommon) { } (void)taosArrayDestroy(pCommon->pList); (void)tsem2_destroy(&pCommon->rsp); - taosThreadMutexDestroy(&pCommon->mutex); + (void)taosThreadMutexDestroy(&pCommon->mutex); taosMemoryFree(pCommon->pTopicName); taosMemoryFree(pCommon); } @@ -3359,7 +3362,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - taosThreadMutexInit(&pCommon->mutex, 0); + (void)taosThreadMutexInit(&pCommon->mutex, 0); pCommon->pTopicName = taosStrdup(pTopic->topicName); pCommon->consumerId = tmq->consumerId; @@ -3545,7 +3548,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ taosWUnLockLatch(&tmq->lock); SMqSeekReq req = {0}; - snprintf(req.subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s:%s", tmq->groupId, tname); + (void)snprintf(req.subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s:%s", tmq->groupId, tname); req.head.vgId = vgId; req.consumerId = tmq->consumerId; From 3bd21d446d6875a1e9bcbe345a02004bc727735c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 23 Jul 2024 14:35:40 +0800 Subject: [PATCH 11/52] fix:[TD-31017]process return value in client for tmq --- source/client/src/clientTmq.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 3f578648fa..65e0d5fe91 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -780,7 +780,7 @@ static void generateTimedTask(int64_t refId, int32_t type) { } } - taosReleaseRef(tmqMgmt.rsetId, refId); + (void)taosReleaseRef(tmqMgmt.rsetId, refId); } void tmqAssignAskEpTask(void* param, void* tmrId) { @@ -2466,7 +2466,7 @@ int32_t tmq_consumer_close(tmq_t* tmq) { code = innerClose(tmq); if(code == 0){ atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); - taosRemoveRef(tmqMgmt.rsetId, tmq->refId); + (void)taosRemoveRef(tmqMgmt.rsetId, tmq->refId); } } From 7497f193c7639473c78e7174a8b5c1c80da8f173 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 Jul 2024 14:47:29 +0800 Subject: [PATCH 12/52] fix(stream): check return value, and do some internal refactor. --- include/libs/executor/storageapi.h | 2 +- source/common/src/tglobal.c | 2 +- source/dnode/mnode/impl/src/mndStream.c | 737 ++++++++++--------- source/dnode/mnode/impl/src/mndStreamHb.c | 62 +- source/dnode/mnode/impl/src/mndStreamTrans.c | 55 +- source/dnode/mnode/impl/src/mndStreamUtil.c | 14 +- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 205 ++++-- source/dnode/vnode/src/tsdb/tsdbRead2.c | 166 ++++- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 58 +- source/libs/executor/inc/tsort.h | 18 +- source/libs/executor/src/cachescanoperator.c | 6 +- source/libs/executor/src/mergeoperator.c | 19 +- source/libs/executor/src/scanoperator.c | 15 +- source/libs/executor/src/sortoperator.c | 44 +- source/libs/executor/src/tsort.c | 552 ++++++++++---- source/libs/stream/src/streamCheckStatus.c | 31 +- source/libs/stream/src/streamMeta.c | 2 +- source/libs/stream/src/streamSched.c | 4 +- 19 files changed, 1360 insertions(+), 634 deletions(-) mode change 100755 => 100644 source/libs/executor/src/mergeoperator.c diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index a746fa8443..05ef14657b 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -195,7 +195,7 @@ typedef struct SStoreCacheReader { int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr, SArray *pFuncTypeList, SColumnInfo* pPkCol, int32_t numOfPks); - void *(*closeReader)(void *pReader); + void (*closeReader)(void *pReader); int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, SArray *pTableUidList); int32_t (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 88b743d905..f126b2d6c9 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -756,7 +756,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1800, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "concurrentCheckpoint", tsMaxConcurrentCheckpoint, 1, 10, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index df8800aee4..31d28d99a4 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -62,6 +62,9 @@ static int32_t mndProcessCheckpointReport(SRpcMsg *pReq); static int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg); static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId, int32_t code); + +static int32_t setStreamAttrInResBlock(SStreamObj *pStream, SSDataBlock *pBlock, int32_t numOfRows); + static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList); static void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo); @@ -156,7 +159,7 @@ void mndCleanupStream(SMnode *pMnode) { taosHashCleanup(execInfo.pTransferStateStreams); taosHashCleanup(execInfo.pChkptStreams); taosHashCleanup(execInfo.pStreamConsensus); - taosThreadMutexDestroy(&execInfo.lock); + (void) taosThreadMutexDestroy(&execInfo.lock); mDebug("mnd stream exec info cleanup"); } @@ -238,8 +241,7 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream) { static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream) { mTrace("stream:%s, perform update action", pOldStream->name); - - atomic_exchange_32(&pOldStream->version, pNewStream->version); + (void) atomic_exchange_32(&pOldStream->version, pNewStream->version); taosWLockLatch(&pOldStream->lock); @@ -303,8 +305,7 @@ static void mndShowStreamTrigger(char *dst, SStreamObj *pStream) { static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) { if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->sourceDB[0] == 0 || pCreate->targetStbFullName[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; - return -1; + return terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; } return 0; } @@ -383,7 +384,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SDbObj *pSourceDb = mndAcquireDb(pMnode, pCreate->sourceDB); if (pSourceDb == NULL) { mInfo("stream:%s failed to create, source db %s not exist since %s", pCreate->name, pObj->sourceDb, terrstr()); - return -1; + return terrno; } pObj->sourceDbUid = pSourceDb->uid; mndReleaseDb(pMnode, pSourceDb); @@ -393,7 +394,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SDbObj *pTargetDb = mndAcquireDbByStb(pMnode, pObj->targetSTbName); if (pTargetDb == NULL) { mInfo("stream:%s failed to create, target db %s not exist since %s", pCreate->name, pObj->targetDb, terrstr()); - return -1; + return terrno; } tstrncpy(pObj->targetDb, pTargetDb->name, TSDB_DB_FNAME_LEN); @@ -503,7 +504,11 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) { if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER) { pTask->ver = SSTREAM_TASK_VER; } - tEncodeStreamTask(&encoder, pTask); + + int32_t code = tEncodeStreamTask(&encoder, pTask); + if (code == -1) { + return TSDB_CODE_INVALID_MSG; + } int32_t size = encoder.pos; int32_t tlen = sizeof(SMsgHead) + size; @@ -511,25 +516,29 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) { void *buf = taosMemoryCalloc(1, tlen); if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } ((SMsgHead *)buf)->vgId = htonl(pTask->info.nodeId); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncoderInit(&encoder, abuf, size); - - tEncodeStreamTask(&encoder, pTask); + code = tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); - int32_t code = setTransAction(pTrans, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0, 0); - if (code != 0) { + if (code == -1) { + code = TSDB_CODE_INVALID_PARA; + mError("failed to encode stream task, code:%s", tstrerror(code)); taosMemoryFree(buf); - return -1; + return code; } - return 0; + code = setTransAction(pTrans, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0, 0); + if (code) { + taosMemoryFree(buf); + } + + return code; } int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) { @@ -548,9 +557,10 @@ int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) { return code; } - if (mndPersistTaskDeployReq(pTrans, pTask) < 0) { + code = mndPersistTaskDeployReq(pTrans, pTask); + if (code) { destroyStreamTaskIter(pIter); - return -1; + return code; } } @@ -566,19 +576,21 @@ int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) { int32_t numOfTasks = taosArrayGetSize(pLevel); for (int32_t j = 0; j < numOfTasks; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); - if (mndPersistTaskDeployReq(pTrans, pTask) < 0) { - return -1; + code = mndPersistTaskDeployReq(pTrans, pTask); + if (code) { + return code; } } } } - return 0; + return code; } int32_t mndPersistStream(STrans *pTrans, SStreamObj *pStream) { - if (mndPersistStreamTasks(pTrans, pStream) < 0) { - return -1; + int32_t code = 0; + if ((code = mndPersistStreamTasks(pTrans, pStream)) < 0) { + return code; } return mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY); @@ -726,7 +738,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { SCMCreateStreamReq createReq = {0}; if (tDeserializeSCMCreateStreamReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -736,7 +748,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { #endif mInfo("stream:%s, start to create stream, sql:%s", createReq.name, createReq.sql); - if (mndCheckCreateStreamReq(&createReq) != 0) { + if ((code = mndCheckCreateStreamReq(&createReq)) != 0) { mError("stream:%s, failed to create since %s", createReq.name, terrstr()); goto _OVER; } @@ -766,12 +778,12 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } // build stream obj from request - if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createReq) < 0) { + if ((code = mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createReq)) < 0) { mError("stream:%s, failed to create since %s", createReq.name, terrstr()); goto _OVER; } - if (doStreamCheck(pMnode, &streamObj) < 0) { + if ((code = doStreamCheck(pMnode, &streamObj)) < 0) { goto _OVER; } @@ -783,7 +795,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { // create stb for stream if (createReq.createStb == STREAM_CREATE_STABLE_TRUE) { - if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { + if ((code = mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user)) < 0) { mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; @@ -793,25 +805,25 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } // schedule stream task for stream obj - if (mndScheduleStream(pMnode, &streamObj, createReq.lastTs, createReq.pVgroupVerList) < 0) { + if ((code = mndScheduleStream(pMnode, &streamObj, createReq.lastTs, createReq.pVgroupVerList)) < 0) { mError("stream:%s, failed to schedule since %s", createReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; } // add stream to trans - if (mndPersistStream(pTrans, &streamObj) < 0) { - mError("stream:%s, failed to schedule since %s", createReq.name, terrstr()); + if ((code = mndPersistStream(pTrans, &streamObj)) < 0) { + mError("stream:%s, failed to persist since %s", createReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; } - if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, streamObj.sourceDb) != 0) { + if ((code = mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, streamObj.sourceDb)) != 0) { mndTransDrop(pTrans); goto _OVER; } - if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, streamObj.targetDb) != 0) { + if ((code = mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, streamObj.targetDb)) != 0) { mndTransDrop(pTrans); goto _OVER; } @@ -824,7 +836,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { streamMutexUnlock(&execInfo.lock); // execute creation - if (mndTransPrepare(pMnode, pTrans) != 0) { + if ((code = mndTransPrepare(pMnode, pTrans)) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); goto _OVER; @@ -833,12 +845,18 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { mndTransDrop(pTrans); SName dbname = {0}; - tNameFromString(&dbname, createReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + code = tNameFromString(&dbname, createReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (code) { + goto _OVER; + } SName name = {0}; - tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - // reuse this function for stream + code = tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (code) { + goto _OVER; + } + // reuse this function for stream if (sql != NULL && sqlLen > 0) { auditRecord(pReq, pMnode->clusterId, "createStream", dbname.dbname, name.dbname, sql, sqlLen); } else { @@ -848,7 +866,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } _OVER: - if (terrno != TSDB_CODE_SUCCESS && terrno != TSDB_CODE_ACTION_IN_PROGRESS) { + if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stream:%s, failed to create since %s", createReq.name, terrstr()); } @@ -860,7 +878,7 @@ _OVER: taosMemoryFreeClear(sql); } - return terrno; + return code; } int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) { @@ -948,7 +966,10 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); - tEncodeStreamCheckpointSourceReq(&encoder, &req); + int32_t pos = tEncodeStreamCheckpointSourceReq(&encoder, &req); + if (pos == -1) { + return TSDB_CODE_INVALID_MSG; + } SMsgHead *pMsgHead = (SMsgHead *)buf; pMsgHead->contLen = htonl(tlen); @@ -1001,7 +1022,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre if (conflict) { mWarn("checkpoint conflict with other trans in %s, ignore the checkpoint for stream:%s %" PRIx64, pStream->sourceDb, pStream->name, pStream->uid); - return -1; + return TSDB_CODE_MND_TRANS_CONFLICT; } STrans *pTrans = NULL; @@ -1014,7 +1035,12 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre goto _ERR; } - mndStreamRegisterTrans(pTrans, MND_STREAM_CHECKPOINT_NAME, pStream->uid); + code = mndStreamRegisterTrans(pTrans, MND_STREAM_CHECKPOINT_NAME, pStream->uid); + if (code) { + mError("failed to register checkpoint trans for stream:%s, checkpointId:%" PRId64, pStream->name, checkpointId); + goto _ERR; + } + mDebug("start to trigger checkpoint for stream:%s, checkpoint: %" PRId64 "", pStream->name, checkpointId); taosWLockLatch(&pStream->lock); @@ -1054,6 +1080,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre } if ((code = mndTransPrepare(pMnode, pTrans)) != TSDB_CODE_SUCCESS) { + code = terrno; mError("failed to prepare checkpoint trans since %s", terrstr()); goto _ERR; } @@ -1066,7 +1093,11 @@ _ERR: int32_t extractStreamNodeList(SMnode *pMnode) { if (taosArrayGetSize(execInfo.pNodeList) == 0) { - extractNodeListFromStream(pMnode, execInfo.pNodeList); + int32_t code = extractNodeListFromStream(pMnode, execInfo.pNodeList); + if (code) { + mError("Failed to extract node list from stream, code:%s", tstrerror(code)); + return code; + } } return taosArrayGetSize(execInfo.pNodeList); @@ -1148,7 +1179,10 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { for (int32_t j = 0; j < taosArrayGetSize(pInvalidList); ++j) { STaskId *pId = taosArrayGet(pInvalidList, j); if (pEntry->id.streamId == pId->streamId) { - taosArrayPush(pInvalidList, &pEntry->id); + void* px = taosArrayPush(pInvalidList, &pEntry->id); + if (px == NULL) { + mError("failed to put stream into invalid list, code:%s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + } break; } } @@ -1247,12 +1281,15 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { streamMutexUnlock(&execInfo.lock); SCheckpointInterval in = {.streamId = pStream->uid, .duration = duration}; - taosArrayPush(pList, &in); - - int32_t currentSize = taosArrayGetSize(pList); - mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %ds(%" PRId64 "s) beyond threshold:%d", - pStream->name, pStream->uid, tsStreamCheckpointInterval, duration / 1000, currentSize); - + void* p = taosArrayPush(pList, &in); + if (p) { + int32_t currentSize = taosArrayGetSize(pList); + mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %ds(%" PRId64 + "s) beyond concurrently launch threshold:%d", + pStream->name, pStream->uid, tsStreamCheckpointInterval, duration / 1000, currentSize); + } else { + mError("failed to record the checkpoint interval info, stream:0x%" PRIx64, pStream->uid); + } sdbRelease(pSdb, pStream); } @@ -1263,9 +1300,14 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { } taosArraySort(pList, streamWaitComparFn); - mndStreamClearFinishedTrans(pMnode, &numOfCheckpointTrans); - int32_t numOfQual = taosArrayGetSize(pList); + code = mndStreamClearFinishedTrans(pMnode, &numOfCheckpointTrans); + if (code) { + mError("failed to clear finish trans, code:%s", tstrerror(code)); + taosArrayDestroy(pList); + return code; + } + int32_t numOfQual = taosArrayGetSize(pList); if (numOfCheckpointTrans > tsMaxConcurrentCheckpoint) { mDebug( "%d stream(s) checkpoint interval longer than %ds, ongoing checkpoint trans:%d reach maximum allowed:%d, new " @@ -1310,9 +1352,9 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { } static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; + SMnode * pMnode = pReq->info.node; SStreamObj *pStream = NULL; - int32_t code = 0; + int32_t code = 0; SMDropStreamReq dropReq = {0}; if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) { @@ -1341,7 +1383,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { if (pStream->smaId != 0) { mDebug("stream:%s, uid:0x%" PRIx64 " try to drop sma related stream", dropReq.name, pStream->uid); - void *pIter = NULL; + void * pIter = NULL; SSmaObj *pSma = NULL; pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void **)&pSma); while (pIter) { @@ -1429,15 +1471,18 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { removeStreamTasksInBuf(pStream, &execInfo); SName name = {0}; - tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - + code = tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); auditRecord(pReq, pMnode->clusterId, "dropStream", "", name.dbname, dropReq.sql, dropReq.sqlLen); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); tFreeMDropStreamReq(&dropReq); - return TSDB_CODE_ACTION_IN_PROGRESS; + if (code == 0) { + return TSDB_CODE_ACTION_IN_PROGRESS; + } else { + return code; + } } int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { @@ -1532,113 +1577,16 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStreamObj *pStream = NULL; + int32_t code = 0; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream); if (pShow->pIter == NULL) break; - SColumnInfoData *pColInfo; - int32_t cols = 0; - - char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName)); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false); - - // create time - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->createTime, false); - - // stream id - char buf[128] = {0}; - int64ToHexStr(pStream->uid, buf, tListLen(buf)); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, buf, false); - - // related fill-history stream id - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - if (pStream->hTaskUid != 0) { - int64ToHexStr(pStream->hTaskUid, buf, tListLen(buf)); - colDataSetVal(pColInfo, numOfRows, buf, false); - } else { - colDataSetVal(pColInfo, numOfRows, buf, true); + code = setStreamAttrInResBlock(pStream, pBlock, numOfRows); + if (code == 0) { + numOfRows++; } - - // related fill-history stream id - char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(sql, pStream->sql, sizeof(sql)); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)sql, false); - - char status[20 + VARSTR_HEADER_SIZE] = {0}; - char status2[20] = {0}; - mndShowStreamStatus(status2, pStream); - STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status)); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&status, false); - - char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(sourceDB, mndGetDbStr(pStream->sourceDb), sizeof(sourceDB)); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&sourceDB, false); - - char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(targetDB, mndGetDbStr(pStream->targetDb), sizeof(targetDB)); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&targetDB, false); - - if (pStream->targetSTbName[0] == 0) { - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, NULL, true); - } else { - char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(targetSTB, mndGetStbStr(pStream->targetSTbName), sizeof(targetSTB)); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&targetSTB, false); - } - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->conf.watermark, false); - - char trigger[20 + VARSTR_HEADER_SIZE] = {0}; - char trigger2[20] = {0}; - mndShowStreamTrigger(trigger2, pStream); - STR_WITH_MAXSIZE_TO_VARSTR(trigger, trigger2, sizeof(trigger)); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&trigger, false); - - // sink_quota - char sinkQuota[20 + VARSTR_HEADER_SIZE] = {0}; - sinkQuota[0] = '0'; - char dstStr[20] = {0}; - STR_TO_VARSTR(dstStr, sinkQuota) - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)dstStr, false); - - // checkpoint interval - char tmp[20 + VARSTR_HEADER_SIZE] = {0}; - sprintf(varDataVal(tmp), "%d sec", tsStreamCheckpointInterval); - varDataSetLen(tmp, strlen(varDataVal(tmp))); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)tmp, false); - - // checkpoint backup type - char backup[20 + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(backup, "none") - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)backup, false); - - // history scan idle - char scanHistoryIdle[20 + VARSTR_HEADER_SIZE] = {0}; - strcpy(scanHistoryIdle, "100a"); - - memset(dstStr, 0, tListLen(dstStr)); - STR_TO_VARSTR(dstStr, scanHistoryIdle) - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)dstStr, false); - - numOfRows++; sdbRelease(pSdb, pStream); } @@ -1651,9 +1599,133 @@ static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } +int32_t setStreamAttrInResBlock(SStreamObj *pStream, SSDataBlock *pBlock, int32_t numOfRows) { + int32_t code = 0; + int32_t cols = 0; + int32_t lino = 0; + + char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName)); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + + code = colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false); + TSDB_CHECK_CODE(code, lino, _end); + + // create time + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->createTime, false); + TSDB_CHECK_CODE(code, lino, _end); + + // stream id + char buf[128] = {0}; + int64ToHexStr(pStream->uid, buf, tListLen(buf)); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, buf, false); + TSDB_CHECK_CODE(code, lino, _end); + + // related fill-history stream id + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + if (pStream->hTaskUid != 0) { + int64ToHexStr(pStream->hTaskUid, buf, tListLen(buf)); + code = colDataSetVal(pColInfo, numOfRows, buf, false); + } else { + code = colDataSetVal(pColInfo, numOfRows, buf, true); + } + TSDB_CHECK_CODE(code, lino, _end); + + // related fill-history stream id + char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(sql, pStream->sql, sizeof(sql)); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)sql, false); + TSDB_CHECK_CODE(code, lino, _end); + + char status[20 + VARSTR_HEADER_SIZE] = {0}; + char status2[20] = {0}; + mndShowStreamStatus(status2, pStream); + STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status)); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&status, false); + TSDB_CHECK_CODE(code, lino, _end); + + char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(sourceDB, mndGetDbStr(pStream->sourceDb), sizeof(sourceDB)); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&sourceDB, false); + TSDB_CHECK_CODE(code, lino, _end); + + char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(targetDB, mndGetDbStr(pStream->targetDb), sizeof(targetDB)); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&targetDB, false); + TSDB_CHECK_CODE(code, lino, _end); + + if (pStream->targetSTbName[0] == 0) { + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, NULL, true); + } else { + char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(targetSTB, mndGetStbStr(pStream->targetSTbName), sizeof(targetSTB)); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&targetSTB, false); + } + TSDB_CHECK_CODE(code, lino, _end); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->conf.watermark, false); + TSDB_CHECK_CODE(code, lino, _end); + + char trigger[20 + VARSTR_HEADER_SIZE] = {0}; + char trigger2[20] = {0}; + mndShowStreamTrigger(trigger2, pStream); + STR_WITH_MAXSIZE_TO_VARSTR(trigger, trigger2, sizeof(trigger)); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&trigger, false); + TSDB_CHECK_CODE(code, lino, _end); + + // sink_quota + char sinkQuota[20 + VARSTR_HEADER_SIZE] = {0}; + sinkQuota[0] = '0'; + char dstStr[20] = {0}; + STR_TO_VARSTR(dstStr, sinkQuota) + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)dstStr, false); + TSDB_CHECK_CODE(code, lino, _end); + + // checkpoint interval + char tmp[20 + VARSTR_HEADER_SIZE] = {0}; + sprintf(varDataVal(tmp), "%d sec", tsStreamCheckpointInterval); + varDataSetLen(tmp, strlen(varDataVal(tmp))); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)tmp, false); + TSDB_CHECK_CODE(code, lino, _end); + + // checkpoint backup type + char backup[20 + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(backup, "none") + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)backup, false); + TSDB_CHECK_CODE(code, lino, _end); + + // history scan idle + char scanHistoryIdle[20 + VARSTR_HEADER_SIZE] = {0}; + strcpy(scanHistoryIdle, "100a"); + + memset(dstStr, 0, tListLen(dstStr)); + STR_TO_VARSTR(dstStr, scanHistoryIdle) + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)dstStr, false); + + _end: + return code; +} + static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDataBlock *pBlock, int32_t numOfRows) { SColumnInfoData *pColInfo; int32_t cols = 0; + int32_t code = 0; + int32_t lino = 0; STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; @@ -1662,7 +1734,7 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS mError("task:0x%" PRIx64 " not exists in any vnodes, streamName:%s, streamId:0x%" PRIx64 " createTs:%" PRId64 " no valid status/stage info", id.taskId, pStream->name, pStream->uid, pStream->createTime); - return -1; + return TSDB_CODE_STREAM_TASK_NOT_EXIST; } // stream name @@ -1670,14 +1742,16 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false); + TSDB_CHECK_CODE(code, lino, _end); // task id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); char idstr[128] = {0}; int64ToHexStr(pTask->id.taskId, idstr, tListLen(idstr)); - colDataSetVal(pColInfo, numOfRows, idstr, false); + code = colDataSetVal(pColInfo, numOfRows, idstr, false); + TSDB_CHECK_CODE(code, lino, _end); // node type char nodeType[20 + VARSTR_HEADER_SIZE] = {0}; @@ -1688,12 +1762,14 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS } else { memcpy(varDataVal(nodeType), "snode", 5); } - colDataSetVal(pColInfo, numOfRows, nodeType, false); + code = colDataSetVal(pColInfo, numOfRows, nodeType, false); + TSDB_CHECK_CODE(code, lino, _end); // node id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); int64_t nodeId = TMAX(pTask->info.nodeId, 0); - colDataSetVal(pColInfo, numOfRows, (const char *)&nodeId, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&nodeId, false); + TSDB_CHECK_CODE(code, lino, _end); // level char level[20 + VARSTR_HEADER_SIZE] = {0}; @@ -1709,7 +1785,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)level, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)level, false); + TSDB_CHECK_CODE(code, lino, _end); // status char status[20 + VARSTR_HEADER_SIZE] = {0}; @@ -1719,11 +1796,13 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS // status pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)status, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)status, false); + TSDB_CHECK_CODE(code, lino, _end); // stage pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pe->stage, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&pe->stage, false); + TSDB_CHECK_CODE(code, lino, _end); // input queue char vbuf[40] = {0}; @@ -1733,7 +1812,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS STR_TO_VARSTR(vbuf, buf); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + TSDB_CHECK_CODE(code, lino, _end); // input total const char *formatTotalMb = "%7.2f MiB"; @@ -1748,7 +1828,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS STR_TO_VARSTR(vbuf, buf); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + TSDB_CHECK_CODE(code, lino, _end); // process throughput const char *formatKb = "%7.2f KiB/s"; @@ -1763,7 +1844,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS STR_TO_VARSTR(vbuf, buf); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + TSDB_CHECK_CODE(code, lino, _end); // output total pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1775,7 +1857,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS memset(vbuf, 0, tListLen(vbuf)); STR_TO_VARSTR(vbuf, buf); - colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + TSDB_CHECK_CODE(code, lino, _end); } // output throughput @@ -1793,7 +1876,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS memset(vbuf, 0, tListLen(vbuf)); STR_TO_VARSTR(vbuf, buf); - colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + TSDB_CHECK_CODE(code, lino, _end); } // output queue @@ -1818,35 +1902,42 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS STR_TO_VARSTR(vbuf, buf); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + TSDB_CHECK_CODE(code, lino, _end); // start_time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startTime, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startTime, false); + TSDB_CHECK_CODE(code, lino, _end); // start id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startCheckpointId, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startCheckpointId, false); + TSDB_CHECK_CODE(code, lino, _end); // start ver pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startCheckpointVer, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startCheckpointVer, false); + TSDB_CHECK_CODE(code, lino, _end); // checkpoint time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (pe->checkpointInfo.latestTime != 0) { - colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestTime, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestTime, false); } else { - colDataSetVal(pColInfo, numOfRows, 0, true); + code = colDataSetVal(pColInfo, numOfRows, 0, true); } + TSDB_CHECK_CODE(code, lino, _end); // checkpoint_id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestId, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestId, false); + TSDB_CHECK_CODE(code, lino, _end); // checkpoint version pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestVer, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestVer, false); + TSDB_CHECK_CODE(code, lino, _end); // checkpoint size pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1854,26 +1945,31 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS // checkpoint backup status pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, 0, true); + code = colDataSetVal(pColInfo, numOfRows, 0, true); + TSDB_CHECK_CODE(code, lino, _end); // ds_err_info pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, 0, true); + code = colDataSetVal(pColInfo, numOfRows, 0, true); + TSDB_CHECK_CODE(code, lino, _end); // history_task_id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (pe->hTaskId != 0) { int64ToHexStr(pe->hTaskId, idstr, tListLen(idstr)); - colDataSetVal(pColInfo, numOfRows, idstr, false); + code = colDataSetVal(pColInfo, numOfRows, idstr, false); } else { - colDataSetVal(pColInfo, numOfRows, 0, true); + code = colDataSetVal(pColInfo, numOfRows, 0, true); } + TSDB_CHECK_CODE(code, lino, _end); // history_task_status pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, 0, true); + code = colDataSetVal(pColInfo, numOfRows, 0, true); + TSDB_CHECK_CODE(code, lino, _end); - return TSDB_CODE_SUCCESS; + _end: + return code; } static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { @@ -1881,6 +1977,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStreamObj *pStream = NULL; + int32_t code = 0; streamMutexLock(&execInfo.lock); mndInitStreamExecInfo(pMnode, &execInfo); @@ -1897,12 +1994,18 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock int32_t count = mndGetNumOfStreamTasks(pStream); if (numOfRows + count > rowsCapacity) { - blockDataEnsureCapacity(pBlock, numOfRows + count); + code = blockDataEnsureCapacity(pBlock, numOfRows + count); + if (code) { + mError("failed to prepare the result block buffer, quit return value"); + taosRUnLockLatch(&pStream->lock); + sdbRelease(pSdb, pStream); + continue; + } } // add row for each task SStreamTaskIter *pIter = NULL; - int32_t code = createStreamTaskIter(pStream, &pIter); + code = createStreamTaskIter(pStream, &pIter); if (code) { taosRUnLockLatch(&pStream->lock); sdbRelease(pSdb, pStream); @@ -2123,7 +2226,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_RESUME_NAME, true); if (conflict) { sdbRelease(pMnode->pSdb, pStream); - return -1; + return terrno; } STrans *pTrans = NULL; @@ -2209,7 +2312,7 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP const SEp *pPrevEp = GET_ACTIVE_EP(&pPrevEntry->epset); char buf[256] = {0}; - epsetToStr(&pCurrent->epset, buf, tListLen(buf)); + (void) epsetToStr(&pCurrent->epset, buf, tListLen(buf)); // ignore this error mDebug("nodeId:%d restart/epset changed detected, old:%s:%d -> new:%s, stageUpdate:%d", pCurrent->nodeId, pPrevEp->fqdn, pPrevEp->port, buf, pPrevEntry->stageUpdated); @@ -2217,14 +2320,21 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP SNodeUpdateInfo updateInfo = {.nodeId = pPrevEntry->nodeId}; epsetAssign(&updateInfo.prevEp, &pPrevEntry->epset); epsetAssign(&updateInfo.newEp, &pCurrent->epset); - taosArrayPush(info.pUpdateNodeList, &updateInfo); + + void* p = taosArrayPush(info.pUpdateNodeList, &updateInfo); + if (p == NULL) { + mError("failed to put update entry into node list, code:%s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + } } // todo handle the snode info if (pCurrent->nodeId != SNODE_HANDLE) { SVgObj *pVgroup = mndAcquireVgroup(pMnode, pCurrent->nodeId); - taosHashPut(info.pDBMap, pVgroup->dbName, strlen(pVgroup->dbName), NULL, 0); + int32_t code = taosHashPut(info.pDBMap, pVgroup->dbName, strlen(pVgroup->dbName), NULL, 0); mndReleaseVgroup(pMnode, pVgroup); + if (code) { + mError("failed to put into dbmap, code:out of memory"); + } } break; @@ -2255,7 +2365,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange if (conflict) { mError("nodeUpdate conflict with other trans, current nodeUpdate ignored"); sdbCancelFetch(pSdb, pIter); - return -1; + return terrno; } } @@ -2274,7 +2384,10 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange return terrno = code; } - mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid); + code = mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid); + if (code) { + mError("failed to register trans, transId:%d, and continue", pTrans->id); + } } void *p = taosHashGet(pChangeInfo->pDBMap, pStream->targetDb, strlen(pStream->targetDb)); @@ -2329,8 +2442,13 @@ static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) { SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; void *pIter = NULL; + int32_t code = 0; SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (pHash == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) { @@ -2340,7 +2458,7 @@ static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) { taosWLockLatch(&pStream->lock); SStreamTaskIter *pTaskIter = NULL; - int32_t code = createStreamTaskIter(pStream, &pTaskIter); + code = createStreamTaskIter(pStream, &pTaskIter); if (code) { taosWUnLockLatch(&pStream->lock); sdbRelease(pSdb, pStream); @@ -2357,7 +2475,10 @@ static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) { SNodeEntry entry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId}; epsetAssign(&entry.epset, &pTask->info.epSet); - taosHashPut(pHash, &entry.nodeId, sizeof(entry.nodeId), &entry, sizeof(entry)); + code = taosHashPut(pHash, &entry.nodeId, sizeof(entry.nodeId), &entry, sizeof(entry)); + if (code) { + break; + } } destroyStreamTaskIter(pTaskIter); @@ -2372,22 +2493,32 @@ static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) { pIter = NULL; while ((pIter = taosHashIterate(pHash, pIter)) != NULL) { SNodeEntry *pEntry = (SNodeEntry *)pIter; - taosArrayPush(pNodeList, pEntry); + + void *p = taosArrayPush(pNodeList, pEntry); + if (p == NULL) { + mError("failed to put entry into node list, nodeId:%d, code: out of memory", pEntry->nodeId); + if (code == 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + continue; + } char buf[256] = {0}; - epsetToStr(&pEntry->epset, buf, tListLen(buf)); + (void) epsetToStr(&pEntry->epset, buf, tListLen(buf)); // ignore this error since it is only for log file mDebug("extract nodeInfo from stream obj, nodeId:%d, %s", pEntry->nodeId, buf); } taosHashCleanup(pHash); mDebug("numOfNodes:%d for stream after extract nodeInfo from stream", (int32_t)taosArrayGetSize(pNodeList)); - return TSDB_CODE_SUCCESS; + return code; } // this function runs by only one thread, so it is not multi-thread safe static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { int32_t code = 0; + bool allReady = true; + SArray *pNodeSnapshot = NULL; int32_t old = atomic_val_compare_exchange_32(&mndNodeCheckSentinel, 0, 1); if (old != 0) { @@ -2411,9 +2542,6 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { return 0; } - bool allReady = true; - SArray *pNodeSnapshot = NULL; - code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot); if (code) { mError("failed to take the vgroup snapshot, ignore it and continue"); @@ -2428,7 +2556,10 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { streamMutexLock(&execInfo.lock); - removeExpiredNodeEntryAndTaskInBuf(pNodeSnapshot); + code = removeExpiredNodeEntryAndTaskInBuf(pNodeSnapshot); + if (code) { + goto _end; + } SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeList, pNodeSnapshot); if (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0) { @@ -2438,7 +2569,12 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { // keep the new vnode snapshot if success if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { - extractNodeListFromStream(pMnode, execInfo.pNodeList); + code = extractNodeListFromStream(pMnode, execInfo.pNodeList); + if (code) { + mError("failed to extract node list from stream, code:%s", tstrerror(code)); + goto _end; + } + execInfo.ts = ts; mDebug("create trans successfully, update cached node list, numOfNodes:%d", (int)taosArrayGetSize(execInfo.pNodeList)); @@ -2449,12 +2585,13 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { mDebug("no update found in nodeList"); } - taosArrayDestroy(pNodeSnapshot); - streamMutexUnlock(&execInfo.lock); - taosArrayDestroy(changeInfo.pUpdateNodeList); taosHashCleanup(changeInfo.pDBMap); + _end: + streamMutexUnlock(&execInfo.lock); + taosArrayDestroy(pNodeSnapshot); + mDebug("end to do stream task node change checking"); atomic_store_32(&mndNodeCheckSentinel, 0); return 0; @@ -2471,8 +2608,7 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) { SMStreamNodeCheckMsg *pMsg = rpcMallocCont(size); SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_NODECHANGE_CHECK, .pCont = pMsg, .contLen = size}; - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - return 0; + return tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { @@ -2496,11 +2632,18 @@ void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) STaskStatusEntry entry = {0}; streamTaskStatusInit(&entry, pTask); - taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry)); - taosArrayPush(pExecNode->pTaskList, &id); - - int32_t num = (int32_t)taosArrayGetSize(pExecNode->pTaskList); - mInfo("s-task:0x%x add into task buffer, total:%d", (int32_t)entry.id.taskId, num); + code = taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry)); + if (code == 0) { + void * px = taosArrayPush(pExecNode->pTaskList, &id); + int32_t num = (int32_t)taosArrayGetSize(pExecNode->pTaskList); + if (px) { + mInfo("s-task:0x%x add into task buffer, total:%d", (int32_t)entry.id.taskId, num); + } else { + mError("s-task:0x%x failed to add into task buffer, total:%d", (int32_t)entry.id.taskId, num); + } + } else { + mError("s-task:0x%x failed to add into task map, since out of memory", (int32_t) entry.id.taskId); + } // add the new vgroups if not added yet bool exist = false; @@ -2516,8 +2659,12 @@ void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) SNodeEntry nodeEntry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId}; epsetAssign(&nodeEntry.epset, &pTask->info.epSet); - taosArrayPush(pExecNode->pNodeList, &nodeEntry); - mInfo("vgId:%d added into nodeList, total:%d", nodeEntry.nodeId, (int)taosArrayGetSize(pExecNode->pNodeList)); + void* px = taosArrayPush(pExecNode->pNodeList, &nodeEntry); + if (px) { + mInfo("vgId:%d added into nodeList, total:%d", nodeEntry.nodeId, (int)taosArrayGetSize(pExecNode->pNodeList)); + } else { + mError("vgId:%d failed to add into nodeList, total:%d", nodeEntry.nodeId, (int)taosArrayGetSize(pExecNode->pNodeList)) + } } } } @@ -2534,10 +2681,15 @@ static void doAddTaskId(SArray *pList, int32_t taskId, int64_t uid, int32_t numO } } - taosArrayPush(pList, &taskId); - + void* p = taosArrayPush(pList, &taskId); int32_t numOfTasks = taosArrayGetSize(pList); - mDebug("stream:0x%" PRIx64 " receive %d reqs for checkpoint, remain:%d", uid, numOfTasks, numOfTotal - numOfTasks); + + if (p) { + mDebug("stream:0x%" PRIx64 " receive %d reqs for checkpoint, remain:%d", uid, numOfTasks, numOfTotal - numOfTasks); + } else { + mError("stream:0x%" PRIx64 " receive %d reqs for checkpoint, failed to added into task list, since out of memory", + uid, numOfTasks); + } } int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { @@ -2549,9 +2701,8 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { if (tDecodeStreamTaskCheckpointReq(&decoder, &req)) { tDecoderClear(&decoder); - terrno = TSDB_CODE_INVALID_MSG; mError("invalid task checkpoint req msg received"); - return -1; + return TSDB_CODE_INVALID_MSG; } tDecoderClear(&decoder); @@ -2572,9 +2723,8 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { void *p = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); if (p == NULL) { mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint req", req.streamId); - terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; streamMutexUnlock(&execInfo.lock); - return -1; + return TSDB_CODE_MND_STREAM_NOT_EXIST; } else { mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet", req.streamId, req.taskId); @@ -2587,8 +2737,10 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { if (pReqTaskList == NULL) { SArray *pList = taosArrayInit(4, sizeof(int32_t)); doAddTaskId(pList, req.taskId, req.streamId, numOfTasks); - taosHashPut(execInfo.pTransferStateStreams, &req.streamId, sizeof(int64_t), &pList, sizeof(void *)); - + code = taosHashPut(execInfo.pTransferStateStreams, &req.streamId, sizeof(int64_t), &pList, sizeof(void *)); + if (code) { + mError("failed to put into transfer state stream map, code: out of memory"); + } pReqTaskList = (SArray **)taosHashGet(execInfo.pTransferStateStreams, &req.streamId, sizeof(req.streamId)); } else { doAddTaskId(*pReqTaskList, req.taskId, req.streamId, numOfTasks); @@ -2600,7 +2752,10 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { mInfo("stream:0x%" PRIx64 " all tasks req checkpoint, start checkpointId:%" PRId64, req.streamId, checkpointId); if (pStream != NULL) { // TODO:handle error - int32_t code = mndProcessStreamCheckpointTrans(pMnode, pStream, checkpointId, 0, false); + code = mndProcessStreamCheckpointTrans(pMnode, pStream, checkpointId, 0, false); + if (code) { + mError("failed to create checkpoint trans, code:%s", strerror(code)); + } } else { // todo: wait for the create stream trans completed, and launch the checkpoint trans // SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); @@ -2608,7 +2763,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { } // remove this entry - taosHashRemove(execInfo.pTransferStateStreams, &req.streamId, sizeof(int64_t)); + (void) taosHashRemove(execInfo.pTransferStateStreams, &req.streamId, sizeof(int64_t)); int32_t numOfStreams = taosHashGetSize(execInfo.pTransferStateStreams); mDebug("stream:0x%" PRIx64 " removed, remain streams:%d fill-history not completed", req.streamId, numOfStreams); @@ -2654,7 +2809,11 @@ static void doAddReportStreamTask(SArray* pList, const SCheckpointReport* pRepor .checkpointId = pReport->checkpointId, .nodeId = pReport->nodeId, }; - taosArrayPush(pList, &info); + + void* p = taosArrayPush(pList, &info); + if (p == NULL) { + mError("failed to put into task list, taskId:0x%x", pReport->taskId); + } } } @@ -2667,9 +2826,8 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) { if (tDecodeStreamTaskChkptReport(&decoder, &req)) { tDecoderClear(&decoder); - terrno = TSDB_CODE_INVALID_MSG; mError("invalid task checkpoint-report msg received"); - return -1; + return TSDB_CODE_INVALID_MSG; } tDecoderClear(&decoder); @@ -2693,7 +2851,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) { mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId); terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; streamMutexUnlock(&execInfo.lock); - return -1; + return terrno; } else { mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet", req.streamId, req.taskId); @@ -2706,7 +2864,11 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) { if (pReqTaskList == NULL) { SArray *pList = taosArrayInit(4, sizeof(STaskChkptInfo)); doAddReportStreamTask(pList, &req); - taosHashPut(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId), &pList, POINTER_BYTES); + + code = taosHashPut(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId), &pList, POINTER_BYTES); + if (code) { + mError("stream:0x%"PRIx64 " failed to put into checkpoint stream", req.streamId); + } pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId)); } else { @@ -2727,7 +2889,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) { streamMutexUnlock(&execInfo.lock); doSendQuickRsp(&pReq->info, sizeof(SMStreamUpdateChkptRsp), req.nodeId, TSDB_CODE_SUCCESS); - return 0; + return code; } static int64_t getConsensusId(int64_t streamId, int32_t numOfTasks, int32_t* pExistedTasks, bool *pAllSame) { @@ -2770,100 +2932,6 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId, pInfo->handle = NULL; // disable auto rsp } -//static int32_t mndProcessConsensusCheckpointId(SRpcMsg *pMsg) { -// SMnode *pMnode = pMsg->info.node; -// SDecoder decoder = {0}; -// -// SRestoreCheckpointInfo req = {0}; -// tDecoderInit(&decoder, pMsg->pCont, pMsg->contLen); -// -// if (tDecodeRestoreCheckpointInfo(&decoder, &req)) { -// tDecoderClear(&decoder); -// terrno = TSDB_CODE_INVALID_MSG; -// mError("invalid task consensus-checkpoint msg received"); -// return -1; -// } -// tDecoderClear(&decoder); -// -// mDebug("receive stream task consensus-checkpoint msg, vgId:%d, s-task:0x%" PRIx64 "-0x%x, checkpointId:%" PRId64, -// req.nodeId, req.streamId, req.taskId, req.checkpointId); -// -// // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. -// streamMutexLock(&execInfo.lock); -// -// // mnode handle the create stream transaction too slow may cause this problem -// SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); -// if (pStream == NULL) { -// mWarn("failed to find the stream:0x%" PRIx64 ", not handle consensus-checkpointId", req.streamId); -// -// // not in meta-store yet, try to acquire the task in exec buffer -// // the checkpoint req arrives too soon before the completion of the create stream trans. -// STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; -// void *p = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); -// if (p == NULL) { -// mError("failed to find the stream:0x%" PRIx64 " in buf, not handle consensus-checkpointId", req.streamId); -// terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; -// streamMutexUnlock(&execInfo.lock); -// -// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); -// return -1; -// } else { -// mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet", -// req.streamId, req.taskId); -// // todo wait for stream is created -// } -// } -// -// mInfo("vgId:%d stream:0x%" PRIx64 " %s meta-stored checkpointId:%" PRId64, req.nodeId, req.streamId, pStream->name, -// pStream->checkpointId); -// -// int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream); -// if ((pStream != NULL) && (pStream->checkpointId == 0)) { // not generated checkpoint yet, return 0 directly -// streamMutexUnlock(&execInfo.lock); -// mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, 0, req.startTs); -// -// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); -// return TSDB_CODE_SUCCESS; -// } -// -// int32_t num = 0; -// int64_t chkId = getConsensusId(req.streamId, numOfTasks, &num); -// -// // some tasks not send hbMsg to mnode yet, wait for 5s. -// if (chkId == -1) { -// mDebug("not all(%d/%d) task(s) send hbMsg yet, wait for a while and check again, s-task:0x%x", req.taskId, num, -// numOfTasks); -// SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, req.streamId, numOfTasks); -// mndAddConsensusTasks(pInfo, &req); -// -// streamMutexUnlock(&execInfo.lock); -// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); -// return 0; -// } -// -// if (chkId == req.checkpointId) { -// mDebug("vgId:%d stream:0x%" PRIx64 " %s consensus-checkpointId is:%" PRId64 ", meta-stored checkpointId:%" PRId64, -// req.nodeId, req.streamId, pStream->name, chkId, pStream->checkpointId); -// mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, chkId, req.startTs); -// -// streamMutexUnlock(&execInfo.lock); -// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); -// return 0; -// } -// -// // wait for 5s and check again -// SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, req.streamId, numOfTasks); -// mndAddConsensusTasks(pInfo, &req); -// -// if (pStream != NULL) { -// mndReleaseStream(pMnode, pStream); -// } -// -// streamMutexUnlock(&execInfo.lock); -// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); -// return 0; -//} - int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; int64_t now = taosGetTimestampMs(); @@ -2921,9 +2989,15 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) { mDebug("s-task:0x%x sendTs:%" PRId64 " wait %.2fs and all tasks have same checkpointId", pe->req.taskId, pe->req.startTs, (now - pe->ts) / 1000.0); ASSERT(chkId <= pe->req.checkpointId); - mndCreateSetConsensusChkptIdTrans(pMnode, pStream, pe->req.taskId, chkId, pe->req.startTs); + code = mndCreateSetConsensusChkptIdTrans(pMnode, pStream, pe->req.taskId, chkId, pe->req.startTs); + if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("failed to create consensus-checkpoint trans, stream:0x%" PRIx64, pStream->uid); + } - taosArrayPush(pList, &pe->req.taskId); + void* p = taosArrayPush(pList, &pe->req.taskId); + if (p == NULL) { + mError("failed to put into task list, taskId:0x%x", pe->req.taskId); + } streamId = pe->req.streamId; } else { mDebug("s-task:0x%x sendTs:%" PRId64 " wait %.2fs already, wait for next round to check", pe->req.taskId, @@ -2951,7 +3025,10 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) { if (taosArrayGetSize(pInfo->pTaskList) == 0) { mndClearConsensusRspEntry(pInfo); ASSERT(streamId != -1); - taosArrayPush(pStreamList, &streamId); + void* p = taosArrayPush(pStreamList, &streamId); + if (p == NULL) { + mError("failed to put into stream list, stream:0x%" PRIx64, streamId); + } } } diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index c5297b5ba8..1ca46f128f 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -57,7 +57,10 @@ void addIntoCheckpointList(SArray *pList, const SFailedCheckpointInfo *pInfo) { } } - taosArrayPush(pList, pInfo); + void* p = taosArrayPush(pList, pInfo); + if (p == NULL) { + mError("failed to push failed checkpoint info checkpointId:%" PRId64 " in list", pInfo->checkpointId); + } } int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { @@ -208,6 +211,8 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) { SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; void *pIter = NULL; + int32_t code = 0; + while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) break; @@ -219,7 +224,17 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) { int32_t contLen = tSerializeSMPauseStreamReq(NULL, 0, &reqPause); void *pHead = rpcMallocCont(contLen); - tSerializeSMPauseStreamReq(pHead, contLen, &reqPause); + if (pHead == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + sdbRelease(pSdb, pStream); + continue; + } + + code = tSerializeSMPauseStreamReq(pHead, contLen, &reqPause); + if (code) { + sdbRelease(pSdb, pStream); + continue; + } SRpcMsg rpcMsg = { .msgType = TDMT_MND_PAUSE_STREAM, @@ -228,14 +243,14 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) { .info = *info, }; - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - mInfo("receive pause stream:%s, %s, %" PRId64 ", because grant expired", pStream->name, reqPause.name, - pStream->uid); + code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + mInfo("receive pause stream:%s, %s, %" PRId64 ", because grant expired, code:%s", pStream->name, reqPause.name, + pStream->uid, tstrerror(code)); } sdbRelease(pSdb, pStream); } - return 0; + return code; } int32_t mndProcessStreamHb(SRpcMsg *pReq) { @@ -267,7 +282,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { pFailedChkpt = taosArrayInit(4, sizeof(SFailedCheckpointInfo)); pOrphanTasks = taosArrayInit(4, sizeof(SOrphanTask)); - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); mndInitStreamExecInfo(pMnode, &execInfo); if (!validateHbMsg(execInfo.pNodeList, req.vgId)) { @@ -276,7 +291,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { code = terrno = TSDB_CODE_INVALID_MSG; doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); return code; } @@ -284,7 +299,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes); if (numOfUpdated > 0) { mDebug("%d stream node(s) need updated from hbMsg(vgId:%d)", numOfUpdated, req.vgId); - setNodeEpsetExpiredFlag(req.pUpdateNodes); + (void) setNodeEpsetExpiredFlag(req.pUpdateNodes); } bool snodeChanged = false; @@ -296,7 +311,10 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { mError("s-task:0x%" PRIx64 " not found in mnode task list", p->id.taskId); SOrphanTask oTask = {.streamId = p->id.streamId, .taskId = p->id.taskId, .nodeId = p->nodeId}; - taosArrayPush(pOrphanTasks, &oTask); + void* px = taosArrayPush(pOrphanTasks, &oTask); + if (px == NULL) { + mError("Failed to put task into list, taskId:0x%" PRIx64, p->id.taskId); + } continue; } @@ -346,7 +364,10 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { addIntoCheckpointList(pFailedChkpt, &info); // remove failed trans from pChkptStreams - taosHashRemove(execInfo.pChkptStreams, &p->id.streamId, sizeof(p->id.streamId)); + code = taosHashRemove(execInfo.pChkptStreams, &p->id.streamId, sizeof(p->id.streamId)); + if (code) { + mError("failed to remove stream:0x%"PRIx64" in checkpoint stream list", p->id.streamId); + } } } @@ -386,7 +407,10 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status", pInfo->checkpointId, pInfo->transId); - mndResetStatusFromCheckpoint(pMnode, pInfo->streamUid, pInfo->transId); + code = mndResetStatusFromCheckpoint(pMnode, pInfo->streamUid, pInfo->transId); + if (code) { + mError("failed to create reset task trans, code:%s", tstrerror(code)); + } } } else { mInfo("not all vgroups are ready, wait for next HB from stream tasks to reset the task status"); @@ -395,20 +419,19 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { // handle the orphan tasks that are invalid but not removed in some vnodes or snode due to some unknown errors. if (taosArrayGetSize(pOrphanTasks) > 0) { - mndDropOrphanTasks(pMnode, pOrphanTasks); + code = mndDropOrphanTasks(pMnode, pOrphanTasks); } if (pMnode != NULL) { // make sure that the unit test case can work mndStreamStartUpdateCheckpointInfo(pMnode); } - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); - terrno = TSDB_CODE_SUCCESS; - doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); + doSendHbMsgRsp(TSDB_CODE_SUCCESS, &pReq->info, req.vgId, req.msgId); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); - return terrno; + return code; } void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg @@ -416,7 +439,10 @@ void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doC if (pMsg != NULL) { int32_t size = sizeof(SMStreamDoCheckpointMsg); SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_UPDATE_CHKPT_EVT, .pCont = pMsg, .contLen = size}; - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + int32_t code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + if (code) { + mError("failed to put into write Queue, code:%s", tstrerror(code)); + } } } diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index c0a869fb77..fe656ba9c4 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -24,8 +24,7 @@ typedef struct SKeyInfo { int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId) { SStreamTransInfo info = { .transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pTransName, .streamId = streamId}; - taosHashPut(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId), &info, sizeof(SStreamTransInfo)); - return 0; + return taosHashPut(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId), &info, sizeof(SStreamTransInfo)); } int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) { @@ -45,7 +44,10 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) SKeyInfo info = {.pKey = pKey, .keyLen = keyLen}; mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name, pEntry->startTime); - taosArrayPush(pList, &info); + void* p = taosArrayPush(pList, &info); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { if (strcmp(pEntry->name, MND_STREAM_CHECKPOINT_NAME) == 0) { num++; @@ -57,7 +59,11 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) int32_t size = taosArrayGetSize(pList); for (int32_t i = 0; i < size; ++i) { SKeyInfo *pKey = taosArrayGet(pList, i); - taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen); + int32_t code = taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen); + if (code != 0) { + taosArrayDestroy(pList); + return code; + } } mDebug("clear %d finished stream-trans, remained:%d, active checkpoint trans:%d", size, @@ -79,25 +85,28 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) // 2. create/drop/reset/update trans are conflict with any other trans. bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock) { if (lock) { - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); } int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans); if (num <= 0) { if (lock) { - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); } return false; } - mndStreamClearFinishedTrans(pMnode, NULL); + int32_t code = mndStreamClearFinishedTrans(pMnode, NULL); + if (code) { + mError("failed to clear finish trans, code:%s", tstrerror(code)); + } SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId)); if (pEntry != NULL) { SStreamTransInfo tInfo = *pEntry; if (lock) { - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); } if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0) { @@ -122,32 +131,36 @@ bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *p } if (lock) { - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); } return false; } int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId) { - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans); if (num <= 0) { - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); return 0; } - mndStreamClearFinishedTrans(pMnode, NULL); + int32_t code = mndStreamClearFinishedTrans(pMnode, NULL); + if (code) { + mError("failed to clear finish trans, code:%s", tstrerror(code)); + } + SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId)); if (pEntry != NULL) { SStreamTransInfo tInfo = *pEntry; - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0 || strcmp(tInfo.name, MND_STREAM_TASK_UPDATE_NAME) == 0 || strcmp(tInfo.name, MND_STREAM_CHKPT_UPDATE_NAME) == 0) { return tInfo.transId; } } else { - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); } return 0; @@ -231,21 +244,21 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status) if (pCommitRaw == NULL) { mError("failed to encode stream since %s", terrstr()); mndTransDrop(pTrans); - return -1; + return terrno; } if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); sdbFreeRaw(pCommitRaw); mndTransDrop(pTrans); - return -1; + return terrno; } if (sdbSetRawStatus(pCommitRaw, status) != 0) { mError("stream trans:%d failed to set raw status:%d since %s", pTrans->id, status, terrstr()); sdbFreeRaw(pCommitRaw); mndTransDrop(pTrans); - return -1; + return terrno; } return 0; @@ -303,8 +316,12 @@ void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo) { void *pKey = taosHashGetKey(pDb, &len); char *p = strndup(pKey, len); - mDebug("clear checkpoint trans in Db:%s", p); - doKillCheckpointTrans(pMnode, pKey, len); + int32_t code = doKillCheckpointTrans(pMnode, pKey, len); + if (code) { + mError("failed to kill trans, transId:%p", pKey) + } else { + mDebug("clear checkpoint trans in Db:%s", p); + } taosMemoryFree(p); } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 0b96626536..e250034184 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -1203,13 +1203,17 @@ void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) { } int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) { - int32_t code = taosHashRemove(pHash, &streamId, sizeof(streamId)); + int32_t code = 0; + int32_t numOfStreams = taosHashGetSize(pHash); + if (numOfStreams == 0) { + return TSDB_CODE_SUCCESS; + } + + code = taosHashRemove(pHash, &streamId, sizeof(streamId)); if (code == 0) { - int32_t numOfStreams = taosHashGetSize(pHash); - mDebug("drop stream:0x%" PRIx64 " in consensus-checkpointId list after new checkpoint generated, remain:%d", - streamId, numOfStreams); + mDebug("drop stream:0x%" PRIx64 " in consensus-checkpointId list, remain:%d", streamId, numOfStreams); } else { - mError("failed to remove stream:0x%"PRIx64" in consensus-checkpointId list", streamId); + mError("failed to remove stream:0x%"PRIx64" in consensus-checkpointId list, remain:%d", streamId, numOfStreams); } return code; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 990d03f940..6dc777b6df 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -185,7 +185,7 @@ int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, SArray *pFuncTypeList, SColumnInfo *pkCol, int32_t numOfPks); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, SArray *pTableUids); -void *tsdbCacherowsReaderClose(void *pReader); +void tsdbCacherowsReaderClose(void *pReader); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); size_t tsdbCacheGetCapacity(SVnode *pVnode); diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 46c3ba4785..e4f17a8b3c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -23,20 +23,27 @@ #define HASTYPE(_type, _t) (((_type) & (_t)) == (_t)) -static void setFirstLastResColToNull(SColumnInfoData* pCol, int32_t row) { - char* buf = taosMemoryCalloc(1, pCol->info.bytes); +static int32_t setFirstLastResColToNull(SColumnInfoData* pCol, int32_t row) { + char* buf = taosMemoryCalloc(1, pCol->info.bytes); + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SFirstLastRes* pRes = (SFirstLastRes*)((char*)buf + VARSTR_HEADER_SIZE); pRes->bytes = 0; pRes->hasResult = true; pRes->isNull = true; varDataSetLen(buf, pCol->info.bytes - VARSTR_HEADER_SIZE); - colDataSetVal(pCol, row, buf, false); + int32_t code = colDataSetVal(pCol, row, buf, false); taosMemoryFree(buf); + + return code; } -static void saveOneRowForLastRaw(SLastCol* pColVal, SCacheRowsReader* pReader, const int32_t slotId, +static int32_t saveOneRowForLastRaw(SLastCol* pColVal, SCacheRowsReader* pReader, const int32_t slotId, SColumnInfoData* pColInfoData, int32_t numOfRows) { SColVal* pVal = &pColVal->colVal; + int32_t code = 0; // allNullRow = false; if (IS_VAR_DATA_TYPE(pColVal->colVal.value.type)) { @@ -46,17 +53,19 @@ static void saveOneRowForLastRaw(SLastCol* pColVal, SCacheRowsReader* pReader, c varDataSetLen(pReader->transferBuf[slotId], pVal->value.nData); memcpy(varDataVal(pReader->transferBuf[slotId]), pVal->value.pData, pVal->value.nData); - colDataSetVal(pColInfoData, numOfRows, pReader->transferBuf[slotId], false); + code = colDataSetVal(pColInfoData, numOfRows, pReader->transferBuf[slotId], false); } } else { - colDataSetVal(pColInfoData, numOfRows, (const char*)&pVal->value.val, !COL_VAL_IS_VALUE(pVal)); + code = colDataSetVal(pColInfoData, numOfRows, (const char*)&pVal->value.val, !COL_VAL_IS_VALUE(pVal)); } - return; + + return code; } static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pReader, const int32_t* slotIds, const int32_t* dstSlotIds, void** pRes, const char* idStr) { int32_t numOfRows = pBlock->info.rows; + int32_t code = 0; if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) { uint64_t ts = TSKEY_MIN; @@ -64,14 +73,33 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p col_id_t colId = -1; SArray* funcTypeBlockArray = taosArrayInit(pReader->numOfCols, sizeof(int32_t)); + if (funcTypeBlockArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } for (int32_t i = 0; i < pReader->numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]); - int32_t funcType = FUNCTION_TYPE_CACHE_LAST; + if (pColInfoData == NULL) { + return TSDB_CODE_INVALID_PARA; + } + int32_t funcType = FUNCTION_TYPE_CACHE_LAST; if (pReader->pFuncTypeList != NULL && taosArrayGetSize(pReader->pFuncTypeList) > i) { - funcType = *(int32_t*)taosArrayGet(pReader->pFuncTypeList, i); - taosArrayInsert(funcTypeBlockArray, dstSlotIds[i], taosArrayGet(pReader->pFuncTypeList, i)); + void* pVal = taosArrayGet(pReader->pFuncTypeList, i); + if (pVal == NULL) { + return TSDB_CODE_INVALID_PARA; + } + funcType = *(int32_t*) pVal; + + pVal = taosArrayGet(pReader->pFuncTypeList, i); + if (pVal == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + void* px = taosArrayInsert(funcTypeBlockArray, dstSlotIds[i], pVal); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } if (slotIds[i] == -1) { @@ -79,15 +107,27 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p colDataSetNULL(pColInfoData, numOfRows); continue; } - setFirstLastResColToNull(pColInfoData, numOfRows); + + code = setFirstLastResColToNull(pColInfoData, numOfRows); + if (code) { + return code; + } continue; } + int32_t slotId = slotIds[i]; SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); - colId = pColVal->colVal.cid; + if (pColVal == NULL) { + return TSDB_CODE_INVALID_PARA; + } + colId = pColVal->colVal.cid; if (FUNCTION_TYPE_CACHE_LAST_ROW == funcType) { - saveOneRowForLastRaw(pColVal, pReader, slotId, pColInfoData, numOfRows); + code = saveOneRowForLastRaw(pColVal, pReader, slotId, pColInfoData, numOfRows); + if (code) { + return code; + } + continue; } @@ -112,13 +152,25 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p // pColInfoData->info.bytes includes the VARSTR_HEADER_SIZE, need to subtract it p->hasResult = true; varDataSetLen(pRes[i], pColInfoData->info.bytes - VARSTR_HEADER_SIZE); - colDataSetVal(pColInfoData, numOfRows, (const char*)pRes[i], false); + code = colDataSetVal(pColInfoData, numOfRows, (const char*)pRes[i], false); + if (code) { + return code; + } } for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx); + if (pCol == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (idx < funcTypeBlockArray->size) { - int32_t funcType = *(int32_t*)taosArrayGet(funcTypeBlockArray, idx); + void* pVal = taosArrayGet(funcTypeBlockArray, idx); + if (pVal == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + int32_t funcType = *(int32_t*)pVal; if (FUNCTION_TYPE_CACHE_LAST_ROW == funcType) { continue; } @@ -128,12 +180,18 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p if (ts == TSKEY_MIN) { colDataSetNULL(pCol, numOfRows); } else { - colDataSetVal(pCol, numOfRows, (const char*)&ts, false); + code = colDataSetVal(pCol, numOfRows, (const char*)&ts, false); + if (code) { + return code; + } } continue; } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && (pCol->info.colId == colId || colId == -1)) { if (p && !p->isNull) { - colDataSetVal(pCol, numOfRows, p->buf, false); + code = colDataSetVal(pCol, numOfRows, p->buf, false); + if (code) { + return code; + } } else { colDataSetNULL(pCol, numOfRows); } @@ -146,15 +204,25 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p } else if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST_ROW)) { for (int32_t i = 0; i < pReader->numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]); + if (pColInfoData == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t slotId = slotIds[i]; if (slotId == -1) { colDataSetNULL(pColInfoData, numOfRows); continue; } - SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); - saveOneRowForLastRaw(pColVal, pReader, slotId, pColInfoData, numOfRows); + SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); + if (pColVal == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + code = saveOneRowForLastRaw(pColVal, pReader, slotId, pColInfoData, numOfRows); + if (code) { + return code; + } } // pBlock->info.rows += allNullRow ? 0 : 1; @@ -164,7 +232,7 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p return TSDB_CODE_INVALID_PARA; } - return TSDB_CODE_SUCCESS; + return code; } static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* idstr) { @@ -206,7 +274,7 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf destroySttBlockReader(pReader->pLDataIterArray, NULL); pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); - return TSDB_CODE_SUCCESS; + return (pReader->pLDataIterArray != NULL) ? TSDB_CODE_SUCCESS : TSDB_CODE_OUT_OF_MEMORY; } int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols, @@ -269,18 +337,22 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, } p->idstr = taosStrdup(idstr); - taosThreadMutexInit(&p->readerMutex, NULL); + code = taosThreadMutexInit(&p->readerMutex, NULL); + if (code) { + tsdbCacherowsReaderClose(p); + return code; + } p->lastTs = INT64_MIN; *pReader = p; - return TSDB_CODE_SUCCESS; + return code; } -void* tsdbCacherowsReaderClose(void* pReader) { +void tsdbCacherowsReaderClose(void* pReader) { SCacheRowsReader* p = pReader; if (p == NULL) { - return NULL; + return; } if (p->pSchema != NULL) { @@ -299,12 +371,12 @@ void* tsdbCacherowsReaderClose(void* pReader) { } if (p->pFileReader) { - tsdbDataFileReaderClose(&p->pFileReader); + (void) tsdbDataFileReaderClose(&p->pFileReader); p->pFileReader = NULL; } taosMemoryFree((void*)p->idstr); - taosThreadMutexDestroy(&p->readerMutex); + (void) taosThreadMutexDestroy(&p->readerMutex); if (p->pTableMap) { void* pe = NULL; @@ -322,7 +394,6 @@ void* tsdbCacherowsReaderClose(void* pReader) { } taosMemoryFree(pReader); - return NULL; } static void freeItem(void* pItem) { @@ -342,8 +413,7 @@ static int32_t tsdbCacheQueryReseek(void* pQHandle) { // just wait for the big all tables' snapshot untaking for now code = TSDB_CODE_VND_QUERY_BUSY; - - taosThreadMutexUnlock(&pReader->readerMutex); + (void)taosThreadMutexUnlock(&pReader->readerMutex); return code; } else if (code == EBUSY) { @@ -360,9 +430,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } SCacheRowsReader* pr = pReader; - int32_t code = TSDB_CODE_SUCCESS; - SArray* pRow = taosArrayInit(TARRAY_SIZE(pr->pCidList), sizeof(SLastCol)); - bool hasRes = false; + + int32_t code = TSDB_CODE_SUCCESS; + bool hasRes = false; + SArray* pRow = taosArrayInit(TARRAY_SIZE(pr->pCidList), sizeof(SLastCol)); + if (pRow == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } void** pRes = taosMemoryCalloc(pr->numOfCols, POINTER_BYTES); if (pRes == NULL) { @@ -370,7 +445,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 goto _end; } - int32_t pkBufLen = (pr->rowKey.numOfPKs > 0)? pr->pkColumn.bytes:0; + int32_t pkBufLen = (pr->rowKey.numOfPKs > 0) ? pr->pkColumn.bytes : 0; for (int32_t j = 0; j < pr->numOfCols; ++j) { int32_t bytes = (slotIds[j] == -1) ? 1 : pr->pSchema->columns[slotIds[j]].bytes; @@ -379,7 +454,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 p->ts = INT64_MIN; } - taosThreadMutexLock(&pr->readerMutex); + (void)taosThreadMutexLock(&pr->readerMutex); code = tsdbTakeReadSnap2((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap); if (code != TSDB_CODE_SUCCESS) { goto _end; @@ -401,7 +476,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 int32_t slotId = slotIds[i]; if (slotId == -1) { SLastCol p = {.rowKey.ts = INT64_MIN, .colVal.value.type = TSDB_DATA_TYPE_BOOL, .colVal.flag = CV_FLAG_NULL}; - taosArrayPush(pLastCols, &p); + void* px = taosArrayPush(pLastCols, &p); + if (px == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } continue; } struct STColumn* pCol = &pr->pSchema->columns[slotId]; @@ -409,8 +488,17 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (IS_VAR_DATA_TYPE(pCol->type)) { p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char)); + if (p.colVal.value.pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + } + + void* px = taosArrayPush(pLastCols, &p); + if (px == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; } - taosArrayPush(pLastCols, &p); } int64_t st = taosGetTimestampUs(); @@ -418,7 +506,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 for (int32_t i = 0; i < pr->numOfTables; ++i) { tb_uid_t uid = pTableList[i].uid; - tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); + code = tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); + if (code) { + goto _end; + } + if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { taosArrayClearEx(pRow, freeItem); continue; @@ -430,7 +522,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 for (int32_t k = 0; k < pr->numOfCols; ++k) { if (slotIds[k] == -1) continue; SLastCol* p = taosArrayGet(pLastCols, k); + if (p == NULL) { + return TSDB_CODE_INVALID_PARA; + } + SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k); + if (pColVal == NULL) { + return TSDB_CODE_INVALID_PARA; + } if (pColVal->rowKey.ts > p->rowKey.ts) { if (!COL_VAL_IS_VALUE(&pColVal->colVal) && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) { @@ -447,7 +546,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 p->rowKey.ts = pColVal->rowKey.ts; if (k == 0) { if (TARRAY_SIZE(pTableUidList) == 0) { - taosArrayPush(pTableUidList, &uid); + void* px = taosArrayPush(pTableUidList, &uid); + if (px == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } } else { taosArraySet(pTableUidList, 0, &uid); } @@ -487,7 +590,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } if (hasRes) { - saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); + code = saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); + if (code) { + goto _end; + } } taosArrayDestroyEx(pLastCols, freeItem); @@ -495,16 +601,27 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) { tb_uid_t uid = pTableList[i].uid; - tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); + if ((code = tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype)) != 0) { + goto _end; + } + if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { taosArrayClearEx(pRow, freeItem); continue; } - saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); + code = saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); + if (code) { + goto _end; + } + taosArrayClearEx(pRow, freeItem); - taosArrayPush(pTableUidList, &uid); + void* px = taosArrayPush(pTableUidList, &uid); + if (px == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } ++pr->tableIndex; if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -521,7 +638,7 @@ _end: pr->pCurFileSet = NULL; } - taosThreadMutexUnlock(&pr->readerMutex); + (void)taosThreadMutexUnlock(&pr->readerMutex); if (pRes != NULL) { for (int32_t j = 0; j < pr->numOfCols; ++j) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 23e396b6a4..95ae9bcdf0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -48,7 +48,7 @@ typedef struct { static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo); static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader); -static int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRow); +static int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRes); static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, STsdbReader* pReader); static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, @@ -73,8 +73,8 @@ static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBloc STsdbReader* pReader); static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost); -static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr, - int8_t* pLevel); +static void getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr, + int8_t* pLevel, STsdb** pTsdb); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); static int32_t doBuildDataBlock(STsdbReader* pReader); static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); @@ -123,7 +123,7 @@ static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_ pKey->pks[0].val = cv.value.val; } else { pKey->pks[0].nData = cv.value.nData; - (void)(void)memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); + (void)memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); } } @@ -439,7 +439,6 @@ static int32_t tsdbUninitReaderLock(STsdbReader* pReader) { tsdbTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); code = taosThreadMutexDestroy(&pReader->readerMutex); - tsdbTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; @@ -450,7 +449,6 @@ static int32_t tsdbAcquireReader(STsdbReader* pReader) { tsdbTrace("tsdb/read: %p, pre-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); code = taosThreadMutexLock(&pReader->readerMutex); - tsdbTrace("tsdb/read: %p, post-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; @@ -541,8 +539,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void } initReaderStatus(&pReader->status); - - pReader->pTsdb = getTsdbByRetentions(pVnode, pCond, pVnode->config.tsdbCfg.retentions, idstr, &level); + getTsdbByRetentions(pVnode, pCond, pVnode->config.tsdbCfg.retentions, idstr, &level, &pReader->pTsdb); pReader->info.suid = pCond->suid; pReader->info.order = pCond->order; @@ -550,6 +547,11 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void pReader->info.window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); pReader->idStr = (idstr != NULL) ? taosStrdup(idstr) : NULL; + if (pReader->idStr == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + pReader->type = pCond->type; pReader->bFilesetDelimited = false; pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket @@ -586,6 +588,11 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void } pReader->status.pPrimaryTsCol = taosArrayGet(pReader->resBlockInfo.pResBlock->pDataBlock, pSup->slotId[0]); + if (pReader->status.pPrimaryTsCol == NULL) { + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + int32_t type = pReader->status.pPrimaryTsCol->info.type; if (type != TSDB_DATA_TYPE_TIMESTAMP) { tsdbError("the first column isn't primary timestamp in result block, actual: %s, %s", tDataTypes[type].name, @@ -651,7 +658,6 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead STableUidList* pList = &pReader->status.uidList; int32_t i = 0; - while (i < TARRAY2_SIZE(pBlkArray)) { pBrinBlk = &pBlkArray->data[i]; if (pBrinBlk->maxTbid.suid < pReader->info.suid) { @@ -824,6 +830,10 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S p1 = taosArrayPush(pTableScanInfoList, &pScanInfo); } else { STableBlockScanInfo** p = taosArrayGetLast(pTableScanInfoList); + if (p == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if ((*p)->uid != uid) { p1 = taosArrayPush(pTableScanInfoList, &pScanInfo); } @@ -1273,6 +1283,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro int32_t rowIndex = 0; SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pColData == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); i += 1; @@ -1288,6 +1302,9 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro colIndex += 1; } else if (pData->cid == pSupInfo->colId[i]) { pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pColData == NULL) { + return TSDB_CODE_INVALID_PARA; + } if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { colDataSetNNULL(pColData, 0, dumpedRows); @@ -1309,6 +1326,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro i += 1; } else { // the specified column does not exist in file block, fill with null data pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pColData == NULL) { + return TSDB_CODE_INVALID_PARA; + } + colDataSetNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1317,6 +1338,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro // fill the mis-matched columns with null value while (i < numOfOutputCols) { pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pColData == NULL) { + return TSDB_CODE_INVALID_PARA; + } + colDataSetNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1448,7 +1473,15 @@ static bool getNeighborBlockOfTable(SDataBlockIter* pBlockIter, SFileDataBlockIn } STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); + if (pTableDataBlockIdx == NULL) { + return TSDB_CODE_INVALID_PARA; + } + SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex); + if (p == NULL) { + return TSDB_CODE_INVALID_PARA; + } + blockInfoToRecord(pRecord, p, pSupInfo); *nextIndex = pBlockInfo->tbBlockIdx + step; @@ -1462,13 +1495,21 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte return TSDB_CODE_FAILED; } - SFileDataBlockInfo fblock = *(SFileDataBlockInfo*)taosArrayGet(pBlockIter->blockList, index); + void* p = taosArrayGet(pBlockIter->blockList, index); + if (p == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + SFileDataBlockInfo fblock = *(SFileDataBlockInfo*) p; pBlockIter->index += step; if (index != pBlockIter->index) { if (index > pBlockIter->index) { for (int32_t i = index - 1; i >= pBlockIter->index; --i) { SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i); + if (pBlockInfo == NULL) { + return TSDB_CODE_INVALID_PARA; + } STableBlockScanInfo* pBlockScanInfo = NULL; code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); @@ -1477,6 +1518,10 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte } STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); + if (pTableDataBlockIdx == NULL) { + return TSDB_CODE_INVALID_PARA; + } + pTableDataBlockIdx->globalIndex = i + 1; taosArraySet(pBlockIter->blockList, i + 1, pBlockInfo); @@ -1484,6 +1529,9 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte } else if (index < pBlockIter->index) { for (int32_t i = index + 1; i <= pBlockIter->index; ++i) { SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i); + if (pBlockInfo == NULL) { + return TSDB_CODE_INVALID_PARA; + } STableBlockScanInfo* pBlockScanInfo = NULL; code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); @@ -1492,8 +1540,11 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte } STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); - pTableDataBlockIdx->globalIndex = i - 1; + if (pTableDataBlockIdx == NULL) { + return TSDB_CODE_INVALID_PARA; + } + pTableDataBlockIdx->globalIndex = i - 1; taosArraySet(pBlockIter->blockList, i - 1, pBlockInfo); } } @@ -1505,7 +1556,11 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte return code; } - STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, fblock.tbBlockIdx); + STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, fblock.tbBlockIdx); + if (pTableDataBlockIdx == NULL) { + return TSDB_CODE_INVALID_PARA; + } + pTableDataBlockIdx->globalIndex = pBlockIter->index; } @@ -2338,7 +2393,7 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan pSttBlockReader->uid = pScanInfo->uid; // second time init stt block reader - if (pScanInfo->cleanSttBlocks && pReader->info.execMode == READER_EXEC_ROWS) { + if (pScanInfo->cleanSttBlocks && (pReader->info.execMode == READER_EXEC_ROWS)) { return !pScanInfo->sttBlockReturned; } @@ -2374,6 +2429,9 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan }; SSttDataInfoForTable info = {.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange))}; + if (info.pKeyRangeList == NULL) { + return false; + } int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info); if (code != TSDB_CODE_SUCCESS) { @@ -2398,6 +2456,10 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan // calculate the time window for data in stt files for (int32_t i = 0; i < taosArrayGetSize(info.pKeyRangeList); ++i) { SSttKeyRange* pKeyRange = taosArrayGet(info.pKeyRangeList, i); + if (pKeyRange == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (pkCompEx(&pScanInfo->sttRange.skey, &pKeyRange->skey) > 0) { tRowKeyAssign(&pScanInfo->sttRange.skey, &pKeyRange->skey); } @@ -2618,6 +2680,9 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock // 1. find the next neighbor block in the scan block list STableDataBlockIdx* tableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, nextIndex); + if (tableDataBlockIdx == NULL) { + return TSDB_CODE_INVALID_PARA; + } // 2. remove it from the scan block list int32_t neighborIndex = tableDataBlockIdx->globalIndex; @@ -2787,6 +2852,9 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t orde taosArrayClear(pBlockScanInfo->delSkyline); } else { pBlockScanInfo->delSkyline = taosArrayInit(4, sizeof(TSDBKEY)); + if (pBlockScanInfo->delSkyline == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } SArray* pSource = pBlockScanInfo->pFileDelData; @@ -3412,6 +3480,9 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { SBlockNumber num = {0}; SArray* pTableList = taosArrayInit(40, POINTER_BYTES); + if (pTableList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } int32_t code = moveToNextFile(pReader, &num, pTableList); if (code != TSDB_CODE_SUCCESS) { @@ -3563,8 +3634,13 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } } -static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idStr, - int8_t* pLevel) { +static void getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idStr, + int8_t* pLevel, STsdb** pTsdb) { + if (pTsdb == NULL) { + return; + } + + *pTsdb = NULL; if (VND_IS_RSMA(pVnode) && !pCond->skipRollup) { int8_t level = 0; int8_t precision = pVnode->config.tsdbCfg.precision; @@ -3573,7 +3649,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SR : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L : 1000000L); - for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) { + for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { SRetention* pRetention = retentions + level; if (pRetention->keep <= 0) { if (level > 0) { @@ -3592,19 +3668,19 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SR if (level == TSDB_RETENTION_L0) { *pLevel = TSDB_RETENTION_L0; tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L0, str); - return VND_RSMA0(pVnode); + *pTsdb = VND_RSMA0(pVnode); } else if (level == TSDB_RETENTION_L1) { *pLevel = TSDB_RETENTION_L1; tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L1, str); - return VND_RSMA1(pVnode); + *pTsdb = VND_RSMA1(pVnode); } else { *pLevel = TSDB_RETENTION_L2; tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L2, str); - return VND_RSMA2(pVnode); + *pTsdb = VND_RSMA2(pVnode); } } - return VND_TSDB(pVnode); + *pTsdb = VND_TSDB(pVnode); } SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) { @@ -3631,6 +3707,10 @@ static int32_t reverseSearchStartPos(const SArray* pDelList, int32_t index, int6 } TSDBKEY* p = taosArrayGet(pDelList, start); + if (p == NULL) { + return TSDB_CODE_INVALID_PARA; + } + while (p->ts >= key && start > 0) { start -= 1; } @@ -3640,6 +3720,10 @@ static int32_t reverseSearchStartPos(const SArray* pDelList, int32_t index, int6 } TSDBKEY* p = taosArrayGet(pDelList, start); + if (p == NULL) { + return TSDB_CODE_INVALID_PARA; + } + while (p->ts <= key && start < num - 1) { start += 1; } @@ -3665,17 +3749,27 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t if (asc) { if (*index >= num - 1) { TSDBKEY* last = taosArrayGetLast(pDelList); - ASSERT(key >= last->ts); + if (last == NULL) { + return false; + } + ASSERT(key >= last->ts); if (key > last->ts) { return false; } else if (key == last->ts) { TSDBKEY* prev = taosArrayGet(pDelList, num - 2); + if (prev == NULL) { + return false; + } + return (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); } } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); TSDBKEY* pNext = taosArrayGet(pDelList, (*index) + 1); + if (pCurrent == NULL || pNext == NULL) { + return false; + } if (key < pCurrent->ts) { return false; @@ -3692,6 +3786,9 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t if ((*index) < num - 1) { pCurrent = taosArrayGet(pDelList, *index); pNext = taosArrayGet(pDelList, (*index) + 1); + if (pCurrent == NULL || pNext == NULL) { + return false; + } // it is not a consecutive deletion range, ignore it if (pCurrent->version == 0 && pNext->version > 0) { @@ -3710,6 +3807,9 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t } else { if (*index <= 0) { TSDBKEY* pFirst = taosArrayGet(pDelList, 0); + if (pFirst == NULL) { + return false; + } if (key < pFirst->ts) { return false; @@ -3721,6 +3821,9 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); TSDBKEY* pPrev = taosArrayGet(pDelList, (*index) - 1); + if (pCurrent == NULL || pPrev == NULL) { + return false; + } if (key > pCurrent->ts) { return false; @@ -3736,6 +3839,9 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t if ((*index) >= 1) { pCurrent = taosArrayGet(pDelList, *index); pPrev = taosArrayGet(pDelList, (*index) - 1); + if (pCurrent == NULL || pPrev == NULL) { + return false; + } // it is not a consecutive deletion range, ignore it if (pCurrent->version > 0 && pPrev->version == 0) { @@ -4229,6 +4335,10 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + if (pColData == NULL) { + return TSDB_CODE_INVALID_PARA; + } + ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; i += 1; } @@ -4238,6 +4348,9 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT if (colId == pSchema->columns[j].colId) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + if (pColInfoData == NULL) { + return TSDB_CODE_INVALID_PARA; + } code = tRowGet(pTSRow, pSchema, j, &colVal); if (code) { @@ -4252,6 +4365,9 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT j += 1; } else if (colId < pSchema->columns[j].colId) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + if (pColInfoData == NULL) { + return TSDB_CODE_INVALID_PARA; + } colDataSetNULL(pColInfoData, outputRowIndex); i += 1; @@ -4263,6 +4379,10 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT // set null value since current column does not exist in the "pSchema" while (i < pSupInfo->numOfCols) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + if (pColInfoData == NULL) { + return TSDB_CODE_INVALID_PARA; + } + colDataSetNULL(pColInfoData, outputRowIndex); i += 1; } @@ -4311,6 +4431,10 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S while (i < numOfOutputCols) { SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pCol == NULL) { + return TSDB_CODE_INVALID_PARA; + } + colDataSetNULL(pCol, outputRowIndex); i += 1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index d7da6cbd9d..64569e63ac 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -122,7 +122,9 @@ void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) { size_t num = taosArrayGetSize(pBuf->pData); for (int32_t i = 0; i < num; ++i) { char** p = taosArrayGet(pBuf->pData, i); - taosMemoryFree(*p); + if (p != NULL) { + taosMemoryFree(*p); + } } taosArrayDestroy(pBuf->pData); @@ -446,8 +448,8 @@ void cleanupInfoForNextFileset(SSHashObj* pTableMap) { // brin records iterator void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList) { - memset(&pIter->block, 0, sizeof(SBrinBlock)); - memset(&pIter->record, 0, sizeof(SBrinRecord)); + (void) memset(&pIter->block, 0, sizeof(SBrinBlock)); + (void) memset(&pIter->record, 0, sizeof(SBrinRecord)); pIter->blockIndex = -1; pIter->recordIndex = -1; @@ -465,6 +467,9 @@ int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) { } pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); + if (pIter->pCurrentBlk == NULL) { + return TSDB_CODE_INVALID_PARA; + } (void) tBrinBlockClear(&pIter->block); int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); @@ -631,6 +636,10 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 for (int32_t k = 0; k < num; ++k) { SFileDataBlockInfo* pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, k); + if (pBlockInfo == NULL) { + return TSDB_CODE_INVALID_PARA; + } + sup.pDataBlockInfo[sup.numOfTables][k] = (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pBlockInfo->blockOffset, .pInfo = pTableScanInfo}; cnt++; @@ -689,6 +698,10 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 int32_t index = sup.indexPerTable[pos]++; SFileDataBlockInfo* pBlockInfo = taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); + if (pBlockInfo == NULL) { + return TSDB_CODE_INVALID_PARA; + } + void* px = taosArrayPush(pBlockIter->blockList, pBlockInfo); if (px == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1208,6 +1221,10 @@ bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlo } SSttKeyRange* pRange = taosArrayGet(pKeyRangeList, 0); + if (pRange == NULL) { + return false; + } + STimeWindow w = {.skey = pRange->skey.ts, .ekey = pRange->ekey.ts}; if (overlapWithTimeWindow(&w, pQueryWindow, pScanInfo, order)) { return false; @@ -1216,6 +1233,9 @@ bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlo for (int32_t i = 0; i < num - 1; ++i) { SSttKeyRange* p1 = taosArrayGet(pKeyRangeList, i); SSttKeyRange* p2 = taosArrayGet(pKeyRangeList, i + 1); + if (p1 == NULL || p2 == NULL) { + return false; + } if (p1->ekey.ts >= p2->skey.ts) { return false; @@ -1237,6 +1257,10 @@ static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const S for (int32_t i = startIndex; i < num; i += 1) { TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i); + if (p == NULL) { + return false; + } + if (p->ts >= pRecord->firstKey.key.ts && p->ts <= pRecord->lastKey.key.ts) { if (p->version >= pRecord->minVer) { return true; @@ -1245,6 +1269,10 @@ static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const S if (p->version >= pRecord->minVer) { if (i < num - 1) { TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1); + if (pnext == NULL) { + return false; + } + if (pnext->ts >= pRecord->firstKey.key.ts) { return true; } @@ -1266,11 +1294,19 @@ static bool doCheckDatablockOverlapWithoutVersion(STableBlockScanInfo* pBlockSca for (int32_t i = startIndex; i < num; i += 1) { TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i); + if (p == NULL) { + return false; + } + if (p->ts >= pRecord->firstKey.key.ts && p->ts <= pRecord->lastKey.key.ts) { return true; } else if (p->ts < pRecord->firstKey.key.ts) { // p->ts < pBlock->minKey.ts if (i < num - 1) { TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1); + if (pnext == NULL) { + return false; + } + if (pnext->ts >= pRecord->firstKey.key.ts) { return true; } @@ -1291,6 +1327,10 @@ bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecor // ts is not overlap TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0); TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline); + if (pFirst == NULL || pLast == NULL) { + return false; + } + if (pRecord->firstKey.key.ts > pLast->ts || pRecord->lastKey.key.ts < pFirst->ts) { return false; } @@ -1302,6 +1342,10 @@ bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecor int32_t index = pBlockScanInfo->fileDelIndex; while (1) { TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index); + if (p == NULL) { + return false; + } + if (p->ts > pRecord->firstKey.key.ts && index > 0) { index -= 1; } else { // find the first point that is smaller than the minKey.ts of dataBlock. @@ -1324,6 +1368,10 @@ bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const // ts is not overlap TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0); TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline); + if (pFirst == NULL || pLast == NULL) { + return false; + } + if (pRecord->firstKey.key.ts > pLast->ts || pRecord->lastKey.key.ts < pFirst->ts) { return false; } @@ -1335,6 +1383,10 @@ bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const int32_t index = pBlockScanInfo->fileDelIndex; while (1) { TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index); + if (p == NULL) { + return false; + } + if (p->ts > pRecord->firstKey.key.ts && index > 0) { index -= 1; } else { // find the first point that is smaller than the minKey.ts of dataBlock. diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index b9118cf00d..c6f3986776 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -77,9 +77,9 @@ typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void* * @param sortBufSize sort memory buf size, for check if heap sort is applicable * @return */ -SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, int32_t type, int32_t pageSize, int32_t numOfPages, +int32_t tsortCreateSortHandle(SArray* pOrderInfo, int32_t type, int32_t pageSize, int32_t numOfPages, SSDataBlock* pBlock, const char* idstr, uint64_t pqMaxRows, uint32_t pqMaxTupleLength, - uint32_t pqSortBufSize); + uint32_t pqSortBufSize, SSortHandle** pHandle); void tsortSetForceUsePQSort(SSortHandle* pHandle); @@ -101,7 +101,7 @@ int32_t tsortOpen(SSortHandle* pHandle); * @param pHandle * @return */ -int32_t tsortClose(SSortHandle* pHandle); +void tsortClose(SSortHandle* pHandle); /** * @@ -116,16 +116,17 @@ int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetc * @param fp * @return */ -int32_t tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp); +void tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp); /** * */ void tsortSetMergeLimit(SSortHandle* pHandle, int64_t mergeLimit); + /** * */ -int32_t tsortSetCompareGroupId(SSortHandle* pHandle, bool compareGroupId); +void tsortSetCompareGroupId(SSortHandle* pHandle, bool compareGroupId); /** * @@ -140,7 +141,7 @@ int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource); * @param pHandle * @return */ -STupleHandle* tsortNextTuple(SSortHandle* pHandle); +int32_t tsortNextTuple(SSortHandle* pHandle, STupleHandle** pTupleHandle); /** * @@ -156,7 +157,7 @@ bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId); * @param colId * @return */ -void* tsortGetValue(STupleHandle* pVHandle, int32_t colId); +void tsortGetValue(STupleHandle* pVHandle, int32_t colId, void** pVal); /** * @@ -197,7 +198,8 @@ void tsortSetAbortCheckFn(SSortHandle* pHandle, bool (*checkFn)(void* param), vo int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsSize); -void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHandle* pTupleHandle); +int32_t tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHandle* pTupleHandle); + /** * @brief comp the tuple with keyBuf, if not equal, new keys will be built in keyBuf, newLen will be stored in keyLen * @param [in] pSortCols cols to comp and build diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index ad7d089da9..bfa9dfd8d9 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -359,7 +359,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - pInfo->pLastrowReader = pReaderFn->closeReader(pInfo->pLastrowReader); + pReaderFn->closeReader(pInfo->pLastrowReader); + pInfo->pLastrowReader = NULL; setOperatorCompleted(pOperator); return NULL; } @@ -378,7 +379,8 @@ void destroyCacheScanOperator(void* param) { tableListDestroy(pInfo->pTableList); if (pInfo->pLastrowReader != NULL) { - pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader); + pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader); + pInfo->pLastrowReader = NULL; } cleanupExprSupp(&pInfo->pseudoExprSup); diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c old mode 100755 new mode 100644 index 2816bae03c..cef1915d2c --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -70,8 +70,12 @@ int32_t openSortMergeOperator(SOperatorInfo* pOperator) { int32_t numOfBufPage = pSortMergeInfo->sortBufSize / pSortMergeInfo->bufPageSize; - pSortMergeInfo->pSortHandle = tsortCreateSortHandle(pSortMergeInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pSortMergeInfo->bufPageSize, numOfBufPage, - pSortMergeInfo->pInputBlock, pTaskInfo->id.str, 0, 0, 0); + pSortMergeInfo->pSortHandle = NULL; + int32_t code = tsortCreateSortHandle(pSortMergeInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pSortMergeInfo->bufPageSize, + numOfBufPage, pSortMergeInfo->pInputBlock, pTaskInfo->id.str, 0, 0, 0, &pSortMergeInfo->pSortHandle); + if (code) { + return code; + } tsortSetFetchRawDataFp(pSortMergeInfo->pSortHandle, sortMergeloadNextDataBlock, NULL, NULL); tsortSetCompareGroupId(pSortMergeInfo->pSortHandle, pInfo->groupMerge); @@ -96,12 +100,17 @@ static void doGetSortedBlockData(SMultiwayMergeOperatorInfo* pInfo, SSortHandle* SSDataBlock* p, bool* newgroup) { SSortMergeInfo* pSortMergeInfo = &pInfo->sortMergeInfo; *newgroup = false; + int32_t code = 0; while (1) { STupleHandle* pTupleHandle = NULL; if (pInfo->groupMerge || pInfo->inputWithGroupId) { if (pSortMergeInfo->prefetchedTuple == NULL) { - pTupleHandle = tsortNextTuple(pHandle); + pTupleHandle = NULL; + code = tsortNextTuple(pHandle, &pTupleHandle); + if (code) { + // todo handle error + } } else { pTupleHandle = pSortMergeInfo->prefetchedTuple; pSortMergeInfo->prefetchedTuple = NULL; @@ -112,11 +121,11 @@ static void doGetSortedBlockData(SMultiwayMergeOperatorInfo* pInfo, SSortHandle* } } } else { - pTupleHandle = tsortNextTuple(pHandle); + code = tsortNextTuple(pHandle, &pTupleHandle); pInfo->groupId = 0; } - if (pTupleHandle == NULL) { + if (pTupleHandle == NULL || (code != 0)) { break; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 520cba92a1..591c225e19 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -5113,8 +5113,13 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) { pInfo->sortBufSize = 2048 * pInfo->bufPageSize; int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, - pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); + pInfo->pSortHandle = NULL; + code = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, + pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0, &pInfo->pSortHandle); + if (code) { + return code; + } + if (pInfo->bSortRowId && numOfTable != 1) { int32_t memSize = 512 * 1024 * 1024; code = tsortSetSortByRowId(pInfo->pSortHandle, memSize); @@ -5247,10 +5252,12 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* STupleHandle* pTupleHandle = NULL; while (1) { while (1) { - pTupleHandle = tsortNextTuple(pHandle); - if (pTupleHandle == NULL) { + pTupleHandle = NULL; + int32_t code = tsortNextTuple(pHandle, &pTupleHandle); + if (pTupleHandle == NULL || code != 0) { break; } + tsortAppendTupleToBlock(pInfo->pSortHandle, pResBlock, pTupleHandle); if (pResBlock->info.rows >= capacity) { break; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 82eebf5310..67b993d702 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -157,7 +157,8 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { if (isNull) { colDataSetNULL(pColInfo, pBlock->info.rows); } else { - char* pData = tsortGetValue(pTupleHandle, i); + char* pData = NULL; + tsortGetValue(pTupleHandle, i, (void**) &pData); if (pData != NULL) { colDataSetVal(pColInfo, pBlock->info.rows, pData, false); } @@ -176,9 +177,13 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { * @retval NULL if no more tuples */ static STupleHandle* nextTupleWithGroupId(SSortHandle* pHandle, SSortOperatorInfo* pInfo, SSDataBlock* pBlock) { + int32_t code = 0; STupleHandle* retTuple = pInfo->pGroupIdCalc->pSavedTuple; if (!retTuple) { - retTuple = tsortNextTuple(pHandle); + code = tsortNextTuple(pHandle, &retTuple); + if (code) { + return NULL; + } } if (retTuple) { @@ -219,6 +224,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SSortOperatorInfo* pInfo) { blockDataCleanup(pDataBlock); + int32_t code = 0; SSDataBlock* p = tsortGetSortedDataBlock(pHandle); if (p == NULL) { return NULL; @@ -231,9 +237,9 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i if (pInfo->pGroupIdCalc) { pTupleHandle = nextTupleWithGroupId(pHandle, pInfo, p); } else { - pTupleHandle = tsortNextTuple(pHandle); + code = tsortNextTuple(pHandle, &pTupleHandle); } - if (pTupleHandle == NULL) { + if (pTupleHandle == NULL || code != 0) { break; } @@ -295,8 +301,12 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) { pInfo->startTs = taosGetTimestampUs(); // pInfo->binfo.pRes is not equalled to the input datablock. - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, NULL, pTaskInfo->id.str, - pInfo->maxRows, pInfo->maxTupleLength, tsPQSortMemThreshold * 1024 * 1024); + pInfo->pSortHandle = NULL; + int32_t code = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, NULL, pTaskInfo->id.str, + pInfo->maxRows, pInfo->maxTupleLength, tsPQSortMemThreshold * 1024 * 1024, &pInfo->pSortHandle); + if (code) { + return code; + } tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator); @@ -305,8 +315,7 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) { ps->onlyRef = true; tsortAddSource(pInfo->pSortHandle, ps); - int32_t code = tsortOpen(pInfo->pSortHandle); - + code = tsortOpen(pInfo->pSortHandle); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, terrno); } @@ -442,8 +451,9 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo blockDataEnsureCapacity(p, capacity); while (1) { - STupleHandle* pTupleHandle = tsortNextTuple(pHandle); - if (pTupleHandle == NULL) { + STupleHandle* pTupleHandle = NULL; + int32_t code = tsortNextTuple(pHandle, &pTupleHandle); + if (pTupleHandle == NULL || code != 0) { break; } @@ -508,8 +518,13 @@ int32_t beginSortGroup(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // pInfo->binfo.pRes is not equalled to the input datablock. - pInfo->pCurrSortHandle = - tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, NULL, pTaskInfo->id.str, 0, 0, 0); + pInfo->pCurrSortHandle = NULL; + + int32_t code = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, NULL, pTaskInfo->id.str, 0, 0, + 0, &pInfo->pCurrSortHandle); + if (code) { + return code; + } tsortSetFetchRawDataFp(pInfo->pCurrSortHandle, fetchNextGroupSortDataBlock, applyScalarFunction, pOperator); @@ -521,13 +536,12 @@ int32_t beginSortGroup(SOperatorInfo* pOperator) { ps->onlyRef = false; tsortAddSource(pInfo->pCurrSortHandle, ps); - int32_t code = tsortOpen(pInfo->pCurrSortHandle); - + code = tsortOpen(pInfo->pCurrSortHandle); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, terrno); } - return TSDB_CODE_SUCCESS; + return code; } int32_t finishSortGroup(SOperatorInfo* pOperator) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 6d88eaef99..33596c4802 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -114,7 +114,7 @@ struct SSortHandle { void* mergeLimitReachedParam; }; -static int32_t destroySortMemFile(SSortHandle* pHandle); +static void destroySortMemFile(SSortHandle* pHandle); static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, int32_t tupleOffset, int32_t rowLen, char** ppRow, bool* pFreeRow); void tsortSetSingleTableMerge(SSortHandle* pHandle) { @@ -226,16 +226,21 @@ void destroyTuple(void* t) { } } - /** * * @param type * @return */ -SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t pageSize, int32_t numOfPages, - SSDataBlock* pBlock, const char* idstr, uint64_t pqMaxRows, uint32_t pqMaxTupleLength, - uint32_t pqSortBufSize) { +int32_t tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t pageSize, int32_t numOfPages, + SSDataBlock* pBlock, const char* idstr, uint64_t pqMaxRows, uint32_t pqMaxTupleLength, + uint32_t pqSortBufSize, SSortHandle** pHandle) { + int32_t code = 0; + *pHandle = NULL; + SSortHandle* pSortHandle = taosMemoryCalloc(1, sizeof(SSortHandle)); + if (pSortHandle == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } pSortHandle->type = type; pSortHandle->pageSize = pageSize; @@ -248,15 +253,24 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page pSortHandle->pqSortBufSize = pqSortBufSize; pSortHandle->pqMaxRows = pqMaxRows; } - pSortHandle->forceUsePQSort = false; + pSortHandle->forceUsePQSort = false; if (pBlock != NULL) { pSortHandle->pDataBlock = createOneDataBlock(pBlock, false); + if (pSortHandle->pDataBlock == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } } pSortHandle->mergeLimit = -1; pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES); + if (pSortHandle->pOrderedSource == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pSortHandle->cmpParam.orderInfo = pSortInfo; pSortHandle->cmpParam.cmpGroupId = false; pSortHandle->cmpParam.sortType = type; @@ -278,9 +292,18 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page if (idstr != NULL) { pSortHandle->idStr = taosStrdup(idstr); + if (pSortHandle->idStr == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } } - return pSortHandle; + *pHandle = pSortHandle; + return code; + +_err: + tsortDestroySortHandle(pSortHandle); + return code; } static int32_t sortComparCleanup(SMsortComparParam* cmpParam) { @@ -336,6 +359,7 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { if (pSortHandle == NULL) { return; } + tsortClose(pSortHandle); if (pSortHandle->pMergeTree != NULL) { tMergeTreeDestroy(&pSortHandle->pMergeTree); @@ -354,6 +378,7 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { if (pSortHandle->pExtRowsMemFile != NULL) { destroySortMemFile(pSortHandle); } + taosArrayDestroy(pSortHandle->pSortInfo); taosArrayDestroy(pSortHandle->aExtRowsOrders); pSortHandle->aExtRowsOrders = NULL; @@ -361,8 +386,8 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { } int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) { - taosArrayPush(pSortHandle->pOrderedSource, &pSource); - return TSDB_CODE_SUCCESS; + void* p = taosArrayPush(pSortHandle->pOrderedSource, &pSource); + return (p != NULL)? TSDB_CODE_SUCCESS:TSDB_CODE_OUT_OF_MEMORY; } static int32_t doAddNewExternalMemSource(SDiskbasedBuf* pBuf, SArray* pAllSources, SSDataBlock* pBlock, @@ -375,7 +400,12 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf* pBuf, SArray* pAllSource pSource->src.pBlock = pBlock; pSource->pageIdList = pPageIdList; - taosArrayPush(pAllSources, &pSource); + + void* p = taosArrayPush(pAllSources, &pSource); + if (p == NULL) { + taosArrayDestroy(pPageIdList); + return TSDB_CODE_OUT_OF_MEMORY; + } (*sourceId) += 1; @@ -410,7 +440,13 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { SArray* pPageIdList = taosArrayInit(4, sizeof(int32_t)); while (start < pDataBlock->info.rows) { int32_t stop = 0; - blockDataSplitRows(pDataBlock, pDataBlock->info.hasVarCol, start, &stop, pHandle->pageSize); + + int32_t code = blockDataSplitRows(pDataBlock, pDataBlock->info.hasVarCol, start, &stop, pHandle->pageSize); + if (code) { + taosArrayDestroy(pPageIdList); + return code; + } + SSDataBlock* p = blockDataExtractBlock(pDataBlock, start, stop - start + 1); if (p == NULL) { taosArrayDestroy(pPageIdList); @@ -425,12 +461,20 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { return terrno; } - taosArrayPush(pPageIdList, &pageId); + void* px = taosArrayPush(pPageIdList, &pageId); + if (px == NULL) { + taosArrayDestroy(pPageIdList); + blockDataDestroy(p); + return TSDB_CODE_OUT_OF_MEMORY; + } int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t); ASSERT(size <= getBufPageSize(pHandle->pBuf)); - blockDataToBuf(pPage, p); + code = blockDataToBuf(pPage, p); + if (code) { + return code; + } setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); @@ -520,23 +564,32 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 return code; } -static void appendOneRowToDataBlock(SSDataBlock* pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { +static int32_t appendOneRowToDataBlock(SSDataBlock* pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { + int32_t code = 0; + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i); bool isNull = colDataIsNull(pSrcColInfo, pSource->info.rows, *rowIndex, NULL); if (isNull) { - colDataSetVal(pColInfo, pBlock->info.rows, NULL, true); + code = colDataSetVal(pColInfo, pBlock->info.rows, NULL, true); + if (code) { + return code; + } } else { if (!pSrcColInfo->pData) continue; char* pData = colDataGetData(pSrcColInfo, *rowIndex); - colDataSetVal(pColInfo, pBlock->info.rows, pData, false); + code = colDataSetVal(pColInfo, pBlock->info.rows, pData, false); + if (code) { + return code; + } } } pBlock->info.rows += 1; *rowIndex += 1; + return code; } static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeTreeInfo* pTree, SSortHandle* pHandle, @@ -609,7 +662,11 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT return TSDB_CODE_SUCCESS; } -static SSDataBlock* getSortedBlockDataInner(SSortHandle* pHandle, SMsortComparParam* cmpParam, int32_t capacity) { +static int32_t getSortedBlockDataInner(SSortHandle* pHandle, SMsortComparParam* cmpParam, int32_t capacity, + SSDataBlock** pRes) { + *pRes = NULL; + + int32_t code = 0; blockDataCleanup(pHandle->pDataBlock); while (1) { @@ -620,20 +677,24 @@ static SSDataBlock* getSortedBlockDataInner(SSortHandle* pHandle, SMsortComparPa int32_t index = tMergeTreeGetChosenIndex(pHandle->pMergeTree); SSortSource* pSource = (*cmpParam).pSources[index]; - appendOneRowToDataBlock(pHandle->pDataBlock, pSource->src.pBlock, &pSource->src.rowIndex); + code = appendOneRowToDataBlock(pHandle->pDataBlock, pSource->src.pBlock, &pSource->src.rowIndex); + if (code) { + return code; + } - int32_t code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources); + code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources); if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; + return terrno = code; } if (pHandle->pDataBlock->info.rows >= capacity) { - return pHandle->pDataBlock; + *pRes = pHandle->pDataBlock; + return code; } } - return (pHandle->pDataBlock->info.rows > 0) ? pHandle->pDataBlock : NULL; + *pRes = (pHandle->pDataBlock->info.rows > 0) ? pHandle->pDataBlock : NULL; + return code; } // TODO: improve this function performance @@ -819,7 +880,10 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { int32_t numOfRows = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize, blockDataGetSerialMetaSize(taosArrayGetSize(pHandle->pDataBlock->pDataBlock))); - blockDataEnsureCapacity(pHandle->pDataBlock, numOfRows); + int32_t code = blockDataEnsureCapacity(pHandle->pDataBlock, numOfRows); + if (code) { + return code; + } // the initial pass + sortPass + final mergePass pHandle->loops = sortPass + 2; @@ -845,7 +909,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { pHandle->cmpParam.numOfSources = end - i * numOfInputSources + 1; - int32_t code = sortComparInit(&pHandle->cmpParam, pHandle->pOrderedSource, i * numOfInputSources, end, pHandle); + code = sortComparInit(&pHandle->cmpParam, pHandle->pOrderedSource, i * numOfInputSources, end, pHandle); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pResList); return code; @@ -866,8 +930,10 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { code = terrno = TSDB_CODE_TSC_QUERY_CANCELLED; return code; } - SSDataBlock* pDataBlock = getSortedBlockDataInner(pHandle, &pHandle->cmpParam, numOfRows); - if (pDataBlock == NULL) { + + SSDataBlock* pDataBlock = NULL; + code = getSortedBlockDataInner(pHandle, &pHandle->cmpParam, numOfRows, &pDataBlock); + if (pDataBlock == NULL || code != 0) { break; } @@ -879,13 +945,21 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { return terrno; } - taosArrayPush(pPageIdList, &pageId); + void* px = taosArrayPush(pPageIdList, &pageId); + if (px == NULL) { + taosArrayDestroy(pResList); + taosArrayDestroy(pPageIdList); + return terrno; + } int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t); ASSERT(size <= getBufPageSize(pHandle->pBuf)); - blockDataToBuf(pPage, pDataBlock); + code= blockDataToBuf(pPage, pDataBlock); + if (code) { + return code; + } setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); @@ -897,7 +971,11 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { } } - sortComparCleanup(&pHandle->cmpParam); + code = sortComparCleanup(&pHandle->cmpParam); + if (code) { + return code; + } + tMergeTreeDestroy(&pHandle->pMergeTree); pHandle->numOfCompletedSources = 0; @@ -910,7 +988,11 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { } tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); - taosArrayAddAll(pHandle->pOrderedSource, pResList); + void* px = taosArrayAddAll(pHandle->pOrderedSource, pResList); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosArrayDestroy(pResList); numOfSorted = taosArrayGetSize(pHandle->pOrderedSource); @@ -960,15 +1042,27 @@ static int32_t createPageBuf(SSortHandle* pHandle) { return 0; } -void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHandle* pTupleHandle) { +int32_t tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHandle* pTupleHandle) { + int32_t code = 0; + if (pHandle->bSortByRowId) { - int32_t regionId = *(int32_t*)tsortGetValue(pTupleHandle, 1); - int32_t offset = *(int32_t*)tsortGetValue(pTupleHandle, 2); - int32_t length = *(int32_t*)tsortGetValue(pTupleHandle, 3); + int32_t *p1, *p2, *p3; + tsortGetValue(pTupleHandle, 1, (void**) &p1); + tsortGetValue(pTupleHandle, 2, (void**) &p2); + tsortGetValue(pTupleHandle, 3, (void**) &p3); + + int32_t regionId = *p1; + int32_t offset = *p2; + int32_t length = *p3; char* buf = NULL; bool bFreeRow = false; - getRowBufFromExtMemFile(pHandle, regionId, offset, length, &buf, &bFreeRow); + + code = getRowBufFromExtMemFile(pHandle, regionId, offset, length, &buf, &bFreeRow); + if (code) { + return code; + } + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); char* isNull = (char*)buf; char* pStart = (char*)buf + sizeof(int8_t) * numOfCols; @@ -976,7 +1070,11 @@ void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHa SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); if (!isNull[i]) { - colDataSetVal(pColInfo, pBlock->info.rows, pStart, false); + code = colDataSetVal(pColInfo, pBlock->info.rows, pStart, false); + if (code) { + return code; + } + if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) { int32_t dataLen = getJsonValueLen(pStart); pStart += dataLen; @@ -1011,9 +1109,13 @@ void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHa if (isNull) { colDataSetNULL(pColInfo, pBlock->info.rows); } else { - char* pData = tsortGetValue(pTupleHandle, i); + char* pData = NULL; + tsortGetValue(pTupleHandle, i, (void**)&pData); if (pData != NULL) { - colDataSetVal(pColInfo, pBlock->info.rows, pData, false); + code = colDataSetVal(pColInfo, pBlock->info.rows, pData, false); + if (code) { + return code; + } } } } @@ -1022,6 +1124,8 @@ void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHa pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag; pBlock->info.rows += 1; } + + return code; } static int32_t blockRowToBuf(SSDataBlock* pBlock, int32_t rowIdx, char* buf) { @@ -1078,7 +1182,10 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i if (pRegion->buf == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET); + + // todo + (void)taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET); + int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize); int32_t ret = taosReadFromCFile(pRegion->buf, readBytes, 1, pMemFile->pTdFile); if (ret != 1) { @@ -1098,7 +1205,9 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i } int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset); memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset, szThisBlock); - taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET); + + // todo + (void) taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET); int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize - (pRegion->bufRegOffset + pRegion->bufLen)); int32_t ret = taosReadFromCFile(pRegion->buf, readBytes, 1, pMemFile->pTdFile); if (ret != 1) { @@ -1131,7 +1240,11 @@ static int32_t createSortMemFile(SSortHandle* pHandle) { } } if (code == TSDB_CODE_SUCCESS) { - taosSetAutoDelFile(pMemFile->memFilePath); + code = taosSetAutoDelFile(pMemFile->memFilePath); + if (code) { + qError("failed to set the auto-delete file attribute"); + return code; + } pMemFile->currRegionId = -1; pMemFile->currRegionOffset = -1; @@ -1145,6 +1258,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) { code = TSDB_CODE_OUT_OF_MEMORY; } } + if (code == TSDB_CODE_SUCCESS) { pMemFile->cacheSize = pHandle->extRowsMemSize; pMemFile->aFileRegions = taosArrayInit(64, sizeof(SSortMemFileRegion)); @@ -1152,6 +1266,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) { code = TSDB_CODE_OUT_OF_MEMORY; } } + if (code == TSDB_CODE_SUCCESS) { pHandle->pExtRowsMemFile = pMemFile; } else { @@ -1159,7 +1274,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) { if (pMemFile->aFileRegions) taosMemoryFreeClear(pMemFile->aFileRegions); if (pMemFile->writeBuf) taosMemoryFreeClear(pMemFile->writeBuf); if (pMemFile->pTdFile) { - taosCloseCFile(pMemFile->pTdFile); + (void) taosCloseCFile(pMemFile->pTdFile); pMemFile->pTdFile = NULL; } taosMemoryFreeClear(pMemFile); @@ -1168,8 +1283,10 @@ static int32_t createSortMemFile(SSortHandle* pHandle) { return code; } -static int32_t destroySortMemFile(SSortHandle* pHandle) { - if (pHandle->pExtRowsMemFile == NULL) return TSDB_CODE_SUCCESS; +static void destroySortMemFile(SSortHandle* pHandle) { + if (pHandle->pExtRowsMemFile == NULL) { + return; + } SSortMemFile* pMemFile = pHandle->pExtRowsMemFile; for (int32_t i = 0; i < taosArrayGetSize(pMemFile->aFileRegions); ++i) { @@ -1182,21 +1299,26 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) { taosMemoryFree(pMemFile->writeBuf); pMemFile->writeBuf = NULL; - taosCloseCFile(pMemFile->pTdFile); + (void) taosCloseCFile(pMemFile->pTdFile); pMemFile->pTdFile = NULL; - taosRemoveFile(pMemFile->memFilePath); + (void) taosRemoveFile(pMemFile->memFilePath); taosMemoryFree(pMemFile); pHandle->pExtRowsMemFile = NULL; - return TSDB_CODE_SUCCESS; } static int32_t tsortOpenRegion(SSortHandle* pHandle) { SSortMemFile* pMemFile = pHandle->pExtRowsMemFile; + int32_t code = 0; + if (pMemFile->currRegionId == -1) { SSortMemFileRegion region = {0}; region.fileOffset = 0; region.bufRegOffset = 0; - taosArrayPush(pMemFile->aFileRegions, ®ion); + void* px = taosArrayPush(pMemFile->aFileRegions, ®ion); + if (px == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + pMemFile->currRegionId = 0; pMemFile->currRegionOffset = 0; pMemFile->writeFileOffset = 0; @@ -1205,12 +1327,15 @@ static int32_t tsortOpenRegion(SSortHandle* pHandle) { SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, pMemFile->currRegionId); regionNew.fileOffset = pRegion->fileOffset + pRegion->regionSize; regionNew.bufRegOffset = 0; - taosArrayPush(pMemFile->aFileRegions, ®ionNew); + void* px = taosArrayPush(pMemFile->aFileRegions, ®ionNew); + if (px == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } ++pMemFile->currRegionId; pMemFile->currRegionOffset = 0; pMemFile->writeFileOffset = regionNew.fileOffset; } - return TSDB_CODE_SUCCESS; + return code; } static int32_t tsortCloseRegion(SSortHandle* pHandle) { @@ -1233,14 +1358,22 @@ static int32_t tsortFinalizeRegions(SSortHandle* pHandle) { SSortMemFile* pMemFile = pHandle->pExtRowsMemFile; size_t numRegions = taosArrayGetSize(pMemFile->aFileRegions); ASSERT(numRegions == (pMemFile->currRegionId + 1)); - if (numRegions == 0) return TSDB_CODE_SUCCESS; + if (numRegions == 0) { + return TSDB_CODE_SUCCESS; + } + int32_t blockReadBytes = (pMemFile->cacheSize / numRegions + 4095) & ~4095; pMemFile->blockSize = blockReadBytes; for (int32_t i = 0; i < numRegions; ++i) { SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, i); + if (pRegion == NULL) { + return TSDB_CODE_INVALID_PARA; + } + pRegion->bufRegOffset = 0; } + taosMemoryFree(pMemFile->writeBuf); pMemFile->writeBuf = NULL; return TSDB_CODE_SUCCESS; @@ -1274,18 +1407,26 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p return TSDB_CODE_SUCCESS; } -static void appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSource, int32_t* rowIndex) { +static int32_t appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSource, int32_t* rowIndex) { int32_t pageId = -1; int32_t offset = -1; int32_t length = -1; - saveBlockRowToExtRowsMemFile(pHandle, pSource, *rowIndex, &pageId, &offset, &length); + int32_t code = 0; + + code = saveBlockRowToExtRowsMemFile(pHandle, pSource, *rowIndex, &pageId, &offset, &length); + if (code) { + return code; + } SSDataBlock* pBlock = pHandle->pDataBlock; SBlockOrderInfo* extRowsTsOrder = taosArrayGet(pHandle->aExtRowsOrders, 0); SColumnInfoData* pSrcTsCol = taosArrayGet(pSource->pDataBlock, extRowsTsOrder->slotId); SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, 0); char* pData = colDataGetData(pSrcTsCol, *rowIndex); - colDataSetVal(pTsCol, pBlock->info.rows, pData, false); + code = colDataSetVal(pTsCol, pBlock->info.rows, pData, false); + if (code) { + return code; + } SColumnInfoData* pRegionIdCol = taosArrayGet(pBlock->pDataBlock, 1); colDataSetInt32(pRegionIdCol, pBlock->info.rows, &pageId); @@ -1304,32 +1445,53 @@ static void appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSource colDataSetNULL(pPkCol, pBlock->info.rows); } else { char* pPkData = colDataGetData(pSrcPkCol, *rowIndex); - colDataSetVal(pPkCol, pBlock->info.rows, pPkData, false); + code = colDataSetVal(pPkCol, pBlock->info.rows, pPkData, false); + if (code) { + return code; + } } } pBlock->info.rows += 1; *rowIndex += 1; + return code; } -static void initRowIdSort(SSortHandle* pHandle) { +static int32_t initRowIdSort(SSortHandle* pHandle) { SBlockOrderInfo* pkOrder = (pHandle->bSortPk) ? taosArrayGet(pHandle->aExtRowsOrders, 1) : NULL; SColumnInfoData* extPkCol = (pHandle->bSortPk) ? taosArrayGet(pHandle->pDataBlock->pDataBlock, pkOrder->slotId) : NULL; SColumnInfoData pkCol = {0}; SSDataBlock* pSortInput = createDataBlock(); + pHandle->pDataBlock = pSortInput; + SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1); - blockDataAppendColInfo(pSortInput, &tsCol); + int32_t code = blockDataAppendColInfo(pSortInput, &tsCol); + if (code) { + return code; + } + SColumnInfoData regionIdCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2); - blockDataAppendColInfo(pSortInput, ®ionIdCol); + code = blockDataAppendColInfo(pSortInput, ®ionIdCol); + if (code) { + return code; + } + SColumnInfoData offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3); - blockDataAppendColInfo(pSortInput, &offsetCol); + code = blockDataAppendColInfo(pSortInput, &offsetCol); + if (code) { + return code; + } + SColumnInfoData lengthCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 4); - blockDataAppendColInfo(pSortInput, &lengthCol); + code = blockDataAppendColInfo(pSortInput, &lengthCol); + if (code) { + return code; + } if (pHandle->bSortPk) { pkCol = createColumnInfoData(extPkCol->info.type, extPkCol->info.bytes, 5); - blockDataAppendColInfo(pSortInput, &pkCol); + code = blockDataAppendColInfo(pSortInput, &pkCol); } blockDataDestroy(pHandle->pDataBlock); @@ -1341,6 +1503,9 @@ static void initRowIdSort(SSortHandle* pHandle) { pHandle->numOfPages = 256; SArray* pOrderInfoList = taosArrayInit(1, sizeof(SBlockOrderInfo)); + if (pOrderInfoList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } int32_t tsOrder = ((SBlockOrderInfo*)taosArrayGet(pHandle->pSortInfo, 0))->order; @@ -1349,7 +1514,10 @@ static void initRowIdSort(SSortHandle* pHandle) { biTs.slotId = 0; biTs.nullFirst = (biTs.order == TSDB_ORDER_ASC); biTs.compFn = getKeyComparFunc(TSDB_DATA_TYPE_TIMESTAMP, biTs.order); - taosArrayPush(pOrderInfoList, &biTs); + void* p = taosArrayPush(pOrderInfoList, &biTs); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } if (pHandle->bSortPk) { SBlockOrderInfo biPk = {0}; @@ -1357,24 +1525,34 @@ static void initRowIdSort(SSortHandle* pHandle) { biPk.slotId = 4; biPk.nullFirst = (biPk.order == TSDB_ORDER_ASC); biPk.compFn = getKeyComparFunc(pkCol.info.type, biPk.order); - taosArrayPush(pOrderInfoList, &biPk); + + void* px = taosArrayPush(pOrderInfoList, &biPk); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } taosArrayDestroy(pHandle->pSortInfo); pHandle->pSortInfo = pOrderInfoList; + return TSDB_CODE_SUCCESS; } int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsMemSize) { pHandle->extRowBytes = blockDataGetRowSize(pHandle->pDataBlock) + taosArrayGetSize(pHandle->pDataBlock->pDataBlock) + sizeof(int32_t); pHandle->extRowsMemSize = extRowsMemSize; pHandle->aExtRowsOrders = taosArrayDup(pHandle->pSortInfo, NULL); - initRowIdSort(pHandle); - if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_DISKSPACE; - qError("create sort mem file failed since %s, tempDir:%s", terrstr(), tsTempDir); - return terrno; + + int32_t code = initRowIdSort(pHandle); + if (code) { + return code; } - int32_t code = createSortMemFile(pHandle); + + if (!osTempSpaceAvailable()) { + qError("create sort mem file failed since %s, tempDir:%s", terrstr(), tsTempDir); + return TSDB_CODE_NO_DISKSPACE; + } + + code = createSortMemFile(pHandle); pHandle->bSortByRowId = true; return code; } @@ -1439,17 +1617,21 @@ static int32_t appendDataBlockToPageBuf(SSortHandle* pHandle, SSDataBlock* blk, if (pPage == NULL) { return terrno; } - taosArrayPush(aPgId, &pageId); + + void* px = taosArrayPush(aPgId, &pageId); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } int32_t size = blockDataGetSize(blk) + sizeof(int32_t) + taosArrayGetSize(blk->pDataBlock) * sizeof(int32_t); ASSERT(size <= getBufPageSize(pHandle->pBuf)); - blockDataToBuf(pPage, blk); + int32_t code = blockDataToBuf(pPage, blk); setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); - return 0; + return code; } static int32_t getPageBufIncForRow(SSDataBlock* pSrcBlock, int32_t srcRowIndex, int32_t dstRowIndex) { @@ -1590,9 +1772,12 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* int32_t pageHeaderSize = sizeof(int32_t) + sizeof(int32_t) * blockDataGetNumOfCols(pHandle->pDataBlock); int32_t rowCap = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize, pageHeaderSize); - blockDataEnsureCapacity(pHandle->pDataBlock, rowCap); - blockDataCleanup(pHandle->pDataBlock); + code = blockDataEnsureCapacity(pHandle->pDataBlock, rowCap); + if (code) { + return code; + } + blockDataCleanup(pHandle->pDataBlock); SBlkMergeSupport sup = {0}; SBlockOrderInfo* pOrigBlockTsOrder = @@ -1606,7 +1791,10 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* (!pHandle->bSortByRowId) ? taosArrayGet(pHandle->pSortInfo, 1) : taosArrayGet(pHandle->aExtRowsOrders, 1); } - initMergeSup(&sup, aBlk, pOrigBlockTsOrder->order, pOrigBlockTsOrder->slotId, pOrigBlockPkOrder); + code = initMergeSup(&sup, aBlk, pOrigBlockTsOrder->order, pOrigBlockTsOrder->slotId, pOrigBlockPkOrder); + if (code) { + return code; + } int32_t totalRows = getTotalRows(aBlk); @@ -1661,11 +1849,19 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* } } - blockDataEnsureCapacity(pHandle->pDataBlock, pHandle->pDataBlock->info.rows + 1); + code = blockDataEnsureCapacity(pHandle->pDataBlock, pHandle->pDataBlock->info.rows + 1); + if (code) { + return code; + } + if (pHandle->bSortByRowId) { - appendToRowIndexDataBlock(pHandle, minBlk, &minRow); + code = appendToRowIndexDataBlock(pHandle, minBlk, &minRow); } else { - appendOneRowToDataBlock(pHandle->pDataBlock, minBlk, &minRow); + code = appendOneRowToDataBlock(pHandle->pDataBlock, minBlk, &minRow); + } + + if (code) { + return code; } blkPgSz += bufInc; @@ -1705,22 +1901,30 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* } SSDataBlock* pMemSrcBlk = createOneDataBlock(pHandle->pDataBlock, false); - doAddNewExternalMemSource(pHandle->pBuf, aExtSrc, pMemSrcBlk, &pHandle->sourceId, aPgId); + code = doAddNewExternalMemSource(pHandle->pBuf, aExtSrc, pMemSrcBlk, &pHandle->sourceId, aPgId); cleanupMergeSup(&sup); tMergeTreeDestroy(&pTree); - return 0; + return code; } -static SSDataBlock* getRowsBlockWithinMergeLimit(const SSortHandle* pHandle, SSHashObj* mTableNumRows, SSDataBlock* pOrigBlk, bool* pExtractedBlock, bool *pSkipBlock) { +static int32_t getRowsBlockWithinMergeLimit(const SSortHandle* pHandle, SSHashObj* mTableNumRows, SSDataBlock* pOrigBlk, + bool* pExtractedBlock, bool* pSkipBlock, SSDataBlock** pRes) { int64_t nRows = 0; int64_t prevRows = 0; + int32_t code = 0; + + *pRes = NULL; + void* pNum = tSimpleHashGet(mTableNumRows, &pOrigBlk->info.id.uid, sizeof(pOrigBlk->info.id.uid)); if (pNum == NULL) { prevRows = 0; nRows = pOrigBlk->info.rows; - tSimpleHashPut(mTableNumRows, &pOrigBlk->info.id.uid, sizeof(pOrigBlk->info.id.uid), &nRows, sizeof(nRows)); + code = tSimpleHashPut(mTableNumRows, &pOrigBlk->info.id.uid, sizeof(pOrigBlk->info.id.uid), &nRows, sizeof(nRows)); + if (code) { + return code; + } } else { prevRows = *(int64_t*)pNum; *(int64_t*)pNum = *(int64_t*)pNum + pOrigBlk->info.rows; @@ -1737,7 +1941,7 @@ static SSDataBlock* getRowsBlockWithinMergeLimit(const SSortHandle* pHandle, SSH if (keepRows == 0) { *pSkipBlock = true; - return pOrigBlk; + *pRes = pOrigBlk; } *pSkipBlock = false; @@ -1749,7 +1953,9 @@ static SSDataBlock* getRowsBlockWithinMergeLimit(const SSortHandle* pHandle, SSH *pExtractedBlock = false; pBlock = pOrigBlk; } - return pBlock; + + *pRes = pBlock; + return code; } static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { @@ -1784,8 +1990,10 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { bool bExtractedBlock = false; bool bSkipBlock = false; if (pBlk != NULL && pHandle->mergeLimit > 0) { - pBlk = getRowsBlockWithinMergeLimit(pHandle, mTableNumRows, pBlk, &bExtractedBlock, &bSkipBlock); - if (bSkipBlock) { + pBlk = NULL; + + code = getRowsBlockWithinMergeLimit(pHandle, mTableNumRows, pBlk, &bExtractedBlock, &bSkipBlock, &pBlk); + if (bSkipBlock || code != 0) { continue; } } @@ -1807,14 +2015,26 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { void* ppBlk = tSimpleHashGet(mUidBlk, &pBlk->info.id.uid, sizeof(pBlk->info.id.uid)); if (ppBlk != NULL) { SSDataBlock* tBlk = *(SSDataBlock**)(ppBlk); - blockDataMerge(tBlk, pBlk); + + code = blockDataMerge(tBlk, pBlk); + if (code) { + return code; + } + if (bExtractedBlock) { blockDataDestroy(pBlk); } } else { SSDataBlock* tBlk = (bExtractedBlock) ? pBlk : createOneDataBlock(pBlk, true); - tSimpleHashPut(mUidBlk, &pBlk->info.id.uid, sizeof(pBlk->info.id.uid), &tBlk, POINTER_BYTES); - taosArrayPush(aBlkSort, &tBlk); + code = tSimpleHashPut(mUidBlk, &pBlk->info.id.uid, sizeof(pBlk->info.id.uid), &tBlk, POINTER_BYTES); + if (code) { + return code; + } + + void* px = taosArrayPush(aBlkSort, &tBlk); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } } @@ -1823,10 +2043,13 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { int64_t p = taosGetTimestampUs(); if (pHandle->bSortByRowId) { - tsortOpenRegion(pHandle); + code = tsortOpenRegion(pHandle); + if (code) { + return code; + } } - code = sortBlocksToExtSource(pHandle, aBlkSort, aExtSrc); + code = sortBlocksToExtSource(pHandle, aBlkSort, aExtSrc); if (code != TSDB_CODE_SUCCESS) { for (int32_t i = 0; i < taosArrayGetSize(aBlkSort); ++i) { blockDataDestroy(taosArrayGetP(aBlkSort, i)); @@ -1834,15 +2057,18 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { taosArrayClear(aBlkSort); break; } + if (pHandle->bSortByRowId) { - tsortCloseRegion(pHandle); + code = tsortCloseRegion(pHandle); // ignore this error code } + int64_t el = taosGetTimestampUs() - p; pHandle->sortElapsed += el; for (int32_t i = 0; i < taosArrayGetSize(aBlkSort); ++i) { blockDataDestroy(taosArrayGetP(aBlkSort, i)); } + taosArrayClear(aBlkSort); szSort = 0; qDebug("source %zu created", taosArrayGetSize(aExtSrc)); @@ -1869,13 +2095,18 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { taosArrayDestroy(aBlkSort); tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); if (!tsortIsClosed(pHandle)) { - taosArrayAddAll(pHandle->pOrderedSource, aExtSrc); + void* px = taosArrayAddAll(pHandle->pOrderedSource, aExtSrc); + if (px == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } + taosArrayDestroy(aExtSrc); tSimpleHashCleanup(mTableNumRows); if (pHandle->bSortByRowId) { - tsortFinalizeRegions(pHandle); + code = tsortFinalizeRegions(pHandle); } + pHandle->type = SORT_SINGLESOURCE_SORT; return code; } @@ -2023,9 +2254,8 @@ static bool tsortOpenForBufMergeSort(SSortHandle* pHandle) { return tMergeTreeCreate(&pHandle->pMergeTree, pHandle->cmpParam.numOfSources, &pHandle->cmpParam, pHandle->comparFn); } -int32_t tsortClose(SSortHandle* pHandle) { - atomic_val_compare_exchange_8(&pHandle->closed, 0, 1); - return TSDB_CODE_SUCCESS; +void tsortClose(SSortHandle* pHandle) { + (void) atomic_val_compare_exchange_8(&pHandle->closed, 0, 1); } bool tsortIsClosed(SSortHandle* pHandle) { @@ -2048,22 +2278,28 @@ int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetc return TSDB_CODE_SUCCESS; } -int32_t tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp) { - pHandle->comparFn = fp; - return TSDB_CODE_SUCCESS; -} - -int32_t tsortSetCompareGroupId(SSortHandle* pHandle, bool compareGroupId) { - pHandle->cmpParam.cmpGroupId = compareGroupId; - return TSDB_CODE_SUCCESS; -} - -static STupleHandle* tsortBufMergeSortNextTuple(SSortHandle* pHandle) { - if (tsortIsClosed(pHandle)) { - return NULL; +void tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp) { + if (pHandle) { + pHandle->comparFn = fp; } +} + +void tsortSetCompareGroupId(SSortHandle* pHandle, bool compareGroupId) { + if (pHandle) { + pHandle->cmpParam.cmpGroupId = compareGroupId; + } +} + +static int32_t tsortBufMergeSortNextTuple(SSortHandle* pHandle, STupleHandle** pTupleHandle) { + *pTupleHandle = NULL; + int32_t code = 0; + + if (tsortIsClosed(pHandle)) { + return code; + } + if (pHandle->cmpParam.numOfSources == pHandle->numOfCompletedSources) { - return NULL; + return code; } // All the data are hold in the buffer, no external sort is invoked. @@ -2071,26 +2307,26 @@ static STupleHandle* tsortBufMergeSortNextTuple(SSortHandle* pHandle) { pHandle->tupleHandle.rowIndex += 1; if (pHandle->tupleHandle.rowIndex == pHandle->pDataBlock->info.rows) { pHandle->numOfCompletedSources = 1; - return NULL; + return code; } - return &pHandle->tupleHandle; + *pTupleHandle = &pHandle->tupleHandle; + return code; } int32_t index = tMergeTreeGetChosenIndex(pHandle->pMergeTree); SSortSource* pSource = pHandle->cmpParam.pSources[index]; if (pHandle->needAdjust) { - int32_t code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources); + code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources); if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; + return code; } } // all sources are completed. if (pHandle->cmpParam.numOfSources == pHandle->numOfCompletedSources) { - return NULL; + return code; } // Get the adjusted value after the loser tree is updated. @@ -2105,7 +2341,8 @@ static STupleHandle* tsortBufMergeSortNextTuple(SSortHandle* pHandle) { pHandle->needAdjust = true; pSource->src.rowIndex += 1; - return &pHandle->tupleHandle; + *pTupleHandle = &pHandle->tupleHandle; + return code; } static bool tsortIsForceUsePQSort(SSortHandle* pHandle) { @@ -2216,12 +2453,20 @@ static int32_t tsortOpenForPQSort(SSortHandle* pHandle) { return TSDB_CODE_SUCCESS; } -static STupleHandle* tsortPQSortNextTuple(SSortHandle* pHandle) { +static int32_t tsortPQSortNextTuple(SSortHandle* pHandle, STupleHandle **pTupleHandle) { + int32_t code = 0; + + *pTupleHandle = NULL; if (pHandle->pDataBlock == NULL) { // when no input stream datablock - return NULL; + return code; } + blockDataCleanup(pHandle->pDataBlock); - blockDataEnsureCapacity(pHandle->pDataBlock, 1); + code = blockDataEnsureCapacity(pHandle->pDataBlock, 1); + if (code) { + return code; + } + // abandon the top tuple if queue size bigger than max size if (taosBQSize(pHandle->pBoundedQueue) == taosBQMaxSize(pHandle->pBoundedQueue) + 1) { taosBQPop(pHandle->pBoundedQueue); @@ -2241,36 +2486,55 @@ static STupleHandle* tsortPQSortNextTuple(SSortHandle* pHandle) { if (!pData) { colDataSetNULL(bdGetColumnInfoData(pHandle->pDataBlock, i), 0); } else { - colDataSetVal(bdGetColumnInfoData(pHandle->pDataBlock, i), 0, pData, false); + code = colDataSetVal(bdGetColumnInfoData(pHandle->pDataBlock, i), 0, pData, false); + if (code) { + return code; + } } } pHandle->pDataBlock->info.rows++; pHandle->tmpRowIdx++; taosBQPop(pHandle->pBoundedQueue); } - if (pHandle->pDataBlock->info.rows == 0) return NULL; + + if (pHandle->pDataBlock->info.rows == 0) { + return code; + } + pHandle->tupleHandle.pBlock = pHandle->pDataBlock; - return &pHandle->tupleHandle; + *pTupleHandle = &pHandle->tupleHandle; + return code; } -static STupleHandle* tsortSingleTableMergeNextTuple(SSortHandle* pHandle) { - if (1 == pHandle->numOfCompletedSources) return NULL; +static int32_t tsortSingleTableMergeNextTuple(SSortHandle* pHandle, STupleHandle** pTupleHandle) { + *pTupleHandle = NULL; + int32_t code = 0; + + if (1 == pHandle->numOfCompletedSources) { + return code; + } + if (pHandle->tupleHandle.pBlock && pHandle->tupleHandle.rowIndex + 1 < pHandle->tupleHandle.pBlock->info.rows) { pHandle->tupleHandle.rowIndex++; } else { - if (pHandle->tupleHandle.rowIndex == -1) return NULL; + if (pHandle->tupleHandle.rowIndex == -1) { + return code; + } + SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0); SSortSource* source = *pSource; SSDataBlock* pBlock = pHandle->fetchfp(source->param); if (!pBlock || pBlock->info.rows == 0) { setCurrentSourceDone(source, pHandle); pHandle->tupleHandle.pBlock = NULL; - return NULL; + return code; } pHandle->tupleHandle.pBlock = pBlock; pHandle->tupleHandle.rowIndex = 0; } - return &pHandle->tupleHandle; + + *pTupleHandle = &pHandle->tupleHandle; + return code; } int32_t tsortOpen(SSortHandle* pHandle) { @@ -2289,13 +2553,18 @@ int32_t tsortOpen(SSortHandle* pHandle) { return tsortOpenForBufMergeSort(pHandle); } -STupleHandle* tsortNextTuple(SSortHandle* pHandle) { - if (pHandle->singleTableMerge) - return tsortSingleTableMergeNextTuple(pHandle); - else if (pHandle->pBoundedQueue) - return tsortPQSortNextTuple(pHandle); - else - return tsortBufMergeSortNextTuple(pHandle); +int32_t tsortNextTuple(SSortHandle* pHandle, STupleHandle** pTupleHandle) { + int32_t code = 0; + + if (pHandle->singleTableMerge) { + code = tsortSingleTableMergeNextTuple(pHandle, pTupleHandle); + } else if (pHandle->pBoundedQueue) { + code = tsortPQSortNextTuple(pHandle, pTupleHandle); + } else { + code = tsortBufMergeSortNextTuple(pHandle, pTupleHandle); + } + + return code; } bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) { @@ -2303,12 +2572,11 @@ bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) { return colDataIsNull_s(pColInfoSrc, pVHandle->rowIndex); } -void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { +void tsortGetValue(STupleHandle* pVHandle, int32_t colIndex, void** pVal) { + *pVal = NULL; SColumnInfoData* pColInfo = TARRAY_GET_ELEM(pVHandle->pBlock->pDataBlock, colIndex); - if (pColInfo->pData == NULL) { - return NULL; - } else { - return colDataGetData(pColInfo, pVHandle->rowIndex); + if (pColInfo->pData != NULL) { + *pVal = colDataGetData(pColInfo, pVHandle->rowIndex); } } diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 55209fc427..66c1941abb 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -36,7 +36,7 @@ static void setCheckDownstreamReqInfo(SStreamTaskCheckReq* pReq, int64_t reqId, static void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, int32_t* numOfFault, int32_t* numOfNotRsp, SArray* pTimeoutList, SArray* pNotReadyList, const char* id); static int32_t addDownstreamFailedStatusResultAsync(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId); -static SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId); +static void findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId, SDownstreamStatusInfo** pStatusInfo); int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int64_t* oldStage) { @@ -402,26 +402,29 @@ void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutput pInfo->stopCheckProcess = 0; } -SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId) { +void findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId, SDownstreamStatusInfo** pStatusInfo) { + if (pStatusInfo == NULL) { + return; + } + + *pStatusInfo = NULL; for (int32_t j = 0; j < taosArrayGetSize(pInfo->pList); ++j) { SDownstreamStatusInfo* p = taosArrayGet(pInfo->pList, j); if (p->taskId == taskId) { - return p; + *pStatusInfo = p; } } - - return NULL; } int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs, int64_t reqId, int32_t* pNotReady, const char* id) { streamMutexLock(&pInfo->checkInfoLock); - SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); + SDownstreamStatusInfo* p = NULL; + findCheckRspStatus(pInfo, taskId, &p); if (p != NULL) { if (reqId != p->reqId) { - stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 - " expired check-rsp recv from downstream task:0x%x, discarded", + stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 " expired check-rsp recv from downstream task:0x%x, discarded", id, reqId, p->reqId, taskId); streamMutexUnlock(&pInfo->checkInfoLock); return TSDB_CODE_FAILED; @@ -495,7 +498,8 @@ void streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, SDownstreamStatusInfo info = {.taskId = taskId, .status = -1, .vgId = vgId, .reqId = reqId, .rspTs = 0}; streamMutexLock(&pInfo->checkInfoLock); - SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); + SDownstreamStatusInfo* p = NULL; + findCheckRspStatus(pInfo, taskId, &p); if (p != NULL) { stDebug("s-task:%s check info to task:0x%x already sent", id, taskId); streamMutexUnlock(&pInfo->checkInfoLock); @@ -598,7 +602,8 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { for (int32_t i = 0; i < numOfTimeout; ++i) { int32_t taskId = *(int32_t*)taosArrayGet(pTimeoutList, i); - SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); + SDownstreamStatusInfo* p = NULL; + findCheckRspStatus(pInfo, taskId, &p); if (p != NULL) { ASSERT(p->status == -1 && p->rspTs == 0); doSendCheckMsg(pTask, p); @@ -613,7 +618,8 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { for (int32_t i = 0; i < numOfTimeout; ++i) { int32_t taskId = *(int32_t*)taosArrayGet(pTimeoutList, i); - SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); + SDownstreamStatusInfo* p = NULL; + findCheckRspStatus(pInfo, taskId, &p); if (p != NULL) { addIntoNodeUpdateList(pTask, p->vgId); stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 100sec, add into nodeUpate list", @@ -640,7 +646,8 @@ void handleNotReadyDownstreamTask(SStreamTask* pTask, SArray* pNotReadyList) { for (int32_t i = 0; i < numOfNotReady; ++i) { int32_t taskId = *(int32_t*)taosArrayGet(pNotReadyList, i); - SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); + SDownstreamStatusInfo* p = NULL; + findCheckRspStatus(pInfo, taskId, &p); if (p != NULL) { p->rspTs = 0; p->status = -1; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 1283f8e20b..43a22e0637 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -726,7 +726,7 @@ int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_ SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask == NULL || streamTaskShouldStop(*ppTask)) { *pTask = NULL; - return TSDB_CODE_FAILED; + return TSDB_CODE_STREAM_TASK_NOT_EXIST; } int32_t ref = atomic_add_fetch_32(&(*ppTask)->refCnt, 1); diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 74f370d199..f9fcf36668 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -104,10 +104,10 @@ void streamTaskResumeHelper(void* param, void* tmrId) { } int32_t code = streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK); + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); if (code) { - stError("s-task:%s sched task failed, code:%s", pTask->id.idStr, strerror(code)); + stError("s-task:%s sched task failed, code:%s, ref:%d", pId->idStr, strerror(code), ref); } else { - int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("trigger to resume s-task:%s after being idled for %dms, ref:%d", pId->idStr, pTask->status.schedIdleTime, ref); From 37fd3d4fa8a545905e43ce998cca25a61a240b0f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 23 Jul 2024 16:29:10 +0800 Subject: [PATCH 13/52] fix:[TD-31017]process return value in client for tmq --- source/client/src/clientTmq.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 65e0d5fe91..7fb39a424b 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -697,7 +697,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us for (int32_t i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (pTopic != NULL) { + if (pTopic == NULL) { code = TSDB_CODE_TMQ_INVALID_TOPIC; taosRUnLockLatch(&tmq->lock); goto end; @@ -707,7 +707,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us numOfVgroups); for (int32_t j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg != NULL) { + if (pVg == NULL) { code = TSDB_CODE_INVALID_PARA; taosRUnLockLatch(&tmq->lock); goto end; @@ -2446,6 +2446,8 @@ static int32_t innerClose(tmq_t* tmq){ } int32_t tmq_unsubscribe(tmq_t* tmq) { if (tmq == NULL) return TSDB_CODE_INVALID_PARA; + + tscInfo("consumer:0x%" PRIx64 " start to unsubscribe consumer, status:%d", tmq->consumerId, tmq->status); int32_t code = 0; if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) { code = innerClose(tmq); @@ -2466,10 +2468,12 @@ int32_t tmq_consumer_close(tmq_t* tmq) { code = innerClose(tmq); if(code == 0){ atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); - (void)taosRemoveRef(tmqMgmt.rsetId, tmq->refId); } } + if (code == 0){ + (void)taosRemoveRef(tmqMgmt.rsetId, tmq->refId); + } return code; } From b873356d8033c04d249b038139ae2db0bac4fb3c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 23 Jul 2024 16:35:46 +0800 Subject: [PATCH 14/52] fix:[TD-31017]process return value in client for tmq --- source/client/src/clientTmq.c | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 7fb39a424b..6a5d34e707 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -29,22 +29,6 @@ #define DEFAULT_HEARTBEAT_INTERVAL 3000 #define DEFAULT_ASKEP_INTERVAL 1000 -#define CLIENT_TMQ_NULL_CHECK(c) \ - do { \ - if (c == NULL) { \ - code = TSDB_CODE_OUT_OF_MEMORY; \ - goto END; \ - } \ - } while (0) - -#define CLIENT_TMQ_RETURN_CHECK(c) \ - do { \ - code = c; \ - if (code != 0) { \ - goto END; \ - } \ - } while (0) - struct SMqMgmt { tmr_h timer; int32_t rsetId; From cd0722fe1f52641d1312d01369ae7e87d4a8a980 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 Jul 2024 16:38:05 +0800 Subject: [PATCH 15/52] fix(stream): check return value. --- source/libs/executor/inc/tsort.h | 8 ++--- source/libs/executor/src/tsort.c | 53 ++++++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index c6f3986776..cf1464ccd7 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -165,13 +165,13 @@ void tsortGetValue(STupleHandle* pVHandle, int32_t colId, void** pVal); * @return */ uint64_t tsortGetGroupId(STupleHandle* pVHandle); -void* tsortGetBlockInfo(STupleHandle* pVHandle); +void tsortGetBlockInfo(STupleHandle* pVHandle, SDataBlockInfo* pInfo); /** * * @param pSortHandle * @return */ -SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle); +int32_t tsortGetSortedDataBlock(const SSortHandle* pSortHandle, SSDataBlock** pBlock); /** * return the sort execution information. @@ -215,8 +215,8 @@ int32_t tsortCompAndBuildKeys(const SArray* pSortCols, char* keyBuf, int32_t* ke */ void tsortSetMergeLimitReachedFp(SSortHandle* pHandle, void (*mergeLimitReached)(uint64_t tableUid, void* param), void* param); -int tsortComparBlockCell(SSDataBlock* pLeftBlock, SSDataBlock* pRightBlock, - int32_t leftRowIndex, int32_t rightRowIndex, void* pOrder); +int tsortComparBlockCell(SSDataBlock* pLeftBlock, SSDataBlock* pRightBlock, int32_t leftRowIndex, int32_t rightRowIndex, + void* pOrder); #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 33596c4802..b4e5a9a01a 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -167,8 +167,17 @@ static void* tupleGetField(char* t, uint32_t colIdx, uint32_t colNum) { return t + *tupleOffset(t, colIdx); } -SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle) { - return createOneDataBlock(pSortHandle->pDataBlock, false); +int32_t tsortGetSortedDataBlock(const SSortHandle* pSortHandle, SSDataBlock** pBlock) { + if (pBlock == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + *pBlock = createOneDataBlock(pSortHandle->pDataBlock, false); + if (*pBlock == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } else { + return TSDB_CODE_SUCCESS; + } } #define AllocatedTupleType 0 @@ -1099,7 +1108,11 @@ int32_t tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupl } pBlock->info.dataLoad = 1; - pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag; + + SDataBlockInfo info = {0}; + tsortGetBlockInfo(pTupleHandle, &info); + + pBlock->info.scanFlag = info.scanFlag; pBlock->info.rows += 1; } else { @@ -1121,7 +1134,10 @@ int32_t tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupl } pBlock->info.dataLoad = 1; - pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag; + SDataBlockInfo info = {0}; + tsortGetBlockInfo(pTupleHandle, &info); + + pBlock->info.scanFlag = info.scanFlag; pBlock->info.rows += 1; } @@ -1463,35 +1479,41 @@ static int32_t initRowIdSort(SSortHandle* pHandle) { SColumnInfoData pkCol = {0}; SSDataBlock* pSortInput = createDataBlock(); - pHandle->pDataBlock = pSortInput; - SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1); int32_t code = blockDataAppendColInfo(pSortInput, &tsCol); if (code) { + blockDataDestroy(pSortInput); return code; } SColumnInfoData regionIdCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2); code = blockDataAppendColInfo(pSortInput, ®ionIdCol); if (code) { + blockDataDestroy(pSortInput); return code; } SColumnInfoData offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3); code = blockDataAppendColInfo(pSortInput, &offsetCol); if (code) { + blockDataDestroy(pSortInput); return code; } SColumnInfoData lengthCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 4); code = blockDataAppendColInfo(pSortInput, &lengthCol); if (code) { + blockDataDestroy(pSortInput); return code; } if (pHandle->bSortPk) { pkCol = createColumnInfoData(extPkCol->info.type, extPkCol->info.bytes, 5); code = blockDataAppendColInfo(pSortInput, &pkCol); + if (code) { + blockDataDestroy(pSortInput); + return code; + } } blockDataDestroy(pHandle->pDataBlock); @@ -1990,12 +2012,13 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { bool bExtractedBlock = false; bool bSkipBlock = false; if (pBlk != NULL && pHandle->mergeLimit > 0) { - pBlk = NULL; - - code = getRowsBlockWithinMergeLimit(pHandle, mTableNumRows, pBlk, &bExtractedBlock, &bSkipBlock, &pBlk); + SSDataBlock* p = NULL; + code = getRowsBlockWithinMergeLimit(pHandle, mTableNumRows, pBlk, &bExtractedBlock, &bSkipBlock, &p); if (bSkipBlock || code != 0) { continue; } + + pBlk = p; } if (pBlk != NULL) { @@ -2416,15 +2439,21 @@ static int32_t tsortOpenForPQSort(SSortHandle* pHandle) { while (1) { // fetch data SSDataBlock* pBlock = pHandle->fetchfp(source->param); - if (NULL == pBlock) break; + if (NULL == pBlock) { + break; + } if (pHandle->beforeFp != NULL) { pHandle->beforeFp(pBlock, pHandle->param); } + if (pHandle->pDataBlock == NULL) { pHandle->pDataBlock = createOneDataBlock(pBlock, false); } - if (pHandle->pDataBlock == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + if (pHandle->pDataBlock == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } size_t colNum = blockDataGetNumOfCols(pBlock); @@ -2581,7 +2610,7 @@ void tsortGetValue(STupleHandle* pVHandle, int32_t colIndex, void** pVal) { } uint64_t tsortGetGroupId(STupleHandle* pVHandle) { return pVHandle->pBlock->info.id.groupId; } -void* tsortGetBlockInfo(STupleHandle* pVHandle) { return &pVHandle->pBlock->info; } +void tsortGetBlockInfo(STupleHandle* pVHandle, SDataBlockInfo* pBlockInfo) { *pBlockInfo = pVHandle->pBlock->info; } SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) { SSortExecInfo info = {0}; From eefee54c6aa64a6f871cd9a8f12d2497a7ff2626 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 Jul 2024 17:31:41 +0800 Subject: [PATCH 16/52] fix(stream): check return value. --- source/libs/executor/inc/operator.h | 4 +- source/libs/executor/inc/querytask.h | 21 +++---- source/libs/executor/src/executor.c | 30 ++++++++-- source/libs/executor/src/executorInt.c | 7 ++- source/libs/executor/src/mergeoperator.c | 6 +- source/libs/executor/src/querytask.c | 65 +++++++++++++--------- source/libs/executor/src/sortoperator.c | 19 ++++--- source/libs/stream/src/streamCheckStatus.c | 7 ++- source/libs/stream/src/streamCheckpoint.c | 42 ++++++++++++++ 9 files changed, 143 insertions(+), 58 deletions(-) diff --git a/source/libs/executor/inc/operator.h b/source/libs/executor/inc/operator.h index 95208545bd..37cdd5d7b3 100644 --- a/source/libs/executor/inc/operator.h +++ b/source/libs/executor/inc/operator.h @@ -140,7 +140,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SHashJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SHashJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle); @@ -164,7 +164,7 @@ SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo SOperatorInfo* createGroupCacheOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SGroupCachePhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SDynQueryCtrlPhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SDynQueryCtrlPhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo); // clang-format on diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h index 48de49f07c..e3bb9a1361 100644 --- a/source/libs/executor/inc/querytask.h +++ b/source/libs/executor/inc/querytask.h @@ -98,16 +98,17 @@ struct SExecTaskInfo { SQueryAutoQWorkerPoolCB* pWorkerCb; }; -void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); -SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI); -void doDestroyTask(SExecTaskInfo* pTaskInfo); -bool isTaskKilled(void* pTaskInfo); -void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); -void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); -int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, - int32_t vgId, char* sql, EOPTR_EXEC_MODEL model); -int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo); -SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo); +void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); +int32_t doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI, + SExecTaskInfo** pTaskInfo); +void doDestroyTask(SExecTaskInfo* pTaskInfo); +bool isTaskKilled(void* pTaskInfo); +void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); +void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); +int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, + int32_t vgId, char* sql, EOPTR_EXEC_MODEL model); +int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo); +int32_t getTableListInfo(const SExecTaskInfo* pTaskInfo, SArray** pList); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 8cf44602f9..8c98df5c8d 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -288,8 +288,10 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id) { if (msg == NULL) { // create raw scan - SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, &pReaderHandle->api); - if (NULL == pTaskInfo) { + SExecTaskInfo* pTaskInfo = NULL; + + int32_t code = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, &pReaderHandle->api, &pTaskInfo); + if (NULL == pTaskInfo || code != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -1450,7 +1452,13 @@ SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SExecTaskInfo* pTaskInfo = tinfo; - SArray* plist = getTableListInfo(pTaskInfo); + SArray* plist = NULL; + + code = getTableListInfo(pTaskInfo, &plist); + if (code || plist == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } // only extract table in the first elements STableListInfo* pTableListInfo = taosArrayGetP(plist, 0); @@ -1502,11 +1510,21 @@ _end: } } -SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo) { - SArray* pArray = taosArrayInit(0, POINTER_BYTES); +int32_t getTableListInfo(const SExecTaskInfo* pTaskInfo, SArray** pList) { + if (pList == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + SArray* pArray = taosArrayInit(0, POINTER_BYTES); + if (pArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SOperatorInfo* pOperator = pTaskInfo->pRoot; extractTableList(pArray, pOperator); - return pArray; + + *pList = pArray; + return TSDB_CODE_SUCCESS; } int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo) { diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index a3e3501114..b974384d85 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -1068,7 +1068,12 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* return TSDB_CODE_OUT_OF_MEMORY; } - SArray* pInfoList = getTableListInfo(pTask); + SArray* pInfoList = NULL; + int32_t code = getTableListInfo(pTask, &pInfoList); + if (code || pInfoList == NULL) { + return code; + } + STableListInfo* pTableListInfo = taosArrayGetP(pInfoList, 0); taosArrayDestroy(pInfoList); diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index cef1915d2c..384b14ce41 100644 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -168,8 +168,10 @@ SSDataBlock* doSortMerge(SOperatorInfo* pOperator) { blockDataCleanup(pDataBlock); if (pSortMergeInfo->pIntermediateBlock == NULL) { - pSortMergeInfo->pIntermediateBlock = tsortGetSortedDataBlock(pHandle); - if (pSortMergeInfo->pIntermediateBlock == NULL) { + pSortMergeInfo->pIntermediateBlock = NULL; + + int32_t code = tsortGetSortedDataBlock(pHandle, &pSortMergeInfo->pIntermediateBlock); + if (pSortMergeInfo->pIntermediateBlock == NULL || code != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index 0bf4ac2b21..ddba556c7a 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -35,38 +35,51 @@ #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) -SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI) { - SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); +int32_t doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI, + SExecTaskInfo** pTaskInfo) { if (pTaskInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_SUCCESS; } - setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pTaskInfo->cost.created = taosGetTimestampUs(); + SExecTaskInfo* p = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - pTaskInfo->execModel = model; - pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); - pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES); - pTaskInfo->storageAPI = *pAPI; + setTaskStatus(p, TASK_NOT_COMPLETED); + p->cost.created = taosGetTimestampUs(); - taosInitRWLatch(&pTaskInfo->lock); + p->execModel = model; + p->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); + p->pResultBlockList = taosArrayInit(128, POINTER_BYTES); + if (p->stopInfo.pStopInfo == NULL || p->pResultBlockList == NULL) { + doDestroyTask(p); + return TSDB_CODE_OUT_OF_MEMORY; + } - pTaskInfo->id.vgId = vgId; - pTaskInfo->id.queryId = queryId; - pTaskInfo->id.taskId = taskId; - pTaskInfo->id.str = taosMemoryMalloc(64); - buildTaskId(taskId, queryId, pTaskInfo->id.str); - pTaskInfo->schemaInfos = taosArrayInit(1, sizeof(SSchemaInfo)); - - return pTaskInfo; + p->storageAPI = *pAPI; + taosInitRWLatch(&p->lock); + + p->id.vgId = vgId; + p->id.queryId = queryId; + p->id.taskId = taskId; + p->id.str = taosMemoryMalloc(64); + buildTaskId(taskId, queryId, p->id.str); + p->schemaInfos = taosArrayInit(1, sizeof(SSchemaInfo)); + if (p->id.str == NULL || p->schemaInfos == NULL) { + doDestroyTask(p); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pTaskInfo = p; + return TSDB_CODE_SUCCESS; } bool isTaskKilled(void* pTaskInfo) { return (0 != ((SExecTaskInfo*)pTaskInfo)->code); } void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; - stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str, &pTaskInfo->storageAPI); + (void) stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str, &pTaskInfo->storageAPI); } void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { @@ -81,10 +94,10 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) { - *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model, &pHandle->api); - if (*pTaskInfo == NULL) { + int32_t code = doCreateTask(pPlan->id.queryId, taskId, vgId, model, &pHandle->api, pTaskInfo); + if (*pTaskInfo == NULL || code != 0) { taosMemoryFree(sql); - return terrno; + return code; } if (pHandle) { @@ -165,12 +178,10 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo } pAPI->metaReaderFn.clearReader(&mr); - schemaInfo.qsw = extractQueriedColumnSchema(pScanNode); - taosArrayPush(pTaskInfo->schemaInfos, &schemaInfo); - - return TSDB_CODE_SUCCESS; + void* p = taosArrayPush(pTaskInfo->schemaInfos, &schemaInfo); + return (p != NULL)? TSDB_CODE_SUCCESS:TSDB_CODE_OUT_OF_MEMORY; } SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 67b993d702..9892114735 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -166,7 +166,11 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { } pBlock->info.dataLoad = 1; - pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag; + + SDataBlockInfo info = {0}; + tsortGetBlockInfo(pTupleHandle, &info); + + pBlock->info.scanFlag = info.scanFlag; pBlock->info.rows += 1; } @@ -224,9 +228,9 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SSortOperatorInfo* pInfo) { blockDataCleanup(pDataBlock); - int32_t code = 0; - SSDataBlock* p = tsortGetSortedDataBlock(pHandle); - if (p == NULL) { + SSDataBlock* p = NULL; + int32_t code = tsortGetSortedDataBlock(pHandle, &p); + if (p == NULL || (code != 0)) { return NULL; } @@ -443,8 +447,9 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo blockDataCleanup(pDataBlock); blockDataEnsureCapacity(pDataBlock, capacity); - SSDataBlock* p = tsortGetSortedDataBlock(pHandle); - if (p == NULL) { + SSDataBlock* p = NULL; + int32_t code = tsortGetSortedDataBlock(pHandle, &p); + if (p == NULL || (code != 0)) { return NULL; } @@ -452,7 +457,7 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo while (1) { STupleHandle* pTupleHandle = NULL; - int32_t code = tsortNextTuple(pHandle, &pTupleHandle); + code = tsortNextTuple(pHandle, &pTupleHandle); if (pTupleHandle == NULL || code != 0) { break; } diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 66c1941abb..625305da03 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -418,13 +418,14 @@ void findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId, SDownstreamStatus int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs, int64_t reqId, int32_t* pNotReady, const char* id) { - streamMutexLock(&pInfo->checkInfoLock); - SDownstreamStatusInfo* p = NULL; + + streamMutexLock(&pInfo->checkInfoLock); findCheckRspStatus(pInfo, taskId, &p); if (p != NULL) { if (reqId != p->reqId) { - stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 " expired check-rsp recv from downstream task:0x%x, discarded", + stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 + " expired check-rsp recv from downstream task:0x%x, discarded", id, reqId, p->reqId, taskId); streamMutexUnlock(&pInfo->checkInfoLock); return TSDB_CODE_FAILED; diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 60019977cc..4dc7b14cc3 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -176,6 +176,10 @@ int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStream int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { SSDataBlock* pDataBlock = taosArrayGet(pBlock->blocks, 0); + if (pDataBlock == NULL) { + return TSDB_CODE_INVALID_PARA; + } + int64_t checkpointId = pDataBlock->info.version; int32_t transId = pDataBlock->info.window.skey; const char* id = pTask->id.idStr; @@ -248,6 +252,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock // check if already recv or not, and duplicated checkpoint-trigger msg recv, discard it for (int32_t i = 0; i < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++i) { STaskCheckpointReadyInfo* p = taosArrayGet(pActiveInfo->pReadyMsgList, i); + if (p == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (p->upstreamTaskId == pBlock->srcTaskId) { ASSERT(p->checkpointId == checkpointId); stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64 @@ -381,6 +389,10 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList); for (int32_t i = 0; i < size; ++i) { STaskDownstreamReadyInfo* p = taosArrayGet(pInfo->pCheckpointReadyRecvList, i); + if (p == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (p->downstreamTaskId == downstreamTaskId) { received = true; break; @@ -420,6 +432,10 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream streamMutexLock(&pInfo->lock); for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); + if (pReadyInfo == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) { pReadyInfo->sendCompleted = 1; stDebug("s-task:%s send checkpoint-ready msg to upstream:0x%x confirmed, checkpointId:%" PRId64 " ts:%" PRId64, @@ -430,6 +446,10 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); + if (pReadyInfo == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (pReadyInfo->sendCompleted == 1) { numOfConfirmed += 1; } @@ -819,6 +839,10 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { bool recved = false; for (int32_t j = 0; j < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++j) { STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, j); + if (pReady == NULL) { + continue; + } + if (pInfo->nodeId == pReady->upstreamNodeId) { recved = true; break; @@ -867,6 +891,9 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) { for (int32_t i = 0; i < size; i++) { SStreamUpstreamEpInfo* pUpstreamTask = taosArrayGet(pNotSendList, i); + if (pUpstreamTask == NULL) { + return TSDB_CODE_INVALID_PARA; + } SRetrieveChkptTriggerReq* pReq = rpcMallocCont(sizeof(SRetrieveChkptTriggerReq)); if (pReq == NULL) { @@ -917,6 +944,10 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { STaskTriggerSendInfo* pSendInfo = taosArrayGet(pInfo->pDispatchTriggerList, i); + if (pSendInfo == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (pSendInfo->nodeId != downstreamNodeId) { continue; } @@ -974,6 +1005,9 @@ void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { } else { for (int32_t i = 0; i < streamTaskGetNumOfDownstream(pTask); ++i) { SVgroupInfo* pVgInfo = taosArrayGet(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos, i); + if (pVgInfo == NULL) { + continue; + } STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId}; void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); @@ -993,6 +1027,10 @@ int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) { streamMutexLock(&pInfo->lock); for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); + if (p == NULL) { + return num; + } + if (p->recved) { num++; } @@ -1009,6 +1047,10 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); + if (p == NULL) { + continue; + } + if (p->nodeId == vgId) { ASSERT(p->recved == false); From fcf32af53f37e6a098d768e6147decf403cec971 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 23 Jul 2024 09:46:40 +0000 Subject: [PATCH 17/52] fix/TD-30989 --- source/dnode/mnode/impl/src/mndVgroup.c | 954 +++++++++++++++--------- source/dnode/mnode/impl/src/mndView.c | 39 +- 2 files changed, 616 insertions(+), 377 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0422bfabff..b26b563e36 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -189,9 +189,17 @@ static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *p int code = -1; pRow = mndVgroupActionDecode(pRaw); - if (pRow == NULL) goto _OVER; + if (pRow == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } pVgroup = sdbGetRowObj(pRow); - if (pVgroup == NULL) goto _OVER; + if (pVgroup == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); if (maxVgId > pVgroup->vgId) { @@ -203,7 +211,7 @@ static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *p _OVER: if (pVgroup) mndVgroupActionDelete(pSdb, pVgroup); taosMemoryFreeClear(pRow); - return code; + TAOS_RETURN(code); } static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup) { @@ -756,28 +764,24 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup if (size < pVgroup->replica) { mError("db:%s, vgId:%d, no enough online dnodes:%d to alloc %d replica", pVgroup->dbName, pVgroup->vgId, size, pVgroup->replica); - terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - return -1; + TAOS_RETURN(TSDB_CODE_MND_NO_ENOUGH_DNODES); } for (int32_t v = 0; v < pVgroup->replica; ++v) { SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SDnodeObj *pDnode = taosArrayGet(pArray, v); if (pDnode == NULL) { - terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - return -1; + TAOS_RETURN(TSDB_CODE_MND_NO_ENOUGH_DNODES); } if (pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) { - terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES; - return -1; + TAOS_RETURN(TSDB_CODE_MND_NO_ENOUGH_VNODES); } int64_t vgMem = mndGetVgroupMemory(pMnode, pDb, pVgroup); if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) { mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d, avail:%" PRId64 " used:%" PRId64, pVgroup->dbName, pVgroup->vgId, vgMem, pDnode->id, pDnode->memAvail, pDnode->memUsed); - terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE; - return -1; + TAOS_RETURN(TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE); } else { pDnode->memUsed += vgMem; } @@ -799,8 +803,13 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup } int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup) { + int32_t code = 0; SArray *pArray = mndBuildDnodesArray(pMnode, 0); - if (pArray == NULL) return -1; + if (pArray == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } pVgroup->vgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); pVgroup->isTsma = 1; @@ -825,12 +834,16 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { pVgroups = taosMemoryCalloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); if (pVgroups == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } pArray = mndBuildDnodesArray(pMnode, 0); - if (pArray == NULL) goto _OVER; + if (pArray == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mInfo("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), pDb->cfg.numOfVgroups, pDb->cfg.numOfVgroups * pDb->cfg.replications); @@ -875,7 +888,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { _OVER: if (code != 0) taosMemoryFree(pVgroups); taosArrayDestroy(pArray); - return code; + TAOS_RETURN(code); } SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) { @@ -901,7 +914,7 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) { SEpSet mndGetVgroupEpsetById(SMnode *pMnode, int32_t vgId) { SEpSet epset = {0}; - SVgObj * pVgroup = mndAcquireVgroup(pMnode, vgId); + SVgObj *pVgroup = mndAcquireVgroup(pMnode, vgId); if (!pVgroup) return epset; for (int32_t v = 0; v < pVgroup->replica; ++v) { @@ -1199,6 +1212,7 @@ static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter) { } static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SArray *pArray) { + int32_t code = 0; taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); @@ -1219,20 +1233,17 @@ static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgro if (used) continue; if (pDnode == NULL) { - terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - return -1; + TAOS_RETURN(TSDB_CODE_MND_NO_ENOUGH_DNODES); } if (pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) { - terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES; - return -1; + TAOS_RETURN(TSDB_CODE_MND_NO_ENOUGH_VNODES); } int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup); if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) { mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d avail:%" PRId64 " used:%" PRId64, pVgroup->dbName, pVgroup->vgId, vgMem, pDnode->id, pDnode->memAvail, pDnode->memUsed); - terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE; - return -1; + TAOS_RETURN(TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE); } else { pDnode->memUsed += vgMem; } @@ -1246,19 +1257,23 @@ static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgro pDnode->numOfVnodes++; SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendRedolog(pTrans, pVgRaw)) != 0) { sdbFreeRaw(pVgRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); - return 0; + TAOS_RETURN(code); } - terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - mError("db:%s, failed to add vnode to vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); - return -1; + code = TSDB_CODE_MND_NO_ENOUGH_DNODES; + mError("db:%s, failed to add vnode to vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, tstrerror(code)); + TAOS_RETURN(code); } static int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SArray *pArray, @@ -1293,9 +1308,9 @@ static int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, STrans *pTrans, SVgObj * _OVER: if (code != 0) { - terrno = TSDB_CODE_APP_ERROR; - mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); - return -1; + code = TSDB_CODE_APP_ERROR; + mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, tstrerror(code)); + TAOS_RETURN(code); } for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { @@ -1304,14 +1319,18 @@ _OVER: } SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); - if (pVgRaw == NULL) return -1; + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { sdbFreeRaw(pVgRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); - return 0; + TAOS_RETURN(code); } static int32_t mndRemoveVnodeFromVgroupWithoutSave(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SArray *pArray, @@ -1346,9 +1365,9 @@ static int32_t mndRemoveVnodeFromVgroupWithoutSave(SMnode *pMnode, STrans *pTran _OVER: if (code != 0) { - terrno = TSDB_CODE_APP_ERROR; - mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); - return -1; + code = TSDB_CODE_APP_ERROR; + mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, tstrerror(code)); + TAOS_RETURN(code); } for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { @@ -1356,10 +1375,11 @@ _OVER: mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); } - return 0; + TAOS_RETURN(code); } int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid) { + int32_t code = 0; STransAction action = {0}; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); @@ -1376,38 +1396,44 @@ int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVg action.msgType = TDMT_DND_CREATE_VNODE; action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } int32_t mndRestoreAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SDnodeObj *pDnode) { + int32_t code = 0; STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); int32_t contLen = 0; void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_VNODE; action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { + int32_t code = 0; STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); @@ -1415,8 +1441,7 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD int32_t contLen = sizeof(SMsgHead); SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } pHead->contLen = htonl(contLen); @@ -1428,30 +1453,34 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD // incorrect redirect result will cause this erro action.retryCode = TSDB_CODE_VND_INVALID_VGROUP_ID; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pHead); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } int32_t mndAddChangeConfigAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pOldVgroup, SVgObj *pNewVgroup, int32_t dnodeId) { + int32_t code = 0; STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pNewVgroup); int32_t contLen = 0; void *pReq = mndBuildAlterVnodeReplicaReq(pMnode, pDb, pNewVgroup, dnodeId, &contLen); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } int32_t totallen = contLen + sizeof(SMsgHead); SMsgHead *pHead = taosMemoryMalloc(totallen); if (pHead == NULL) { taosMemoryFree(pReq); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } pHead->contLen = htonl(totallen); @@ -1464,73 +1493,93 @@ int32_t mndAddChangeConfigAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SV action.contLen = totallen; action.msgType = TDMT_SYNC_CONFIG_CHANGE; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pHead); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } static int32_t mndAddAlterVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, int32_t srcVgId, SVgObj *pVgroup) { + int32_t code = 0; STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); int32_t contLen = 0; void *pReq = mndBuildAlterVnodeHashRangeReq(pMnode, srcVgId, pVgroup, &contLen); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_ALTER_HASHRANGE; action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } mInfo("trans:%d, add alter vnode hash range action for from vgId:%d to vgId:%d", pTrans->id, srcVgId, pVgroup->vgId); - return 0; + TAOS_RETURN(code); } int32_t mndAddAlterVnodeConfigAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { + int32_t code = 0; STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); int32_t contLen = 0; void *pReq = mndBuildAlterVnodeConfigReq(pMnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_ALTER_CONFIG; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } int32_t mndAddNewVgPrepareAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVg) { + int32_t code = 0; SSdbRaw *pRaw = mndVgroupActionEncode(pVg); - if (pRaw == NULL) goto _err; + if (pRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _err; + } - if (mndTransAppendPrepareLog(pTrans, pRaw) != 0) goto _err; + TAOS_CHECK_GOTO(mndTransAppendPrepareLog(pTrans, pRaw), NULL, _err); (void)sdbSetRawStatus(pRaw, SDB_STATUS_CREATING); pRaw = NULL; - return 0; + TAOS_RETURN(code); _err: sdbFreeRaw(pRaw); - return -1; + TAOS_RETURN(code); } int32_t mndAddAlterVnodeReplicaAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) { + int32_t code = 0; SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); - if (pDnode == NULL) return -1; + if (pDnode == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -1538,23 +1587,32 @@ int32_t mndAddAlterVnodeReplicaAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD int32_t contLen = 0; void *pReq = mndBuildAlterVnodeReplicaReq(pMnode, pDb, pVgroup, dnodeId, &contLen); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_ALTER_REPLICA; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } int32_t mndAddCheckLearnerCatchupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) { + int32_t code = 0; SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); - if (pDnode == NULL) return -1; + if (pDnode == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -1562,7 +1620,11 @@ int32_t mndAddCheckLearnerCatchupAction(SMnode *pMnode, STrans *pTrans, SDbObj * int32_t contLen = 0; void *pReq = mndBuildCheckLearnCatchupReq(pMnode, pDb, pVgroup, dnodeId, &contLen); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.pCont = pReq; action.contLen = contLen; @@ -1570,17 +1632,22 @@ int32_t mndAddCheckLearnerCatchupAction(SMnode *pMnode, STrans *pTrans, SDbObj * action.acceptableCode = TSDB_CODE_VND_ALREADY_IS_VOTER; action.retryCode = TSDB_CODE_VND_NOT_CATCH_UP; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } int32_t mndAddAlterVnodeTypeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) { + int32_t code = 0; SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); - if (pDnode == NULL) return -1; + if (pDnode == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -1588,7 +1655,11 @@ int32_t mndAddAlterVnodeTypeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, int32_t contLen = 0; void *pReq = mndBuildAlterVnodeReplicaReq(pMnode, pDb, pVgroup, dnodeId, &contLen); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.pCont = pReq; action.contLen = contLen; @@ -1596,22 +1667,27 @@ int32_t mndAddAlterVnodeTypeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, action.acceptableCode = TSDB_CODE_VND_ALREADY_IS_VOTER; action.retryCode = TSDB_CODE_VND_NOT_CATCH_UP; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } int32_t mndRestoreAddAlterVnodeTypeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SDnodeObj *pDnode) { + int32_t code = 0; STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); int32_t contLen = 0; void *pReq = mndBuildAlterVnodeReplicaReq(pMnode, pDb, pVgroup, pDnode->id, &contLen); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.pCont = pReq; action.contLen = contLen; @@ -1619,18 +1695,23 @@ int32_t mndRestoreAddAlterVnodeTypeAction(SMnode *pMnode, STrans *pTrans, SDbObj action.acceptableCode = TSDB_CODE_VND_ALREADY_IS_VOTER; action.retryCode = TSDB_CODE_VND_NOT_CATCH_UP; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } static int32_t mndAddDisableVnodeWriteAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) { + int32_t code = 0; SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); - if (pDnode == NULL) return -1; + if (pDnode == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -1638,32 +1719,45 @@ static int32_t mndAddDisableVnodeWriteAction(SMnode *pMnode, STrans *pTrans, SDb int32_t contLen = 0; void *pReq = mndBuildDisableVnodeWriteReq(pMnode, pDb, pVgroup->vgId, &contLen); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_DISABLE_WRITE; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo) { + int32_t code = 0; STransAction action = {0}; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; + if (pDnode == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); int32_t contLen = 0; void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.pCont = pReq; action.contLen = contLen; @@ -1671,23 +1765,24 @@ int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgOb action.acceptableCode = TSDB_CODE_VND_NOT_EXIST; if (isRedo) { - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } } else { - if (mndTransAppendUndoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendUndoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } } - return 0; + TAOS_RETURN(code); } int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vnIndex, SArray *pArray, bool force, bool unsafe) { - SVgObj newVg = {0}; + int32_t code = 0; + SVgObj newVg = {0}; memcpy(&newVg, pVgroup, sizeof(SVgObj)); mInfo("vgId:%d, vgroup info before move, replica:%d", newVg.vgId, newVg.replica); @@ -1702,12 +1797,12 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, if (newVg.replica == 1) { #endif mInfo("vgId:%d, will add 1 vnode, replca:%d", pVgroup->vgId, newVg.replica); - if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1; + TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray)); for (int32_t i = 0; i < newVg.replica - 1; ++i) { - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId)); } - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1]) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; + TAOS_CHECK_RETURN(mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1])); + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg)); mInfo("vgId:%d, will remove 1 vnode, replca:2", pVgroup->vgId); newVg.replica--; @@ -1716,19 +1811,23 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid)); { SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRaw) != 0) { + if (pRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendRedolog(pTrans, pRaw)) != 0) { sdbFreeRaw(pRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); } - if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1; + TAOS_CHECK_RETURN(mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true)); for (int32_t i = 0; i < newVg.replica; ++i) { - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId)); } - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg)); #if 1 } #else @@ -1761,33 +1860,36 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, #endif } else { mInfo("vgId:%d, will add 1 vnode and force remove 1 vnode", pVgroup->vgId); - if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1; + TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray)); newVg.replica--; // SVnodeGid del = newVg.vnodeGid[vnIndex]; newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica]; memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid)); { SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRaw) != 0) { + if (pRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendRedolog(pTrans, pRaw)) != 0) { sdbFreeRaw(pRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); } for (int32_t i = 0; i < newVg.replica; ++i) { if (i != vnIndex) { - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId)); } } - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[vnIndex]) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; + TAOS_CHECK_RETURN(mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[vnIndex])); + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg)); if (newVg.replica == 1) { if (force && !unsafe) { - terrno = TSDB_CODE_VND_META_DATA_UNSAFE_DELETE; - return -1; + TAOS_RETURN(TSDB_CODE_VND_META_DATA_UNSAFE_DELETE); } SSdb *pSdb = pMnode->pSdb; @@ -1799,10 +1901,10 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, if (pIter == NULL) break; if (strcmp(pStb->db, pDb->name) == 0) { - if (mndSetForceDropCreateStbRedoActions(pMnode, pTrans, &newVg, pStb) != 0) { + if ((code = mndSetForceDropCreateStbRedoActions(pMnode, pTrans, &newVg, pStb)) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pStb); - return -1; + TAOS_RETURN(code); } } @@ -1815,10 +1917,14 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, { SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { + if (pRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendCommitlog(pTrans, pRaw)) != 0) { sdbFreeRaw(pRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); } @@ -1827,13 +1933,17 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, for (int32_t i = 0; i < newVg.replica; ++i) { mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); } - return 0; + TAOS_RETURN(code); } int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t delDnodeId, bool force, bool unsafe) { int32_t code = 0; SArray *pArray = mndBuildDnodesArray(pMnode, delDnodeId); - if (pArray == NULL) return -1; + if (pArray == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } void *pIter = NULL; while (1) { @@ -1866,11 +1976,12 @@ int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t del } taosArrayDestroy(pArray); - return code; + TAOS_RETURN(code); } static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t newDnodeId) { + int32_t code = 0; mInfo("vgId:%d, will add 1 vnode, replica:%d dnode:%d", pVgroup->vgId, pVgroup->replica, newDnodeId); // assoc dnode @@ -1881,34 +1992,39 @@ static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDb pGid->nodeRole = TAOS_SYNC_ROLE_LEARNER; SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendRedolog(pTrans, pVgRaw)) != 0) { sdbFreeRaw(pVgRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); // learner for (int32_t i = 0; i < pVgroup->replica - 1; ++i) { - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId)); } - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pGid) != 0) return -1; + TAOS_CHECK_RETURN(mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pGid)); // voter pGid->nodeRole = TAOS_SYNC_ROLE_VOTER; - if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pDb, pVgroup, pGid->dnodeId) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeTypeAction(pMnode, pTrans, pDb, pVgroup, pGid->dnodeId)); for (int32_t i = 0; i < pVgroup->replica - 1; ++i) { - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId)); } // confirm - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup)); - return 0; + TAOS_RETURN(code); } static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t delDnodeId) { + int32_t code = 0; mInfo("vgId:%d, will remove 1 vnode, replica:%d dnode:%d", pVgroup->vgId, pVgroup->replica, delDnodeId); SVnodeGid *pGid = NULL; @@ -1928,20 +2044,24 @@ static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, S memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid)); SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendRedolog(pTrans, pVgRaw)) != 0) { sdbFreeRaw(pVgRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); - if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1; + TAOS_CHECK_RETURN(mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true)); for (int32_t i = 0; i < pVgroup->replica; ++i) { - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId)); } - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup)); - return 0; + TAOS_RETURN(code); } static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup, SDnodeObj *pNew1, @@ -1951,10 +2071,14 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, STrans *pTrans = NULL; pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "red-vgroup"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mndTransSetDbName(pTrans, pVgroup->dbName, NULL); - if (mndTransCheckConflictWithCompact(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransCheckConflictWithCompact(pMnode, pTrans), NULL, _OVER); mndTransSetSerial(pTrans); mInfo("trans:%d, used to redistribute vgroup, vgId:%d", pTrans->id, pVgroup->vgId); @@ -1972,7 +2096,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, if (numOfVnodes >= pNew1->numOfSupportVnodes) { mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew1->id, numOfVnodes, pNew1->numOfSupportVnodes); - terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES; + code = TSDB_CODE_MND_NO_ENOUGH_VNODES; goto _OVER; } @@ -1980,14 +2104,14 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, if (pNew1->memAvail - vgMem - pNew1->memUsed <= 0) { mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d avail:%" PRId64 " used:%" PRId64, pVgroup->dbName, pVgroup->vgId, vgMem, pNew1->id, pNew1->memAvail, pNew1->memUsed); - terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE; + code = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE; goto _OVER; } else { pNew1->memUsed += vgMem; } - if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew1->id) != 0) goto _OVER; - if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld1->id) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew1->id), NULL, _OVER); + TAOS_CHECK_GOTO(mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld1->id), NULL, _OVER); } if (pNew2 != NULL && pOld2 != NULL) { @@ -1995,20 +2119,20 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, if (numOfVnodes >= pNew2->numOfSupportVnodes) { mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew2->id, numOfVnodes, pNew2->numOfSupportVnodes); - terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES; + code = TSDB_CODE_MND_NO_ENOUGH_VNODES; goto _OVER; } int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup); if (pNew2->memAvail - vgMem - pNew2->memUsed <= 0) { mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d avail:%" PRId64 " used:%" PRId64, pVgroup->dbName, pVgroup->vgId, vgMem, pNew2->id, pNew2->memAvail, pNew2->memUsed); - terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE; + code = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE; goto _OVER; } else { pNew2->memUsed += vgMem; } - if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew2->id) != 0) goto _OVER; - if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld2->id) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew2->id), NULL, _OVER); + TAOS_CHECK_GOTO(mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld2->id), NULL, _OVER); } if (pNew3 != NULL && pOld3 != NULL) { @@ -2016,26 +2140,30 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, if (numOfVnodes >= pNew3->numOfSupportVnodes) { mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew3->id, numOfVnodes, pNew3->numOfSupportVnodes); - terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES; + code = TSDB_CODE_MND_NO_ENOUGH_VNODES; goto _OVER; } int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup); if (pNew3->memAvail - vgMem - pNew3->memUsed <= 0) { mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d avail:%" PRId64 " used:%" PRId64, pVgroup->dbName, pVgroup->vgId, vgMem, pNew3->id, pNew3->memAvail, pNew3->memUsed); - terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE; + code = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE; goto _OVER; } else { pNew3->memUsed += vgMem; } - if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew3->id) != 0) goto _OVER; - if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld3->id) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew3->id), NULL, _OVER); + TAOS_CHECK_GOTO(mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld3->id), NULL, _OVER); } { SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL) goto _OVER; - if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { + if (pRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + if ((code = mndTransAppendCommitlog(pTrans, pRaw)) != 0) { sdbFreeRaw(pRaw); goto _OVER; } @@ -2047,13 +2175,13 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); } - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; _OVER: mndTransDrop(pTrans); mndReleaseDb(pMnode, pDb); - return code; + TAOS_RETURN(code); } static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { @@ -2075,24 +2203,32 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { SRedistributeVgroupReq req = {0}; if (tDeserializeSRedistributeVgroupReq(pReq->pCont, pReq->contLen, &req) != 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } mInfo("vgId:%d, start to redistribute vgroup to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) { + if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP)) != 0) { goto _OVER; } pVgroup = mndAcquireVgroup(pMnode, req.vgId); - if (pVgroup == NULL) goto _OVER; + if (pVgroup == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } pDb = mndAcquireDb(pMnode, pVgroup->dbName); - if (pDb == NULL) goto _OVER; + if (pDb == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } if (pVgroup->replica == 1) { if (req.dnodeId1 <= 0 || req.dnodeId2 > 0 || req.dnodeId3 > 0) { - terrno = TSDB_CODE_MND_INVALID_REPLICA; + code = TSDB_CODE_MND_INVALID_REPLICA; goto _OVER; } @@ -2103,16 +2239,24 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } pNew1 = mndAcquireDnode(pMnode, req.dnodeId1); - if (pNew1 == NULL) goto _OVER; + if (pNew1 == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } if (!mndIsDnodeOnline(pNew1, curMs)) { - terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + code = TSDB_CODE_MND_HAS_OFFLINE_DNODE; goto _OVER; } pOld1 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[0].dnodeId); - if (pOld1 == NULL) goto _OVER; + if (pOld1 == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } if (!mndIsDnodeOnline(pOld1, curMs)) { - terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + code = TSDB_CODE_MND_HAS_OFFLINE_DNODE; goto _OVER; } @@ -2120,12 +2264,12 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } else if (pVgroup->replica == 3) { if (req.dnodeId1 <= 0 || req.dnodeId2 <= 0 || req.dnodeId3 <= 0) { - terrno = TSDB_CODE_MND_INVALID_REPLICA; + code = TSDB_CODE_MND_INVALID_REPLICA; goto _OVER; } if (req.dnodeId1 == req.dnodeId2 || req.dnodeId1 == req.dnodeId3 || req.dnodeId2 == req.dnodeId3) { - terrno = TSDB_CODE_MND_INVALID_REPLICA; + code = TSDB_CODE_MND_INVALID_REPLICA; goto _OVER; } @@ -2167,54 +2311,78 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { if (newDnodeId[0] != 0) { pNew1 = mndAcquireDnode(pMnode, newDnodeId[0]); - if (pNew1 == NULL) goto _OVER; + if (pNew1 == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } if (!mndIsDnodeOnline(pNew1, curMs)) { - terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + code = TSDB_CODE_MND_HAS_OFFLINE_DNODE; goto _OVER; } } if (newDnodeId[1] != 0) { pNew2 = mndAcquireDnode(pMnode, newDnodeId[1]); - if (pNew2 == NULL) goto _OVER; + if (pNew2 == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } if (!mndIsDnodeOnline(pNew2, curMs)) { - terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + code = TSDB_CODE_MND_HAS_OFFLINE_DNODE; goto _OVER; } } if (newDnodeId[2] != 0) { pNew3 = mndAcquireDnode(pMnode, newDnodeId[2]); - if (pNew3 == NULL) goto _OVER; + if (pNew3 == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } if (!mndIsDnodeOnline(pNew3, curMs)) { - terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + code = TSDB_CODE_MND_HAS_OFFLINE_DNODE; goto _OVER; } } if (oldDnodeId[0] != 0) { pOld1 = mndAcquireDnode(pMnode, oldDnodeId[0]); - if (pOld1 == NULL) goto _OVER; + if (pOld1 == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } if (!mndIsDnodeOnline(pOld1, curMs)) { - terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + code = TSDB_CODE_MND_HAS_OFFLINE_DNODE; goto _OVER; } } if (oldDnodeId[1] != 0) { pOld2 = mndAcquireDnode(pMnode, oldDnodeId[1]); - if (pOld2 == NULL) goto _OVER; + if (pOld2 == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } if (!mndIsDnodeOnline(pOld2, curMs)) { - terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + code = TSDB_CODE_MND_HAS_OFFLINE_DNODE; goto _OVER; } } if (oldDnodeId[2] != 0) { pOld3 = mndAcquireDnode(pMnode, oldDnodeId[2]); - if (pOld3 == NULL) goto _OVER; + if (pOld3 == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } if (!mndIsDnodeOnline(pOld3, curMs)) { - terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + code = TSDB_CODE_MND_HAS_OFFLINE_DNODE; goto _OVER; } } @@ -2228,7 +2396,7 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { code = mndRedistributeVgroup(pMnode, pReq, pDb, pVgroup, pNew1, pOld1, pNew2, pOld2, pNew3, pOld3); } else { - terrno = TSDB_CODE_MND_REQ_REJECTED; + code = TSDB_CODE_MND_REQ_REJECTED; goto _OVER; } @@ -2242,7 +2410,7 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("vgId:%d, failed to redistribute to dnode %d:%d:%d since %s", req.vgId, req.dnodeId1, req.dnodeId2, - req.dnodeId3, terrstr()); + req.dnodeId3, tstrerror(code)); } mndReleaseDnode(pMnode, pNew1); @@ -2255,7 +2423,7 @@ _OVER: mndReleaseDb(pMnode, pDb); tFreeSRedistributeVgroupReq(&req); - return code; + TAOS_RETURN(code); } static void *mndBuildSForceBecomeFollowerReq(SMnode *pMnode, SVgObj *pVgroup, int32_t dnodeId, int32_t *pContLen) { @@ -2286,8 +2454,13 @@ static void *mndBuildSForceBecomeFollowerReq(SMnode *pMnode, SVgObj *pVgroup, in } int32_t mndAddBalanceVgroupLeaderAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t dnodeId) { + int32_t code = 0; SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); - if (pDnode == NULL) return -1; + if (pDnode == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -2295,22 +2468,27 @@ int32_t mndAddBalanceVgroupLeaderAction(SMnode *pMnode, STrans *pTrans, SVgObj * int32_t contLen = 0; void *pReq = mndBuildSForceBecomeFollowerReq(pMnode, pVgroup, dnodeId, &contLen); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_SYNC_FORCE_FOLLOWER; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } int32_t mndAddVgroupBalanceToTrans(SMnode *pMnode, SVgObj *pVgroup, STrans *pTrans) { - SSdb *pSdb = pMnode->pSdb; + int32_t code = 0; + SSdb *pSdb = pMnode->pSdb; int32_t vgid = pVgroup->vgId; int8_t replica = pVgroup->replica; @@ -2342,15 +2520,17 @@ int32_t mndAddVgroupBalanceToTrans(SMnode *pMnode, SVgObj *pVgroup, STrans *pTra if (exist && online) { mInfo("trans:%d, vgid:%d leader to dnode:%d", pTrans->id, vgid, dnodeId); - if (mndAddBalanceVgroupLeaderAction(pMnode, pTrans, pVgroup, dnodeId) != 0) { + if ((code = mndAddBalanceVgroupLeaderAction(pMnode, pTrans, pVgroup, dnodeId)) != 0) { mError("trans:%d, vgid:%d failed to be balanced to dnode:%d", pTrans->id, vgid, dnodeId); - return -1; + TAOS_RETURN(code); } SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); if (pDb == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; mError("trans:%d, vgid:%d failed to be balanced to dnode:%d, because db not exist", pTrans->id, vgid, dnodeId); - return -1; + TAOS_RETURN(code); } mndReleaseDb(pMnode, pDb); @@ -2359,7 +2539,7 @@ int32_t mndAddVgroupBalanceToTrans(SMnode *pMnode, SVgObj *pVgroup, STrans *pTra online); } - return 0; + TAOS_RETURN(code); } extern int32_t mndProcessVgroupBalanceLeaderMsgImp(SRpcMsg *pReq); @@ -2392,8 +2572,7 @@ static int32_t mndCheckDnodeMemory(SMnode *pMnode, SDbObj *pOldDb, SDbObj *pNewD if (pDnode->memAvail - pDnode->memUsed <= 0) { mError("db:%s, vgId:%d, no enough memory in dnode:%d, avail:%" PRId64 " used:%" PRId64, pNewVgroup->dbName, pNewVgroup->vgId, pDnode->id, pDnode->memAvail, pDnode->memUsed); - terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE; - return -1; + TAOS_RETURN(TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE); } else if (inVgroup) { mInfo("db:%s, vgId:%d, memory in dnode:%d, avail:%" PRId64 " used:%" PRId64, pNewVgroup->dbName, pNewVgroup->vgId, pDnode->id, pDnode->memAvail, pDnode->memUsed); @@ -2405,11 +2584,12 @@ static int32_t mndCheckDnodeMemory(SMnode *pMnode, SDbObj *pOldDb, SDbObj *pNewD int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup, SArray *pArray, SVgObj *pNewVgroup) { + int32_t code = 0; memcpy(pNewVgroup, pVgroup, sizeof(SVgObj)); if (pVgroup->replica <= 0 || pVgroup->replica == pNewDb->cfg.replications) { - if (mndAddAlterVnodeConfigAction(pMnode, pTrans, pNewDb, pVgroup) != 0) return -1; - if (mndCheckDnodeMemory(pMnode, pOldDb, pNewDb, pNewVgroup, pVgroup, pArray) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeConfigAction(pMnode, pTrans, pNewDb, pVgroup)); + TAOS_CHECK_RETURN(mndCheckDnodeMemory(pMnode, pOldDb, pNewDb, pNewVgroup, pVgroup, pArray)); return 0; } @@ -2420,78 +2600,78 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb pVgroup->vnodeGid[0].dnodeId); // add second - if (mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray) != 0) return -1; + TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray)); // learner stage pNewVgroup->vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; pNewVgroup->vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0) - return -1; + TAOS_CHECK_RETURN( + mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId)); - if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &pNewVgroup->vnodeGid[1]) != 0) return -1; + TAOS_CHECK_RETURN(mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &pNewVgroup->vnodeGid[1])); // follower stage pNewVgroup->vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; - if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[1].dnodeId) != 0) return -1; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0) - return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeTypeAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[1].dnodeId)); + TAOS_CHECK_RETURN( + mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId)); - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup)); // add third - if (mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray) != 0) return -1; + TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray)); pNewVgroup->vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; pNewVgroup->vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; pNewVgroup->vnodeGid[2].nodeRole = TAOS_SYNC_ROLE_VOTER; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0) - return -1; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[1].dnodeId) != 0) - return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &pNewVgroup->vnodeGid[2]) != 0) return -1; + TAOS_CHECK_RETURN( + mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId)); + TAOS_CHECK_RETURN( + mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[1].dnodeId)); + TAOS_CHECK_RETURN(mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &pNewVgroup->vnodeGid[2])); - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup)); } else if (pNewVgroup->replica == 3 && pNewDb->cfg.replications == 1) { mInfo("db:%s, vgId:%d, will remove 2 vnodes, vn:0 dnode:%d vn:1 dnode:%d vn:2 dnode:%d", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId, pVgroup->vnodeGid[1].dnodeId, pVgroup->vnodeGid[2].dnodeId); SVnodeGid del1 = {0}; SVnodeGid del2 = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, pTrans, pNewVgroup, pArray, &del1) != 0) return -1; - if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &del1, true) != 0) return -1; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0) - return -1; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[1].dnodeId) != 0) - return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup) != 0) return -1; + TAOS_CHECK_RETURN(mndRemoveVnodeFromVgroup(pMnode, pTrans, pNewVgroup, pArray, &del1)); + TAOS_CHECK_RETURN(mndAddDropVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &del1, true)); + TAOS_CHECK_RETURN( + mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId)); + TAOS_CHECK_RETURN( + mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[1].dnodeId)); + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup)); - if (mndRemoveVnodeFromVgroup(pMnode, pTrans, pNewVgroup, pArray, &del2) != 0) return -1; - if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &del2, true) != 0) return -1; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0) - return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup) != 0) return -1; + TAOS_CHECK_RETURN(mndRemoveVnodeFromVgroup(pMnode, pTrans, pNewVgroup, pArray, &del2)); + TAOS_CHECK_RETURN(mndAddDropVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &del2, true)); + TAOS_CHECK_RETURN( + mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId)); + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup)); } else if (pNewVgroup->replica == 1 && pNewDb->cfg.replications == 2) { mInfo("db:%s, vgId:%d, will add 1 vnode, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); // add second - if (mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray) != 0) return -1; + TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray)); // learner stage pNewVgroup->vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; pNewVgroup->vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0) - return -1; + TAOS_CHECK_RETURN( + mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId)); - if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &pNewVgroup->vnodeGid[1]) != 0) return -1; + TAOS_CHECK_RETURN(mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &pNewVgroup->vnodeGid[1])); // follower stage pNewVgroup->vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; - if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[1].dnodeId) != 0) return -1; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0) - return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeTypeAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[1].dnodeId)); + TAOS_CHECK_RETURN( + mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId)); - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup)); } else { return -1; } @@ -2500,25 +2680,30 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb { SSdbRaw *pVgRaw = mndVgroupActionEncode(pNewVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendCommitlog(pTrans, pVgRaw)) != 0) { sdbFreeRaw(pVgRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); } - return 0; + TAOS_RETURN(code); } int32_t mndBuildRaftAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup, SArray *pArray) { - SVgObj newVgroup = {0}; + int32_t code = 0; + SVgObj newVgroup = {0}; memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); if (pVgroup->replica <= 0 || pVgroup->replica == pNewDb->cfg.replications) { - if (mndAddAlterVnodeConfigAction(pMnode, pTrans, pNewDb, pVgroup) != 0) return -1; - if (mndCheckDnodeMemory(pMnode, pOldDb, pNewDb, &newVgroup, pVgroup, pArray) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeConfigAction(pMnode, pTrans, pNewDb, pVgroup)); + TAOS_CHECK_RETURN(mndCheckDnodeMemory(pMnode, pOldDb, pNewDb, &newVgroup, pVgroup, pArray)); return 0; } @@ -2532,22 +2717,22 @@ int32_t mndBuildRaftAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pO pVgroup->vnodeGid[0].dnodeId); // add second - if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1; + TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray)); // add third - if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1; + TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray)); // add learner stage newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER; newVgroup.vnodeGid[2].nodeRole = TAOS_SYNC_ROLE_LEARNER; - if (mndAddChangeConfigAction(pMnode, pTrans, pNewDb, pVgroup, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) - return -1; + TAOS_CHECK_RETURN( + mndAddChangeConfigAction(pMnode, pTrans, pNewDb, pVgroup, &newVgroup, newVgroup.vnodeGid[0].dnodeId)); mInfo("trans:%d, vgId:%d, add change config, syncConfChangeVer:%d, version:%d, replica:%d", pTrans->id, pVgroup->vgId, newVgroup.syncConfChangeVer, pVgroup->version, pVgroup->replica); - if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1; + TAOS_CHECK_RETURN(mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1])); mInfo("trans:%d, vgId:%d, create vnode, syncConfChangeVer:%d, version:%d, replica:%d", pTrans->id, pVgroup->vgId, newVgroup.syncConfChangeVer, pVgroup->version, pVgroup->replica); - if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[2]) != 0) return -1; + TAOS_CHECK_RETURN(mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[2])); mInfo("trans:%d, vgId:%d, create vnode, syncConfChangeVer:%d, version:%d, replica:%d", pTrans->id, pVgroup->vgId, newVgroup.syncConfChangeVer, pVgroup->version, pVgroup->replica); @@ -2555,33 +2740,37 @@ int32_t mndBuildRaftAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pO newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; newVgroup.vnodeGid[2].nodeRole = TAOS_SYNC_ROLE_VOTER; - if (mndAddCheckLearnerCatchupAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0) - return -1; - if (mndAddCheckLearnerCatchupAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[2].dnodeId) != 0) - return -1; + TAOS_CHECK_RETURN( + mndAddCheckLearnerCatchupAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId)); + TAOS_CHECK_RETURN( + mndAddCheckLearnerCatchupAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[2].dnodeId)); // change raft type newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; newVgroup.vnodeGid[2].nodeRole = TAOS_SYNC_ROLE_LEARNER; - if (mndAddChangeConfigAction(pMnode, pTrans, pNewDb, pVgroup, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) - return -1; + TAOS_CHECK_RETURN( + mndAddChangeConfigAction(pMnode, pTrans, pNewDb, pVgroup, &newVgroup, newVgroup.vnodeGid[0].dnodeId)); - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup)); newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; newVgroup.vnodeGid[2].nodeRole = TAOS_SYNC_ROLE_VOTER; - if (mndAddChangeConfigAction(pMnode, pTrans, pNewDb, pVgroup, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) - return -1; + TAOS_CHECK_RETURN( + mndAddChangeConfigAction(pMnode, pTrans, pNewDb, pVgroup, &newVgroup, newVgroup.vnodeGid[0].dnodeId)); - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup)); SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendRedolog(pTrans, pVgRaw)) != 0) { sdbFreeRaw(pVgRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); } else if (newVgroup.replica == 3 && pNewDb->cfg.replications == 1) { @@ -2589,38 +2778,46 @@ int32_t mndBuildRaftAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pO pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId, pVgroup->vnodeGid[1].dnodeId, pVgroup->vnodeGid[2].dnodeId); SVnodeGid del1 = {0}; - if (mndRemoveVnodeFromVgroupWithoutSave(pMnode, pTrans, &newVgroup, pArray, &del1) != 0) return -1; + TAOS_CHECK_RETURN(mndRemoveVnodeFromVgroupWithoutSave(pMnode, pTrans, &newVgroup, pArray, &del1)); - if (mndAddChangeConfigAction(pMnode, pTrans, pNewDb, pVgroup, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) - return -1; + TAOS_CHECK_RETURN( + mndAddChangeConfigAction(pMnode, pTrans, pNewDb, pVgroup, &newVgroup, newVgroup.vnodeGid[0].dnodeId)); - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup)); - if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del1, true) != 0) return -1; + TAOS_CHECK_RETURN(mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del1, true)); SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendRedolog(pTrans, pVgRaw)) != 0) { sdbFreeRaw(pVgRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); SVnodeGid del2 = {0}; - if (mndRemoveVnodeFromVgroupWithoutSave(pMnode, pTrans, &newVgroup, pArray, &del2) != 0) return -1; + TAOS_CHECK_RETURN(mndRemoveVnodeFromVgroupWithoutSave(pMnode, pTrans, &newVgroup, pArray, &del2)); - if (mndAddChangeConfigAction(pMnode, pTrans, pNewDb, pVgroup, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) - return -1; + TAOS_CHECK_RETURN( + mndAddChangeConfigAction(pMnode, pTrans, pNewDb, pVgroup, &newVgroup, newVgroup.vnodeGid[0].dnodeId)); - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; + TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup)); - if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del2, true) != 0) return -1; + TAOS_CHECK_RETURN(mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del2, true)); pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendRedolog(pTrans, pVgRaw)) != 0) { sdbFreeRaw(pVgRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); } else { @@ -2631,20 +2828,25 @@ int32_t mndBuildRaftAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pO { SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendCommitlog(pTrans, pVgRaw)) != 0) { sdbFreeRaw(pVgRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); } - return 0; + TAOS_RETURN(code); } int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *db, SVgObj *pVgroup, SDnodeObj *pDnode) { - SVgObj newVgroup = {0}; + int32_t code = 0; + SVgObj newVgroup = {0}; memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); mInfo("db:%s, vgId:%d, restore vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); @@ -2657,7 +2859,7 @@ int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj selected = i; } } - if (mndAddCreateVnodeAction(pMnode, pTrans, db, &newVgroup, &newVgroup.vnodeGid[selected]) != 0) return -1; + TAOS_CHECK_RETURN(mndAddCreateVnodeAction(pMnode, pTrans, db, &newVgroup, &newVgroup.vnodeGid[selected])); } else if (newVgroup.replica == 2 || newVgroup.replica == 3) { for (int i = 0; i < newVgroup.replica; i++) { if (newVgroup.vnodeGid[i].dnodeId == pDnode->id) { @@ -2666,25 +2868,29 @@ int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_VOTER; } } - if (mndRestoreAddCreateVnodeAction(pMnode, pTrans, db, &newVgroup, pDnode) != 0) return -1; + TAOS_CHECK_RETURN(mndRestoreAddCreateVnodeAction(pMnode, pTrans, db, &newVgroup, pDnode)); for (int i = 0; i < newVgroup.replica; i++) { newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_VOTER; if (newVgroup.vnodeGid[i].dnodeId == pDnode->id) { } } - if (mndRestoreAddAlterVnodeTypeAction(pMnode, pTrans, db, &newVgroup, pDnode) != 0) return -1; + TAOS_CHECK_RETURN(mndRestoreAddAlterVnodeTypeAction(pMnode, pTrans, db, &newVgroup, pDnode)); } SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendCommitlog(pTrans, pVgRaw)) != 0) { sdbFreeRaw(pVgRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); - return 0; + TAOS_RETURN(code); } static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { @@ -2694,29 +2900,39 @@ static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, typedef int32_t (*FpTransActionCb)(STrans *pTrans, SSdbRaw *pRaw); static int32_t mndAddVgStatusAction(STrans *pTrans, SVgObj *pVg, ESdbStatus vgStatus, ETrnStage stage) { + int32_t code = 0; FpTransActionCb appendActionCb = (stage == TRN_STAGE_COMMIT_ACTION) ? mndTransAppendCommitlog : mndTransAppendRedolog; SSdbRaw *pRaw = mndVgroupActionEncode(pVg); - if (pRaw == NULL) goto _err; - if (appendActionCb(pTrans, pRaw) != 0) goto _err; + if (pRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _err; + } + if ((code = appendActionCb(pTrans, pRaw)) != 0) goto _err; (void)sdbSetRawStatus(pRaw, vgStatus); pRaw = NULL; - return 0; + TAOS_RETURN(code); _err: sdbFreeRaw(pRaw); - return -1; + TAOS_RETURN(code); } static int32_t mndAddDbStatusAction(STrans *pTrans, SDbObj *pDb, ESdbStatus dbStatus, ETrnStage stage) { + int32_t code = 0; FpTransActionCb appendActionCb = (stage == TRN_STAGE_COMMIT_ACTION) ? mndTransAppendCommitlog : mndTransAppendRedolog; SSdbRaw *pRaw = mndDbActionEncode(pDb); - if (pRaw == NULL) goto _err; - if (appendActionCb(pTrans, pRaw) != 0) goto _err; + if (pRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _err; + } + if ((code = appendActionCb(pTrans, pRaw)) != 0) goto _err; (void)sdbSetRawStatus(pRaw, dbStatus); pRaw = NULL; - return 0; + TAOS_RETURN(code); _err: sdbFreeRaw(pRaw); - return -1; + TAOS_RETURN(code); } int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) { @@ -2735,11 +2951,11 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro // } int32_t numOfStreams = 0; - if (mndGetNumOfStreams(pMnode, pDb->name, &numOfStreams) != 0) { + if ((code = mndGetNumOfStreams(pMnode, pDb->name, &numOfStreams)) != 0) { goto _OVER; } if (numOfStreams > 0) { - terrno = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; + code = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; goto _OVER; } @@ -2759,7 +2975,11 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro } pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "split-vgroup"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mndTransSetSerial(pTrans); mInfo("trans:%d, used to split vgroup, vgId:%d", pTrans->id, pVgroup->vgId); @@ -2774,31 +2994,36 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro } if (newVg1.replica == 1) { - if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg1, pArray) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddVnodeToVgroup(pMnode, pTrans, &newVg1, pArray), NULL, _OVER); newVg1.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId), NULL, + _OVER); + TAOS_CHECK_GOTO(mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]), NULL, _OVER); newVg1.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; - if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId) != 0) goto _OVER; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddAlterVnodeTypeAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId), NULL, _OVER); + TAOS_CHECK_GOTO(mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId), NULL, + _OVER); - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1), NULL, _OVER); } else if (newVg1.replica == 3) { SVnodeGid del1 = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVg1, pArray, &del1) != 0) goto _OVER; - if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVg1, pArray, &del1), NULL, _OVER); + TAOS_CHECK_GOTO(mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true), NULL, _OVER); + TAOS_CHECK_GOTO(mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId), NULL, + _OVER); + TAOS_CHECK_GOTO(mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId), NULL, + _OVER); } else { goto _OVER; } for (int32_t i = 0; i < newVg1.replica; ++i) { - if (mndAddDisableVnodeWriteAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[i].dnodeId) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddDisableVnodeWriteAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[i].dnodeId), NULL, + _OVER); } - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1), NULL, _OVER); SVgObj newVg2 = {0}; memcpy(&newVg2, &newVg1, sizeof(SVgObj)); @@ -2826,63 +3051,66 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); int32_t srcVgId = newVg1.vgId; newVg1.vgId = maxVgId; - if (mndAddNewVgPrepareAction(pMnode, pTrans, &newVg1) != 0) goto _OVER; - if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, srcVgId, &newVg1) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddNewVgPrepareAction(pMnode, pTrans, &newVg1), NULL, _OVER); + TAOS_CHECK_GOTO(mndAddAlterVnodeHashRangeAction(pMnode, pTrans, srcVgId, &newVg1), NULL, _OVER); maxVgId++; srcVgId = newVg2.vgId; newVg2.vgId = maxVgId; - if (mndAddNewVgPrepareAction(pMnode, pTrans, &newVg2) != 0) goto _OVER; - if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, srcVgId, &newVg2) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddNewVgPrepareAction(pMnode, pTrans, &newVg2), NULL, _OVER); + TAOS_CHECK_GOTO(mndAddAlterVnodeHashRangeAction(pMnode, pTrans, srcVgId, &newVg2), NULL, _OVER); - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg2) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1), NULL, _OVER); + TAOS_CHECK_GOTO(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg2), NULL, _OVER); - if (mndAddVgStatusAction(pTrans, &newVg1, SDB_STATUS_READY, TRN_STAGE_REDO_ACTION) < 0) goto _OVER; - if (mndAddVgStatusAction(pTrans, &newVg2, SDB_STATUS_READY, TRN_STAGE_REDO_ACTION) < 0) goto _OVER; - if (mndAddVgStatusAction(pTrans, pVgroup, SDB_STATUS_DROPPED, TRN_STAGE_REDO_ACTION) < 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddVgStatusAction(pTrans, &newVg1, SDB_STATUS_READY, TRN_STAGE_REDO_ACTION), NULL, _OVER); + TAOS_CHECK_GOTO(mndAddVgStatusAction(pTrans, &newVg2, SDB_STATUS_READY, TRN_STAGE_REDO_ACTION), NULL, _OVER); + TAOS_CHECK_GOTO(mndAddVgStatusAction(pTrans, pVgroup, SDB_STATUS_DROPPED, TRN_STAGE_REDO_ACTION), NULL, _OVER); // update db status memcpy(&dbObj, pDb, sizeof(SDbObj)); if (dbObj.cfg.pRetensions != NULL) { dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); - if (dbObj.cfg.pRetensions == NULL) goto _OVER; + if (dbObj.cfg.pRetensions == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } } dbObj.vgVersion++; dbObj.updateTime = taosGetTimestampMs(); dbObj.cfg.numOfVgroups++; - if (mndAddDbStatusAction(pTrans, &dbObj, SDB_STATUS_READY, TRN_STAGE_REDO_ACTION) < 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddDbStatusAction(pTrans, &dbObj, SDB_STATUS_READY, TRN_STAGE_REDO_ACTION), NULL, _OVER); // adjust vgroup replica if (pDb->cfg.replications != newVg1.replica) { SVgObj tmpGroup = {0}; - if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray, &tmpGroup) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray, &tmpGroup), NULL, _OVER); } else { - if (mndAddVgStatusAction(pTrans, &newVg1, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddVgStatusAction(pTrans, &newVg1, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION), NULL, _OVER); } if (pDb->cfg.replications != newVg2.replica) { SVgObj tmpGroup = {0}; - if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray, &tmpGroup) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray, &tmpGroup), NULL, _OVER); } else { - if (mndAddVgStatusAction(pTrans, &newVg2, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddVgStatusAction(pTrans, &newVg2, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION), NULL, _OVER); } - if (mndAddVgStatusAction(pTrans, pVgroup, SDB_STATUS_DROPPED, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddVgStatusAction(pTrans, pVgroup, SDB_STATUS_DROPPED, TRN_STAGE_COMMIT_ACTION), NULL, _OVER); // commit db status dbObj.vgVersion++; dbObj.updateTime = taosGetTimestampMs(); - if (mndAddDbStatusAction(pTrans, &dbObj, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddDbStatusAction(pTrans, &dbObj, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION), NULL, _OVER); - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; _OVER: taosArrayDestroy(pArray); mndTransDrop(pTrans); taosArrayDestroy(dbObj.cfg.pRetensions); - return code; + TAOS_RETURN(code); } extern int32_t mndProcessSplitVgroupMsgImp(SRpcMsg *pReq); @@ -2895,22 +3123,27 @@ int32_t mndProcessSplitVgroupMsgImp(SRpcMsg *pReq) { return 0; } static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SDnodeObj *pSrc, SDnodeObj *pDst) { - SVgObj newVg = {0}; + int32_t code = 0; + SVgObj newVg = {0}; memcpy(&newVg, pVgroup, sizeof(SVgObj)); mInfo("vgId:%d, vgroup info before balance, replica:%d", newVg.vgId, newVg.replica); for (int32_t i = 0; i < newVg.replica; ++i) { mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); } - if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pDst->id) != 0) return -1; - if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pSrc->id) != 0) return -1; + TAOS_CHECK_RETURN(mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pDst->id)); + TAOS_CHECK_RETURN(mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pSrc->id)); { SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { + if (pRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendCommitlog(pTrans, pRaw)) != 0) { sdbFreeRaw(pRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); } @@ -2919,7 +3152,7 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD for (int32_t i = 0; i < newVg.replica; ++i) { mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); } - return 0; + TAOS_RETURN(code); } static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst, @@ -2952,6 +3185,8 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); if (pDb == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; mError("vgId:%d, balance vgroup can't find db obj dbName:%s", pVgroup->vgId, pVgroup->dbName); goto _OUT; } @@ -2986,7 +3221,11 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { if (pBalancedVgroups == NULL) goto _OVER; pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "balance-vgroup"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mndTransSetSerial(pTrans); mInfo("trans:%d, used to balance vgroup", pTrans->id); @@ -3035,7 +3274,7 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { _OVER: taosHashCleanup(pBalancedVgroups); mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { @@ -3047,12 +3286,12 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { SBalanceVgroupReq req = {0}; if (tDeserializeSBalanceVgroupReq(pReq->pCont, pReq->contLen, &req) != 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } mInfo("start to balance vgroup"); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) { + if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP)) != 0) { goto _OVER; } @@ -3062,7 +3301,7 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { if (pIter == NULL) break; if (!mndIsDnodeOnline(pDnode, curMs)) { sdbCancelFetch(pMnode->pSdb, pIter); - terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + code = TSDB_CODE_MND_HAS_OFFLINE_DNODE; mError("failed to balance vgroup since %s, dnode:%d", terrstr(), pDnode->id); sdbRelease(pMnode->pSdb, pDnode); goto _OVER; @@ -3072,7 +3311,11 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { } pArray = mndBuildDnodesArray(pMnode, 0); - if (pArray == NULL) goto _OVER; + if (pArray == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } if (taosArrayGetSize(pArray) < 2) { mInfo("no need to balance vgroup since dnode num less than 2"); @@ -3085,12 +3328,12 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("failed to balance vgroup since %s", terrstr()); + mError("failed to balance vgroup since %s", tstrerror(code)); } taosArrayDestroy(pArray); tFreeSBalanceVgroupReq(&req); - return code; + TAOS_RETURN(code); } bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; } @@ -3135,27 +3378,32 @@ static void *mndBuildCompactVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgrou static int32_t mndAddCompactVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs, STimeWindow tw) { + int32_t code = 0; STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); int32_t contLen = 0; void *pReq = mndBuildCompactVnodeReq(pMnode, pDb, pVgroup, &contLen, compactTs, tw); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_COMPACT; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs, STimeWindow tw) { - if (mndAddCompactVnodeAction(pMnode, pTrans, pDb, pVgroup, compactTs, tw) != 0) return -1; + TAOS_CHECK_RETURN(mndAddCompactVnodeAction(pMnode, pTrans, pDb, pVgroup, compactTs, tw)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndView.c b/source/dnode/mnode/impl/src/mndView.c index d53e740736..f4138b2afe 100755 --- a/source/dnode/mnode/impl/src/mndView.c +++ b/source/dnode/mnode/impl/src/mndView.c @@ -43,18 +43,15 @@ int32_t mndInitView(SMnode *pMnode) { #endif } -void mndCleanupView(SMnode *pMnode) { - mDebug("mnd view cleanup"); -} +void mndCleanupView(SMnode *pMnode) { mDebug("mnd view cleanup"); } int32_t mndProcessCreateViewReq(SRpcMsg *pReq) { #ifndef TD_ENTERPRISE return TSDB_CODE_OPS_NOT_SUPPORT; #else - SCMCreateViewReq createViewReq = {0}; + SCMCreateViewReq createViewReq = {0}; if (tDeserializeSCMCreateViewReq(pReq->pCont, pReq->contLen, &createViewReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_MSG); } mInfo("start to create view:%s, sql:%s", createViewReq.fullname, createViewReq.sql); @@ -65,17 +62,16 @@ int32_t mndProcessCreateViewReq(SRpcMsg *pReq) { int32_t mndProcessDropViewReq(SRpcMsg *pReq) { #ifndef TD_ENTERPRISE - return TSDB_CODE_OPS_NOT_SUPPORT; + return TSDB_CODE_OPS_NOT_SUPPORT; #else - SCMDropViewReq dropViewReq = {0}; - if (tDeserializeSCMDropViewReq(pReq->pCont, pReq->contLen, &dropViewReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - mInfo("start to drop view:%s, sql:%s", dropViewReq.name, dropViewReq.sql); - - return mndProcessDropViewReqImpl(&dropViewReq, pReq); + SCMDropViewReq dropViewReq = {0}; + if (tDeserializeSCMDropViewReq(pReq->pCont, pReq->contLen, &dropViewReq) != 0) { + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + mInfo("start to drop view:%s, sql:%s", dropViewReq.name, dropViewReq.sql); + + return mndProcessDropViewReqImpl(&dropViewReq, pReq); #endif } @@ -83,18 +79,16 @@ int32_t mndProcessGetViewMetaReq(SRpcMsg *pReq) { #ifndef TD_ENTERPRISE return TSDB_CODE_OPS_NOT_SUPPORT; #else - SViewMetaReq req = {0}; + SViewMetaReq req = {0}; if (tDeserializeSViewMetaReq(pReq->pCont, pReq->contLen, &req) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_MSG); } return mndProcessViewMetaReqImpl(&req, pReq); -#endif +#endif } - int32_t mndRetrieveView(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { #ifndef TD_ENTERPRISE return 0; @@ -107,6 +101,3 @@ void mndCancelGetNextView(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } - - - From 47a16a94565c054e4c8656e91730f885bdff2357 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 23 Jul 2024 09:56:13 +0000 Subject: [PATCH 18/52] fix/TD-30989 --- source/libs/monitor/src/monMain.c | 38 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 3389780916..6cba9b0727 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -15,17 +15,17 @@ #define _DEFAULT_SOURCE #include "monInt.h" +#include "taos_monitor.h" #include "taoserror.h" +#include "tglobal.h" #include "thttp.h" #include "ttime.h" -#include "taos_monitor.h" -#include "tglobal.h" SMonitor tsMonitor = {0}; -char* tsMonUri = "/report"; -char* tsMonFwUri = "/general-metric"; -char* tsMonSlowLogUri = "/slow-sql-detail-batch"; -char* tsMonFwBasicUri = "/taosd-cluster-basic"; +char *tsMonUri = "/report"; +char *tsMonFwUri = "/general-metric"; +char *tsMonSlowLogUri = "/slow-sql-detail-batch"; +char *tsMonFwBasicUri = "/taosd-cluster-basic"; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { taosThreadMutexLock(&tsMonitor.lock); @@ -54,8 +54,7 @@ int32_t monGetLogs(SMonLogs *logs) { taosArrayClear(tsMonitor.logs); taosThreadMutexUnlock(&tsMonitor.lock); if (logs->logs == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } return 0; } @@ -105,8 +104,7 @@ void monSetBmInfo(SMonBmInfo *pInfo) { int32_t monInit(const SMonCfg *pCfg) { tsMonitor.logs = taosArrayInit(16, sizeof(SMonLogItem)); if (tsMonitor.logs == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } tsMonitor.cfg = *pCfg; @@ -146,6 +144,7 @@ static void monCleanupMonitorInfo(SMonInfo *pMonitor) { } static SMonInfo *monCreateMonitorInfo() { + terrno = 0; SMonInfo *pMonitor = taosMemoryCalloc(1, sizeof(SMonInfo)); if (pMonitor == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -274,11 +273,11 @@ static void monGenClusterJsonBasic(SMonInfo *pMonitor) { SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; - //tjsonAddStringToObject(pMonitor->pJson, "first_ep", pInfo->first_ep); + // tjsonAddStringToObject(pMonitor->pJson, "first_ep", pInfo->first_ep); tjsonAddStringToObject(pMonitor->pJson, "first_ep", tsFirst); tjsonAddDoubleToObject(pMonitor->pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id); tjsonAddStringToObject(pMonitor->pJson, "cluster_version", pInfo->version); - //tjsonAddDoubleToObject(pMonitor->pJson, "monitor_interval", pInfo->monitor_interval); + // tjsonAddDoubleToObject(pMonitor->pJson, "monitor_interval", pInfo->monitor_interval); } static void monGenVgroupJson(SMonInfo *pMonitor) { @@ -554,9 +553,9 @@ static void monGenLogJson(SMonInfo *pMonitor) { if (tjsonAddItemToArray(pSummaryJson, pLogTrace) != 0) tjsonDelete(pLogTrace); } -void monSendReport(SMonInfo *pMonitor){ +void monSendReport(SMonInfo *pMonitor) { char *pCont = tjsonToString(pMonitor->pJson); - if(tsMonitorLogProtocol){ + if (tsMonitorLogProtocol) { uInfoL("report cont:\n%s", pCont); } if (pCont != NULL) { @@ -591,7 +590,7 @@ void monGenAndSendReport() { SMonInfo *pMonitor = monCreateMonitorInfo(); if (pMonitor == NULL) return; - if(!tsMonitorForceV2){ + if (!tsMonitorForceV2) { monGenBasicJson(pMonitor); monGenClusterJson(pMonitor); monGenVgroupJson(pMonitor); @@ -602,8 +601,7 @@ void monGenAndSendReport() { monGenLogJson(pMonitor); monSendReport(pMonitor); - } - else{ + } else { monGenClusterInfoTable(pMonitor); monGenVgroupInfoTable(pMonitor); monGenDnodeInfoTable(pMonitor); @@ -624,10 +622,10 @@ void monGenAndSendReport() { monCleanupMonitorInfo(pMonitor); } -void monSendContent(char *pCont, const char* uri) { +void monSendContent(char *pCont, const char *uri) { if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; - if(tsMonitorLogProtocol){ - if (pCont != NULL){ + if (tsMonitorLogProtocol) { + if (pCont != NULL) { uInfoL("report client cont:\n%s\n", pCont); } } From aaf01291cfcee2238cb92ba9146055336fa98f5f Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 23 Jul 2024 18:05:56 +0800 Subject: [PATCH 19/52] enh: refactor return code --- include/os/osMemory.h | 10 + include/util/tutil.h | 14 +- source/common/src/trow.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 9 +- source/dnode/mnode/impl/src/mndUser.c | 485 ++++++++++++----------- 6 files changed, 274 insertions(+), 248 deletions(-) diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 7f2963f15a..0cb4534d83 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -65,6 +65,16 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size); } \ } while (0) +#define TAOS_MEMORY_REALLOC(ptr, len) \ + do { \ + void *tmp = taosMemoryRealloc(ptr, (len)); \ + if (tmp) { \ + (ptr) = tmp; \ + } else { \ + taosMemoryFreeClear(ptr); \ + } \ + } while (0) + #ifdef __cplusplus } #endif diff --git a/include/util/tutil.h b/include/util/tutil.h index 0e79480bfe..823ffba23b 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -157,18 +157,18 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, return (terrno = (code)); \ } while (0) -#define TAOS_CHECK_RETURN(CMD) \ - do { \ - int32_t code = (CMD); \ - if (code != TSDB_CODE_SUCCESS) { \ - TAOS_RETURN(code); \ - } \ +#define TAOS_CHECK_RETURN(CMD) \ + do { \ + int32_t code = (CMD); \ + if (code < TSDB_CODE_SUCCESS) { \ + TAOS_RETURN(code); \ + } \ } while (0) #define TAOS_CHECK_GOTO(CMD, LINO, LABEL) \ do { \ code = (CMD); \ - if (code != TSDB_CODE_SUCCESS) { \ + if (code < TSDB_CODE_SUCCESS) { \ if (LINO) { \ *((int32_t *)(LINO)) = __LINE__; \ } \ diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 075f29454f..7234e5c5f6 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -535,7 +535,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r _exit: taosMemoryFreeClear(varBuf); - if (code != 0) { + if (code < 0) { if (isAlloc) { taosMemoryFreeClear(*ppRow); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 498ccd7c0c..ccbfeb5aa5 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -182,7 +182,7 @@ int32_t dmInitVars(SDnode *pDnode) { code = 0; strncpy(tsEncryptKey, tsAuthCode, 16); - if(code != 0) { + if (code != 0) { if(code == -1){ terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY; dError("machine code changed, can't get crypt key"); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index da041bc0dd..a6d37a5566 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -735,8 +735,9 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj } static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, SUserObj *pUser) { - int32_t code = -1; - SDbObj dbObj = {0}; + int32_t code = 0; + SUserObj newUserObj = {0}; + SDbObj dbObj = {0}; memcpy(dbObj.name, pCreate->db, TSDB_DB_FNAME_LEN); memcpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); dbObj.createdTime = taosGetTimestampMs(); @@ -816,7 +817,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, } // add database privileges for user - SUserObj newUserObj = {0}, *pNewUserDuped = NULL; + SUserObj *pNewUserDuped = NULL; if (!pUser->superUser) { TAOS_CHECK_GOTO(mndUserDupObj(pUser, &newUserObj), NULL, _OVER); taosHashPut(newUserObj.readDbs, dbObj.name, strlen(dbObj.name) + 1, dbObj.name, TSDB_FILENAME_LEN); @@ -845,8 +846,6 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, TAOS_CHECK_GOTO(mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups), NULL, _OVER); TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); - code = 0; - _OVER: taosMemoryFree(pVgroups); mndUserFreeObj(&newUserObj); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index add46bf1d6..1e6d09df44 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -96,7 +96,7 @@ (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_SUBSCRIBE_PRIV(_priv)) static int32_t createDefaultIpWhiteList(SIpWhiteList **ppWhiteList); -static int32_t createIpWhiteList(void *buf, int32_t len, SIpWhiteList **ppWhiteList); +static int32_t createIpWhiteList(void *buf, int32_t len, SIpWhiteList **ppWhiteList); static bool updateIpWhiteList(SIpWhiteList *pOld, SIpWhiteList *pNew); static bool isIpWhiteListEqual(SIpWhiteList *a, SIpWhiteList *b); static bool isIpRangeEqual(SIpV4Range *a, SIpV4Range *b); @@ -121,7 +121,7 @@ static int32_t mndRetrieveUsersFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter); -static int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab); +static int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab); static int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq); static int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type, bool *pUpdate); @@ -154,6 +154,7 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { int32_t code = 0; int32_t lino = 0; bool update = true; + SArray *fqdns = NULL; taosThreadRwlockWrlock(&ipWhiteMgt.rw); SIpWhiteList **ppList = taosHashGet(ipWhiteMgt.pIpWhiteTab, user, strlen(user)); @@ -186,7 +187,12 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { } } } - SArray *fqdns = mndGetAllDnodeFqdns(pMnode); // TODO: update this line after refactor api + + fqdns = mndGetAllDnodeFqdns(pMnode); // TODO: update this line after refactor api + if (fqdns == NULL) { + update = false; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } for (int i = 0; i < taosArrayGetSize(fqdns); i++) { char *fqdn = taosArrayGetP(fqdns, i); @@ -198,12 +204,6 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { update |= upd; } - for (int i = 0; i < taosArrayGetSize(fqdns); i++) { - char *fqdn = taosArrayGetP(fqdns, i); - taosMemoryFree(fqdn); - } - taosArrayDestroy(fqdns); - // for (int i = 0; i < taosArrayGetSize(pUserNames); i++) { // taosMemoryFree(taosArrayGetP(pUserNames, i)); // } @@ -213,7 +213,8 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { _OVER: taosThreadRwlockUnlock(&ipWhiteMgt.rw); - if (code != 0) { + taosArrayDestroyP(fqdns, (FDelete)taosMemoryFree); + if (code < 0) { mError("failed to update ip white list for user: %s at line %d since %s", user, lino, tstrerror(code)); } TAOS_RETURN(code); @@ -301,6 +302,8 @@ static int32_t ipWhiteMgtUpdateAll(SMnode *pMnode) { destroyIpWhiteTab(pOld); TAOS_RETURN(0); } + +#if 0 void ipWhiteMgtUpdate2(SMnode *pMnode) { taosThreadRwlockWrlock(&ipWhiteMgt.rw); @@ -308,13 +311,19 @@ void ipWhiteMgtUpdate2(SMnode *pMnode) { taosThreadRwlockUnlock(&ipWhiteMgt.rw); } +#endif int64_t mndGetIpWhiteVer(SMnode *pMnode) { int64_t ver = 0; + int32_t code = 0; taosThreadRwlockWrlock(&ipWhiteMgt.rw); if (ipWhiteMgt.ver == 0) { // get user and dnode ip white list - ipWhiteMgtUpdateAll(pMnode); + if ((code = ipWhiteMgtUpdateAll(pMnode)) != 0) { + taosThreadRwlockUnlock(&ipWhiteMgt.rw); + mError("%s failed to update ip white list since %s", __func__, tstrerror(code)); + return ver; + } ipWhiteMgt.ver = taosGetTimestampMs(); } ver = ipWhiteMgt.ver; @@ -342,7 +351,7 @@ int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8 if (type == IP_WHITE_ADD) { if (pList == NULL) { SIpWhiteList *pNewList = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range)); - if(pNewList == NULL) { + if (pNewList == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } memcpy(pNewList->pIpRange, &range, sizeof(SIpV4Range)); @@ -384,7 +393,7 @@ int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8 int32_t idx = 0; int32_t sz = sizeof(SIpWhiteList) + sizeof(SIpV4Range) * (pList->num - 1); SIpWhiteList *pNewList = taosMemoryCalloc(1, sz); - if(pNewList == NULL) { + if (pNewList == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } for (int i = 0; i < pList->num; i++) { @@ -412,7 +421,7 @@ int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8 _OVER: if (pUpdate) *pUpdate = update; - if (code != 0) { + if (code < 0) { mError("failed to update ip-white-list for user: %s, fqdn: %s at line %d since %s", user, fqdn, lino, tstrerror(code)); } @@ -459,9 +468,10 @@ int32_t mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_ TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } memcpy(keyDup, key, klen); - code = mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, keyDup, fqdn, type, &update); - if (code != 0) { - update = false; + bool upd = false; + code = mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, keyDup, fqdn, type, &upd); + update |= upd; + if (code < 0) { taosMemoryFree(keyDup); TAOS_CHECK_GOTO(code, &lino, _OVER); } @@ -473,7 +483,7 @@ int32_t mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_ _OVER: if (update) ipWhiteMgt.ver++; if (lock) taosThreadRwlockUnlock(&ipWhiteMgt.rw); - if (code != 0) { + if (code < 0) { mError("failed to update ip-white-list for user: %s, fqdn: %s at line %d since %s", user, fqdn, lino, tstrerror(code)); } @@ -508,7 +518,6 @@ static int64_t ipWhiteMgtFillMsg(SUpdateIpWhite *pUpdate) { pUser->pIpRanges = taosMemoryCalloc(1, list->num * sizeof(SIpV4Range)); if (pUser->pIpRanges == NULL) { taosThreadRwlockUnlock(&ipWhiteMgt.rw); - tFreeSUpdateIpWhiteReq(pUpdate); TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pUser->pIpRanges, list->pIpRange, list->num * sizeof(SIpV4Range)); @@ -549,7 +558,7 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } pUserNames = taosArrayInit(8, sizeof(void *)); - if(pUserNames == NULL) { + if (pUserNames == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } @@ -564,7 +573,7 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { sdbCancelFetch(pSdb, pIter); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - if(taosHashPut(pIpWhiteTab, pUser->user, strlen(pUser->user), &pWhiteList, sizeof(void *)) != 0) { + if (taosHashPut(pIpWhiteTab, pUser->user, strlen(pUser->user), &pWhiteList, sizeof(void *)) != 0) { taosMemoryFree(pWhiteList); sdbRelease(pSdb, pUser); sdbCancelFetch(pSdb, pIter); @@ -572,12 +581,12 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { } char *name = taosStrdup(pUser->user); - if(name == NULL) { + if (name == NULL) { sdbRelease(pSdb, pUser); sdbCancelFetch(pSdb, pIter); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - if(taosArrayPush(pUserNames, &name) == NULL){ + if (taosArrayPush(pUserNames, &name) == NULL) { taosMemoryFree(name); sdbRelease(pSdb, pUser); sdbCancelFetch(pSdb, pIter); @@ -606,7 +615,10 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { } } - fqdns = mndGetAllDnodeFqdns(pMnode); // TODO: refactor this line after refactor api + fqdns = mndGetAllDnodeFqdns(pMnode); // TODO: refactor this line after refactor api + if (fqdns == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } for (int i = 0; i < taosArrayGetSize(fqdns); i++) { char *fqdn = taosArrayGetP(fqdns, i); @@ -618,18 +630,10 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { } _OVER: - for (int i = 0; i < taosArrayGetSize(fqdns); i++) { - char *fqdn = taosArrayGetP(fqdns, i); - taosMemoryFree(fqdn); - } - taosArrayDestroy(fqdns); + taosArrayDestroyP(fqdns, taosMemoryFree); + taosArrayDestroyP(pUserNames, taosMemoryFree); - for (int i = 0; i < taosArrayGetSize(pUserNames); i++) { - taosMemoryFree(taosArrayGetP(pUserNames, i)); - } - taosArrayDestroy(pUserNames); - - if (code != 0) { + if (code < 0) { mError("failed to fetch all ip white list at line %d since %s", lino, tstrerror(code)); destroyIpWhiteTab(pIpWhiteTab); pIpWhiteTab = NULL; @@ -725,9 +729,12 @@ int32_t convertIpWhiteListToStr(SIpWhiteList *pList, char **buf) { return 0; } *buf = taosMemoryCalloc(1, pList->num * 36); + if (*buf == NULL) { + return 0; + } int32_t len = ipRangeListToStr(pList->pIpRange, pList->num, *buf); if (len == 0) { - taosMemoryFree(*buf); + taosMemoryFreeClear(*buf); return 0; } return strlen(*buf); @@ -753,7 +760,7 @@ int32_t tSerializeIpWhiteList(void *buf, int32_t len, SIpWhiteList *pList, uint3 tlen = encoder.pos; _OVER: tEncoderClear(&encoder); - if (code != 0) { + if (code < 0) { mError("failed to serialize ip white list at line %d since %s", lino, tstrerror(code)); } if (pLen) *pLen = tlen; @@ -778,7 +785,7 @@ int32_t tDerializeIpWhileList(void *buf, int32_t len, SIpWhiteList *pList) { _OVER: tEndDecode(&decoder); tDecoderClear(&decoder); - if(code != 0) { + if (code < 0) { mError("failed to deserialize ip white list at line %d since %s", lino, tstrerror(code)); } @@ -805,7 +812,7 @@ static int32_t createIpWhiteList(void *buf, int32_t len, SIpWhiteList **ppList) _OVER: tEndDecode(&decoder); tDecoderClear(&decoder); - if (code != 0) { + if (code < 0) { taosMemoryFreeClear(p); mError("failed to create ip white list at line %d since %s", lino, tstrerror(code)); } @@ -1152,7 +1159,6 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_INT32(pRaw, dataPos, len, _OVER); SDB_SET_BINARY(pRaw, dataPos, buf, len, _OVER); - SDB_SET_INT64(pRaw, dataPos, pUser->ipWhiteListVer, _OVER); @@ -1161,7 +1167,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { _OVER: taosMemoryFree(buf); - if (code != 0) { + if (code < 0) { mError("user:%s, failed to encode user action to raw:%p at line %d since %s", pUser->user, pRaw, lino, tstrerror(code)); sdbFreeRaw(pRaw); @@ -1178,6 +1184,8 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t lino = 0; SSdbRow *pRow = NULL; SUserObj *pUser = NULL; + char *key = NULL; + char *value = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) { @@ -1299,7 +1307,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t keyLen = 0; SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); - char *key = taosMemoryCalloc(keyLen, sizeof(char)); + TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } @@ -1308,29 +1316,23 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); - char *value = taosMemoryCalloc(valuelen, sizeof(char)); + TAOS_MEMORY_REALLOC(value, valuelen * sizeof(char)); if (value == NULL) { - taosMemoryFree(key); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) if (taosHashPut(pUser->readTbs, key, keyLen, value, valuelen) != 0) { - taosMemoryFree(key); - taosMemoryFree(value); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - - taosMemoryFree(key); - taosMemoryFree(value); } for (int32_t i = 0; i < numOfWriteTbs; ++i) { int32_t keyLen = 0; SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); - char *key = taosMemoryCalloc(keyLen, sizeof(char)); + TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } @@ -1339,22 +1341,16 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); - char *value = taosMemoryCalloc(valuelen, sizeof(char)); + TAOS_MEMORY_REALLOC(value, valuelen * sizeof(char)); if (value == NULL) { - taosMemoryFree(key); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) if (taosHashPut(pUser->writeTbs, key, keyLen, value, valuelen) != 0) { - taosMemoryFree(key); - taosMemoryFree(value); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - - taosMemoryFree(key); - taosMemoryFree(value); } if (sver >= 6) { @@ -1362,7 +1358,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t keyLen = 0; SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); - char *key = taosMemoryCalloc(keyLen, sizeof(char)); + TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } @@ -1371,29 +1367,23 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); - char *value = taosMemoryCalloc(valuelen, sizeof(char)); + TAOS_MEMORY_REALLOC(value, valuelen * sizeof(char)); if (value == NULL) { - taosMemoryFree(key); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) if (taosHashPut(pUser->alterTbs, key, keyLen, value, valuelen) != 0) { - taosMemoryFree(key); - taosMemoryFree(value); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - - taosMemoryFree(key); - taosMemoryFree(value); } for (int32_t i = 0; i < numOfReadViews; ++i) { int32_t keyLen = 0; SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); - char *key = taosMemoryCalloc(keyLen, sizeof(char)); + TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } @@ -1402,29 +1392,23 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); - char *value = taosMemoryCalloc(valuelen, sizeof(char)); + TAOS_MEMORY_REALLOC(value, valuelen * sizeof(char)); if (value == NULL) { - taosMemoryFree(key); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) if (taosHashPut(pUser->readViews, key, keyLen, value, valuelen) != 0) { - taosMemoryFree(key); - taosMemoryFree(value); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - - taosMemoryFree(key); - taosMemoryFree(value); } for (int32_t i = 0; i < numOfWriteViews; ++i) { int32_t keyLen = 0; SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); - char *key = taosMemoryCalloc(keyLen, sizeof(char)); + TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } @@ -1433,29 +1417,23 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); - char *value = taosMemoryCalloc(valuelen, sizeof(char)); + TAOS_MEMORY_REALLOC(value, valuelen * sizeof(char)); if (value == NULL) { - taosMemoryFree(key); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) if (taosHashPut(pUser->writeViews, key, keyLen, value, valuelen) != 0) { - taosMemoryFree(key); - taosMemoryFree(value); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - - taosMemoryFree(key); - taosMemoryFree(value); } for (int32_t i = 0; i < numOfAlterViews; ++i) { int32_t keyLen = 0; SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); - char *key = taosMemoryCalloc(keyLen, sizeof(char)); + TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } @@ -1464,22 +1442,16 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); - char *value = taosMemoryCalloc(valuelen, sizeof(char)); + TAOS_MEMORY_REALLOC(value, valuelen * sizeof(char)); if (value == NULL) { - taosMemoryFree(key); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) if (taosHashPut(pUser->alterViews, key, keyLen, value, valuelen) != 0) { - taosMemoryFree(key); - taosMemoryFree(value); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - - taosMemoryFree(key); - taosMemoryFree(value); } } @@ -1487,7 +1459,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t keyLen = 0; SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); - char *key = taosMemoryCalloc(keyLen, sizeof(char)); + TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } @@ -1498,10 +1470,8 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &ref, _OVER); if (taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref)) != 0) { - taosMemoryFree(key); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - taosMemoryFree(key); } } // decoder white list @@ -1509,31 +1479,32 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t len = 0; SDB_GET_INT32(pRaw, dataPos, &len, _OVER); - char *buf = taosMemoryMalloc(len); - if(buf == NULL) { + TAOS_MEMORY_REALLOC(key, len); + if (key == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - SDB_GET_BINARY(pRaw, dataPos, buf, len, _OVER); + SDB_GET_BINARY(pRaw, dataPos, key, len, _OVER); - code = createIpWhiteList(buf, len, &pUser->pIpWhiteList); - taosMemoryFree(buf); + TAOS_CHECK_GOTO(createIpWhiteList(key, len, &pUser->pIpWhiteList), &lino, _OVER); SDB_GET_INT64(pRaw, dataPos, &pUser->ipWhiteListVer, _OVER); } if (pUser->pIpWhiteList == NULL) { - createDefaultIpWhiteList(&pUser->pIpWhiteList); + TAOS_CHECK_GOTO(createDefaultIpWhiteList(&pUser->pIpWhiteList), &lino, _OVER); pUser->ipWhiteListVer = taosGetTimestampMs(); } SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) taosInitRWLatch(&pUser->lock); - terrno = 0; - _OVER: - if (terrno != 0) { - mError("user:%s, failed to decode from raw:%p since %s", pUser == NULL ? "null" : pUser->user, pRaw, terrstr()); + taosMemoryFree(key); + taosMemoryFree(value); + if (code < 0) { + terrno = code; + mError("user:%s, failed to decode at line %d from raw:%p since %s", pUser == NULL ? "null" : pUser->user, lino, + pRaw, tstrerror(code)); if (pUser != NULL) { taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); @@ -1587,7 +1558,6 @@ int32_t mndDupTableHash(SHashObj *pOld, SHashObj **ppNew) { if ((code = taosHashPut(*ppNew, key, keyLen, tb, valueLen)) != 0) { taosHashCancelIterate(pOld, tb); taosHashCleanup(*ppNew); - code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; // TODO: remove this line after terrno is removed TAOS_RETURN(code); } tb = taosHashIterate(pOld, tb); @@ -1612,7 +1582,6 @@ int32_t mndDupUseDbHash(SHashObj *pOld, SHashObj **ppNew) { if ((code = taosHashPut(*ppNew, key, keyLen, db, sizeof(*db))) != 0) { taosHashCancelIterate(pOld, db); taosHashCleanup(*ppNew); - code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; // TODO: remove this line after terrno is removed TAOS_RETURN(code); } db = taosHashIterate(pOld, db); @@ -1700,7 +1669,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { TSWAP(pOld->useDbs, pNew->useDbs); int32_t sz = sizeof(SIpWhiteList) + pNew->pIpWhiteList->num * sizeof(SIpV4Range); - pOld->pIpWhiteList = taosMemoryRealloc(pOld->pIpWhiteList, sz); + TAOS_MEMORY_REALLOC(pOld->pIpWhiteList, sz); if (pOld->pIpWhiteList == NULL) { taosWUnLockLatch(&pOld->lock); return TSDB_CODE_OUT_OF_MEMORY; @@ -1756,7 +1725,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate TAOS_CHECK_RETURN(createDefaultIpWhiteList(&userObj.pIpWhiteList)); } else { SHashObj *pUniqueTab = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); - if(pUniqueTab == NULL){ + if (pUniqueTab == NULL) { TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } int32_t dummpy = 0; @@ -1783,8 +1752,8 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate taosHashCleanup(pUniqueTab); TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - void *pIter = taosHashIterate(pUniqueTab, NULL); - int32_t i = 0; + void *pIter = taosHashIterate(pUniqueTab, NULL); + int32_t i = 0; while (pIter) { size_t len = 0; SIpV4Range *key = taosHashGetKey(pIter, &len); @@ -1858,50 +1827,38 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { #endif if (createReq.isImport != 1) { - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER), &lino, _OVER); } else { if (strcmp(pReq->info.conn.user, "root") != 0) { mError("The operation is not permitted, user:%s", pReq->info.conn.user); - code = TSDB_CODE_MND_NO_RIGHTS; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_NO_RIGHTS, &lino, _OVER); } } if (createReq.user[0] == 0) { - code = TSDB_CODE_MND_INVALID_USER_FORMAT; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_USER_FORMAT, &lino, _OVER); } if (createReq.pass[0] == 0) { - code = TSDB_CODE_MND_INVALID_PASS_FORMAT; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_PASS_FORMAT, &lino, _OVER); } if (createReq.isImport != 1) { if (strlen(createReq.pass) >= TSDB_PASSWORD_LEN) { - code = TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG, &lino, _OVER); } } code = mndAcquireUser(pMnode, createReq.user, &pUser); if (pUser != NULL) { - code = TSDB_CODE_MND_USER_ALREADY_EXIST; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_USER_ALREADY_EXIST, &lino, _OVER); } code = mndAcquireUser(pMnode, pReq->info.conn.user, &pOperUser); if (pOperUser == NULL) { - code = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_MND_NO_USER_FROM_CONN, &lino, _OVER); } - if ((terrno = grantCheck(TSDB_GRANT_USER)) != 0) { - code = terrno; - goto _OVER; - } TAOS_CHECK_GOTO(grantCheck(TSDB_GRANT_USER), &lino, _OVER); code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); @@ -1920,8 +1877,8 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { auditRecord(pReq, pMnode->clusterId, operation, "", createReq.user, detail, strlen(detail)); _OVER: - if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("user:%s, failed to create since %s", createReq.user, terrstr()); + if (code < 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("user:%s, failed to create at line %d since %s", createReq.user, lino, tstrerror(code)); } mndReleaseUser(pMnode, pUser); @@ -1963,14 +1920,14 @@ int32_t mndProcessGetUserWhiteListReq(SRpcMsg *pReq) { } contLen = tSerializeSGetUserWhiteListRsp(pRsp, contLen, &wlRsp); - if(contLen < 0) { + if (contLen < 0) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } _OVER: mndReleaseUser(pMnode, pUser); tFreeSGetUserWhiteListRsp(&wlRsp); - if (code != 0) { + if (code < 0) { mError("user:%s, failed to get whitelist at line %d since %s", wlReq.user, lino, tstrerror(code)); rpcFreeCont(pRsp); pRsp = NULL; @@ -2016,7 +1973,7 @@ int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq) { } _OVER: - if (code != 0) { + if (code < 0) { mError("failed to process retrieve ip white request at line %d since %s", lino, tstrerror(code)); rpcFreeCont(pRsp); pRsp = NULL; @@ -2074,7 +2031,6 @@ static int32_t mndDupObjHash(SHashObj *pOld, int32_t dataLen, SHashObj **ppNew) while (db != NULL) { int32_t len = strlen(db) + 1; if ((code = taosHashPut(*ppNew, db, len, db, dataLen)) != 0) { - if (terrno != 0) code = terrno; // TODO: remove this line after terrno is removed taosHashCancelIterate(pOld, db); taosHashCleanup(*ppNew); TAOS_RETURN(code); @@ -2181,7 +2137,7 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname); if (pDb == NULL) { mndReleaseDb(pMnode, pDb); - TAOS_RETURN(terrno); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } if ((code = taosHashPut(pNewUser->readDbs, pAlterReq->objname, len, pAlterReq->objname, TSDB_DB_FNAME_LEN)) != 0) { @@ -2336,7 +2292,7 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode } _OVER: - if (code != 0) { + if (code < 0) { mError("user:%s, failed to alter user privileges at line %d since %s", pAlterReq->user, lino, terrstr()); } TAOS_RETURN(code); @@ -2450,7 +2406,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { bool localHost = false; SIpWhiteList *pNew = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range) * num); - if(pNew == NULL) { + if (pNew == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } @@ -2541,8 +2497,8 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { } _OVER: - if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("user:%s, failed to alter since %s", alterReq.user, terrstr()); + if (code < 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("user:%s, failed to alter at line %d since %s", alterReq.user, lino, tstrerror(code)); } tFreeSAlterUserReq(&alterReq); @@ -2604,7 +2560,7 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { auditRecord(pReq, pMnode->clusterId, "dropUser", "", dropReq.user, dropReq.sql, dropReq.sqlLen); _OVER: - if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + if (code < 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("user:%s, failed to drop at line %d since %s", dropReq.user, lino, tstrerror(code)); } @@ -2644,7 +2600,7 @@ static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq) { _OVER: mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); - if (code != 0) { + if (code < 0) { mError("user:%s, failed to get auth at line %d since %s", authReq.user, lino, tstrerror(code)); rpcFreeCont(pRsp); pRsp = NULL; @@ -2666,7 +2622,9 @@ static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl SUserObj *pUser = NULL; int32_t cols = 0; int8_t flag = 0; - char *pWrite; + char *pWrite = NULL; + char *buf = NULL; + char *varstr = NULL; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_USER, pShow->pIter, (void **)&pUser); @@ -2676,47 +2634,49 @@ static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->pMeta->pSchemas[cols].bytes); - colDataSetVal(pColInfo, numOfRows, (const char *)name, false); + COL_DATA_SET_VAL_GOTO((const char *)name, false, pUser, _exit); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->superUser, false); + COL_DATA_SET_VAL_GOTO((const char *)&pUser->superUser, false, pUser, _exit); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->enable, false); + COL_DATA_SET_VAL_GOTO((const char *)&pUser->enable, false, pUser, _exit); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->sysInfo, false); + COL_DATA_SET_VAL_GOTO((const char *)&pUser->sysInfo, false, pUser, _exit); cols++; flag = pUser->createdb ? 1 : 0; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataSetVal(pColInfo, numOfRows, (const char *)&flag, false); + COL_DATA_SET_VAL_GOTO((const char *)&flag, false, pUser, _exit); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->createdTime, false); + COL_DATA_SET_VAL_GOTO((const char *)&pUser->createdTime, false, pUser, _exit); cols++; - char *buf = NULL; int32_t tlen = convertIpWhiteListToStr(pUser->pIpWhiteList, &buf); // int32_t tlen = mndFetchIpWhiteList(pUser->pIpWhiteList, &buf); if (tlen != 0) { - char *varstr = taosMemoryCalloc(1, VARSTR_HEADER_SIZE + tlen); + TAOS_MEMORY_REALLOC(varstr, VARSTR_HEADER_SIZE + tlen); + if (varstr == NULL) { + sdbRelease(pSdb, pUser); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } varDataSetLen(varstr, tlen); memcpy(varDataVal(varstr), buf, tlen); pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataSetVal(pColInfo, numOfRows, (const char *)varstr, false); + COL_DATA_SET_VAL_GOTO((const char *)varstr, false, pUser, _exit); - taosMemoryFree(varstr); - taosMemoryFree(buf); + taosMemoryFreeClear(buf); } else { pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataSetVal(pColInfo, numOfRows, (const char *)NULL, true); + COL_DATA_SET_VAL_GOTO((const char *)NULL, true, pUser, _exit); } numOfRows++; @@ -2725,7 +2685,9 @@ static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pShow->numOfRows += numOfRows; _exit: - if (code != 0) { + taosMemoryFreeClear(buf); + taosMemoryFreeClear(varstr); + if (code < 0) { uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); TAOS_RETURN(code); } @@ -2738,10 +2700,13 @@ static int32_t mndRetrieveUsersFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; SUserObj *pUser = NULL; + int32_t code = 0; + int32_t lino = 0; int32_t cols = 0; int8_t flag = 0; - char *pWrite; - int32_t code = 0; + char *pWrite = NULL; + char *buf = NULL; + char *varstr = NULL; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_USER, pShow->pIter, (void **)&pUser); @@ -2751,58 +2716,52 @@ static int32_t mndRetrieveUsersFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->pMeta->pSchemas[cols].bytes); - code = colDataSetVal(pColInfo, numOfRows, (const char *)name, false); - if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code)); + COL_DATA_SET_VAL_GOTO((const char *)name, false, pUser, _exit); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - code = colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->superUser, false); - if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code)); + COL_DATA_SET_VAL_GOTO((const char *)&pUser->superUser, false, pUser, _exit); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - code = colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->enable, false); - if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code)); + COL_DATA_SET_VAL_GOTO((const char *)&pUser->enable, false, pUser, _exit); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - code = colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->sysInfo, false); - if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code)); + COL_DATA_SET_VAL_GOTO((const char *)&pUser->sysInfo, false, pUser, _exit); cols++; flag = pUser->createdb ? 1 : 0; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - code = colDataSetVal(pColInfo, numOfRows, (const char *)&flag, false); - if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code)); + COL_DATA_SET_VAL_GOTO((const char *)&flag, false, pUser, _exit); // mInfo("pUser->pass:%s", pUser->pass); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); char pass[TSDB_PASSWORD_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(pass, pUser->pass, pShow->pMeta->pSchemas[cols].bytes); - code = colDataSetVal(pColInfo, numOfRows, (const char *)pass, false); - if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code)); + COL_DATA_SET_VAL_GOTO((const char *)pass, false, pUser, _exit); cols++; - char *buf = NULL; int32_t tlen = convertIpWhiteListToStr(pUser->pIpWhiteList, &buf); // int32_t tlen = mndFetchIpWhiteList(pUser->pIpWhiteList, &buf); if (tlen != 0) { - char *varstr = taosMemoryCalloc(1, VARSTR_HEADER_SIZE + tlen); + TAOS_MEMORY_REALLOC(varstr, VARSTR_HEADER_SIZE + tlen); + if (varstr == NULL) { + sdbRelease(pSdb, pUser); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } varDataSetLen(varstr, tlen); memcpy(varDataVal(varstr), buf, tlen); pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - code = colDataSetVal(pColInfo, numOfRows, (const char *)varstr, false); - if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code)); + COL_DATA_SET_VAL_GOTO((const char *)varstr, false, pUser, _exit); - taosMemoryFree(varstr); - taosMemoryFree(buf); + taosMemoryFreeClear(buf); } else { pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - code = colDataSetVal(pColInfo, numOfRows, (const char *)NULL, true); - if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code)); + COL_DATA_SET_VAL_GOTO((const char *)NULL, true, pUser, _exit); } numOfRows++; @@ -2810,6 +2769,13 @@ static int32_t mndRetrieveUsersFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock } pShow->numOfRows += numOfRows; +_exit: + taosMemoryFreeClear(buf); + taosMemoryFreeClear(varstr); + if (code < 0) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + TAOS_RETURN(code); + } #endif return numOfRows; } @@ -2819,22 +2785,26 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int32_t *numOfRows, char *user, - SShowObj *pShow) { +static int32_t mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int32_t *pNumOfRows, SSdb *pSdb, + SUserObj *pUser, SShowObj *pShow, char **condition, char **sql) { char *value = taosHashIterate(hash, NULL); + char *user = pUser->user; + int32_t code = 0; + int32_t lino = 0; int32_t cols = 0; + int32_t numOfRows = *pNumOfRows; while (value != NULL) { cols = 0; char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, user, pShow->pMeta->pSchemas[cols].bytes); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)userName, false); + COL_DATA_SET_VAL_GOTO((const char *)userName, false, NULL, _exit); char privilege[20] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(privilege, priType, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)privilege, false); + COL_DATA_SET_VAL_GOTO((const char *)privilege, false, NULL, _exit); size_t keyLen = 0; void *key = taosHashGetKey(value, &keyLen); @@ -2844,67 +2814,87 @@ static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int3 char dbNameContent[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(dbNameContent, dbName, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)dbNameContent, false); + COL_DATA_SET_VAL_GOTO((const char *)dbNameContent, false, NULL, _exit); char tableName[TSDB_TABLE_NAME_LEN] = {0}; mndExtractTbNameFromStbFullName(key, tableName, TSDB_TABLE_NAME_LEN); char tableNameContent[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(tableNameContent, tableName, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)tableNameContent, false); + COL_DATA_SET_VAL_GOTO((const char *)tableNameContent, false, NULL, _exit); if (strcmp("t", value) != 0 && strcmp("v", value) != 0) { SNode *pAst = NULL; int32_t sqlLen = 0; size_t bufSz = strlen(value) + 1; - char *sql = taosMemoryMalloc(bufSz + 1); - char *obj = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + if (bufSz < 5) bufSz = 5; + TAOS_MEMORY_REALLOC(*sql, bufSz + 1); + if (*sql == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + TAOS_MEMORY_REALLOC(*condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + if ((*condition) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } - if (sql != NULL && obj != NULL && nodesStringToNode(value, &pAst) == 0) { - nodesNodeToSQL(pAst, sql, bufSz, &sqlLen); + if (nodesStringToNode(value, &pAst) == 0) { + nodesNodeToSQL(pAst, *sql, bufSz, &sqlLen); nodesDestroyNode(pAst); } else { sqlLen = 5; - sprintf(sql, "error"); + sprintf(*sql, "error"); } - STR_WITH_MAXSIZE_TO_VARSTR(obj, sql, pShow->pMeta->pSchemas[cols].bytes); + STR_WITH_MAXSIZE_TO_VARSTR((*condition), (*sql), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)obj, false); - taosMemoryFree(obj); - taosMemoryFree(sql); + COL_DATA_SET_VAL_GOTO((const char *)(*condition), false, NULL, _exit); char notes[2] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)notes, false); + COL_DATA_SET_VAL_GOTO((const char *)notes, false, NULL, _exit); } else { - char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); - STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); + TAOS_MEMORY_REALLOC(*condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + if ((*condition) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + STR_WITH_MAXSIZE_TO_VARSTR((*condition), "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)condition, false); - taosMemoryFree(condition); + COL_DATA_SET_VAL_GOTO((const char *)(*condition), false, NULL, _exit); char notes[64 + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(notes, value[0] == 'v' ? "view" : "", sizeof(notes)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)notes, false); + COL_DATA_SET_VAL_GOTO((const char *)notes, false, NULL, _exit); } - (*numOfRows)++; + numOfRows++; value = taosHashIterate(hash, value); } + *pNumOfRows = numOfRows; +_exit: + if (code < 0) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + sdbRelease(pSdb, pUser); + } + TAOS_RETURN(code); } static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { int32_t code = 0; + int32_t lino = 0; SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SUserObj *pUser = NULL; int32_t cols = 0; - char *pWrite; + char *pWrite = NULL; + char *condition = NULL; + char *sql = NULL; bool fetchNextUser = pShow->restore ? false : true; pShow->restore = false; @@ -2966,11 +2956,14 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + TAOS_MEMORY_REALLOC(condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + if (condition == NULL) { + sdbRelease(pSdb, pUser); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); - taosMemoryFree(condition); char notes[2] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes)); @@ -3006,7 +2999,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + TAOS_MEMORY_REALLOC(condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + if (condition == NULL) { + sdbRelease(pSdb, pUser); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); @@ -3047,7 +3044,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + TAOS_MEMORY_REALLOC(condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + if (condition == NULL) { + sdbRelease(pSdb, pUser); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); @@ -3062,17 +3063,17 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock db = taosHashIterate(pUser->writeDbs, db); } - mndLoopHash(pUser->readTbs, "read", pBlock, &numOfRows, pUser->user, pShow); + TAOS_CHECK_EXIT(mndLoopHash(pUser->readTbs, "read", pBlock, &numOfRows, pSdb, pUser, pShow, &condition, &sql)); - mndLoopHash(pUser->writeTbs, "write", pBlock, &numOfRows, pUser->user, pShow); + TAOS_CHECK_EXIT(mndLoopHash(pUser->writeTbs, "write", pBlock, &numOfRows, pSdb, pUser, pShow, &condition, &sql)); - mndLoopHash(pUser->alterTbs, "alter", pBlock, &numOfRows, pUser->user, pShow); + TAOS_CHECK_EXIT(mndLoopHash(pUser->alterTbs, "alter", pBlock, &numOfRows, pSdb, pUser, pShow, &condition, &sql)); - mndLoopHash(pUser->readViews, "read", pBlock, &numOfRows, pUser->user, pShow); + TAOS_CHECK_EXIT(mndLoopHash(pUser->readViews, "read", pBlock, &numOfRows, pSdb, pUser, pShow, &condition, &sql)); - mndLoopHash(pUser->writeViews, "write", pBlock, &numOfRows, pUser->user, pShow); + TAOS_CHECK_EXIT(mndLoopHash(pUser->writeViews, "write", pBlock, &numOfRows, pSdb, pUser, pShow, &condition, &sql)); - mndLoopHash(pUser->alterViews, "alter", pBlock, &numOfRows, pUser->user, pShow); + TAOS_CHECK_EXIT(mndLoopHash(pUser->alterViews, "alter", pBlock, &numOfRows, pSdb, pUser, pShow, &condition, &sql)); char *topic = taosHashIterate(pUser->topics, NULL); while (topic != NULL) { @@ -3098,7 +3099,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + TAOS_MEMORY_REALLOC(condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + if (condition == NULL) { + sdbRelease(pSdb, pUser); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); @@ -3117,6 +3122,13 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock } pShow->numOfRows += numOfRows; +_exit: + taosMemoryFreeClear(condition); + taosMemoryFreeClear(sql); + if (code < 0) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + TAOS_RETURN(code); + } return numOfRows; } @@ -3127,13 +3139,17 @@ static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter) { int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen, int64_t ipWhiteListVer) { + int32_t code = 0; + int32_t lino = 0; + int32_t rspLen = 0; + void *pRsp = NULL; SUserAuthBatchRsp batchRsp = {0}; + batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp)); if (batchRsp.pArray == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - int32_t code = 0; for (int32_t i = 0; i < numOfUses; ++i) { SUserObj *pUser = NULL; code = mndAcquireUser(pMnode, pUsers[i].user, &pUser); @@ -3141,7 +3157,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ if (TSDB_CODE_MND_USER_NOT_EXIST == code) { SGetUserAuthRsp rsp = {.dropped = 1}; memcpy(rsp.user, pUsers[i].user, TSDB_USER_LEN); - taosArrayPush(batchRsp.pArray, &rsp); + (void)taosArrayPush(batchRsp.pArray, &rsp); } mError("user:%s, failed to auth user since %s", pUsers[i].user, terrstr()); continue; @@ -3158,7 +3174,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ if (code) { mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&rsp); - goto _OVER; + TAOS_CHECK_GOTO(code, &lino, _OVER); } (void)taosArrayPush(batchRsp.pArray, &rsp); @@ -3173,25 +3189,25 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ return 0; } - int32_t rspLen = tSerializeSUserAuthBatchRsp(NULL, 0, &batchRsp); - void *pRsp = taosMemoryMalloc(rspLen); - if (pRsp == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); + rspLen = tSerializeSUserAuthBatchRsp(NULL, 0, &batchRsp); + if (rspLen <= 0) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } + pRsp = taosMemoryMalloc(rspLen); + if (pRsp == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + } + TAOS_CHECK_GOTO(tSerializeSUserAuthBatchRsp(pRsp, rspLen, &batchRsp), &lino, _OVER); +_OVER: + tFreeSUserAuthBatchRsp(&batchRsp); + if (code < 0) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + taosMemoryFreeClear(pRsp); + rspLen = 0; } - tSerializeSUserAuthBatchRsp(pRsp, rspLen, &batchRsp); - *ppRsp = pRsp; *pRspLen = rspLen; - tFreeSUserAuthBatchRsp(&batchRsp); - return 0; - -_OVER: - - *ppRsp = NULL; - *pRspLen = 0; - - tFreeSUserAuthBatchRsp(&batchRsp); TAOS_RETURN(code); } @@ -3219,7 +3235,7 @@ int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); if (pCommitRaw == NULL || (code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) { - code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; // TODO: remove this line when terrno is removed + code = TSDB_CODE_OUT_OF_MEMORY; break; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -3229,6 +3245,7 @@ int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { sdbRelease(pSdb, pUser); } +_OVER: if (pUser != NULL) sdbRelease(pSdb, pUser); if (pIter != NULL) sdbCancelFetch(pSdb, pIter); mndUserFreeObj(&newUser); @@ -3261,7 +3278,7 @@ int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb) { SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); if (pCommitRaw == NULL || (code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) { - code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; // TODO: remove this line when terrno is removed + code = TSDB_CODE_OUT_OF_MEMORY; break; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -3303,7 +3320,7 @@ int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view) { SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); if (pCommitRaw == NULL || (code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) { - code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; // TODO: remove this line when terrno is removed + code = TSDB_CODE_OUT_OF_MEMORY; break; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -3316,7 +3333,7 @@ int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view) { if (pUser != NULL) sdbRelease(pSdb, pUser); if (pIter != NULL) sdbCancelFetch(pSdb, pIter); mndUserFreeObj(&newUser); - TAOS_RETURN(code); + TAOS_RETURN(code); } int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { @@ -3342,7 +3359,7 @@ int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { (void)taosHashRemove(newUser.topics, topic, len); SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); if (pCommitRaw == NULL || (code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) { - code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; // TODO: remove this line when terrno is removed + code = TSDB_CODE_OUT_OF_MEMORY; break; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); From 4557cf05cdbbfe0775fcb89727d5ded9dec9a5c1 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 23 Jul 2024 18:40:58 +0800 Subject: [PATCH 20/52] enh: refactor return code --- include/util/tutil.h | 4 +- source/dnode/mnode/impl/src/mndGrant.c | 13 ++++- source/dnode/mnode/impl/src/mndUser.c | 66 +++++++++++++------------- 3 files changed, 45 insertions(+), 38 deletions(-) diff --git a/include/util/tutil.h b/include/util/tutil.h index 823ffba23b..05a2397139 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -160,7 +160,7 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, #define TAOS_CHECK_RETURN(CMD) \ do { \ int32_t code = (CMD); \ - if (code < TSDB_CODE_SUCCESS) { \ + if (code != TSDB_CODE_SUCCESS) { \ TAOS_RETURN(code); \ } \ } while (0) @@ -168,7 +168,7 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, #define TAOS_CHECK_GOTO(CMD, LINO, LABEL) \ do { \ code = (CMD); \ - if (code < TSDB_CODE_SUCCESS) { \ + if (code != TSDB_CODE_SUCCESS) { \ if (LINO) { \ *((int32_t *)(LINO)) = __LINE__; \ } \ diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index ed22d9b497..af60bc9e4b 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -25,12 +25,16 @@ pColInfo = taosArrayGet(pBlock->pDataBlock, cols); \ src = (display); \ STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); \ - colDataSetVal(pColInfo, numOfRows, tmp, false); \ + COL_DATA_SET_VAL_GOTO(tmp, false, NULL, _exit); \ } while (0) static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; + int32_t code = 0; + int32_t lino = 0; char tmp[32]; if (pShow->numOfRows < 1) { @@ -38,7 +42,7 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); const char *src = TD_PRODUCT_NAME; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataSetVal(pColInfo, numOfRows, tmp, false); + COL_DATA_SET_VAL_GOTO(tmp, false, NULL, _exit); GRANT_ITEM_SHOW("unlimited"); GRANT_ITEM_SHOW("limited"); @@ -52,6 +56,11 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl } pShow->numOfRows += numOfRows; +_exit: + if (code != 0) { + mError("failed to retrieve grant at line %d since %s", lino, tstrerror(code)); + TAOS_RETURN(code); + } return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 1e6d09df44..055af836b4 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -2579,25 +2579,25 @@ static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq) { SGetUserAuthReq authReq = {0}; SGetUserAuthRsp authRsp = {0}; - TAOS_CHECK_GOTO(tDeserializeSGetUserAuthReq(pReq->pCont, pReq->contLen, &authReq), &lino, _OVER); + TAOS_CHECK_EXIT(tDeserializeSGetUserAuthReq(pReq->pCont, pReq->contLen, &authReq)); mTrace("user:%s, start to get auth", authReq.user); - TAOS_CHECK_GOTO(mndAcquireUser(pMnode, authReq.user, &pUser), &lino, _OVER); + TAOS_CHECK_EXIT(mndAcquireUser(pMnode, authReq.user, &pUser)); - TAOS_CHECK_GOTO(mndSetUserAuthRsp(pMnode, pUser, &authRsp), &lino, _OVER); + TAOS_CHECK_EXIT(mndSetUserAuthRsp(pMnode, pUser, &authRsp)); contLen = tSerializeSGetUserAuthRsp(NULL, 0, &authRsp); if (contLen < 0) { - TAOS_CHECK_GOTO(contLen, &lino, _OVER); + TAOS_CHECK_EXIT(contLen); } pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } - TAOS_CHECK_GOTO(tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp), &lino, _OVER); + TAOS_CHECK_EXIT(tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp)); -_OVER: +_exit: mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); if (code < 0) { @@ -2939,22 +2939,22 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)userName, false); + COL_DATA_SET_VAL_GOTO((const char *)userName, false, pUser, _exit); char privilege[20] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(privilege, "all", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false); + COL_DATA_SET_VAL_GOTO((const char *)privilege, false, pUser, _exit); char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(objName, "all", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); + COL_DATA_SET_VAL_GOTO((const char *)objName, false, pUser, _exit); char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); + COL_DATA_SET_VAL_GOTO((const char *)tableName, false, pUser, _exit); TAOS_MEMORY_REALLOC(condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); if (condition == NULL) { @@ -2963,12 +2963,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock } STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + COL_DATA_SET_VAL_GOTO((const char *)condition, false, pUser, _exit); char notes[2] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)notes, false); + COL_DATA_SET_VAL_GOTO((const char *)notes, false, pUser, _exit); numOfRows++; } @@ -2979,12 +2979,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)userName, false); + COL_DATA_SET_VAL_GOTO((const char *)userName, false, pUser, _exit); char privilege[20] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(privilege, "read", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false); + COL_DATA_SET_VAL_GOTO((const char *)privilege, false, pUser, _exit); SName name = {0}; char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -2992,12 +2992,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock tNameGetDbName(&name, varDataVal(objName)); varDataSetLen(objName, strlen(varDataVal(objName))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); + COL_DATA_SET_VAL_GOTO((const char *)objName, false, pUser, _exit); char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); + COL_DATA_SET_VAL_GOTO((const char *)tableName, false, pUser, _exit); TAOS_MEMORY_REALLOC(condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); if (condition == NULL) { @@ -3006,13 +3006,13 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock } STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); - taosMemoryFree(condition); + COL_DATA_SET_VAL_GOTO((const char *)condition, false, pUser, _exit); + char notes[2] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)notes, false); + COL_DATA_SET_VAL_GOTO((const char *)notes, false, pUser, _exit); numOfRows++; db = taosHashIterate(pUser->readDbs, db); @@ -3024,12 +3024,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)userName, false); + COL_DATA_SET_VAL_GOTO((const char *)userName, false, pUser, _exit); char privilege[20] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(privilege, "write", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false); + COL_DATA_SET_VAL_GOTO((const char *)privilege, false, pUser, _exit); SName name = {0}; char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -3037,12 +3037,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock tNameGetDbName(&name, varDataVal(objName)); varDataSetLen(objName, strlen(varDataVal(objName))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); + COL_DATA_SET_VAL_GOTO((const char *)objName, false, pUser, _exit); char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); + COL_DATA_SET_VAL_GOTO((const char *)tableName, false, pUser, _exit); TAOS_MEMORY_REALLOC(condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); if (condition == NULL) { @@ -3051,13 +3051,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock } STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); - taosMemoryFree(condition); + COL_DATA_SET_VAL_GOTO((const char *)condition, false, pUser, _exit); char notes[2] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)notes, false); + COL_DATA_SET_VAL_GOTO((const char *)notes, false, pUser, _exit); numOfRows++; db = taosHashIterate(pUser->writeDbs, db); @@ -3081,23 +3080,23 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)userName, false); + COL_DATA_SET_VAL_GOTO((const char *)userName, false, pUser, _exit); char privilege[20] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(privilege, "subscribe", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false); + COL_DATA_SET_VAL_GOTO((const char *)privilege, false, pUser, _exit); char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0}; tstrncpy(varDataVal(topicName), mndGetDbStr(topic), TSDB_TOPIC_NAME_LEN - 2); varDataSetLen(topicName, strlen(varDataVal(topicName))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false); + COL_DATA_SET_VAL_GOTO((const char *)topicName, false, pUser, _exit); char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); + COL_DATA_SET_VAL_GOTO((const char *)tableName, false, pUser, _exit); TAOS_MEMORY_REALLOC(condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); if (condition == NULL) { @@ -3106,13 +3105,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock } STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); - taosMemoryFree(condition); + COL_DATA_SET_VAL_GOTO((const char *)condition, false, pUser, _exit); char notes[2] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)notes, false); + COL_DATA_SET_VAL_GOTO((const char *)notes, false, pUser, _exit); numOfRows++; topic = taosHashIterate(pUser->topics, topic); From 4d8064c950251925a2998b51729f6dc88e891ba9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 Jul 2024 19:16:06 +0800 Subject: [PATCH 21/52] fix(tsdb):fix invalid return code. --- source/dnode/mnode/impl/src/mndStream.c | 15 +++++++------ source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 22 +++++++++++++++++--- source/util/src/tcompression.c | 4 ++-- tests/script/tsim/parser/last_both_query.sim | 1 + 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 31d28d99a4..b3e81665e3 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -255,14 +255,13 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream } int32_t mndAcquireStream(SMnode *pMnode, char *streamName, SStreamObj **pStream) { - terrno = 0; - + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; (*pStream) = sdbAcquire(pSdb, SDB_STREAM, streamName); if ((*pStream) == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { - terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; + code = TSDB_CODE_MND_STREAM_NOT_EXIST; } - return terrno; + return code; } void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { @@ -754,19 +753,19 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } code = mndAcquireStream(pMnode, createReq.name, &pStream); - if (pStream != NULL || code == 0) { + if (pStream != NULL && code == 0) { if (createReq.igExists) { mInfo("stream:%s, already exist, ignore exist is set", createReq.name); goto _OVER; } else { - terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; + code = TSDB_CODE_MND_STREAM_ALREADY_EXIST; goto _OVER; } - } else if (terrno != TSDB_CODE_MND_STREAM_NOT_EXIST) { + } else if (code != TSDB_CODE_MND_STREAM_NOT_EXIST) { goto _OVER; } - if ((terrno = grantCheck(TSDB_GRANT_STREAMS)) < 0) { + if ((code = grantCheck(TSDB_GRANT_STREAMS)) < 0) { goto _OVER; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 97da98e9c6..38a2dd3ab2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -89,8 +89,8 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p if (pVal == NULL) { return TSDB_CODE_INVALID_PARA; } - funcType = *(int32_t*) pVal; + funcType = *(int32_t*) pVal; pVal = taosArrayGet(pReader->pFuncTypeList, i); if (pVal == NULL) { return TSDB_CODE_INVALID_PARA; @@ -471,6 +471,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 int32_t bytes = (slotIds[j] == -1) ? 1 : pr->pSchema->columns[slotIds[j]].bytes; pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + bytes + pkBufLen + VARSTR_HEADER_SIZE); + if (pRes[j] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]); p->ts = INT64_MIN; } @@ -512,7 +517,12 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 for (int32_t j = 0; j < pr->rowKey.numOfPKs; j++) { p.rowKey.pks[j].type = pr->pkColumn.type; if (IS_VAR_DATA_TYPE(pr->pkColumn.type)) { + p.rowKey.pks[j].pData = taosMemoryCalloc(1, pr->pkColumn.bytes); + if (p.rowKey.pks[j].pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } } } } @@ -538,7 +548,9 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 tb_uid_t uid = pTableList[i].uid; code = tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); - if (code) { + if (code == -1) {// fix the invalid return code + code = 0; + } else if (code != 0) { goto _end; } @@ -642,7 +654,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 tb_uid_t uid = pTableList[i].uid; if ((code = tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype)) != 0) { - goto _end; + if (code == -1) {// fix the invalid return code + code = 0; + } else if (code != 0) { + goto _end; + } } if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 884d7ea1b6..d8656c0f60 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -252,7 +252,7 @@ int32_t l2ComressInitImpl_xz(char *lossyColumns, float fPrecision, double dPreci } int32_t l2CompressImpl_xz(const char *const input, const int32_t inputSize, char *const output, int32_t outputSize, const char type, int8_t lvl) { - size_t len = FL2_compress(output + 1, outputSize - 1, input, inputSize, lvl); + size_t len = 0;//FL2_compress(output + 1, outputSize - 1, input, inputSize, lvl); if (len > inputSize) { output[0] = 0; memcpy(output + 1, input, inputSize); @@ -264,7 +264,7 @@ int32_t l2CompressImpl_xz(const char *const input, const int32_t inputSize, char int32_t l2DecompressImpl_xz(const char *const input, const int32_t compressedSize, char *const output, int32_t outputSize, const char type) { if (input[0] == 1) { - return FL2_decompress(output, outputSize, input + 1, compressedSize - 1); + return 0;//FL2_decompress(output, outputSize, input + 1, compressedSize - 1); } else if (input[0] == 0) { memcpy(output, input + 1, compressedSize - 1); return compressedSize - 1; diff --git a/tests/script/tsim/parser/last_both_query.sim b/tests/script/tsim/parser/last_both_query.sim index 5f86412199..b082351e0b 100644 --- a/tests/script/tsim/parser/last_both_query.sim +++ b/tests/script/tsim/parser/last_both_query.sim @@ -16,6 +16,7 @@ endi sql select last(f1) from tb1 if $rows != 1 then + print expect 1, actual $rows return -1 endi if $data00 != 6 then From d8700998dc8d6ab2dc5ef17b49843290d0522d9e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 Jul 2024 19:17:00 +0800 Subject: [PATCH 22/52] fix(util): fix error --- source/util/src/tcompression.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index d8656c0f60..884d7ea1b6 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -252,7 +252,7 @@ int32_t l2ComressInitImpl_xz(char *lossyColumns, float fPrecision, double dPreci } int32_t l2CompressImpl_xz(const char *const input, const int32_t inputSize, char *const output, int32_t outputSize, const char type, int8_t lvl) { - size_t len = 0;//FL2_compress(output + 1, outputSize - 1, input, inputSize, lvl); + size_t len = FL2_compress(output + 1, outputSize - 1, input, inputSize, lvl); if (len > inputSize) { output[0] = 0; memcpy(output + 1, input, inputSize); @@ -264,7 +264,7 @@ int32_t l2CompressImpl_xz(const char *const input, const int32_t inputSize, char int32_t l2DecompressImpl_xz(const char *const input, const int32_t compressedSize, char *const output, int32_t outputSize, const char type) { if (input[0] == 1) { - return 0;//FL2_decompress(output, outputSize, input + 1, compressedSize - 1); + return FL2_decompress(output, outputSize, input + 1, compressedSize - 1); } else if (input[0] == 0) { memcpy(output, input + 1, compressedSize - 1); return compressedSize - 1; From 994a08ad41a73fb30aeccedd9b202b529ef42c5d Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 23 Jul 2024 19:30:55 +0800 Subject: [PATCH 23/52] enh: refactor return code --- source/dnode/vnode/src/sma/smaOpen.c | 119 +++++++++++----------- source/dnode/vnode/src/sma/smaRollup.c | 32 +++--- source/dnode/vnode/src/sma/smaTimeRange.c | 89 +++++++--------- 3 files changed, 113 insertions(+), 127 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 29854b4441..19f0c4a964 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -16,38 +16,34 @@ #include "sma.h" #include "tsdb.h" -static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration); +static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration, + int32_t *days); static int32_t smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type); static int32_t rsmaRestore(SSma *pSma); -#define SMA_SET_KEEP_CFG(v, l) \ - do { \ - SRetention *r = &pCfg->retentions[l]; \ - int64_t keep = -1; \ - convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE, &keep); \ - pKeepCfg->keep2 = (int32_t)keep; \ - pKeepCfg->keep0 = pKeepCfg->keep2; \ - pKeepCfg->keep1 = pKeepCfg->keep2; \ - pKeepCfg->days = smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days); \ - pKeepCfg->keepTimeOffset = 0; \ +#define SMA_SET_KEEP_CFG(v, l) \ + do { \ + SRetention *r = &pCfg->retentions[l]; \ + int64_t keep = -1; \ + TAOS_CHECK_EXIT(convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE, &keep)); \ + pKeepCfg->keep2 = (int32_t)keep; \ + pKeepCfg->keep0 = pKeepCfg->keep2; \ + pKeepCfg->keep1 = pKeepCfg->keep2; \ + TAOS_CHECK_EXIT(smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days, &pKeepCfg->days)); \ + pKeepCfg->keepTimeOffset = 0; \ } while (0) -#define SMA_OPEN_RSMA_IMPL(v, l, force) \ - do { \ - SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \ - if (!RETENTION_VALID(l, r)) { \ - if (l == 0) { \ - code = TSDB_CODE_INVALID_PARA; \ - TSDB_CHECK_CODE(code, lino, _exit); \ - } \ - break; \ - } \ - code = smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \ - TSDB_CHECK_CODE(code, lino, _exit); \ - if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback, force) < 0) { \ - code = terrno; \ - TSDB_CHECK_CODE(code, lino, _exit); \ - } \ +#define SMA_OPEN_RSMA_IMPL(v, l, force) \ + do { \ + SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \ + if (!RETENTION_VALID(l, r)) { \ + if (l == 0) { \ + TAOS_CHECK_EXIT(TSDB_CODE_INVALID_PARA); \ + } \ + break; \ + } \ + TAOS_CHECK_EXIT(smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l)); \ + TAOS_CHECK_EXIT(tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback, force)); \ } while (0) /** @@ -59,51 +55,61 @@ static int32_t rsmaRestore(SSma *pSma); * @param level * @param precision * @param duration + * @param days * @return int32_t */ -static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration) { - int32_t code = TSDB_CODE_SUCCESS; +static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration, + int32_t *days) { + int32_t code = 0; + int32_t lino = 0; int64_t freqDuration = -1; int64_t keepDuration = -1; - code = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE, &freqDuration); - code = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE, &keepDuration); - int32_t days = duration; // min + TAOS_CHECK_EXIT( + convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE, &freqDuration)); + TAOS_CHECK_EXIT( + convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE, &keepDuration)); + *days = duration; // min - if (days < freqDuration) { - days = freqDuration; + if (*days < freqDuration) { + *days = freqDuration; } - if (days > keepDuration) { - days = keepDuration; + if (*days > keepDuration) { + *days = keepDuration; } if (level < TSDB_RETENTION_L1 || level > TSDB_RETENTION_L2) { goto _exit; } - code = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE, &freqDuration); - code = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE, &keepDuration); + TAOS_CHECK_EXIT(convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE, &freqDuration)); + TAOS_CHECK_EXIT(convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE, &keepDuration)); int32_t nFreqTimes = (r + level)->freq / (60 * 1000); // use 60s for freq of 1st level - days *= (nFreqTimes > 1 ? nFreqTimes : 1); + *days *= (nFreqTimes > 1 ? nFreqTimes : 1); - if (days < freqDuration) { - days = freqDuration; + if (*days < freqDuration) { + *days = freqDuration; } int32_t maxKeepDuration = TMIN(keepDuration, TSDB_MAX_DURATION_PER_FILE); - if (days > maxKeepDuration) { - days = maxKeepDuration; + if (*days > maxKeepDuration) { + *days = maxKeepDuration; } _exit: - smaInfo("vgId:%d, evaluated duration for level %d is %d, raw val:%d", TD_VID(pVnode), level + 1, days, duration); - return days; + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } else { + smaInfo("vgId:%d, evaluated duration for level %d is %d, raw val:%d", TD_VID(pVnode), level + 1, *days, duration); + } + TAOS_RETURN(code); } int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type) { - terrno = 0; + int32_t code = 0; + int32_t lino = 0; pKeepCfg->precision = pCfg->precision; switch (type) { case TSDB_TYPE_RSMA_L0: @@ -116,10 +122,14 @@ int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int ty SMA_SET_KEEP_CFG(pVnode, 2); break; default: - terrno = TSDB_CODE_APP_ERROR; + code = TSDB_CODE_APP_ERROR; break; } - return terrno; +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + TAOS_RETURN(code); } int32_t smaOpen(SVnode *pVnode, int8_t rollback, bool force) { @@ -129,8 +139,7 @@ int32_t smaOpen(SVnode *pVnode, int8_t rollback, bool force) { SSma *pSma = taosMemoryCalloc(1, sizeof(SSma)); if (!pSma) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } pVnode->pSma = pSma; @@ -152,18 +161,14 @@ int32_t smaOpen(SVnode *pVnode, int8_t rollback, bool force) { } // restore the rsma - if (tdRSmaRestore(pSma, RSMA_RESTORE_REBOOT, pVnode->state.committed, rollback) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } + TAOS_CHECK_EXIT(tdRSmaRestore(pSma, RSMA_RESTORE_REBOOT, pVnode->state.committed, rollback)); } _exit: if (code) { smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); - terrno = code; } - return code; + TAOS_RETURN(code); } int32_t smaClose(SSma *pSma) { @@ -190,7 +195,7 @@ int32_t smaClose(SSma *pSma) { */ int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback) { if (!VND_IS_RSMA(pSma->pVnode)) { - return TSDB_CODE_RSMA_INVALID_ENV; + TAOS_RETURN(TSDB_CODE_RSMA_INVALID_ENV); } return tdRSmaProcessRestoreImpl(pSma, type, committedVer, rollback); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 823f65a9fd..ad7eb24f28 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -123,8 +123,7 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); if (*pStore == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; @@ -132,12 +131,13 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd) { SRSmaInfo *pRSmaInfo = NULL; + int32_t code = 0; if (!suid || !tbUids) { - terrno = TSDB_CODE_INVALID_PTR; + code = TSDB_CODE_INVALID_PTR; smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), suid ? *suid : -1, - terrstr()); - return TSDB_CODE_FAILED; + tstrerror(code)); + TAOS_RETURN(code); } int32_t nTables = taosArrayGetSize(tbUids); @@ -151,17 +151,17 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, if (!pRSmaInfo) { smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); - terrno = TSDB_CODE_RSMA_INVALID_STAT; - return TSDB_CODE_FAILED; + code = TSDB_CODE_RSMA_INVALID_STAT; + TAOS_RETURN(code); } for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pRSmaInfo->taskInfo[i]) { - if ((terrno = qUpdateTableListForStreamScanner(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) { + if ((code = qUpdateTableListForStreamScanner(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i, - terrstr()); - return TSDB_CODE_FAILED; + tstrerror(code)); + TAOS_RETURN(code); } smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p. suid:%" PRIi64 " uid:%" PRIi64 "nTables:%d level %d", @@ -170,26 +170,25 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, } tdReleaseRSmaInfo(pSma, pRSmaInfo); - return TSDB_CODE_SUCCESS; + TAOS_RETURN(code); } int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) { + int32_t code = 0; if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) { return TSDB_CODE_SUCCESS; } - if (tdUpdateTbUidListImpl(pSma, &pStore->suid, pStore->tbUids, isAdd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_FAILED; - } + TAOS_CHECK_RETURN(tdUpdateTbUidListImpl(pSma, &pStore->suid, pStore->tbUids, isAdd)); void *pIter = NULL; while ((pIter = taosHashIterate(pStore->uidHash, pIter))) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); SArray *pTbUids = *(SArray **)pIter; - if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids, isAdd) != TSDB_CODE_SUCCESS) { + if ((code = tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids, isAdd)) != TSDB_CODE_SUCCESS) { taosHashCancelIterate(pStore->uidHash, pIter); - return TSDB_CODE_FAILED; + TAOS_RETURN(code); } } return TSDB_CODE_SUCCESS; @@ -206,6 +205,7 @@ int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) { */ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_uid_t uid) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + int32_t code = 0; // only applicable to rollup SMA ctables if (!pEnv) { diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 201e496140..da99a2c9a2 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -32,19 +32,19 @@ int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) { smaError("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(code)); } - return code; + TAOS_RETURN(code); } int32_t tdProcessTSmaCreate(SSma *pSma, int64_t ver, const char *msg) { int32_t code = tdProcessTSmaCreateImpl(pSma, ver, msg); - return code; + TAOS_RETURN(code); } int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { int32_t code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days); - return code; + TAOS_RETURN(code); } /** @@ -70,8 +70,8 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; int64_t sInterval = -1; - code = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND, &sInterval); - if (TSDB_CODE_SUCCESS != code || 0 == sInterval) { + TAOS_CHECK_EXIT(convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND, &sInterval)); + if (0 == sInterval) { *days = pTsdbCfg->days; goto _exit; } @@ -80,10 +80,7 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c *days = pTsdbCfg->days; } else { int64_t mInterval = -1; - code = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE, &mInterval); - if (TSDB_CODE_SUCCESS != code) { - goto _exit; - } + TAOS_CHECK_EXIT(convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE, &mInterval)); int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2; if (daysPerFile > SMA_STORAGE_MINUTES_MAX) { @@ -103,7 +100,7 @@ _exit: smaDebug("vgId:%d, succeed to get tsma days %d", pCfg->vgId, *days); } tDecoderClear(&coder); - return code; + TAOS_RETURN(code); } /** @@ -123,10 +120,7 @@ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t ver, const char *pMsg if (TD_VID(pSma->pVnode) == pCfg->dstVgId) { // create tsma meta in dstVgId - if (metaCreateTSma(SMA_META(pSma), ver, pCfg) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } + TAOS_CHECK_EXIT(metaCreateTSma(SMA_META(pSma), ver, pCfg)); // create stable to save tsma result in dstVgId tNameFromString(&stbFullName, pCfg->dstTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -135,28 +129,24 @@ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t ver, const char *pMsg pReq.schemaRow = pCfg->schemaRow; pReq.schemaTag = pCfg->schemaTag; - if (metaCreateSTable(SMA_META(pSma), ver, &pReq) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } + TAOS_CHECK_EXIT(metaCreateSTable(SMA_META(pSma), ver, &pReq)); } else { - code = terrno = TSDB_CODE_TSMA_INVALID_STAT; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_EXIT(TSDB_CODE_TSMA_INVALID_STAT); } _exit: if (code) { smaError("vgId:%d, failed at line %d to create sma index %s %" PRIi64 " on stb:%" PRIi64 ", dstSuid:%" PRIi64 - " dstTb:%s dstVg:%d", + " dstTb:%s dstVg:%d since %s", SMA_VID(pSma), lino, pCfg->indexName, pCfg->indexUid, pCfg->tableUid, pCfg->dstTbUid, pReq.name, - pCfg->dstVgId); + pCfg->dstVgId, tstrerror(code)); } else { smaDebug("vgId:%d, success to create sma index %s %" PRIi64 " on stb:%" PRIi64 ", dstSuid:%" PRIi64 " dstTb:%s dstVg:%d", SMA_VID(pSma), pCfg->indexName, pCfg->indexUid, pCfg->tableUid, pCfg->dstTbUid, pReq.name, pCfg->dstVgId); } - return code; + TAOS_RETURN(code); } int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema, int64_t suid, @@ -167,6 +157,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * int32_t len = 0; SSubmitReq2 *pReq = NULL; SArray *tagArray = NULL; + SHashObj *pTableIndexMap = NULL; int32_t numOfBlocks = taosArrayGetSize(pBlocks); @@ -174,18 +165,18 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)); if (!tagArray || !pReq) { - code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)); if (pReq->aSubmitTbData == NULL) { - code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } - SHashObj *pTableIndexMap = - taosHashInit(numOfBlocks, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + pTableIndexMap = taosHashInit(numOfBlocks, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pTableIndexMap == NULL) { + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + } // SSubmitTbData req for (int32_t i = 0; i < numOfBlocks; ++i) { @@ -193,8 +184,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * if (pDataBlock->info.type == STREAM_DELETE_RESULT) { pDeleteReq->suid = suid; pDeleteReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); - code = tqBuildDeleteReq(pVnode->pTq, stbFullName, pDataBlock, pDeleteReq, "", true); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_EXIT(tqBuildDeleteReq(pVnode->pTq, stbFullName, pDataBlock, pDeleteReq, "", true)); continue; } @@ -202,11 +192,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1; - code = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true, &tbData.pCreateTbReq); - if (code) { - smaError("failed to build create-table req, code:%d", code); - continue; - } + TAOS_CHECK_EXIT(buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true, &tbData.pCreateTbReq)); { uint64_t groupId = pDataBlock->info.id.groupId; @@ -221,7 +207,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * taosArrayPush(pReq->aSubmitTbData, &tbData); int32_t size = (int32_t)taosArrayGetSize(pReq->aSubmitTbData) - 1; - taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); + TAOS_CHECK_EXIT(taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size))); } else { code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, INT64_MIN, ""); if (code != TSDB_CODE_SUCCESS) { @@ -237,15 +223,13 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * } } - taosHashCleanup(pTableIndexMap); - // encode tEncodeSize(tEncodeSubmitReq, pReq, len, code); if (TSDB_CODE_SUCCESS == code) { SEncoder encoder; len += sizeof(SSubmitReq2Msg); if (!(pBuf = rpcMallocCont(len))) { - code = terrno; + code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } @@ -253,9 +237,8 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * ((SSubmitReq2Msg *)pBuf)->header.contLen = htonl(len); ((SSubmitReq2Msg *)pBuf)->version = htobe64(1); tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); - if (tEncodeSubmitReq(&encoder, pReq) < 0) { + if ((code = tEncodeSubmitReq(&encoder, pReq)) < 0) { tEncoderClear(&encoder); - code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } tEncoderClear(&encoder); @@ -263,6 +246,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * _exit: taosArrayDestroy(tagArray); + taosHashCleanup(pTableIndexMap); if (pReq != NULL) { tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); @@ -276,7 +260,7 @@ _exit: if (ppData) *ppData = pBuf; if (pLen) *pLen = len; } - return code; + TAOS_RETURN(code); } static int32_t tsmaProcessDelReq(SSma *pSma, int64_t indexUid, SBatchDeleteReq *pDelReq) { @@ -290,7 +274,7 @@ static int32_t tsmaProcessDelReq(SSma *pSma, int64_t indexUid, SBatchDeleteReq * void *pBuf = rpcMallocCont(len + sizeof(SMsgHead)); if (!pBuf) { - code = terrno; + code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } @@ -313,7 +297,7 @@ _exit: indexUid, tstrerror(code)); } - return code; + TAOS_RETURN(code); } /** @@ -350,16 +334,15 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char pTsmaStat = SMA_STAT_TSMA(pStat); if (!pTsmaStat->pTSma) { - terrno = 0; STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); if (!pTSma) { - code = terrno ? terrno : TSDB_CODE_TSMA_INVALID_PTR; + code = TSDB_CODE_TSMA_INVALID_PTR; TSDB_CHECK_CODE(code, lino, _exit); } pTsmaStat->pTSma = pTSma; pTsmaStat->pTSchema = metaGetTbTSchema(SMA_META(pSma), pTSma->dstTbUid, -1, 1); if (!pTsmaStat->pTSchema) { - code = terrno ? terrno : TSDB_CODE_TSMA_INVALID_PTR; + code = TSDB_CODE_TSMA_INVALID_PTR; TSDB_CHECK_CODE(code, lino, _exit); } } @@ -378,15 +361,13 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen); TSDB_CHECK_CODE(code, lino, _exit); - if ((terrno = tsmaProcessDelReq(pSma, indexUid, &deleteReq)) != 0) { - goto _exit; - } + TAOS_CHECK_EXIT(tsmaProcessDelReq(pSma, indexUid, &deleteReq)); #if 0 if (!strncasecmp("td.tsma.rst.tb", pTsmaStat->pTSma->dstTbName, 14)) { - terrno = TSDB_CODE_APP_ERROR; + code = TSDB_CODE_APP_ERROR; smaError("vgId:%d, tsma insert for smaIndex %" PRIi64 " failed since %s, %s", SMA_VID(pSma), indexUid, - pTsmaStat->pTSma->indexUid, tstrerror(terrno), pTsmaStat->pTSma->dstTbName); + pTsmaStat->pTSma->indexUid, tstrerror(code), pTsmaStat->pTSma->dstTbName); goto _err; } #endif @@ -405,5 +386,5 @@ _exit: smaError("vgId:%d, %s failed at line %d since %s, smaIndex:%" PRIi64, SMA_VID(pSma), __func__, lino, tstrerror(code), indexUid); } - return code; + TAOS_RETURN(code); } From d8993c16eeab721cc10b2ca845f1d1d3ead43ad9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 Jul 2024 19:32:05 +0800 Subject: [PATCH 24/52] fix(stream): update test case. --- tests/system-test/1-insert/drop.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index 493e1491b8..b467276fce 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -146,11 +146,15 @@ class TDTestCase: tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)') tdSql.execute(f'create table tb using {stbname} tags(1)') tdSql.execute(f'create stream {stream_name} trigger at_once ignore expired 0 into stb as select * from {self.dbname}.{stbname} partition by tbname') + time.sleep(5) + tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"') print(tdSql.queryResult) tdSql.checkEqual(tdSql.queryResult[0][4],f'create stream {stream_name} trigger at_once ignore expired 0 into stb as select * from {self.dbname}.{stbname} partition by tbname') tdSql.execute(f'drop stream {stream_name}') tdSql.execute(f'create stream {stream_name} trigger at_once ignore expired 0 into stb1 as select * from tb') + time.sleep(5) + tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"') tdSql.checkEqual(tdSql.queryResult[0][4],f'create stream {stream_name} trigger at_once ignore expired 0 into stb1 as select * from tb') tdSql.execute(f'drop database {self.dbname}') From 923c7912014cf70fa7036c65a2ca93dc6e73d1a7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 Jul 2024 19:53:32 +0800 Subject: [PATCH 25/52] other: update test cases. --- source/dnode/vnode/src/sma/smaRollup.c | 2 +- tests/system-test/0-others/test_hot_refresh_configurations.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 823f65a9fd..515272ddeb 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -1442,7 +1442,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SMA_VID(pSma), pItem->level, pRSmaInfo->suid); } break; case TASK_TRIGGER_STAT_INIT: { - smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid::%" PRIi64 " since stat is init", + smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is init", SMA_VID(pSma), pItem->level, pRSmaInfo->suid); } break; default: { diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index 71f6290469..15f291787a 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -123,8 +123,8 @@ class TDTestCase: { "name": "checkpointInterval", "alias": "tsStreamCheckpointInterval", - "values": [60, 1000, 1200], - "except_values": [59, 1201] + "values": [60, 1000, 1800], + "except_values": [59, 1801] }, { "name": "trimVDbIntervalSec", From cd000ba9817f65938c5960080231e62520a72ab6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 Jul 2024 00:08:39 +0800 Subject: [PATCH 26/52] fix(stream): set correct check return value. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 32 +++++++------------------ source/libs/executor/src/executor.c | 6 ++--- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 95ae9bcdf0..4aaa6c696c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -547,7 +547,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void pReader->info.window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); pReader->idStr = (idstr != NULL) ? taosStrdup(idstr) : NULL; - if (pReader->idStr == NULL) { + if (idstr != NULL && pReader->idStr == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _end; } @@ -4618,19 +4618,16 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi STimeWindow window = pCond->twindows; SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); int32_t code = 0; + int32_t lino = 0; int32_t capacity = pConf->tsdbCfg.maxRows; if (pResBlock != NULL) { code = blockDataEnsureCapacity(pResBlock, capacity); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); } code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); // check for query time window STsdbReader* pReader = *ppReader; @@ -4654,9 +4651,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi // here we only need one more row, so the capacity is set to be ONE. code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[0], 1, pResBlock, idstr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); if (order == TSDB_ORDER_ASC) { pCond->twindows.skey = window.ekey + 1; @@ -4668,9 +4663,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi pCond->order = order; code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[1], 1, pResBlock, idstr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); } // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here. @@ -4691,9 +4684,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi if (pReader->info.pSchema != NULL) { code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _err); } pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); @@ -4706,18 +4697,13 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi tSimpleHashSetFreeFp(pReader->pSchemaMap, freeSchemaFunc); if (pReader->info.pSchema != NULL) { code = updateBlockSMAInfo(pReader->info.pSchema, &pReader->suppInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); } STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader; code = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables, &pReader->status.pTableMap); - if (code != TSDB_CODE_SUCCESS) { - *ppReader = NULL; - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); if (pReader->status.pLDataIterArray == NULL) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 8c98df5c8d..cf539ba8f1 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1306,9 +1306,9 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanInfo->scanTimes = 0; if (pScanBaseInfo->dataReader == NULL) { - int32_t code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen( - pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, pScanInfo->pResBlock, - (void**)&pScanBaseInfo->dataReader, id, NULL); + code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, + &keyInfo, 1, pScanInfo->pResBlock, + (void**)&pScanBaseInfo->dataReader, id, NULL); if (code != TSDB_CODE_SUCCESS) { qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); terrno = code; From 9bfb0603d8342873b4f8bedfa30ccccb96bf1c0b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 Jul 2024 00:27:22 +0800 Subject: [PATCH 27/52] test: update the test cases. --- source/libs/stream/src/streamMeta.c | 8 +++++--- tests/system-test/0-others/backquote_check.py | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 43a22e0637..7b410501ca 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -539,9 +539,11 @@ void streamMetaClear(SStreamMeta* pMeta) { streamMetaReleaseTask(pMeta, p); } - int32_t code = taosRemoveRef(streamBackendId, pMeta->streamBackendRid); - if (code) { - stError("vgId:%d remove stream backend Ref failed, rid:%"PRId64, pMeta->vgId, pMeta->streamBackendRid); + if (pMeta->streamBackendRid != 0) { + int32_t code = taosRemoveRef(streamBackendId, pMeta->streamBackendRid); + if (code) { + stError("vgId:%d remove stream backend Ref failed, rid:%" PRId64, pMeta->vgId, pMeta->streamBackendRid); + } } taosHashClear(pMeta->pTasksMap); diff --git a/tests/system-test/0-others/backquote_check.py b/tests/system-test/0-others/backquote_check.py index b35701f1df..2858373ced 100644 --- a/tests/system-test/0-others/backquote_check.py +++ b/tests/system-test/0-others/backquote_check.py @@ -67,11 +67,15 @@ class TDTestCase: tdSql.execute(f'use {self.dbname}') tdSql.execute(f'create stable {self.stbname} (ts timestamp,c0 int) tags(t0 int)') tdSql.execute(f'create stream `{self.streamname}` into `{self.streamtb}` as select count(*) from {self.stbname} interval(10s);') + + time.sleep(15) tdSql.query('show streams') tdSql.checkEqual(tdSql.queryResult[0][0],self.streamname) tdSql.execute(f'drop stream {self.streamname}') tdSql.execute(f'drop stable {self.streamtb}') tdSql.execute(f'create stream {self.streamname} into `{self.streamtb}` as select count(*) from {self.stbname} interval(10s);') + + time.sleep(10) tdSql.query('show streams') tdSql.checkEqual(tdSql.queryResult[0][0],self.streamname) tdSql.execute(f'drop stream `{self.streamname}`') From d34ed134b3502d1588a236df7607999565842d1c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 Jul 2024 00:44:52 +0800 Subject: [PATCH 28/52] fix(tsdb): set the correct tsdbReader --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 4aaa6c696c..a0063cbf29 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -3669,14 +3669,17 @@ static void getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRet *pLevel = TSDB_RETENTION_L0; tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L0, str); *pTsdb = VND_RSMA0(pVnode); + return; } else if (level == TSDB_RETENTION_L1) { *pLevel = TSDB_RETENTION_L1; tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L1, str); *pTsdb = VND_RSMA1(pVnode); + return; } else { *pLevel = TSDB_RETENTION_L2; tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L2, str); *pTsdb = VND_RSMA2(pVnode); + return; } } From 7b49327463fd0373104c0bc175f47af07e0242db Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 24 Jul 2024 10:09:24 +0800 Subject: [PATCH 29/52] enh: refactor return code --- source/dnode/vnode/src/sma/smaRollup.c | 416 +++++++++++------------- source/dnode/vnode/src/tsdb/tsdbWrite.c | 4 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 2 +- 3 files changed, 191 insertions(+), 231 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ad7eb24f28..ddf7abf0a5 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -38,24 +38,24 @@ SSmaMgmt smaMgmt = { typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; -static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); -static void tdUidStoreDestory(STbUidStore *pStore); -static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd); -static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, - int8_t idx); -static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int64_t version, int32_t inputType, - SRSmaInfo *pInfo, ERsmaExecType type, int8_t level); -static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid); -static void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); -static void tdFreeRSmaSubmitItems(SArray *pItems, int32_t type); -static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo); -static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, SRSmaInfo *pInfo, - int32_t execType, int8_t *streamFlushed); -static void tdRSmaFetchTrigger(void *param, void *tmrId); -static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); -static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables); -static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int8_t type, int64_t qTaskFileVer); -static int32_t tdRSmaRestoreTSDataReload(SSma *pSma); +static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); +static void tdUidStoreDestory(STbUidStore *pStore); +static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd); +static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, + int8_t idx); +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int64_t version, int32_t inputType, + SRSmaInfo *pInfo, ERsmaExecType type, int8_t level); +static int32_t tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid, SRSmaInfo **ppRSmaInfo); +static void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); +static void tdFreeRSmaSubmitItems(SArray *pItems, int32_t type); +static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo); +static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, SRSmaInfo *pInfo, + int32_t execType, int8_t *streamFlushed); +static void tdRSmaFetchTrigger(void *param, void *tmrId); +static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); +static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables); +static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int8_t type, int64_t qTaskFileVer); +static int32_t tdRSmaRestoreTSDataReload(SSma *pSma); struct SRSmaQTaskInfoItem { int32_t len; @@ -147,11 +147,10 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, return TSDB_CODE_SUCCESS; } - pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, *suid); + code = tdAcquireRSmaInfoBySuid(pSma, *suid, &pRSmaInfo); - if (!pRSmaInfo) { + if (code != 0) { smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); - code = TSDB_CODE_RSMA_INVALID_STAT; TAOS_RETURN(code); } @@ -215,8 +214,7 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SHashObj *infoHash = NULL; if (!pStat || !(infoHash = RSMA_INFO_HASH(pStat))) { - terrno = TSDB_CODE_RSMA_INVALID_STAT; - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_RSMA_INVALID_STAT); } // info cached when create rsma stable and return directly for non-rsma ctables @@ -225,14 +223,12 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui } if (!(*ppStore)) { - if (tdUidStoreInit(ppStore) < 0) { - return TSDB_CODE_FAILED; - } + TAOS_CHECK_RETURN(tdUidStoreInit(ppStore)); } - if (tdUidStorePut(*ppStore, suid, &uid) < 0) { + if ((code = tdUidStorePut(*ppStore, suid, &uid)) < 0) { *ppStore = tdUidStoreFree(*ppStore); - return TSDB_CODE_FAILED; + TAOS_RETURN(code); } return TSDB_CODE_SUCCESS; @@ -262,6 +258,7 @@ static void tdRSmaTaskRemove(SStreamMeta *pMeta, int64_t streamId, int32_t taskI static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx) { + int32_t code = 0; if ((param->qmsgLen > 0) && param->qmsg[idx]) { SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); SRetention *pRetention = SMA_RETENTION(pSma); @@ -275,18 +272,20 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat if (!taosCheckExistFile(taskInfDir)) { char *s = taosStrdup(taskInfDir); + if (!s) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } if (taosMulMkDir(s) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(errno); taosMemoryFree(s); - return TSDB_CODE_FAILED; + TAOS_RETURN(code); } taosMemoryFree(s); } SStreamTask *pStreamTask = taosMemoryCalloc(1, sizeof(*pStreamTask)); if (!pStreamTask) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } pItem->pStreamTask = pStreamTask; pStreamTask->id.taskId = 0; @@ -294,24 +293,20 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat pStreamTask->chkInfo.startTs = taosGetTimestampMs(); pStreamTask->pMeta = pVnode->pTq->pStreamMeta; pStreamTask->exec.qmsg = taosMemoryMalloc(strlen(RSMA_EXEC_TASK_FLAG) + 1); + if (!pStreamTask->exec.qmsg) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } sprintf(pStreamTask->exec.qmsg, "%s", RSMA_EXEC_TASK_FLAG); pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta); tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id); - int32_t code = streamCreateStateMachine(pStreamTask); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TAOS_CHECK_RETURN(streamCreateStateMachine(pStreamTask)); - code = streamTaskCreateActiveChkptInfo(&pStreamTask->chkInfo.pActiveInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TAOS_CHECK_RETURN(streamTaskCreateActiveChkptInfo(&pStreamTask->chkInfo.pActiveInfo)); pStreamState = streamStateOpen(taskInfDir, pStreamTask, pStreamTask->id.streamId, pStreamTask->id.taskId); if (!pStreamState) { - terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_RSMA_STREAM_STATE_OPEN); } pItem->pStreamState = pStreamState; @@ -321,12 +316,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat initStorageAPI(&handle.api); pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode), 0); if (!pRSmaInfo->taskInfo[idx]) { - terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_RSMA_QTASKINFO_CREATE); } if (!(pItem->pResList = taosArrayInit(1, POINTER_BYTES))) { - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } if (pItem->fetchResultVer < pItem->submitReqVer) { @@ -349,7 +343,9 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat pItem->level = idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2; SRSmaRef rsmaRef = {.refId = pStat->refId, .suid = pRSmaInfo->suid}; - taosHashPut(smaMgmt.refHash, &pItem, POINTER_BYTES, &rsmaRef, sizeof(rsmaRef)); + if (taosHashPut(smaMgmt.refHash, &pItem, POINTER_BYTES, &rsmaRef, sizeof(rsmaRef)) != 0) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); @@ -359,7 +355,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat TD_VID(pVnode), pItem->pStreamTask, pRSmaInfo->suid, (int8_t)(idx + 1), pStreamTask->chkInfo.checkpointId, pItem->submitReqVer, pItem->fetchResultVer, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } /** @@ -372,20 +368,14 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat * @return int32_t */ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName) { - int32_t code; + int32_t code = 0; + int32_t lino = 0; if ((param->qmsgLen[0] == 0) && (param->qmsgLen[1] == 0)) { smaDebug("vgId:%d, no qmsg1/qmsg2 for rollup table %s %" PRIi64, SMA_VID(pSma), tbName, suid); return TSDB_CODE_SUCCESS; } -#if 0 - if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } -#endif - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; @@ -399,41 +389,34 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con // from write queue: single thead pRSmaInfo = (SRSmaInfo *)taosMemoryCalloc(1, sizeof(SRSmaInfo)); if (!pRSmaInfo) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, -1, 1); if (!pTSchema) { - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - goto _err; + TAOS_CHECK_EXIT(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION); } pRSmaInfo->pSma = pSma; pRSmaInfo->pTSchema = pTSchema; pRSmaInfo->suid = suid; T_REF_INIT_VAL(pRSmaInfo, 1); - code = taosOpenQueue(&pRSmaInfo->queue); - if (code) goto _err; + TAOS_CHECK_EXIT(taosOpenQueue(&pRSmaInfo->queue)); - code = taosAllocateQall(&pRSmaInfo->qall); - if (code) goto _err; + TAOS_CHECK_EXIT(taosAllocateQall(&pRSmaInfo->qall)); - if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0 || - tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 1) < 0) { - goto _err; + TAOS_CHECK_EXIT(tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0)); + TAOS_CHECK_EXIT(tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 1)); + + TAOS_CHECK_EXIT(taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo))); + +_exit: + if (code != 0) { + tdFreeRSmaInfo(pSma, pRSmaInfo); + } else { + smaDebug("vgId:%d, register rsma info succeed for table %" PRIi64, SMA_VID(pSma), suid); } - - if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != 0) { - goto _err; - } - - smaDebug("vgId:%d, register rsma info succeed for table %" PRIi64, SMA_VID(pSma), suid); - - return TSDB_CODE_SUCCESS; -_err: - tdFreeRSmaInfo(pSma, pRSmaInfo); - return TSDB_CODE_FAILED; + TAOS_RETURN(code); } /** @@ -480,11 +463,13 @@ int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { return TSDB_CODE_SUCCESS; } + int32_t code = 0; SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); + SRSmaInfo *pRSmaInfo = NULL; - SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pReq->suid); + code = tdAcquireRSmaInfoBySuid(pSma, pReq->suid, &pRSmaInfo); - if (!pRSmaInfo) { + if (code != 0) { smaWarn("vgId:%d, drop rsma for stable %s %" PRIi64 " failed no rsma in hash", TD_VID(pVnode), pReq->name, pReq->suid); return TSDB_CODE_SUCCESS; @@ -519,12 +504,11 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) if (uid) { if (!pStore->tbUids) { if (!(pStore->tbUids = taosArrayInit(1, sizeof(tb_uid_t)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } } if (!taosArrayPush(pStore->tbUids, uid)) { - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } } } else { @@ -532,32 +516,29 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) if (!pStore->uidHash) { pStore->uidHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); if (!pStore->uidHash) { - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } } if (uid) { SArray *uidArray = taosHashGet(pStore->uidHash, &suid, sizeof(tb_uid_t)); if (uidArray && ((uidArray = *(SArray **)uidArray))) { - taosArrayPush(uidArray, uid); + if (!taosArrayPush(uidArray, uid)) { + taosArrayDestroy(uidArray); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } else { SArray *pUidArray = taosArrayInit(1, sizeof(tb_uid_t)); if (!pUidArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } if (!taosArrayPush(pUidArray, uid)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; taosArrayDestroy(pUidArray); - return TSDB_CODE_FAILED; - } - if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) { - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } + TAOS_CHECK_RETURN(taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray))); } } else { - if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) { - return TSDB_CODE_FAILED; - } + TAOS_CHECK_RETURN(taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0)); } } return TSDB_CODE_SUCCESS; @@ -600,9 +581,7 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { if (pReq) { SSubmitReq2 *pSubmitReq = (SSubmitReq2 *)pReq; // spin lock for race condition during insert data - if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { - return TSDB_CODE_FAILED; - } + TAOS_CHECK_RETURN(tsdbInsertData(pTsdb, version, pSubmitReq, NULL)); } return TSDB_CODE_SUCCESS; @@ -612,13 +591,9 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq2 *pMsg, STbUidStore *pStore) { SArray *pSubmitTbData = pMsg ? pMsg->aSubmitTbData : NULL; int32_t size = taosArrayGetSize(pSubmitTbData); - terrno = TSDB_CODE_SUCCESS; - for (int32_t i = 0; i < size; ++i) { SSubmitTbData *pData = TARRAY_GET_ELEM(pSubmitTbData, i); - if ((terrno = tdUidStorePut(pStore, pData->suid, NULL)) < 0) { - return -1; - } + TAOS_CHECK_RETURN(tdUidStorePut(pStore, pData->suid, NULL)); } return 0; @@ -645,7 +620,7 @@ int32_t smaRetention(SSma *pSma, int64_t now) { } _end: - return code; + TAOS_RETURN(code); } static int32_t tdRSmaProcessDelReq(SSma *pSma, int64_t suid, int8_t level, SBatchDeleteReq *pDelReq) { @@ -659,13 +634,17 @@ static int32_t tdRSmaProcessDelReq(SSma *pSma, int64_t suid, int8_t level, SBatc void *pBuf = rpcMallocCont(len + sizeof(SMsgHead)); if (!pBuf) { - code = terrno; + code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } SEncoder encoder; tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len); - tEncodeSBatchDeleteReq(&encoder, pDelReq); + if ((code = tEncodeSBatchDeleteReq(&encoder, pDelReq)) < 0) { + tEncoderClear(&encoder); + rpcFreeCont(pBuf); + TSDB_CHECK_CODE(code, lino, _exit); + } tEncoderClear(&encoder); ((SMsgHead *)pBuf)->vgId = TD_VID(pSma->pVnode); @@ -682,7 +661,7 @@ _exit: SMA_VID(pSma), lino, suid, level, tstrerror(code)); } - return code; + TAOS_RETURN(code); } static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, SRSmaInfo *pInfo, @@ -717,7 +696,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma SBatchDeleteReq deleteReq = {.suid = suid, .level = pItem->level}; deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); if (!deleteReq.deleteReqs) { - code = terrno; + code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } code = tqBuildDeleteReq(pSma->pVnode->pTq, NULL, output, &deleteReq, "", true); @@ -753,21 +732,15 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); SSubmitReq2 *pReq = NULL; - if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) { - code = terrno ? terrno : TSDB_CODE_RSMA_RESULT; - TSDB_CHECK_CODE(code, lino, _exit); - } + TAOS_CHECK_EXIT( + buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid)); - if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { - if (terrno == TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE) { + if (pReq && (code = tdProcessSubmitReq(sinkTsdb, output->info.version, pReq)) < 0) { + if (code == TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE) { // TODO: reconfigure SSubmitReq2 - } else { - if (terrno == 0) terrno = TSDB_CODE_RSMA_RESULT; - code = terrno; } tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); - taosMemoryFree(pReq); - pReq = NULL; + taosMemoryFreeClear(pReq); TSDB_CHECK_CODE(code, lino, _exit); } @@ -812,14 +785,12 @@ _exit: */ static int32_t tdExecuteRSmaImplAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid) { - int32_t code; + int32_t code = 0; + int32_t lino = 0; int32_t size = RSMA_EXEC_MSG_HLEN + len; // header + payload void *qItem; - code = taosAllocateQitem(size, DEF_QITEM, 0, (void **)&qItem); - if (code) { - return code; - } + TAOS_CHECK_RETURN(taosAllocateQitem(size, DEF_QITEM, 0, (void **)&qItem)); void *pItem = qItem; @@ -830,7 +801,7 @@ static int32_t tdExecuteRSmaImplAsync(SSma *pSma, int64_t version, const void *p *(int64_t *)pItem = version; memcpy(POINTER_SHIFT(pItem, sizeof(int64_t)), pMsg, len); - taosWriteQitem(pInfo->queue, qItem); + TAOS_CHECK_RETURN(taosWriteQitem(pInfo->queue, qItem)); pInfo->lastRecv = taosGetTimestampMs(); @@ -863,10 +834,10 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { SSubmitMsgIter msgIter = {0}; SSubmitBlkIter blkIter = {0}; STSRow *row = NULL; - if (tInitSubmitMsgIter(pReq, &msgIter) < 0) return -1; + TAOS_CHECK_RETURN(tInitSubmitMsgIter(pReq, &msgIter)); while (true) { SSubmitBlk *pBlock = NULL; - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + TAOS_CHECK_RETURN(tGetSubmitMsgNext(&msgIter, &pBlock)); if (pBlock == NULL) break; tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) { @@ -892,6 +863,7 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { */ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int64_t version, int32_t inputType, SRSmaInfo *pInfo, ERsmaExecType type, int8_t level) { + int32_t code = 0; int32_t idx = level - 1; void *qTaskInfo = RSMA_INFO_QTASK(pInfo, idx); SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx); @@ -902,25 +874,24 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, return TSDB_CODE_SUCCESS; } if (!pInfo->pTSchema) { - terrno = TSDB_CODE_INVALID_PTR; smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, pInfo->suid); - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_INVALID_PTR); } smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p, suid:%" PRIu64 ", nMsg:%d, submitReqVer:%" PRIi64 ", inputType:%d", SMA_VID(pSma), level, RSMA_INFO_QTASK(pInfo, idx), pInfo->suid, msgSize, version, inputType); - if ((terrno = qSetSMAInput(qTaskInfo, pMsg, msgSize, inputType)) < 0) { - smaError("vgId:%d, rsma %" PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); - return TSDB_CODE_FAILED; + if ((code = qSetSMAInput(qTaskInfo, pMsg, msgSize, inputType)) < 0) { + smaError("vgId:%d, rsma %" PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(code)); + TAOS_RETURN(TSDB_CODE_FAILED); } atomic_store_64(&pItem->submitReqVer, version); - terrno = tdRSmaExecAndSubmitResult(pSma, qTaskInfo, pItem, pInfo, STREAM_NORMAL, NULL); + TAOS_CHECK_RETURN(tdRSmaExecAndSubmitResult(pSma, qTaskInfo, pItem, pInfo, STREAM_NORMAL, NULL)); - return terrno ? TSDB_CODE_FAILED : TDB_CODE_SUCCESS; + TAOS_RETURN(code); } /** @@ -928,26 +899,23 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, * * @param pSma * @param suid - * @return SRSmaInfo* */ -static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { +static int32_t tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid, SRSmaInfo **ppRSmaInfo) { int32_t code = 0; int32_t lino = 0; SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = NULL; SRSmaInfo *pRSmaInfo = NULL; - terrno = 0; + *ppRSmaInfo = NULL; if (!pEnv) { - terrno = TSDB_CODE_RSMA_INVALID_ENV; - return NULL; + TAOS_RETURN(TSDB_CODE_RSMA_INVALID_ENV); } pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); if (!pStat || !RSMA_INFO_HASH(pStat)) { - terrno = TSDB_CODE_RSMA_INVALID_STAT; - return NULL; + TAOS_RETURN(TSDB_CODE_RSMA_INVALID_STAT); } taosRLockLatch(SMA_ENV_LOCK(pEnv)); @@ -955,20 +923,19 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { if (pRSmaInfo && (pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { if (RSMA_INFO_IS_DEL(pRSmaInfo)) { taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); - return NULL; + TAOS_RETURN(TSDB_CODE_RSMA_INVALID_STAT); } tdRefRSmaInfo(pSma, pRSmaInfo); taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); if (ASSERTS(pRSmaInfo->suid == suid, "suid:%" PRIi64 " != %" PRIi64, pRSmaInfo->suid, suid)) { - terrno = TSDB_CODE_APP_ERROR; - return NULL; + TAOS_RETURN(TSDB_CODE_APP_ERROR); } - return pRSmaInfo; + TAOS_RETURN(code); } taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); - return NULL; + TAOS_RETURN(TSDB_CODE_RSMA_INVALID_STAT); } static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { @@ -989,16 +956,19 @@ static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { */ static int32_t tdExecuteRSmaAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, tb_uid_t suid) { - SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, suid); - if (!pRSmaInfo) { + int32_t code = 0; + SRSmaInfo *pRSmaInfo = NULL; + + code = tdAcquireRSmaInfoBySuid(pSma, suid, &pRSmaInfo); + if (code != 0) { smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); // return success } if (inputType == STREAM_INPUT__DATA_SUBMIT || inputType == STREAM_INPUT__REF_DATA_BLOCK) { - if (tdExecuteRSmaImplAsync(pSma, version, pMsg, len, inputType, pRSmaInfo, suid) < 0) { + if ((code = tdExecuteRSmaImplAsync(pSma, version, pMsg, len, inputType, pRSmaInfo, suid)) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); - return TSDB_CODE_FAILED; + TAOS_RETURN(code); } if (smaMgmt.tmrHandle) { SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, 0); @@ -1011,11 +981,11 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, int64_t version, const void *pMsg, } } } else { - terrno = TSDB_CODE_APP_ERROR; + code = TSDB_CODE_APP_ERROR; tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, execute rsma, failed for suid:%" PRIu64 " since %s, type:%d", SMA_VID(pSma), suid, - tstrerror(terrno), inputType); - return TSDB_CODE_FAILED; + tstrerror(code), inputType); + TAOS_RETURN(code); } tdReleaseRSmaInfo(pSma, pRSmaInfo); @@ -1025,57 +995,56 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg, int32_t len) { if (!SMA_RSMA_ENV(pSma)) return TSDB_CODE_SUCCESS; - if ((terrno = atomic_load_32(&SMA_RSMA_STAT(pSma)->execStat))) { - smaError("vgId:%d, failed to process rsma submit since invalid exec code: %s", SMA_VID(pSma), terrstr()); - goto _err; + int32_t code = 0; + if ((code = atomic_load_32(&SMA_RSMA_STAT(pSma)->execStat))) { + smaError("vgId:%d, failed to process rsma submit since invalid exec code: %s", SMA_VID(pSma), tstrerror(code)); + goto _exit; } STbUidStore uidStore = {0}; - if (tdFetchSubmitReqSuids(pReq, &uidStore) < 0) { - smaError("vgId:%d, failed to process rsma submit fetch suid since: %s", SMA_VID(pSma), terrstr()); - goto _err; + if ((code = tdFetchSubmitReqSuids(pReq, &uidStore)) < 0) { + smaError("vgId:%d, failed to process rsma submit fetch suid since: %s", SMA_VID(pSma), tstrerror(code)); + goto _exit; } if (uidStore.suid != 0) { - if (tdExecuteRSmaAsync(pSma, version, pMsg, len, STREAM_INPUT__DATA_SUBMIT, uidStore.suid) < 0) { - smaError("vgId:%d, failed to process rsma submit exec 1 since: %s", SMA_VID(pSma), terrstr()); - goto _err; + if ((code = tdExecuteRSmaAsync(pSma, version, pMsg, len, STREAM_INPUT__DATA_SUBMIT, uidStore.suid)) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 1 since: %s", SMA_VID(pSma), tstrerror(code)); + goto _exit; } void *pIter = NULL; while ((pIter = taosHashIterate(uidStore.uidHash, pIter))) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - if (tdExecuteRSmaAsync(pSma, version, pMsg, len, STREAM_INPUT__DATA_SUBMIT, *pTbSuid) < 0) { - smaError("vgId:%d, failed to process rsma submit exec 2 since: %s", SMA_VID(pSma), terrstr()); + if ((code = tdExecuteRSmaAsync(pSma, version, pMsg, len, STREAM_INPUT__DATA_SUBMIT, *pTbSuid)) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 2 since: %s", SMA_VID(pSma), tstrerror(code)); taosHashCancelIterate(uidStore.uidHash, pIter); - goto _err; + goto _exit; } } } +_exit: tdUidStoreDestory(&uidStore); - return TSDB_CODE_SUCCESS; -_err: - tdUidStoreDestory(&uidStore); - return terrno; + TAOS_RETURN(code); } int32_t tdProcessRSmaDelete(SSma *pSma, int64_t version, void *pReq, void *pMsg, int32_t len) { if (!SMA_RSMA_ENV(pSma)) return TSDB_CODE_SUCCESS; - if ((terrno = atomic_load_32(&SMA_RSMA_STAT(pSma)->execStat))) { - smaError("vgId:%d, failed to process rsma delete since invalid exec code: %s", SMA_VID(pSma), terrstr()); - goto _err; + int32_t code = 0; + if ((code = atomic_load_32(&SMA_RSMA_STAT(pSma)->execStat))) { + smaError("vgId:%d, failed to process rsma delete since invalid exec code: %s", SMA_VID(pSma), tstrerror(code)); + goto _exit; } SDeleteRes *pDelRes = pReq; - if (tdExecuteRSmaAsync(pSma, version, pMsg, len, STREAM_INPUT__REF_DATA_BLOCK, pDelRes->suid) < 0) { - smaError("vgId:%d, failed to process rsma submit exec 1 since: %s", SMA_VID(pSma), terrstr()); - goto _err; + if ((code = tdExecuteRSmaAsync(pSma, version, pMsg, len, STREAM_INPUT__REF_DATA_BLOCK, pDelRes->suid)) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 1 since: %s", SMA_VID(pSma), tstrerror(code)); + goto _exit; } - return TSDB_CODE_SUCCESS; -_err: - return terrno; +_exit: + TAOS_RETURN(code); } /** @@ -1099,10 +1068,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { TSDB_CHECK_CODE(code, lino, _exit); } - if (vnodeGetStbIdList(pSma->pVnode, 0, suidList) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } + TAOS_CHECK_EXIT(vnodeGetStbIdList(pSma->pVnode, 0, suidList)); int64_t arrSize = taosArrayGetSize(suidList); @@ -1146,10 +1112,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { " qmsgLen:%" PRIi32, TD_VID(pVnode), suid, i, param->maxdelay[i], param->watermark[i], param->qmsgLen[i]); } - if (tdRSmaProcessCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } + TAOS_CHECK_EXIT(tdRSmaProcessCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name)); #if 0 // reload all ctbUids for suid uidStore.suid = suid; @@ -1180,7 +1143,7 @@ _exit: metaReaderClear(&mr); taosArrayDestroy(suidList); tdUidStoreDestory(&uidStore); - return code; + TAOS_RETURN(code); } /** @@ -1188,20 +1151,16 @@ _exit: */ int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback) { int32_t code = 0; + int32_t lino = 0; int64_t nTables = 0; // step 1: init env - if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TDB_INIT_FAILED; - goto _err; - } + TAOS_CHECK_EXIT(tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP)); // step 2: iterate all stables to restore the rsma env - if ((code = tdRSmaRestoreQTaskInfoInit(pSma, &nTables)) < 0) { - goto _err; - } + TAOS_CHECK_EXIT(tdRSmaRestoreQTaskInfoInit(pSma, &nTables)); -_err: +_exit: if (code) { smaError("vgId:%d, restore rsma task %" PRIi8 "from qtaskf %" PRIi64 " failed since %s", SMA_VID(pSma), type, qtaskFileVer, tstrerror(code)); @@ -1210,7 +1169,7 @@ _err: type, qtaskFileVer, nTables); } - return code; + TAOS_RETURN(code); } int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { @@ -1323,9 +1282,8 @@ _checkpoint: } streamMetaWLock(pMeta); - if (streamMetaSaveTask(pMeta, pTask)) { + if ((code = streamMetaSaveTask(pMeta, pTask)) != 0) { streamMetaWUnLock(pMeta); - code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; taosHashCancelIterate(pInfoHash, infoHash); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1338,9 +1296,9 @@ _checkpoint: } if (pMeta) { streamMetaWLock(pMeta); - if (streamMetaCommit(pMeta)) { + if ((code = streamMetaCommit(pMeta)) != 0) { streamMetaWUnLock(pMeta); - code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; + if (code == -1) code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } streamMetaWUnLock(pMeta); @@ -1354,8 +1312,7 @@ _exit: smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } - terrno = code; - return code; + TAOS_RETURN(code); } /** @@ -1370,6 +1327,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaStat *pStat = NULL; SRSmaInfo *pRSmaInfo = NULL; SRSmaInfoItem *pItem = NULL; + int32_t code = 0; if (!(pRSmaRef = taosHashGet(smaMgmt.refHash, ¶m, POINTER_BYTES))) { smaDebug("rsma fetch task not start since rsma info item:%p not exist in refHash:%p, rsetId:%d", param, @@ -1386,7 +1344,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { pSma = pStat->pSma; - if (!(pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaRef->suid))) { + if ((code = tdAcquireRSmaInfoBySuid(pSma, pRSmaRef->suid, &pRSmaInfo)) != 0) { smaDebug("rsma fetch task not start since rsma info not exist, rsetId:%d refId:%" PRIi64 ")", smaMgmt.rsetId, pRSmaRef->refId); // pRSmaRef freed in taosHashRemove tdReleaseSmaRef(smaMgmt.rsetId, pRSmaRef->refId); @@ -1484,6 +1442,8 @@ static void tdFreeRSmaSubmitItems(SArray *pItems, int32_t type) { * @return int32_t */ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo) { + int32_t code = 0; + int32_t lino = 0; SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; for (int8_t i = 1; i <= TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, i - 1); @@ -1512,12 +1472,10 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo) { pItem->nScanned = 0; - if ((terrno = qSetSMAInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) { - goto _err; - } - if (tdRSmaExecAndSubmitResult(pSma, taskInfo, pItem, pInfo, STREAM_GET_ALL, NULL) < 0) { - atomic_store_32(&SMA_RSMA_STAT(pSma)->execStat, terrno); - goto _err; + TAOS_CHECK_EXIT(qSetSMAInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)); + if ((code = tdRSmaExecAndSubmitResult(pSma, taskInfo, pItem, pInfo, STREAM_GET_ALL, NULL)) < 0) { + atomic_store_32(&SMA_RSMA_STAT(pSma)->execStat, code); + goto _exit; } smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi32 " maxDelay:%d, fetch finished", @@ -1529,10 +1487,8 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo) { } } -_end: - return TSDB_CODE_SUCCESS; -_err: - return TSDB_CODE_FAILED; +_exit: + TAOS_RETURN(code); } static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SArray *pSubmitArr, ERsmaExecType type) { @@ -1541,6 +1497,8 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA int32_t nSubmit = 0; int32_t nDelete = 0; int64_t version = 0; + int32_t code = 0; + int32_t lino = 0; SPackedData packData; @@ -1559,23 +1517,21 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA version = packData.ver; if (!taosArrayPush(pSubmitArr, &packData)) { taosFreeQitem(msg); - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } ++nSubmit; } else if (inputType == STREAM_INPUT__REF_DATA_BLOCK) { _resume_delete: version = RSMA_EXEC_MSG_VER(msg); - if ((terrno = tqExtractDelDataBlock(RSMA_EXEC_MSG_BODY(msg), RSMA_EXEC_MSG_LEN(msg), version, - &packData.pDataBlock, 1))) { + if ((code = tqExtractDelDataBlock(RSMA_EXEC_MSG_BODY(msg), RSMA_EXEC_MSG_LEN(msg), version, + &packData.pDataBlock, 1))) { taosFreeQitem(msg); - goto _err; + TAOS_CHECK_EXIT(code); } if (packData.pDataBlock && !taosArrayPush(pSubmitArr, &packData)) { taosFreeQitem(msg); - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } taosFreeQitem(msg); if (packData.pDataBlock) { @@ -1593,9 +1549,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA ASSERTS(size > 0, "size is %d", size); int32_t inputType = nSubmit > 0 ? STREAM_INPUT__MERGED_SUBMIT : STREAM_INPUT__REF_DATA_BLOCK; for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { - if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, version, inputType, pInfo, type, i) < 0) { - goto _err; - } + TAOS_CHECK_EXIT(tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, version, inputType, pInfo, type, i)); } tdFreeRSmaSubmitItems(pSubmitArr, inputType); nSubmit = 0; @@ -1612,7 +1566,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA _rtn: return TSDB_CODE_SUCCESS; -_err: +_exit: atomic_store_32(&SMA_RSMA_STAT(pSma)->execStat, terrno); smaError("vgId:%d, batch exec for suid:%" PRIi64 " execType:%d size:%d failed since %s", SMA_VID(pSma), pInfo->suid, type, (int32_t)taosArrayGetSize(pSubmitArr), terrstr()); @@ -1626,7 +1580,7 @@ _err: break; } } - return TSDB_CODE_FAILED; + TAOS_RETURN(code); } /** @@ -1677,7 +1631,10 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { taosReadAllQitems(pInfo->queue, pInfo->qall); // queue has mutex lock int32_t qallItemSize = taosQallItemSize(pInfo->qall); if (qallItemSize > 0) { - tdRSmaBatchExec(pSma, pInfo, pInfo->qall, pSubmitArr, type); + if ((code = tdRSmaBatchExec(pSma, pInfo, pInfo->qall, pSubmitArr, type)) != 0) { + taosHashCancelIterate(infoHash, pIter); + TSDB_CHECK_CODE(code, lino, _exit); + } smaDebug("vgId:%d, batchSize:%d, execType:%" PRIi32, SMA_VID(pSma), qallItemSize, type); } @@ -1693,7 +1650,10 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { TSDB_CHECK_CODE(code, lino, _exit); } - tdRSmaFetchAllResult(pSma, pInfo); + if ((code = tdRSmaFetchAllResult(pSma, pInfo)) != 0) { + taosHashCancelIterate(infoHash, pIter); + TSDB_CHECK_CODE(code, lino, _exit); + } if (0 == atomic_sub_fetch_32(&pRSmaStat->nFetchAll, 1)) { atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); @@ -1742,5 +1702,5 @@ _exit: if (code) { smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } - return code; + TAOS_RETURN(code); } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 0b32c96d2e..a35068163f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -34,7 +34,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 int32_t numOfRows = 0; if (ASSERTS(pTsdb->mem != NULL, "vgId:%d, mem is NULL", TD_VID(pTsdb->pVnode))) { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PTR); } arrSize = taosArrayGetSize(pMsg->aSubmitTbData); @@ -42,7 +42,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 // scan and convert if ((code = tsdbScanAndConvertSubmitMsg(pTsdb, pMsg)) < 0) { if (code != TSDB_CODE_TDB_TABLE_RECONFIGURE) { - tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(terrno)); + tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); } return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index f7b618d18b..6b46ddd8df 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -601,7 +601,7 @@ int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) { int32_t code = TSDB_CODE_SUCCESS; SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); if (!pCur) { - return TSDB_CODE_FAILED; + return TSDB_CODE_OUT_OF_MEMORY; } while (1) { From ee03a7eea79caf7c277935715c92af778038e137 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 Jul 2024 13:29:15 +0800 Subject: [PATCH 30/52] fix(stream): add some logs. --- source/dnode/mnode/impl/src/mndStreamUtil.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index e250034184..5a17d659cd 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -744,8 +744,9 @@ int32_t mndInitExecInfo() { void removeExpiredNodeInfo(const SArray *pNodeSnapshot) { SArray *pValidList = taosArrayInit(4, sizeof(SNodeEntry)); int32_t size = taosArrayGetSize(pNodeSnapshot); + int32_t oldSize = taosArrayGetSize(execInfo.pNodeList); - for (int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeList); ++i) { + for (int32_t i = 0; i < oldSize; ++i) { SNodeEntry *p = taosArrayGet(execInfo.pNodeList, i); for (int32_t j = 0; j < size; ++j) { @@ -763,7 +764,8 @@ void removeExpiredNodeInfo(const SArray *pNodeSnapshot) { taosArrayDestroy(execInfo.pNodeList); execInfo.pNodeList = pValidList; - mDebug("remain %d valid node entries after clean expired nodes info", (int32_t)taosArrayGetSize(pValidList)); + mDebug("remain %d valid node entries after clean expired nodes info, prev size:%d", + (int32_t)taosArrayGetSize(pValidList), oldSize); } int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) { From 69fa2a187ae7f37258d7e7d6aa74f437dcd3bf80 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Jul 2024 13:58:17 +0800 Subject: [PATCH 31/52] enh: refactor return code --- include/util/tRealloc.h | 2 +- include/util/tarray2.h | 2 +- source/dnode/qnode/src/qnode.c | 15 ++++--- source/dnode/snode/src/snode.c | 1 - source/dnode/vnode/src/inc/tsdb.h | 1 - source/dnode/vnode/src/tsdb/tsdbFSet2.c | 12 +++-- source/dnode/vnode/src/tsdb/tsdbFile2.c | 2 +- source/dnode/vnode/src/tsdb/tsdbIter.c | 4 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 47 ++++++-------------- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 6 +-- source/dnode/vnode/src/tsdb/tsdbUtil.c | 51 ++++++++++++---------- source/dnode/vnode/src/vnd/vnodeBufPool.c | 6 +-- source/util/src/tarray.c | 6 +-- 13 files changed, 73 insertions(+), 82 deletions(-) diff --git a/include/util/tRealloc.h b/include/util/tRealloc.h index 3229c53039..b830fac284 100644 --- a/include/util/tRealloc.h +++ b/include/util/tRealloc.h @@ -41,7 +41,7 @@ static FORCE_INLINE int32_t tRealloc(uint8_t **ppBuf, int64_t size) { pBuf = (uint8_t *)taosMemoryRealloc(pBuf, bsize + sizeof(int64_t)); if (pBuf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } diff --git a/include/util/tarray2.h b/include/util/tarray2.h index 2e9b0c7cb5..0d1ceded6c 100644 --- a/include/util/tarray2.h +++ b/include/util/tarray2.h @@ -55,7 +55,7 @@ static FORCE_INLINE int32_t tarray2_make_room(void *arr, int32_t expSize, int32_ capacity <<= 1; } void *p = taosMemoryRealloc(a->data, capacity * eleSize); - if (p == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (p == NULL) return terrno; a->capacity = capacity; a->data = p; return 0; diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index b0fcd3400d..e80cbf1391 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -13,17 +13,17 @@ * along with this program. If not, see . */ -#include "tqueue.h" #include "executor.h" #include "qndInt.h" #include "query.h" #include "qworker.h" +#include "tqueue.h" int32_t qndOpen(const SQnodeOpt *pOption, SQnode **pQnode) { *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); if (NULL == *pQnode) { qError("calloc SQnode failed"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (*pQnode)->qndId = QNODE_HANDLE; @@ -75,9 +75,9 @@ int32_t qndPreprocessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { return qWorkerPreprocessQueryMsg(pQnode->pQuery, pMsg, false); } -int32_t qndProcessQueryMsg(SQnode *pQnode, SQueueInfo* pInfo, SRpcMsg *pMsg) { +int32_t qndProcessQueryMsg(SQnode *pQnode, SQueueInfo *pInfo, SRpcMsg *pMsg) { int32_t code = -1; - int64_t ts = pInfo->timestamp; + int64_t ts = pInfo->timestamp; SReadHandle handle = {.pMsgCb = &pQnode->msgCb, .pWorkerCb = pInfo->workerCb}; qTrace("message in qnode queue is processing"); @@ -113,6 +113,9 @@ int32_t qndProcessQueryMsg(SQnode *pQnode, SQueueInfo* pInfo, SRpcMsg *pMsg) { terrno = TSDB_CODE_APP_ERROR; } - if (code == 0) return TSDB_CODE_ACTION_IN_PROGRESS; - return code; + if (code == 0) { + return TSDB_CODE_ACTION_IN_PROGRESS; + } else { + return code; + } } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 3f2ebccdba..5e536e5fbf 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -64,7 +64,6 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { int32_t code = 0; SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode)); if (pSnode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d496853cd5..978821f890 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -244,7 +244,6 @@ void tsdbMemTableDestroy(SMemTable *pMemTable, bool proactive); STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid); int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode); int32_t tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive); -SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable); // STbDataIter int32_t tsdbTbDataIterCreate(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, STbDataIter **ppIter); void *tsdbTbDataIterDestroy(STbDataIter *pIter); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 2c0662b83b..305ce6b56f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -18,14 +18,14 @@ int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl) { if (!(lvl[0] = taosMemoryMalloc(sizeof(SSttLvl)))) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } lvl[0]->level = level; TARRAY2_INIT(lvl[0]->fobjArr); return 0; } -static void tsdbSttLvlClearFObj(void *data) { tsdbTFileObjUnref(*(STFileObj **)data); } +static void tsdbSttLvlClearFObj(void *data) { TAOS_UNUSED(tsdbTFileObjUnref(*(STFileObj **)data)); } int32_t tsdbSttLvlClear(SSttLvl **lvl) { if (lvl[0] != NULL) { @@ -451,7 +451,9 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) { fset[0] = taosMemoryCalloc(1, sizeof(STFileSet)); - if (fset[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (fset[0] == NULL) { + return terrno; + } fset[0]->fid = fid; fset[0]->maxVerValid = VERSION_MAX; @@ -543,7 +545,9 @@ int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_ int32_t tsdbTFileSetRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STFileSetRange **fsr) { fsr[0] = taosMemoryCalloc(1, sizeof(*fsr[0])); - if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (fsr[0] == NULL) { + return terrno; + } fsr[0]->fid = fset1->fid; fsr[0]->sver = sver; fsr[0]->ever = ever; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index a500074a65..52f1ef72ee 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -227,7 +227,7 @@ int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f) { int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) { fobj[0] = taosMemoryMalloc(sizeof(*fobj[0])); if (!fobj[0]) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } taosThreadMutexInit(&fobj[0]->mutex, NULL); diff --git a/source/dnode/vnode/src/tsdb/tsdbIter.c b/source/dnode/vnode/src/tsdb/tsdbIter.c index 42afe1cdc0..d97a5153f3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbIter.c +++ b/source/dnode/vnode/src/tsdb/tsdbIter.c @@ -507,7 +507,7 @@ int32_t tsdbIterOpen(const STsdbIterConfig *config, STsdbIter **iter) { iter[0] = taosMemoryCalloc(1, sizeof(*iter[0])); if (iter[0] == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } iter[0]->type = config->type; @@ -677,7 +677,7 @@ int32_t tsdbIterMergerOpen(const TTsdbIterArray *iterArray, SIterMerger **merger merger[0] = taosMemoryCalloc(1, sizeof(*merger[0])); if (merger[0] == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } merger[0]->isTomb = isTomb; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index e5bcf65bd7..3114c0fa04 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -54,7 +54,7 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { pMemTable = (SMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable)); if (pMemTable == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _err; } taosInitRWLatch(&pMemTable->latch); @@ -71,7 +71,7 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { pMemTable->nBucket = MEM_MIN_HASH; pMemTable->aBucket = (STbData **)taosMemoryCalloc(pMemTable->nBucket, sizeof(STbData *)); if (pMemTable->aBucket == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosMemoryFree(pMemTable); goto _err; } @@ -174,7 +174,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid // do delete SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData)); if (pDelData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _err; } pDelData->version = version; @@ -195,7 +195,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pMemTable->minVer = TMIN(pMemTable->minVer, version); pMemTable->maxVer = TMAX(pMemTable->maxVer, version); - tsdbCacheDel(pTsdb, suid, uid, sKey, eKey); + TAOS_UNUSED(tsdbCacheDel(pTsdb, suid, uid, sKey, eKey)); tsdbTrace("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 " at version %" PRId64, @@ -214,7 +214,7 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, STsdbRowKey *pFrom, int8_t backwa (*ppIter) = (STbDataIter *)taosMemoryCalloc(1, sizeof(STbDataIter)); if ((*ppIter) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -329,7 +329,7 @@ static int32_t tsdbMemTableRehash(SMemTable *pMemTable) { int32_t nBucket = pMemTable->nBucket * 2; STbData **aBucket = (STbData **)taosMemoryCalloc(nBucket, sizeof(STbData *)); if (aBucket == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -369,7 +369,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid ASSERT(pPool != NULL); pTbData = vnodeBufPoolMallocAligned(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); if (pTbData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } pTbData->suid = suid; @@ -520,7 +520,7 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN ASSERT(0); } if (pNode == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -589,7 +589,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, // copy and construct block data SBlockData *pBlockData = vnodeBufPoolMalloc(pPool, sizeof(*pBlockData)); if (pBlockData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -599,7 +599,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, pBlockData->aUid = NULL; pBlockData->aVersion = vnodeBufPoolMalloc(pPool, aColData[0].nData); if (pBlockData->aVersion == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } for (int32_t i = 0; i < pBlockData->nRow; i++) { // todo: here can be optimized @@ -608,7 +608,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, pBlockData->aTSKEY = vnodeBufPoolMalloc(pPool, aColData[0].nData); if (pBlockData->aTSKEY == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } memcpy(pBlockData->aTSKEY, aColData[0].pData, aColData[0].nData); @@ -616,7 +616,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, pBlockData->nColData = nColData - 1; pBlockData->aColData = vnodeBufPoolMalloc(pPool, sizeof(SColData) * pBlockData->nColData); if (pBlockData->aColData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -661,7 +661,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, } if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) { - tsdbCacheColFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, pBlockData); + TAOS_UNUSED(tsdbCacheColFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, pBlockData)); } // SMemTable @@ -720,7 +720,7 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, pTbData->maxKey = key.key.ts; } if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) { - tsdbCacheRowFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, version, nRow, aRow); + TAOS_UNUSED(tsdbCacheRowFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, version, nRow, aRow)); } // SMemTable @@ -780,22 +780,3 @@ static FORCE_INLINE int32_t tbDataPCmprFn(const void *p1, const void *p2) { return 0; } - -SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) { - SArray *aTbDataP = taosArrayInit(pMemTable->nTbData, sizeof(STbData *)); - if (aTbDataP == NULL) goto _exit; - - for (int32_t iBucket = 0; iBucket < pMemTable->nBucket; iBucket++) { - STbData *pTbData = pMemTable->aBucket[iBucket]; - - while (pTbData) { - taosArrayPush(aTbDataP, &pTbData); - pTbData = pTbData->next; - } - } - - taosArraySort(aTbDataP, tbDataPCmprFn); - -_exit: - return aTbDataP; -} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index f515cd5352..56364fbc6a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -42,7 +42,6 @@ static int32_t tsdbTFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { STsdbFSetPartition* tsdbFSetPartitionCreate() { STsdbFSetPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbFSetPartition)); if (pSP == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } for (int32_t i = 0; i < TSDB_FSET_RANGE_TYP_MAX; i++) { @@ -140,7 +139,6 @@ static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition** STsdbFSetPartList* tsdbFSetPartListCreate() { STsdbFSetPartList* pList = taosMemoryCalloc(1, sizeof(STsdbFSetPartList)); if (pList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } TARRAY2_INIT(pList); @@ -160,7 +158,7 @@ int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); if (pDiff == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _err; } TARRAY2_INIT(pDiff); @@ -169,7 +167,7 @@ int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray TARRAY2_FOREACH(pList, part) { STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); if (r == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _err; } int64_t maxVerValid = -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 66db7baab9..a1ca8ccda2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -154,14 +154,16 @@ int32_t tPutMapData(uint8_t *p, SMapData *pMapData) { int32_t tGetMapData(uint8_t *p, SMapData *pMapData, int32_t *decodedSize) { int32_t n = 0; + int32_t code; int32_t offset; tMapDataReset(pMapData); n += tGetI32v(p + n, &pMapData->nItem); if (pMapData->nItem) { - if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) { - return TSDB_CODE_OUT_OF_MEMORY; + code = tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem); + if (code) { + return code; } int32_t lOffset = 0; @@ -172,8 +174,9 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData, int32_t *decodedSize) { } n += tGetI32v(p + n, &pMapData->nData); - if (tRealloc(&pMapData->pData, pMapData->nData)) { - return TSDB_CODE_OUT_OF_MEMORY; + code = tRealloc(&pMapData->pData, pMapData->nData); + if (code) { + return code; } memcpy(pMapData->pData, p + n, pMapData->nData); n += pMapData->nData; @@ -753,7 +756,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) *pColVal = COL_VAL_VALUE(pTColumn->colId, ((SValue){.type = pTColumn->type, .val = key.ts})); if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; return code; } @@ -766,7 +769,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) continue; } else if (pTSchema->columns[jCol].colId > pTColumn->colId) { if (taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } continue; } @@ -778,7 +781,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) pColVal->value.pData = NULL; code = tRealloc(&pColVal->value.pData, pColVal->value.nData); if (code) { - return TSDB_CODE_OUT_OF_MEMORY; + return code; } if (pColVal->value.nData) { @@ -787,14 +790,14 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) } if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } for (; iCol < pMerger->pTSchema->numOfCols; ++iCol) { pTColumn = &pMerger->pTSchema->columns[iCol]; if (taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -866,7 +869,7 @@ int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pSchema) { pMerger->pTSchema = pSchema; pMerger->pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal)); if (pMerger->pArray == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } else { return TSDB_CODE_SUCCESS; } @@ -1036,11 +1039,11 @@ int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx, TSDBKEY *pItem1 = taosArrayGet(aSkyline, sidx * 2); TSDBKEY *pItem2 = taosArrayGet(aSkyline, sidx * 2 + 1); if (taosArrayPush(pSkyline, &pItem1) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pSkyline, &pItem2) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } else { SArray *pSkyline1 = NULL; @@ -1048,10 +1051,13 @@ int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx, midx = (sidx + eidx) / 2; pSkyline1 = taosArrayInit((midx - sidx + 1) * 2, POINTER_BYTES); + if (pSkyline1 == NULL) { + return terrno; + } pSkyline2 = taosArrayInit((eidx - midx) * 2, POINTER_BYTES); - if (pSkyline1 == NULL || pSkyline1 == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _clear; + if (pSkyline2 == NULL) { + taosArrayDestroy(pSkyline1); + return terrno; } code = tsdbBuildDeleteSkylineImpl(aSkyline, sidx, midx, pSkyline1); @@ -1076,25 +1082,25 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr int32_t dataNum = eidx - sidx + 1; SArray *aTmpSkyline = taosArrayInit(dataNum * 2, sizeof(TSDBKEY)); if (aTmpSkyline == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } SArray *pSkyline = taosArrayInit(dataNum * 2, POINTER_BYTES); if (pSkyline == NULL) { taosArrayDestroy(aTmpSkyline); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } taosArrayClear(aSkyline); for (int32_t i = sidx; i <= eidx; ++i) { pDelData = (SDelData *)taosArrayGet(aDelData, i); if (taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->sKey, .version = pDelData->version}) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _clear; } if (taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->eKey, .version = 0}) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _clear; } } @@ -1106,7 +1112,7 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr for (int32_t i = 0; i < skylineNum; ++i) { TSDBKEY *p = taosArrayGetP(pSkyline, i); if (taosArrayPush(aSkyline, p) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _clear; } } @@ -1196,7 +1202,7 @@ static int32_t tBlockDataAdjustColData(SBlockData *pBlockData, int32_t nColData) } else if (pBlockData->nColData < nColData) { SColData *aColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SBlockData) * nColData); if (aColData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -1284,7 +1290,7 @@ int32_t tBlockDataAddColData(SBlockData *pBlockData, int16_t cid, int8_t type, i SColData *newColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SColData) * (pBlockData->nColData + 1)); if (newColData == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pBlockData->aColData = newColData; @@ -1452,6 +1458,7 @@ int32_t tBlockDataCompress(SBlockData *bData, void *pCompr, SBuffer *buffers, SB SColCompressInfo *pInfo = pCompr; code = tsdbGetColCmprAlgFromSet(pInfo->pColCmpr, 1, &pInfo->defaultCmprAlg); + TAOS_UNUSED(code); SDiskDataHdr hdr = { .delimiter = TSDB_FILE_DLMT, diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index ad183839d7..f3c2693b75 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -140,8 +140,9 @@ void *vnodeBufPoolMallocAligned(SVBufPool *pPool, int size) { // allocate a new node pNode = taosMemoryMalloc(sizeof(*pNode) + size); if (pNode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - if (pPool->lock) taosThreadSpinUnlock(pPool->lock); + if (pPool->lock) { + taosThreadSpinUnlock(pPool->lock); + } return NULL; } @@ -173,7 +174,6 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { // allocate a new node pNode = taosMemoryMalloc(sizeof(*pNode) + size); if (pNode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; if (pPool->lock) taosThreadSpinUnlock(pPool->lock); return NULL; } diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index ec5e41f256..cb46a9f80e 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -18,9 +18,9 @@ #include "tcoding.h" // todo refactor API -#define BOUNDARY_SIZE 1024*1024*1024 // 1G +#define BOUNDARY_SIZE 1024 * 1024 * 1024 // 1G #define BOUNDARY_SMALL_FACTOR 1.2 -#define BOUNDARY_BIG_FACTOR 2 +#define BOUNDARY_BIG_FACTOR 2 SArray* taosArrayInit(size_t size, size_t elemSize) { if (elemSize == 0) { @@ -198,7 +198,7 @@ void* taosArrayGet(const SArray* pArray, size_t index) { } if (index >= pArray->size) { - uError("index is out of range, current:%" PRIzu " max:%"PRIzu, index, pArray->size); + uError("index is out of range, current:%" PRIzu " max:%" PRIzu, index, pArray->size); return NULL; } From 45e00afad1954fd92fe3b79bac3f44d2fd37b0b9 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 24 Jul 2024 14:20:59 +0800 Subject: [PATCH 32/52] tsdb/cache: remove unused block index cache --- source/dnode/vnode/src/tsdb/tsdbCache.c | 134 +----------------------- 1 file changed, 1 insertion(+), 133 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 8574ff85c3..9dfa3ea496 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -22,40 +22,6 @@ #define ROCKS_BATCH_SIZE (4096) -#if 0 -static int32_t tsdbOpenBICache(STsdb *pTsdb) { - int32_t code = 0; - SLRUCache *pCache = taosLRUCacheInit(10 * 1024 * 1024, 0, .5); - if (pCache == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - taosLRUCacheSetStrictCapacity(pCache, false); - - taosThreadMutexInit(&pTsdb->biMutex, NULL); - -_err: - pTsdb->biCache = pCache; - return code; -} - -static void tsdbCloseBICache(STsdb *pTsdb) { - SLRUCache *pCache = pTsdb->biCache; - if (pCache) { - int32_t elems = taosLRUCacheGetElems(pCache); - tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems); - taosLRUCacheEraseUnrefEntries(pCache); - elems = taosLRUCacheGetElems(pCache); - tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems); - - taosLRUCacheCleanup(pCache); - - taosThreadMutexDestroy(&pTsdb->biMutex); - } -} -#endif - static int32_t tsdbOpenBCache(STsdb *pTsdb) { int32_t code = 0; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; @@ -1760,14 +1726,6 @@ int32_t tsdbOpenCache(STsdb *pTsdb) { goto _err; } -#if 0 - code = tsdbOpenBICache(pTsdb); - if (code != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } -#endif - code = tsdbOpenBCache(pTsdb); if (code != TSDB_CODE_SUCCESS) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1808,9 +1766,6 @@ void tsdbCloseCache(STsdb *pTsdb) { taosThreadMutexDestroy(&pTsdb->lruMutex); } -#if 0 - tsdbCloseBICache(pTsdb); -#endif tsdbCloseBCache(pTsdb); tsdbClosePgCache(pTsdb); tsdbCloseRocksCache(pTsdb); @@ -3302,7 +3257,7 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC STsdbRowKey rowKey = {0}; tsdbRowGetKey(pRow, &rowKey); - if (lastRowKey.key.ts == TSKEY_MAX) { // first time + if (lastRowKey.key.ts == TSKEY_MAX) { // first time lastRowKey = rowKey; for (int16_t iCol = noneCol; iCol < nCols; ++iCol) { @@ -3591,93 +3546,6 @@ int32_t tsdbCacheGetElems(SVnode *pVnode) { return elems; } -#if 0 -static void getBICacheKey(int32_t fid, int64_t commitID, char *key, int *len) { - struct { - int32_t fid; - int64_t commitID; - } biKey = {0}; - - biKey.fid = fid; - biKey.commitID = commitID; - - *len = sizeof(biKey); - memcpy(key, &biKey, *len); -} - -static int32_t tsdbCacheLoadBlockIdx(SDataFReader *pFileReader, SArray **aBlockIdx) { - SArray *pArray = taosArrayInit(8, sizeof(SBlockIdx)); - int32_t code = tsdbReadBlockIdx(pFileReader, pArray); - - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(pArray); - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } - - *aBlockIdx = pArray; - - return code; -} - -static void deleteBICache(const void *key, size_t keyLen, void *value, void *ud) { - (void)ud; - SArray *pArray = (SArray *)value; - - taosArrayDestroy(pArray); -} - -int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHandle **handle) { - int32_t code = 0; - char key[128] = {0}; - int keyLen = 0; - - getBICacheKey(pFileReader->pSet->fid, pFileReader->pSet->pHeadF->commitID, key, &keyLen); - LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (!h) { - STsdb *pTsdb = pFileReader->pTsdb; - taosThreadMutexLock(&pTsdb->biMutex); - - h = taosLRUCacheLookup(pCache, key, keyLen); - if (!h) { - SArray *pArray = NULL; - code = tsdbCacheLoadBlockIdx(pFileReader, &pArray); - // if table's empty or error, return code of -1 - if (code != TSDB_CODE_SUCCESS || pArray == NULL) { - taosThreadMutexUnlock(&pTsdb->biMutex); - - *handle = NULL; - return 0; - } - - size_t charge = pArray->capacity * pArray->elemSize + sizeof(*pArray); - _taos_lru_deleter_t deleter = deleteBICache; - LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, pArray, charge, deleter, &h, TAOS_LRU_PRIORITY_LOW, NULL); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } - } - - taosThreadMutexUnlock(&pTsdb->biMutex); - } - - tsdbTrace("bi cache:%p, ref", pCache); - *handle = h; - - return code; -} - -int32_t tsdbBICacheRelease(SLRUCache *pCache, LRUHandle *h) { - int32_t code = 0; - - taosLRUCacheRelease(pCache, h, false); - tsdbTrace("bi cache:%p, release", pCache); - - return code; -} -#endif - // block cache static void getBCacheKey(int32_t fid, int64_t commitID, int64_t blkno, char *key, int *len) { struct { From 12371207300dd8d82a6dbc9c2637ee2b5839e1a0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 Jul 2024 14:31:16 +0800 Subject: [PATCH 33/52] fix(stream): ignore error. --- source/dnode/mnode/impl/src/mndStream.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b3e81665e3..2c3f2c59c7 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2474,10 +2474,7 @@ static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) { SNodeEntry entry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId}; epsetAssign(&entry.epset, &pTask->info.epSet); - code = taosHashPut(pHash, &entry.nodeId, sizeof(entry.nodeId), &entry, sizeof(entry)); - if (code) { - break; - } + (void)taosHashPut(pHash, &entry.nodeId, sizeof(entry.nodeId), &entry, sizeof(entry)); } destroyStreamTaskIter(pTaskIter); From 45b1bbf0ed9cd44a10109785e4bc9a6dd4c97d1f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 24 Jul 2024 14:34:27 +0800 Subject: [PATCH 34/52] use new macros from bcache & pgcache opening --- source/dnode/vnode/src/tsdb/tsdbCache.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 9dfa3ea496..6506daacbd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -23,13 +23,12 @@ #define ROCKS_BATCH_SIZE (4096) static int32_t tsdbOpenBCache(STsdb *pTsdb) { - int32_t code = 0; + int32_t code = 0, lino = 0; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; int64_t szBlock = tsS3BlockSize <= 1024 ? 1024 : tsS3BlockSize; SLRUCache *pCache = taosLRUCacheInit((int64_t)tsS3BlockCacheSize * szBlock * szPage, 0, .5); if (pCache == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err); } taosLRUCacheSetStrictCapacity(pCache, false); @@ -39,7 +38,11 @@ static int32_t tsdbOpenBCache(STsdb *pTsdb) { pTsdb->bCache = pCache; _err: - return code; + if (code) { + tsdbError("tsdb/bcache: vgId:%d, open failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); + } + + TAOS_RETURN(code); } static void tsdbCloseBCache(STsdb *pTsdb) { @@ -58,23 +61,27 @@ static void tsdbCloseBCache(STsdb *pTsdb) { } static int32_t tsdbOpenPgCache(STsdb *pTsdb) { - int32_t code = 0; + int32_t code = 0, lino = 0; // SLRUCache *pCache = taosLRUCacheInit(10 * 1024 * 1024, 0, .5); int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; SLRUCache *pCache = taosLRUCacheInit((int64_t)tsS3PageCacheSize * szPage, 0, .5); if (pCache == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err); } taosLRUCacheSetStrictCapacity(pCache, false); taosThreadMutexInit(&pTsdb->pgMutex, NULL); -_err: pTsdb->pgCache = pCache; - return code; + +_err: + if (code) { + tsdbError("tsdb/pgcache: vgId:%d, open failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); + } + + TAOS_RETURN(code); } static void tsdbClosePgCache(STsdb *pTsdb) { From 3cb462cdf987db7d1f993ec257148d6efda5595f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 Jul 2024 14:34:45 +0800 Subject: [PATCH 35/52] refactor: do some internal refactor. --- source/dnode/mnode/impl/src/mndStream.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 2c3f2c59c7..d864326ed0 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -56,7 +56,7 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger); static int32_t mndProcessNodeCheck(SRpcMsg *pReq); static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg); -static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList); +static int32_t refreshNodeListFromExistedStreams(SMnode *pMnode, SArray *pNodeList); static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq); static int32_t mndProcessCheckpointReport(SRpcMsg *pReq); static int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg); @@ -1092,7 +1092,7 @@ _ERR: int32_t extractStreamNodeList(SMnode *pMnode) { if (taosArrayGetSize(execInfo.pNodeList) == 0) { - int32_t code = extractNodeListFromStream(pMnode, execInfo.pNodeList); + int32_t code = refreshNodeListFromExistedStreams(pMnode, execInfo.pNodeList); if (code) { mError("Failed to extract node list from stream, code:%s", tstrerror(code)); return code; @@ -2437,7 +2437,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange return code; } -static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) { +static int32_t refreshNodeListFromExistedStreams(SMnode *pMnode, SArray *pNodeList) { SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; void *pIter = NULL; @@ -2565,7 +2565,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { // keep the new vnode snapshot if success if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { - code = extractNodeListFromStream(pMnode, execInfo.pNodeList); + code = refreshNodeListFromExistedStreams(pMnode, execInfo.pNodeList); if (code) { mError("failed to extract node list from stream, code:%s", tstrerror(code)); goto _end; From 3e6d3c541680ec679b801e4c92ba6438d5574189 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 Jul 2024 14:35:55 +0800 Subject: [PATCH 36/52] refactor: do some internal refactor. --- source/dnode/mnode/impl/src/mndStream.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index d864326ed0..24f757ebe6 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2443,6 +2443,8 @@ static int32_t refreshNodeListFromExistedStreams(SMnode *pMnode, SArray *pNodeLi void *pIter = NULL; int32_t code = 0; + mDebug("start to refresh node list by existed streams"); + SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (pHash == NULL) { return TSDB_CODE_OUT_OF_MEMORY; From 8f04f32b9637dbdf8c25d6ece92cce36779dad9f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Jul 2024 14:41:45 +0800 Subject: [PATCH 37/52] make it compile --- source/dnode/qnode/src/qnode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index e80cbf1391..3a326657c9 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -13,11 +13,12 @@ * along with this program. If not, see . */ +#include "tqueue.h" + #include "executor.h" #include "qndInt.h" #include "query.h" #include "qworker.h" -#include "tqueue.h" int32_t qndOpen(const SQnodeOpt *pOption, SQnode **pQnode) { *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); From e0cd502afe43ad1769c8251d56303fba623facd2 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 24 Jul 2024 14:43:31 +0800 Subject: [PATCH 38/52] remove unused code --- source/dnode/vnode/src/tsdb/tsdbCache.c | 377 ------------------------ 1 file changed, 377 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 6506daacbd..91345bf001 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1788,320 +1788,6 @@ static void getTableCacheKey(tb_uid_t uid, int cacheType, char *key, int *len) { *len = sizeof(uint64_t); } -#ifdef BUILD_NO_CALL -static void deleteTableCacheLast(const void *key, size_t keyLen, void *value, void *ud) { - (void)ud; - SArray *pLastArray = (SArray *)value; - int16_t nCol = taosArrayGetSize(pLastArray); - for (int16_t iCol = 0; iCol < nCol; ++iCol) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLastArray, iCol); - if (IS_VAR_DATA_TYPE(pLastCol->colVal.type) && pLastCol->colVal.value.nData > 0) { - taosMemoryFree(pLastCol->colVal.value.pData); - } - } - - taosArrayDestroy(value); -} - -int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { - int32_t code = 0; - - char key[32] = {0}; - int keyLen = 0; - - // getTableCacheKey(uid, "lr", key, &keyLen); - getTableCacheKey(uid, 0, key, &keyLen); - LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); - bool invalidate = false; - int16_t nCol = taosArrayGetSize(pLast); - - for (int16_t iCol = 0; iCol < nCol; ++iCol) { - SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); - if (eKey >= tTsVal->ts) { - invalidate = true; - break; - } - } - - taosLRUCacheRelease(pCache, h, invalidate); - if (invalidate) { - taosLRUCacheErase(pCache, key, keyLen); - } - } - - return code; -} - -int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { - int32_t code = 0; - - char key[32] = {0}; - int keyLen = 0; - - // getTableCacheKey(uid, "l", key, &keyLen); - getTableCacheKey(uid, 1, key, &keyLen); - LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); - bool invalidate = false; - int16_t nCol = taosArrayGetSize(pLast); - - for (int16_t iCol = 0; iCol < nCol; ++iCol) { - SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); - if (eKey >= tTsVal->ts) { - invalidate = true; - break; - } - } - - taosLRUCacheRelease(pCache, h, invalidate); - if (invalidate) { - taosLRUCacheErase(pCache, key, keyLen); - } - } - - return code; -} - -int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup) { - int32_t code = 0; - STSRow *cacheRow = NULL; - char key[32] = {0}; - int keyLen = 0; - - // getTableCacheKey(uid, "lr", key, &keyLen); - getTableCacheKey(uid, 0, key, &keyLen); - LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - TSKEY keyTs = TSDBROW_TS(row); - bool invalidate = false; - - SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); - int16_t nCol = taosArrayGetSize(pLast); - int16_t iCol = 0; - - if (nCol <= 0) { - nCol = pTSchema->numOfCols; - - STColumn *pTColumn = &pTSchema->columns[0]; - SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); - if (taosArrayPush(pLast, &(SLastCol){.ts = keyTs, .colVal = tColVal}) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; - } - - for (iCol = 1; iCol < nCol; ++iCol) { - SColVal colVal = {0}; - tsdbRowGetColVal(row, pTSchema, iCol, &colVal); - - SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; - if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { - lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; - } - memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); - } - - if (taosArrayPush(pLast, &lastCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; - } - } - - goto _invalidate; - } - - if (nCol != pTSchema->numOfCols) { - invalidate = true; - goto _invalidate; - } - - SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); - if (keyTs > tTsVal->ts) { - STColumn *pTColumn = &pTSchema->columns[0]; - SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); - - taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = tColVal}); - } - - for (++iCol; iCol < nCol; ++iCol) { - SLastCol *tTsVal1 = (SLastCol *)taosArrayGet(pLast, iCol); - if (keyTs >= tTsVal1->ts) { - SColVal *tColVal = &tTsVal1->colVal; - - SColVal colVal = {0}; - tsdbRowGetColVal(row, pTSchema, iCol, &colVal); - - if (colVal.cid != tColVal->cid) { - invalidate = true; - goto _invalidate; - } - - if (!COL_VAL_IS_NONE(&colVal)) { - if (keyTs == tTsVal1->ts && !COL_VAL_IS_NONE(tColVal)) { - invalidate = true; - - break; - } else { // new inserting key is greater than cached, update cached entry - SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; - if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); - if (pLastCol->colVal.value.nData > 0 && NULL != pLastCol->colVal.value.pData) - taosMemoryFree(pLastCol->colVal.value.pData); - - lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; - } - memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); - } - - taosArraySet(pLast, iCol, &lastCol); - } - } - } - } - - _invalidate: - taosMemoryFreeClear(pTSchema); - - taosLRUCacheRelease(pCache, h, invalidate); - if (invalidate) { - taosLRUCacheErase(pCache, key, keyLen); - } - } - - return code; -} - -int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb) { - int32_t code = 0; - STSRow *cacheRow = NULL; - char key[32] = {0}; - int keyLen = 0; - - // getTableCacheKey(uid, "l", key, &keyLen); - getTableCacheKey(uid, 1, key, &keyLen); - LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - TSKEY keyTs = TSDBROW_TS(row); - bool invalidate = false; - - SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); - int16_t nCol = taosArrayGetSize(pLast); - int16_t iCol = 0; - - if (nCol <= 0) { - nCol = pTSchema->numOfCols; - - STColumn *pTColumn = &pTSchema->columns[0]; - SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); - if (taosArrayPush(pLast, &(SLastCol){.ts = keyTs, .colVal = tColVal}) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; - } - - for (iCol = 1; iCol < nCol; ++iCol) { - SColVal colVal = {0}; - tsdbRowGetColVal(row, pTSchema, iCol, &colVal); - - SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; - if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { - lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; - } - memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); - } - - if (taosArrayPush(pLast, &lastCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; - } - } - - goto _invalidate; - } - - if (nCol != pTSchema->numOfCols) { - invalidate = true; - goto _invalidate; - } - - SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); - if (keyTs > tTsVal->ts) { - STColumn *pTColumn = &pTSchema->columns[0]; - SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); - - taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = tColVal}); - } - - for (++iCol; iCol < nCol; ++iCol) { - SLastCol *tTsVal1 = (SLastCol *)taosArrayGet(pLast, iCol); - if (keyTs >= tTsVal1->ts) { - SColVal *tColVal = &tTsVal1->colVal; - - SColVal colVal = {0}; - tsdbRowGetColVal(row, pTSchema, iCol, &colVal); - - if (colVal.cid != tColVal->cid) { - invalidate = true; - goto _invalidate; - } - - if (COL_VAL_IS_VALUE(&colVal)) { - if (keyTs == tTsVal1->ts && COL_VAL_IS_VALUE(tColVal)) { - invalidate = true; - - break; - } else { - SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; - if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); - if (pLastCol->colVal.value.nData > 0 && NULL != pLastCol->colVal.value.pData) - taosMemoryFree(pLastCol->colVal.value.pData); - - lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; - } - memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); - } - - taosArraySet(pLast, iCol, &lastCol); - } - } - } - } - - _invalidate: - taosMemoryFreeClear(pTSchema); - - taosLRUCacheRelease(pCache, h, invalidate); - if (invalidate) { - taosLRUCacheErase(pCache, key, keyLen); - } - } - - return code; -} -#endif - static tb_uid_t getTableSuidByUid(tb_uid_t uid, STsdb *pTsdb) { tb_uid_t suid = 0; @@ -2146,53 +1832,6 @@ static int32_t getTableDelDataFromTbData(STbData *pTbData, SArray *aDelData) { return code; } -#ifdef BUILD_NO_CALL -static int32_t getTableDelData(STbData *pMem, STbData *pIMem, SDelFReader *pDelReader, SDelIdx *pDelIdx, - SArray *aDelData) { - int32_t code = 0; - - if (pDelIdx) { - code = getTableDelDataFromDelIdx(pDelReader, pDelIdx, aDelData); - if (code) goto _err; - } - - if (pMem) { - code = getTableDelDataFromTbData(pMem, aDelData); - if (code) goto _err; - } - - if (pIMem) { - code = getTableDelDataFromTbData(pIMem, aDelData); - if (code) goto _err; - } - -_err: - return code; -} - -static int32_t getTableDelSkyline(STbData *pMem, STbData *pIMem, SDelFReader *pDelReader, SDelIdx *pDelIdx, - SArray *aSkyline) { - int32_t code = 0; - SArray *aDelData = NULL; - - aDelData = taosArrayInit(32, sizeof(SDelData)); - code = getTableDelData(pMem, pIMem, pDelReader, pDelIdx, aDelData); - if (code) goto _err; - - size_t nDelData = taosArrayGetSize(aDelData); - if (nDelData > 0) { - code = tsdbBuildDeleteSkyline(aDelData, 0, (int32_t)(nDelData - 1), aSkyline); - if (code) goto _err; - } - -_err: - if (aDelData) { - taosArrayDestroy(aDelData); - } - return code; -} -#endif - static void freeTableInfoFunc(void *param) { void **p = (void **)param; taosMemoryFreeClear(*p); @@ -3154,22 +2793,6 @@ _err: return code; } -#ifdef BUILD_NO_CALL -static int32_t initLastColArray(STSchema *pTSchema, SArray **ppColArray) { - SArray *pColArray = taosArrayInit(pTSchema->numOfCols, sizeof(SLastCol)); - if (NULL == pColArray) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pTSchema->numOfCols; ++i) { - SLastCol col = {.ts = 0, .colVal = COL_VAL_NULL(pTSchema->columns[i].colId, pTSchema->columns[i].type)}; - taosArrayPush(pColArray, &col); - } - *ppColArray = pColArray; - return TSDB_CODE_SUCCESS; -} -#endif - static int32_t initLastColArrayPartial(STSchema *pTSchema, SArray **ppColArray, int16_t *slotIds, int nCols) { SArray *pColArray = taosArrayInit(nCols, sizeof(SLastCol)); if (NULL == pColArray) { From 4281289eef4283f0992169a6395e197b5221c080 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 23 Jul 2024 15:06:14 +0800 Subject: [PATCH 39/52] feat: (errcode) tconfig.c --- include/util/tconfig.h | 8 +- include/util/tutil.h | 4 +- source/common/src/tglobal.c | 13 +- source/common/src/tmisce.c | 10 +- source/util/src/tconfig.c | 350 +++++++++++++++++------------------ source/util/test/cfgTest.cpp | 11 +- 6 files changed, 191 insertions(+), 205 deletions(-) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 8373e00085..f109153384 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -101,7 +101,7 @@ typedef struct { typedef struct SConfig SConfig; typedef struct SConfigIter SConfigIter; -SConfig *cfgInit(); +int32_t cfgInit(SConfig **ppCfg); int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr); int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair void cfgCleanup(SConfig *pCfg); @@ -110,7 +110,7 @@ SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype, bool lock); int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer); -SConfigIter *cfgCreateIter(SConfig *pConf); +int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter); SConfigItem *cfgNextIter(SConfigIter *pIter); void cfgDestroyIter(SConfigIter *pIter); void cfgLock(SConfig *pCfg); @@ -131,8 +131,8 @@ int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, const char *cfgStypeStr(ECfgSrcType type); const char *cfgDtypeStr(ECfgDataType type); -void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); -void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); +int32_t cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); +int32_t cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump); diff --git a/include/util/tutil.h b/include/util/tutil.h index 5af79dfc49..a41f7d5860 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -152,9 +152,9 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, #define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member))) -#define TAOS_RETURN(code) \ +#define TAOS_RETURN(CODE) \ do { \ - return (terrno = (code)); \ + return (terrno = (CODE)); \ } while (0) #define TAOS_CHECK_RETURN(CMD) \ diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index bffc0b5557..c80d96d189 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1263,8 +1263,8 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { if (tsCfg == NULL) osDefaultInit(); - SConfig *pCfg = cfgInit(); - if (pCfg == NULL) return -1; + SConfig *pCfg = NULL; + TAOS_CHECK_RETURN(cfgInit(&pCfg)); if (tsc) { tsLogEmbedded = 0; @@ -1325,8 +1325,8 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * SArray *pArgs) { if (tsCfg == NULL) osDefaultInit(); - SConfig *pCfg = cfgInit(); - if (pCfg == NULL) return -1; + SConfig *pCfg = NULL; + TAOS_CHECK_RETURN(cfgInit(&pCfg)); if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; @@ -1371,10 +1371,7 @@ static int32_t taosCheckGlobalCfg() { static int32_t cfgInitWrapper(SConfig **pCfg) { if (*pCfg == NULL) { - *pCfg = cfgInit(); - if (*pCfg == NULL) { - return terrno; - } + TAOS_CHECK_RETURN(cfgInit(pCfg)); } return 0; } diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index 154fcc3f6b..dd371c6a2a 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -270,11 +270,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { TAOS_CHECK_GOTO(blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)), NULL, _exit); - pIter = cfgCreateIter(pConf); - if (pIter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TAOS_CHECK_GOTO(code, NULL, _exit); - } + TAOS_CHECK_GOTO(cfgCreateIter(pConf, &pIter), NULL, _exit); cfgLock(pConf); locked = 1; @@ -296,7 +292,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t valueLen = 0; - cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); + TAOS_CHECK_GOTO(cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen), NULL, _exit); varDataSetLen(value, valueLen); pColInfo = taosArrayGet(pBlock->pDataBlock, col++); @@ -308,7 +304,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, value, false), NULL, _exit); char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; - cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen); + TAOS_CHECK_GOTO(cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen), NULL, _exit); varDataSetLen(scope, valueLen); pColInfo = taosArrayGet(pBlock->pDataBlock, col++); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 268b0b8497..83a185adae 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -42,22 +42,21 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); extern char **environ; -SConfig *cfgInit() { +int32_t cfgInit(SConfig ** ppCfg) { SConfig *pCfg = taosMemoryCalloc(1, sizeof(SConfig)); if (pCfg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } pCfg->array = taosArrayInit(32, sizeof(SConfigItem)); if (pCfg->array == NULL) { taosMemoryFree(pCfg); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } taosThreadMutexInit(&pCfg->lock, NULL); - return pCfg; + *ppCfg = pCfg; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr) { @@ -73,7 +72,7 @@ int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr) { case CFG_STYPE_ENV_CMD: return cfgLoadFromEnvCmd(pCfg, (const char **)sourceStr); default: - return -1; + return TSDB_CODE_INVALID_PARA; } } @@ -82,11 +81,11 @@ int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs) { for (int32_t i = 0; i < size; ++i) { SConfigPair *pPair = taosArrayGet(pArgs, i); if (cfgSetItem(pCfg, pPair->name, pPair->value, CFG_STYPE_ARG_LIST, true) != 0) { - return -1; + return TSDB_CODE_INVALID_PARA; } } - return 0; + return TSDB_CODE_SUCCESS; } void cfgItemFreeVal(SConfigItem *pItem) { @@ -126,28 +125,27 @@ static int32_t cfgCheckAndSetConf(SConfigItem *pItem, const char *conf) { pItem->str = taosStrdup(conf); if (pItem->str == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { char fullDir[PATH_MAX] = {0}; if (taosExpandDir(inputDir, fullDir, PATH_MAX) != 0) { - uError("failed to expand dir:%s", inputDir); - return -1; + int32_t code = TAOS_SYSTEM_ERROR(errno); + uError("failed to expand dir:%s since %s", inputDir, tstrerror(code)); + TAOS_RETURN(code); } taosMemoryFreeClear(pItem->str); pItem->str = taosStrdup(fullDir); if (pItem->str == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { @@ -166,97 +164,88 @@ static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType sty static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) { int32_t ival; - int32_t code = taosStrHumanToInt32(value, &ival); - if (code != TSDB_CODE_SUCCESS) return code; + TAOS_CHECK_RETURN(taosStrHumanToInt32(value, &ival)); if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax); - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } pItem->i32 = ival; pItem->stype = stype; - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { int64_t ival; - int32_t code = taosStrHumanToInt64(value, &ival); - if (code != TSDB_CODE_SUCCESS) return code; + TAOS_CHECK_RETURN(taosStrHumanToInt64(value, &ival)); if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax); - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } pItem->i64 = ival; pItem->stype = stype; - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType stype) { double dval; - int32_t code = parseCfgReal(value, &dval); + TAOS_CHECK_RETURN(parseCfgReal(value, &dval)); if (dval < pItem->fmin || dval > pItem->fmax) { uError("cfg:%s, type:%s src:%s value:%f out of range[%f, %f]", pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), dval, pItem->fmin, pItem->fmax); - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } pItem->fval = (float)dval; pItem->stype = stype; - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType stype) { char *tmp = taosStrdup(value); if (tmp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s", pItem->name, cfgDtypeStr(pItem->dtype), - cfgStypeStr(stype), value, terrstr()); - return -1; + cfgStypeStr(stype), value, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } taosMemoryFreeClear(pItem->str); pItem->str = tmp; pItem->stype = stype; - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - if (cfgCheckAndSetDir(pItem, value) != 0) { + int32_t code = cfgCheckAndSetDir(pItem, value); + if (TSDB_CODE_SUCCESS != code) { uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s", pItem->name, cfgDtypeStr(pItem->dtype), - cfgStypeStr(stype), value, terrstr()); - return -1; + cfgStypeStr(stype), value, tstrerror(code)); + TAOS_RETURN(code); } pItem->stype = stype; - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t doSetConf(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - if (cfgCheckAndSetConf(pItem, value) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = cfgCheckAndSetConf(pItem, value); + if (TSDB_CODE_SUCCESS != code) { uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s", pItem->name, cfgDtypeStr(pItem->dtype), - cfgStypeStr(stype), value, terrstr()); - return -1; + cfgStypeStr(stype), value, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } pItem->stype = stype; - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - int32_t code = doSetConf(pItem, value, stype); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - + TAOS_CHECK_RETURN(doSetConf(pItem, value, stype)); osSetTimezone(value); - return code; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, const char *level, const char *primary, @@ -267,16 +256,15 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, if (pItem == NULL) { taosThreadMutexUnlock(&pCfg->lock); - return -1; + TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } if (pItem->array == NULL) { pItem->array = taosArrayInit(16, sizeof(SDiskCfg)); if (pItem->array == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; taosThreadMutexUnlock(&pCfg->lock); - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } } @@ -287,34 +275,32 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, cfg.disable = disable ? atoi(disable) : 0; void *ret = taosArrayPush(pItem->array, &cfg); if (ret == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; taosThreadMutexUnlock(&pCfg->lock); - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } pItem->stype = stype; taosThreadMutexUnlock(&pCfg->lock); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool resetArray) { SConfigItem *pDebugFlagItem = cfgGetItem(pCfg, "debugFlag"); if (resetArray) { // reset - if (pDebugFlagItem == NULL) return -1; + if (pDebugFlagItem == NULL) TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); // logflag names that should 'not' be set by 'debugFlag' if (pDebugFlagItem->array == NULL) { pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar)); if (pDebugFlagItem->array == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } } taosArrayClear(pDebugFlagItem->array); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } // update @@ -322,9 +308,11 @@ static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool rese if (pDebugFlagItem->array != NULL) { SLogVar logVar = {0}; strncpy(logVar.name, name, TSDB_LOG_VAR_LEN - 1); - taosArrayPush(pDebugFlagItem->array, &logVar); + if (NULL == taosArrayPush(pDebugFlagItem->array, &logVar)) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype, bool lock) { @@ -337,9 +325,8 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy SConfigItem *pItem = cfgGetItem(pCfg, name); if (pItem == NULL) { - terrno = TSDB_CODE_CFG_NOT_FOUND; taosThreadMutexUnlock(&pCfg->lock); - return -1; + TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } switch (pItem->dtype) { @@ -382,7 +369,7 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy } case CFG_DTYPE_NONE: default: - terrno = TSDB_CODE_INVALID_CFG; + code = TSDB_CODE_INVALID_CFG; break; } @@ -390,7 +377,7 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy taosThreadMutexUnlock(&pCfg->lock); } - return code; + TAOS_RETURN(code); } SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) { @@ -403,7 +390,6 @@ SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) { } } - terrno = TSDB_CODE_CFG_NOT_FOUND; return NULL; } @@ -420,27 +406,25 @@ void cfgUnLock(SConfig *pCfg) { } int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer) { - ECfgDynType dynType = isServer ? CFG_DYN_SERVER : CFG_DYN_CLIENT; + ECfgDynType dynType = isServer ? CFG_DYN_SERVER : CFG_DYN_CLIENT; cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); if (!pItem || (pItem->dynScope & dynType) == 0) { uError("failed to config:%s, not support update this config", name); - terrno = TSDB_CODE_INVALID_CFG; cfgUnLock(pCfg); - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_CFG); } switch (pItem->dtype) { - case CFG_DTYPE_STRING:{ - if(strcasecmp(name, "slowLogScope") == 0){ - char* tmp = taosStrdup(pVal); - if(taosSetSlowLogScope(tmp) < 0){ - terrno = TSDB_CODE_INVALID_CFG; + case CFG_DTYPE_STRING: { + if (strcasecmp(name, "slowLogScope") == 0) { + char *tmp = taosStrdup(pVal); + if (taosSetSlowLogScope(tmp) < 0) { cfgUnLock(pCfg); taosMemoryFree(tmp); - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_CFG); } taosMemoryFree(tmp); } @@ -449,9 +433,8 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p int32_t ival = (int32_t)atoi(pVal); if (ival != 0 && ival != 1) { uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, cfgDtypeStr(pItem->dtype), ival); - terrno = TSDB_CODE_OUT_OF_RANGE; cfgUnLock(pCfg); - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } } break; case CFG_DTYPE_INT32: { @@ -464,9 +447,8 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s value:%d out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax); - terrno = TSDB_CODE_OUT_OF_RANGE; cfgUnLock(pCfg); - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } } break; case CFG_DTYPE_INT64: { @@ -474,30 +456,28 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p int32_t code = taosStrHumanToInt64(pVal, &ival); if (code != TSDB_CODE_SUCCESS) { cfgUnLock(pCfg); - return code; + TAOS_RETURN(code); } if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax); - terrno = TSDB_CODE_OUT_OF_RANGE; cfgUnLock(pCfg); - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } } break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: { - double dval; + double dval; int32_t code = parseCfgReal(pVal, &dval); if (code != TSDB_CODE_SUCCESS) { cfgUnLock(pCfg); - return code; + TAOS_RETURN(code); } if (dval < pItem->fmin || dval > pItem->fmax) { uError("cfg:%s, type:%s value:%f out of range[%f, %f]", pItem->name, cfgDtypeStr(pItem->dtype), dval, pItem->fmin, pItem->fmax); - terrno = TSDB_CODE_OUT_OF_RANGE; cfgUnLock(pCfg); - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } } break; default: @@ -505,15 +485,14 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p } cfgUnLock(pCfg); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { pItem->stype = CFG_STYPE_DEFAULT; pItem->name = taosStrdup(name); if (pItem->name == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } int32_t size = taosArrayGetSize(pCfg->array); @@ -521,7 +500,7 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { SConfigItem *existItem = taosArrayGet(pCfg->array, i); if (existItem != NULL && strcmp(existItem->name, pItem->name) == 0) { taosMemoryFree(pItem->name); - return TSDB_CODE_INVALID_CFG; + TAOS_RETURN(TSDB_CODE_INVALID_CFG); } } @@ -535,11 +514,10 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { } taosMemoryFree(pItem->name); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope) { @@ -550,8 +528,7 @@ int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scop int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope) { if (defaultVal < minval || defaultVal > maxval) { - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } SConfigItem item = {.dtype = CFG_DTYPE_INT32, @@ -566,8 +543,7 @@ int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope) { if (defaultVal < minval || defaultVal > maxval) { - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } SConfigItem item = {.dtype = CFG_DTYPE_INT64, @@ -582,8 +558,7 @@ int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope, int8_t dynScope) { if (defaultVal < minval || defaultVal > maxval) { - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, @@ -599,45 +574,32 @@ int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, in SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope, .dynScope = dynScope}; item.str = taosStrdup(defaultVal); if (item.str == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } return cfgAddItem(pCfg, &item, name); } int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { SConfigItem item = {.dtype = CFG_DTYPE_DIR, .scope = scope, .dynScope = dynScope}; - if (cfgCheckAndSetDir(&item, defaultVal) != 0) { - return -1; - } - + TAOS_CHECK_RETURN(cfgCheckAndSetDir(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope, .dynScope = dynScope}; - if (cfgCheckAndSetConf(&item, defaultVal) != 0) { - return -1; - } - + TAOS_CHECK_RETURN(cfgCheckAndSetConf(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope, .dynScope = dynScope}; - if (cfgCheckAndSetConf(&item, defaultVal) != 0) { - return -1; - } - + TAOS_CHECK_RETURN(cfgCheckAndSetConf(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope, .dynScope = dynScope}; - if (cfgCheckAndSetConf(&item, defaultVal) != 0) { - return -1; - } - + TAOS_CHECK_RETURN(cfgCheckAndSetConf(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } @@ -693,7 +655,7 @@ const char *cfgDtypeStr(ECfgDataType type) { } } -void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen) { +int32_t cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen) { int32_t len = 0; switch (pItem->dtype) { case CFG_DTYPE_BOOL: @@ -719,14 +681,19 @@ void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p break; } + if (len < 0) { + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + } + if (len > bufSize) { len = bufSize; } *pLen = len; + TAOS_RETURN(TSDB_CODE_SUCCESS); } -void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen) { +int32_t cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen) { int32_t len = 0; switch (pItem->scope) { case CFG_SCOPE_SERVER: @@ -740,11 +707,16 @@ void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p break; } + if (len < 0) { + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + } + if (len > bufSize) { len = bufSize; } *pLen = len; + TAOS_RETURN(TSDB_CODE_SUCCESS); } void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump) { @@ -919,14 +891,14 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { char **pEnv = environ; line[1023] = 0; - if (pEnv == NULL) return 0; + if (pEnv == NULL) TAOS_RETURN(TSDB_CODE_SUCCESS); while (*pEnv != NULL) { name = value = value2 = value3 = value4 = NULL; olen = vlen = vlen2 = vlen3 = vlen4 = 0; strncpy(line, *pEnv, sizeof(line) - 1); pEnv++; - taosEnvToCfg(line, line); + (void)taosEnvToCfg(line, line); paGetToken(line, &name, &olen); if (olen == 0) continue; @@ -943,21 +915,21 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { if (vlen3 != 0) { value3[vlen3] = 0; paGetToken(value3 + vlen3 + 1, &value4, &vlen4); - if(vlen4 != 0) value4[vlen4] = 0; + if (vlen4 != 0) value4[vlen4] = 0; } } code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR, true); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, value4, CFG_STYPE_ENV_VAR); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; } } uInfo("load from env variables cfg success"); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { @@ -965,11 +937,11 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { int32_t olen, vlen, vlen2, vlen3, vlen4; int32_t code = 0; int32_t index = 0; - if (envCmd == NULL) return 0; + if (envCmd == NULL) TAOS_RETURN(TSDB_CODE_SUCCESS); while (envCmd[index] != NULL) { strncpy(buf, envCmd[index], sizeof(buf) - 1); buf[sizeof(buf) - 1] = 0; - taosEnvToCfg(buf, buf); + (void)taosEnvToCfg(buf, buf); index++; name = value = value2 = value3 = value4 = NULL; @@ -990,21 +962,21 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { if (vlen3 != 0) { value3[vlen3] = 0; paGetToken(value3 + vlen3 + 1, &value4, &vlen4); - if(vlen4 != 0) value4[vlen4] = 0; + if (vlen4 != 0) value4[vlen4] = 0; } } code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD, true); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, value4, CFG_STYPE_ENV_CMD); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; } } uInfo("load from env cmd cfg success"); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { @@ -1017,20 +989,19 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { if (envFile != NULL && strlen(envFile) > 0) { if (!taosCheckExistFile(envFile)) { uError("failed to load env file:%s", envFile); - return -1; + TAOS_RETURN(TSDB_CODE_NOT_FOUND); } filepath = envFile; } else { if (!taosCheckExistFile(filepath)) { uInfo("env file:%s not load", filepath); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } } TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } while (!taosEOFFile(pFile)) { @@ -1042,7 +1013,7 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { break; } if (line[_bytes - 1] == '\n') line[_bytes - 1] = 0; - taosEnvToCfg(line, line); + (void)taosEnvToCfg(line, line); paGetToken(line, &name, &olen); if (olen == 0) continue; @@ -1059,23 +1030,23 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { if (vlen3 != 0) { value3[vlen3] = 0; paGetToken(value3 + vlen3 + 1, &value4, &vlen4); - if(vlen4 != 0) value4[vlen4] = 0; + if (vlen4 != 0) value4[vlen4] = 0; } } code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE, true); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, value4, CFG_STYPE_ENV_FILE); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; } } taosCloseFile(&pFile); uInfo("load from env cfg file %s success", filepath); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { @@ -1087,13 +1058,13 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { // success when the file does not exist + code = TAOS_SYSTEM_ERROR(errno); if (errno == ENOENT) { - terrno = TAOS_SYSTEM_ERROR(errno); - uInfo("failed to load from cfg file %s since %s, use default parameters", filepath, terrstr()); - return 0; + uInfo("failed to load from cfg file %s since %s, use default parameters", filepath, tstrerror(code)); + TAOS_RETURN(TSDB_CODE_SUCCESS); } else { - uError("failed to load from cfg file %s since %s", filepath, terrstr()); - return -1; + uError("failed to load from cfg file %s since %s", filepath, tstrerror(code)); + TAOS_RETURN(code); } } @@ -1134,7 +1105,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } code = cfgSetItem(pConfig, name, newValue, CFG_STYPE_CFG_FILE, true); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; } else { paGetToken(value + vlen + 1, &value2, &vlen2); if (vlen2 != 0) { @@ -1148,12 +1119,12 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE, true); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; } if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, value4, CFG_STYPE_CFG_FILE); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; } size_t len = strlen(name); @@ -1161,18 +1132,18 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { const size_t debugFlagLen = strlen(debugFlagStr); if (len >= debugFlagLen && strcasecmp(name + len - debugFlagLen, debugFlagStr) == 0) { code = cfgUpdateDebugFlagItem(pConfig, name, len == debugFlagLen); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; } } taosCloseFile(&pFile); - if (code == 0 || (code != 0 && terrno == TSDB_CODE_CFG_NOT_FOUND)) { + if (TSDB_CODE_SUCCESS == code || TSDB_CODE_CFG_NOT_FOUND == code) { uInfo("load from cfg file %s success", filepath); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } else { - uError("failed to load from cfg file %s since %s", filepath, terrstr()); - return -1; + uError("failed to load from cfg file %s since %s", filepath, tstrerror(code)); + TAOS_RETURN(code); } } @@ -1247,46 +1218,52 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { int32_t code = 0; if (url == NULL || strlen(url) == 0) { uInfo("apoll url not load"); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } char *p = strchr(url, ':'); if (p == NULL) { uError("fail to load apoll url: %s, unknown format", url); - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } p++; + SJson *pJson = NULL; if (strncmp(url, "jsonFile", 8) == 0) { char *filepath = p; if (!taosCheckExistFile(filepath)) { uError("failed to load json file:%s", filepath); - return -1; + TAOS_RETURN(TSDB_CODE_NOT_FOUND); } TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } size_t fileSize = taosLSeekFile(pFile, 0, SEEK_END); char *buf = taosMemoryMalloc(fileSize); + if (!buf) { + taosCloseFile(&pFile); + uError("load json file error: %s, failed to alloc memory", filepath); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + taosLSeekFile(pFile, 0, SEEK_SET); if (taosReadFile(pFile, buf, fileSize) <= 0) { taosCloseFile(&pFile); uError("load json file error: %s", filepath); taosMemoryFreeClear(buf); - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); } taosCloseFile(&pFile); - SJson *pJson = tjsonParse(buf); + pJson = tjsonParse(buf); if (NULL == pJson) { const char *jsonParseError = tjsonGetError(); if (jsonParseError != NULL) { uError("load json file parse error: %s", jsonParseError); } taosMemoryFreeClear(buf); - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); } taosMemoryFreeClear(buf); @@ -1295,13 +1272,17 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { cJSON *item = tjsonGetArrayItem(pJson, i); if (item == NULL) break; char *itemName = NULL, *itemValueString = NULL; - tjsonGetObjectName(item, &itemName); - tjsonGetObjectName(item, &itemName); - tjsonGetObjectValueString(item, &itemValueString); + TAOS_CHECK_GOTO(tjsonGetObjectName(item, &itemName), NULL, _err_json); + TAOS_CHECK_GOTO(tjsonGetObjectValueString(item, &itemValueString), NULL, _err_json); if (itemValueString != NULL && itemName != NULL) { size_t itemNameLen = strlen(itemName); size_t itemValueStringLen = strlen(itemValueString); cfgLineBuf = taosMemoryMalloc(itemNameLen + itemValueStringLen + 2); + if (NULL == cfgLineBuf) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err_json; + } + memcpy(cfgLineBuf, itemName, itemNameLen); cfgLineBuf[itemNameLen] = ' '; memcpy(&cfgLineBuf[itemNameLen + 1], itemValueString, itemValueStringLen); @@ -1327,11 +1308,11 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { } code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL, true); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, value4, CFG_STYPE_APOLLO_URL); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; } } } @@ -1341,16 +1322,20 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { // } else if (strncmp(url, "etcdUrl", 7) == 0) { } else { uError("Unsupported url: %s", url); - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } uInfo("load from apoll url not implemented yet"); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); + +_err_json: + tjsonDelete(pJson); + TAOS_RETURN(code); } int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl) { int32_t index = 0; - if (envCmd == NULL) return 0; + if (envCmd == NULL) TAOS_RETURN(TSDB_CODE_SUCCESS); while (envCmd[index] != NULL) { if (strncmp(envCmd[index], "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(envCmd[index], '='); @@ -1362,7 +1347,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl } memcpy(apolloUrl, p, TMIN(strlen(p) + 1, PATH_MAX)); uInfo("get apollo url from env cmd success"); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } } index++; @@ -1384,7 +1369,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl } memcpy(apolloUrl, p, TMIN(strlen(p) + 1, PATH_MAX)); uInfo("get apollo url from env variables success, apolloUrl=%s", apolloUrl); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } } } @@ -1393,13 +1378,13 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl if (envFile != NULL && strlen(envFile) > 0) { if (!taosCheckExistFile(envFile)) { uError("failed to load env file:%s", envFile); - return -1; + TAOS_RETURN(TSDB_CODE_NOT_FOUND); } filepath = envFile; } else { if (!taosCheckExistFile(filepath)) { uInfo("env file:%s not load", filepath); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } } int64_t _bytes; @@ -1422,7 +1407,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl memcpy(apolloUrl, p, TMIN(strlen(p) + 1, PATH_MAX)); taosCloseFile(&pFile); uInfo("get apollo url from env file success"); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } } } @@ -1430,7 +1415,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl } uInfo("fail get apollo url from cmd env file"); - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } struct SConfigIter { @@ -1438,15 +1423,16 @@ struct SConfigIter { SConfig *pConf; }; -SConfigIter *cfgCreateIter(SConfig *pConf) { - SConfigIter* pIter = taosMemoryCalloc(1, sizeof(SConfigIter)); +int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter) { + SConfigIter *pIter = taosMemoryCalloc(1, sizeof(SConfigIter)); if (pIter == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } pIter->pConf = pConf; - return pIter; + + *ppIter = pIter; + TAOS_RETURN(TSDB_CODE_SUCCESS); } SConfigItem *cfgNextIter(SConfigIter* pIter) { @@ -1463,4 +1449,4 @@ void cfgDestroyIter(SConfigIter *pIter) { } taosMemoryFree(pIter); -} \ No newline at end of file +} diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp index 9f8645b14c..33ababef10 100644 --- a/source/util/test/cfgTest.cpp +++ b/source/util/test/cfgTest.cpp @@ -51,7 +51,10 @@ TEST_F(CfgTest, 01_Str) { } TEST_F(CfgTest, 02_Basic) { - SConfig *pConfig = cfgInit(); + SConfig *pConfig = NULL; + int32_t code = cfgInit(&pConfig); + + ASSERT_EQ(code, TSDB_CODE_SUCCESS); ASSERT_NE(pConfig, nullptr); EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 0), 0); @@ -66,7 +69,11 @@ TEST_F(CfgTest, 02_Basic) { int32_t size = cfgGetSize(pConfig); SConfigItem* pItem = NULL; - SConfigIter* pIter = cfgCreateIter(pConfig); + SConfigIter *pIter = NULL; + code = cfgCreateIter(pConfig, &pIter); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + ASSERT_NE(pIter, nullptr); + while((pItem = cfgNextIter(pIter)) != NULL) { switch (pItem->dtype) { case CFG_DTYPE_BOOL: From 586d3cae64e19b76f00e90a23f2671e01ee773eb Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 24 Jul 2024 16:08:12 +0800 Subject: [PATCH 40/52] enh: refactor return code --- include/util/tutil.h | 12 ++-- source/dnode/mgmt/node_util/src/dmFile.c | 4 +- source/dnode/mnode/impl/src/mndProfile.c | 85 ++++++++++++++++-------- source/dnode/mnode/impl/src/mndUser.c | 20 +++--- 4 files changed, 75 insertions(+), 46 deletions(-) diff --git a/include/util/tutil.h b/include/util/tutil.h index 05a2397139..0e79480bfe 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -157,18 +157,18 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, return (terrno = (code)); \ } while (0) -#define TAOS_CHECK_RETURN(CMD) \ - do { \ - int32_t code = (CMD); \ +#define TAOS_CHECK_RETURN(CMD) \ + do { \ + int32_t code = (CMD); \ if (code != TSDB_CODE_SUCCESS) { \ - TAOS_RETURN(code); \ - } \ + TAOS_RETURN(code); \ + } \ } while (0) #define TAOS_CHECK_GOTO(CMD, LINO, LABEL) \ do { \ code = (CMD); \ - if (code != TSDB_CODE_SUCCESS) { \ + if (code != TSDB_CODE_SUCCESS) { \ if (LINO) { \ *((int32_t *)(LINO)) = __LINE__; \ } \ diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 93b6d976c2..eee6ab3a41 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -429,9 +429,7 @@ int32_t dmUpdateEncryptKey(char *key, bool toLogFile) { } } - if ((code = tGetMachineId(&machineId)) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(tGetMachineId(&machineId), &lino, _OVER); TAOS_CHECK_GOTO(generateEncryptCode(key, machineId, &encryptCode), &lino, _OVER); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 387fcb90bc..f7b509b1cc 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -229,26 +229,23 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { SUserObj *pUser = NULL; SDbObj *pDb = NULL; SConnObj *pConn = NULL; - int32_t code = -1; + int32_t code = 0; SConnectReq connReq = {0}; char ip[24] = {0}; const STraceId *trace = &pReq->info.traceId; if ((code = tDeserializeSConnectReq(pReq->pCont, pReq->contLen, &connReq)) != 0) { - terrno = (-1 == code ? TSDB_CODE_INVALID_MSG : code); goto _OVER; } if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 3)) != 0) { mGError("version not compatible. client version: %s, server version: %s", connReq.sVer, version); - terrno = code; goto _OVER; } - code = -1; taosIp2String(pReq->info.conn.clientIp, ip); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { - mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); + if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONNECT)) != 0) { + mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, tstrerror(code)); goto _OVER; } @@ -271,22 +268,22 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { if (pDb == NULL) { if (0 != strcmp(connReq.db, TSDB_INFORMATION_SCHEMA_DB) && (0 != strcmp(connReq.db, TSDB_PERFORMANCE_SCHEMA_DB))) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; + code = TSDB_CODE_MND_DB_NOT_EXIST; mGError("user:%s, failed to login from %s while use db:%s since %s", pReq->info.conn.user, ip, connReq.db, - terrstr()); + tstrerror(code)); goto _OVER; } } - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb), NULL, _OVER); } pConn = mndCreateConn(pMnode, pReq->info.conn.user, connReq.connType, pReq->info.conn.clientIp, pReq->info.conn.clientPort, connReq.pid, connReq.app, connReq.startTime); if (pConn == NULL) { - mGError("user:%s, failed to login from %s while create connection since %s", pReq->info.conn.user, ip, terrstr()); + code = terrno; + mGError("user:%s, failed to login from %s while create connection since %s", pReq->info.conn.user, ip, + tstrerror(code)); goto _OVER; } @@ -316,10 +313,19 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { mndGetMnodeEpSet(pMnode, &connectRsp.epSet); int32_t contLen = tSerializeSConnectRsp(NULL, 0, &connectRsp); - if (contLen < 0) goto _OVER; + if (contLen < 0) { + TAOS_CHECK_GOTO(contLen, NULL, _OVER); + } void *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) goto _OVER; - tSerializeSConnectRsp(pRsp, contLen, &connectRsp); + if (pRsp == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); + } + + contLen = tSerializeSConnectRsp(pRsp, contLen, &connectRsp); + if (contLen < 0) { + rpcFreeCont(pRsp); + TAOS_CHECK_GOTO(contLen, NULL, _OVER); + } pReq->info.rspLen = contLen; pReq->info.rsp = pRsp; @@ -339,7 +345,7 @@ _OVER: mndReleaseDb(pMnode, pDb); mndReleaseConn(pMnode, pConn, true); - return code; + TAOS_RETURN(code); } static int32_t mndSaveQueryList(SConnObj *pConn, SQueryHbReqBasic *pBasic) { @@ -656,6 +662,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { int32_t code = 0; + int32_t lino = 0; SMnode *pMnode = pReq->info.node; SClientHbBatchReq batchReq = {0}; @@ -675,6 +682,9 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { SClientHbBatchRsp batchRsp = {0}; batchRsp.svrTimestamp = taosGetTimestampSec(); batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp)); + if (batchRsp.rsps == NULL) { + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + } batchRsp.monitorParas.tsEnableMonitor = tsEnableMonitor; batchRsp.monitorParas.tsMonitorInterval = tsMonitorInterval; batchRsp.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold; @@ -687,7 +697,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { for (int i = 0; i < sz; i++) { SClientHbReq *pHbReq = taosArrayGet(batchReq.reqs, i); if (pHbReq->connKey.connType == CONN_TYPE__QUERY) { - mndProcessQueryHeartBeat(pMnode, pReq, pHbReq, &batchRsp, &obj); + TAOS_CHECK_EXIT(mndProcessQueryHeartBeat(pMnode, pReq, pHbReq, &batchRsp, &obj)); } else if (pHbReq->connKey.connType == CONN_TYPE__TMQ) { SClientHbRsp *pRsp = mndMqHbBuildRsp(pMnode, pHbReq); if (pRsp != NULL) { @@ -699,12 +709,22 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq); int32_t tlen = tSerializeSClientHbBatchRsp(NULL, 0, &batchRsp); - void *buf = rpcMallocCont(tlen); - tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp); - - tFreeClientHbBatchRsp(&batchRsp); + if (tlen < 0) { + TAOS_CHECK_EXIT(tlen); + } + void *buf = rpcMallocCont(tlen); + if (!buf) { + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + } + tlen = tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp); + if (tlen < 0) { + rpcFreeCont(buf); + TAOS_CHECK_EXIT(tlen); + } pReq->info.rspLen = tlen; pReq->info.rsp = buf; +_exit: + tFreeClientHbBatchRsp(&batchRsp); taosArrayDestroy(obj.pQnodeList); @@ -771,24 +791,31 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { } static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) { - int32_t code = -1; + int32_t code = 0; + int32_t lino = 0; SServerVerRsp rsp = {0}; tstrncpy(rsp.ver, version, sizeof(rsp.ver)); int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp); - if (contLen < 0) goto _over; + if (contLen < 0) { + TAOS_CHECK_EXIT(contLen); + } void *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) goto _over; - tSerializeSServerVerRsp(pRsp, contLen, &rsp); + if (pRsp == NULL) { + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + } + contLen = tSerializeSServerVerRsp(pRsp, contLen, &rsp); + if (contLen < 0) { + rpcFreeCont(pRsp); + TAOS_CHECK_EXIT(contLen); + } pReq->info.rspLen = contLen; pReq->info.rsp = pRsp; - code = 0; +_exit: -_over: - - return code; + TAOS_RETURN(code); } static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 0e4730a731..f05baf25bd 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1958,8 +1958,7 @@ int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq) { len = tSerializeSUpdateIpWhite(NULL, 0, &ipWhite); if (len < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - TAOS_CHECK_GOTO(code, &lino, _OVER); + TAOS_CHECK_GOTO(len, &lino, _OVER); } pRsp = rpcMallocCont(len); @@ -1968,8 +1967,7 @@ int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq) { } len = tSerializeSUpdateIpWhite(pRsp, len, &ipWhite); if (len < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - TAOS_CHECK_GOTO(code, &lino, _OVER); + TAOS_CHECK_GOTO(len, &lino, _OVER); } _OVER: @@ -2597,7 +2595,10 @@ static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq) { TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } - TAOS_CHECK_EXIT(tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp)); + contLen =tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp); + if(contLen < 0) { + TAOS_CHECK_EXIT(contLen); + } _exit: mndReleaseUser(pMnode, pUser); @@ -3190,14 +3191,17 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ } rspLen = tSerializeSUserAuthBatchRsp(NULL, 0, &batchRsp); - if (rspLen <= 0) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + if (rspLen < 0) { + TAOS_CHECK_GOTO(rspLen, &lino, _OVER); } pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - TAOS_CHECK_GOTO(tSerializeSUserAuthBatchRsp(pRsp, rspLen, &batchRsp), &lino, _OVER); + rspLen = tSerializeSUserAuthBatchRsp(pRsp, rspLen, &batchRsp); + if (rspLen < 0) { + TAOS_CHECK_GOTO(rspLen, &lino, _OVER); + } _OVER: tFreeSUserAuthBatchRsp(&batchRsp); if (code < 0) { From 3c41b5a98c599e7d52a6a3e493d0ac4d6ff2c8fb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 24 Jul 2024 17:22:00 +0800 Subject: [PATCH 41/52] use new macros --- source/dnode/vnode/src/tsdb/tsdbCache.c | 599 ++++++++++++------------ 1 file changed, 302 insertions(+), 297 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 91345bf001..459ad39819 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -39,7 +39,8 @@ static int32_t tsdbOpenBCache(STsdb *pTsdb) { _err: if (code) { - tsdbError("tsdb/bcache: vgId:%d, open failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); + tsdbError("tsdb/bcache: vgId:%d, %s failed at line %d since %s.", TD_VID(pTsdb->pVnode), __func__, lino, + tstrerror(code)); } TAOS_RETURN(code); @@ -159,12 +160,11 @@ static int myCmp(void *state, const char *a, size_t alen, const char *b, size_t } static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { - int32_t code = 0; + int32_t code = 0, lino = 0; rocksdb_comparator_t *cmp = rocksdb_comparator_create(NULL, myCmpDestroy, myCmp, myCmpName); if (NULL == cmp) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } rocksdb_cache_t *cache = rocksdb_cache_create_lru(5 * 1024 * 1024); @@ -175,8 +175,7 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { rocksdb_options_t *options = rocksdb_options_create(); if (NULL == options) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err); } rocksdb_options_set_create_if_missing(options, 1); @@ -189,15 +188,13 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { rocksdb_writeoptions_t *writeoptions = rocksdb_writeoptions_create(); if (NULL == writeoptions) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err2); } rocksdb_writeoptions_disable_WAL(writeoptions, 1); rocksdb_readoptions_t *readoptions = rocksdb_readoptions_create(); if (NULL == readoptions) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err3; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err3); } char *err = NULL; @@ -209,14 +206,12 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); rocksdb_free(err); - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err4; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err4); } rocksdb_flushoptions_t *flushoptions = rocksdb_flushoptions_create(); if (NULL == flushoptions) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err5; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err5); } rocksdb_writebatch_t *writebatch = rocksdb_writebatch_create(); @@ -235,7 +230,7 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { pTsdb->rCache.pTSchema = NULL; - return code; + TAOS_RETURN(code); _err5: rocksdb_close(pTsdb->rCache.db); @@ -249,7 +244,8 @@ _err2: rocksdb_cache_destroy(cache); _err: rocksdb_comparator_destroy(cmp); - return code; + + TAOS_RETURN(code); } static void tsdbCloseRocksCache(STsdb *pTsdb) { @@ -344,24 +340,26 @@ static int32_t tsdbCacheDeserializeV0(char const *value, SLastCol *pLastCol) { } } -static SLastCol *tsdbCacheDeserialize(char const *value, size_t size) { +static int32_t tsdbCacheDeserialize(char const *value, size_t size, SLastCol **ppLastCol) { if (!value) { - return NULL; + return TSDB_CODE_INVALID_PARA; } SLastCol *pLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); if (NULL == pLastCol) { - return NULL; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } int32_t offset = tsdbCacheDeserializeV0(value, pLastCol); if (offset == size) { // version 0 - return pLastCol; + *ppLastCol = pLastCol; + + TAOS_RETURN(TSDB_CODE_SUCCESS); } else if (offset > size) { - terrno = TSDB_CODE_INVALID_DATA_FMT; taosMemoryFreeClear(pLastCol); - return NULL; + + TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); } // version @@ -387,12 +385,14 @@ static SLastCol *tsdbCacheDeserialize(char const *value, size_t size) { } if (offset > size) { - terrno = TSDB_CODE_INVALID_DATA_FMT; taosMemoryFreeClear(pLastCol); - return NULL; + + TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); } - return pLastCol; + *ppLastCol = pLastCol; + + TAOS_RETURN(TSDB_CODE_SUCCESS); } /* @@ -430,7 +430,7 @@ static int32_t tsdbCacheSerializeV0(char const *value, SLastCol *pLastCol) { return 0; } -static void tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size) { +static int32_t tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size) { *size = sizeof(SLastColV0); if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) { *size += pLastCol->colVal.value.nData; @@ -445,6 +445,9 @@ static void tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size) { } *value = taosMemoryMalloc(*size); + if (NULL == *value) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } int32_t offset = tsdbCacheSerializeV0(*value, pLastCol); @@ -467,16 +470,22 @@ static void tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size) { offset += pLastCol->rowKey.pks[i].nData; } } + + TAOS_RETURN(TSDB_CODE_SUCCESS); } static void tsdbCachePutBatch(SLastCol *pLastCol, const void *key, size_t klen, SCacheFlushState *state) { + int32_t code = 0; STsdb *pTsdb = state->pTsdb; SRocksCache *rCache = &pTsdb->rCache; rocksdb_writebatch_t *wb = rCache->writebatch; char *rocks_value = NULL; size_t vlen = 0; - tsdbCacheSerialize(pLastCol, &rocks_value, &vlen); + code = tsdbCacheSerialize(pLastCol, &rocks_value, &vlen); + if (code) { + tsdbError("tsdb/cache: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + } taosThreadMutexLock(&rCache->rMutex); @@ -534,25 +543,31 @@ int32_t tsdbCacheCommit(STsdb *pTsdb) { if (NULL != err) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); rocksdb_free(err); - code = -1; + code = TSDB_CODE_FAILED; } - return code; + TAOS_RETURN(code); } -static void reallocVarDataVal(SValue *pValue) { +static int32_t reallocVarDataVal(SValue *pValue) { if (IS_VAR_DATA_TYPE(pValue->type)) { uint8_t *pVal = pValue->pData; if (pValue->nData > 0) { - pValue->pData = taosMemoryMalloc(pValue->nData); + uint8_t *p = taosMemoryMalloc(pValue->nData); + if (!p) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + pValue->pData = p; memcpy(pValue->pData, pVal, pValue->nData); } else { pValue->pData = NULL; } } + + TAOS_RETURN(TSDB_CODE_SUCCESS); } -static void reallocVarData(SColVal *pColVal) { reallocVarDataVal(&pColVal->value); } +static int32_t reallocVarData(SColVal *pColVal) { return reallocVarDataVal(&pColVal->value); } static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud) { SLastCol *pLastCol = (SLastCol *)value; @@ -587,6 +602,9 @@ static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, i SLastCol *pLastCol = &noneCol; SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); + if (!pTmpLastCol) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } *pTmpLastCol = *pLastCol; pLastCol = pTmpLastCol; @@ -595,13 +613,13 @@ static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, i for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) { SValue *pValue = &pLastCol->rowKey.pks[i]; if (IS_VAR_DATA_TYPE(pValue->type)) { - reallocVarDataVal(pValue); + TAOS_CHECK_RETURN(reallocVarDataVal(pValue)); charge += pValue->nData; } } if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) { - reallocVarData(&pLastCol->colVal); + TAOS_CHECK_RETURN(reallocVarData(&pLastCol->colVal)); charge += pLastCol->colVal.value.nData; } @@ -609,20 +627,10 @@ static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, i LRUStatus status = taosLRUCacheInsert(pCache, pLastKey, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); if (status != TAOS_LRU_STATUS_OK) { - code = -1; + // code = -1; } - /* - // store result back to rocks cache - char *value = NULL; - size_t vlen = 0; - tsdbCacheSerialize(pLastCol, &value, &vlen); - SLastKey *key = pLastKey; - size_t klen = ROCKS_KEY_LEN; - rocksdb_writebatch_put(wb, (char *)key, klen, value, vlen); - taosMemoryFree(value); - */ - return code; + TAOS_RETURN(code); } int32_t tsdbCacheCommitNoLock(STsdb *pTsdb) { @@ -641,10 +649,10 @@ int32_t tsdbCacheCommitNoLock(STsdb *pTsdb) { if (NULL != err) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); rocksdb_free(err); - code = -1; + code = TSDB_CODE_FAILED; } - return code; + TAOS_RETURN(code); } static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, bool hasPrimaryKey) { @@ -681,13 +689,21 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; { - SLastCol *pLastCol = tsdbCacheDeserialize(values_list[0], values_list_sizes[0]); + SLastCol *pLastCol = NULL; + code = tsdbCacheDeserialize(values_list[0], values_list_sizes[0], &pLastCol); + if (code) { + tsdbError("tsdb/cache: vgId:%d, deserialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + } if (NULL != pLastCol) { rocksdb_writebatch_delete(wb, keys_list[0], klen); } taosMemoryFreeClear(pLastCol); - pLastCol = tsdbCacheDeserialize(values_list[1], values_list_sizes[1]); + pLastCol = NULL; + code = tsdbCacheDeserialize(values_list[1], values_list_sizes[1], &pLastCol); + if (code) { + tsdbError("tsdb/cache: vgId:%d, deserialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + } if (NULL != pLastCol) { rocksdb_writebatch_delete(wb, keys_list[1], klen); } @@ -724,7 +740,7 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, taosMemoryFree(values_list); taosMemoryFree(values_list_sizes); - return code; + TAOS_RETURN(code); } int32_t tsdbCacheNewTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWrapper *pSchemaRow) { @@ -745,8 +761,8 @@ int32_t tsdbCacheNewTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWrap code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTSchema); if (code != TSDB_CODE_SUCCESS) { taosThreadMutexUnlock(&pTsdb->lruMutex); - terrno = code; - return -1; + + TAOS_RETURN(code); } for (int i = 0; i < pTSchema->numOfCols; ++i) { @@ -762,7 +778,7 @@ int32_t tsdbCacheNewTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWrap taosThreadMutexUnlock(&pTsdb->lruMutex); - return code; + TAOS_RETURN(code); } int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWrapper *pSchemaRow) { @@ -788,8 +804,9 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra STSchema *pTSchema = NULL; code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTSchema); if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return -1; + taosThreadMutexUnlock(&pTsdb->lruMutex); + + TAOS_RETURN(code); } bool hasPrimayKey = false; @@ -811,7 +828,7 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra taosThreadMutexUnlock(&pTsdb->lruMutex); - return code; + TAOS_RETURN(code); } int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { @@ -824,9 +841,11 @@ int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { STSchema *pTSchema = NULL; code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, suid, -1, &pTSchema); if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return -1; + taosThreadMutexUnlock(&pTsdb->lruMutex); + + TAOS_RETURN(code); } + for (int i = 0; i < TARRAY_SIZE(uids); ++i) { int64_t uid = ((tb_uid_t *)TARRAY_DATA(uids))[i]; @@ -850,7 +869,7 @@ int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { taosThreadMutexUnlock(&pTsdb->lruMutex); - return code; + TAOS_RETURN(code); } int32_t tsdbCacheNewNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, int8_t col_type) { @@ -865,7 +884,7 @@ int32_t tsdbCacheNewNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, int8_t taosThreadMutexUnlock(&pTsdb->lruMutex); //(void)tsdbCacheCommit(pTsdb); - return code; + TAOS_RETURN(code); } int32_t tsdbCacheDropNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey) { @@ -881,7 +900,7 @@ int32_t tsdbCacheDropNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, bool h taosThreadMutexUnlock(&pTsdb->lruMutex); - return code; + TAOS_RETURN(code); } int32_t tsdbCacheNewSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, int8_t col_type) { @@ -900,7 +919,7 @@ int32_t tsdbCacheNewSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, int8_t taosThreadMutexUnlock(&pTsdb->lruMutex); //(void)tsdbCacheCommit(pTsdb); - return code; + TAOS_RETURN(code); } int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, bool hasPrimayKey) { @@ -920,7 +939,7 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, bool taosThreadMutexUnlock(&pTsdb->lruMutex); - return code; + TAOS_RETURN(code); } typedef struct { @@ -955,7 +974,8 @@ static int32_t tsdbCacheUpdateValue(SValue *pOld, SValue *pNew) { } taosMemoryFreeClear(pFree); - return 0; + + TAOS_RETURN(TSDB_CODE_SUCCESS); } static void tsdbCacheUpdateLastCol(SLastCol *pLastCol, SRowKey *pRowKey, SColVal *pColVal) { @@ -981,10 +1001,10 @@ static void tsdbCacheUpdateLastCol(SLastCol *pLastCol, SRowKey *pRowKey, SColVal static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray *updCtxArray) { if (!updCtxArray || TARRAY_SIZE(updCtxArray) == 0) { - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } - int32_t code = 0; + int32_t code = 0, lino = 0; int num_keys = TARRAY_SIZE(updCtxArray); SArray *remainCols = NULL; @@ -1049,7 +1069,11 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray SRowKey *pRowKey = &updCtx->tsdbRowKey.key; SColVal *pColVal = &updCtx->colVal; - SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i], values_list_sizes[i]); + SLastCol *pLastCol = NULL; + code = tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol); + if (code) { + tsdbError("tsdb/cache: vgId:%d, deserialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + } SLastCol *PToFree = pLastCol; if (IS_LAST_KEY(idxKey->key) && !COL_VAL_IS_VALUE(pColVal)) { @@ -1067,7 +1091,10 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray char *value = NULL; size_t vlen = 0; SLastCol lastColTmp = {.rowKey = *pRowKey, .colVal = *pColVal}; - tsdbCacheSerialize(&lastColTmp, &value, &vlen); + code = tsdbCacheSerialize(&lastColTmp, &value, &vlen); + if (code) { + tsdbError("tsdb/cache: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + } taosThreadMutexLock(&pTsdb->rCache.rMutex); @@ -1084,13 +1111,13 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) { SValue *pValue = &pLastCol->rowKey.pks[i]; if (IS_VAR_DATA_TYPE(pValue->type)) { - reallocVarDataVal(pValue); + TAOS_CHECK_GOTO(reallocVarDataVal(pValue), &lino, _exit); charge += pValue->nData; } } if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) { - reallocVarData(&pLastCol->colVal); + TAOS_CHECK_GOTO(reallocVarData(&pLastCol->colVal), &lino, _exit); charge += pLastCol->colVal.value.nData; } @@ -1117,15 +1144,19 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray taosArrayDestroy(remainCols); } +_exit: taosThreadMutexUnlock(&pTsdb->lruMutex); -_exit: - return code; + if (code) { + tsdbError("tsdb/cache: vgId:%d, update failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); + } + + TAOS_RETURN(code); } int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow, SRow **aRow) { - int32_t code = 0; + int32_t code = 0, lino = 0; // 1. prepare last TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version}; @@ -1135,11 +1166,7 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 SArray *ctxArray = NULL; SSHashObj *iColHash = NULL; - code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - goto _exit; - } + TAOS_CHECK_GOTO(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema), &lino, _exit); TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version}; int32_t nCol = pTSchema->numOfCols; @@ -1200,7 +1227,8 @@ _exit: taosMemoryFreeClear(pTSchema); taosArrayDestroy(ctxArray); tSimpleHashCleanup(iColHash); - return code; + + TAOS_RETURN(code); } int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlockData *pBlockData) { @@ -1212,11 +1240,7 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo int32_t sver = TSDBROW_SVERSION(&lRow); SArray *ctxArray = NULL; - code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - goto _exit; - } + TAOS_CHECK_RETURN(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema)); ctxArray = taosArrayInit(pBlockData->nColData, sizeof(SLastUpdateCtx)); @@ -1272,7 +1296,8 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo _exit: taosMemoryFreeClear(pTSchema); taosArrayDestroy(ctxArray); - return 0; + + TAOS_RETURN(code); } static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, @@ -1362,7 +1387,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr .colVal = COL_VAL_NONE(idxKey->key.cid, pr->pSchema->columns[slotIds[i]].type)}; if (!pLastCol) { pLastCol = &noneCol; - reallocVarData(&pLastCol->colVal); + TAOS_CHECK_RETURN(reallocVarData(&pLastCol->colVal)); } taosArraySet(pLastArray, idxKey->idx, pLastCol); @@ -1383,12 +1408,12 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) { SValue *pValue = &pLastCol->rowKey.pks[i]; if (IS_VAR_DATA_TYPE(pValue->type)) { - reallocVarDataVal(pValue); + TAOS_CHECK_RETURN(reallocVarDataVal(pValue)); charge += pValue->nData; } } if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) { - reallocVarData(&pLastCol->colVal); + TAOS_CHECK_RETURN(reallocVarData(&pLastCol->colVal)); charge += pLastCol->colVal.value.nData; } @@ -1402,7 +1427,10 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr wb = pTsdb->rCache.rwritebatch; char *value = NULL; size_t vlen = 0; - tsdbCacheSerialize(pLastCol, &value, &vlen); + code = tsdbCacheSerialize(pLastCol, &value, &vlen); + if (code) { + tsdbError("tsdb/cache: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + } SLastKey *key = &idxKey->key; size_t klen = ROCKS_KEY_LEN; @@ -1428,7 +1456,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr taosMemoryFree(slotIds); - return code; + TAOS_RETURN(code); } static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SArray *remainCols, @@ -1461,7 +1489,11 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA SLRUCache *pCache = pTsdb->lruCache; for (int i = 0, j = 0; i < num_keys && j < TARRAY_SIZE(remainCols); ++i) { - SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i], values_list_sizes[i]); + SLastCol *pLastCol = NULL; + code = tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol); + if (code) { + tsdbError("tsdb/cache: vgId:%d, deserialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + } SLastCol *PToFree = pLastCol; SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[j]; if (pLastCol) { @@ -1473,12 +1505,12 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) { SValue *pValue = &pLastCol->rowKey.pks[i]; if (IS_VAR_DATA_TYPE(pValue->type)) { - reallocVarDataVal(pValue); + TAOS_CHECK_RETURN(reallocVarDataVal(pValue)); charge += pValue->nData; } } if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) { - reallocVarData(&pLastCol->colVal); + TAOS_CHECK_RETURN(reallocVarData(&pLastCol->colVal)); charge += pLastCol->colVal.value.nData; } @@ -1490,9 +1522,9 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA SLastCol lastCol = *pLastCol; for (int8_t i = 0; i < lastCol.rowKey.numOfPKs; i++) { - reallocVarDataVal(&lastCol.rowKey.pks[i]); + TAOS_CHECK_RETURN(reallocVarDataVal(&lastCol.rowKey.pks[i])); } - reallocVarData(&lastCol.colVal); + TAOS_CHECK_RETURN(reallocVarData(&lastCol.colVal)); taosArraySet(pLastArray, idxKey->idx, &lastCol); taosArrayRemove(remainCols, j); @@ -1515,7 +1547,7 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA code = tsdbCacheLoadFromRaw(pTsdb, uid, pLastArray, remainCols, pr, ltype); } - return code; + TAOS_RETURN(code); } int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { @@ -1545,9 +1577,9 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache SLastCol lastCol = *pLastCol; for (int8_t j = 0; j < lastCol.rowKey.numOfPKs; j++) { - reallocVarDataVal(&lastCol.rowKey.pks[j]); + TAOS_CHECK_RETURN(reallocVarDataVal(&lastCol.rowKey.pks[j])); } - reallocVarData(&lastCol.colVal); + TAOS_CHECK_RETURN(reallocVarData(&lastCol.colVal)); taosArrayPush(pLastArray, &lastCol); taosLRUCacheRelease(pCache, h, false); @@ -1574,9 +1606,17 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache SLastCol lastCol = *pLastCol; for (int8_t j = 0; j < lastCol.rowKey.numOfPKs; j++) { - reallocVarDataVal(&lastCol.rowKey.pks[j]); + code = reallocVarDataVal(&lastCol.rowKey.pks[j]); + if (code) { + taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_RETURN(code); + } + } + code = reallocVarData(&lastCol.colVal); + if (code) { + taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_RETURN(code); } - reallocVarData(&lastCol.colVal); taosArraySet(pLastArray, idxKey->idx, &lastCol); taosLRUCacheRelease(pCache, h, false); @@ -1597,7 +1637,7 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache } } - return code; + TAOS_RETURN(code); } int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) { @@ -1605,11 +1645,8 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE // fetch schema STSchema *pTSchema = NULL; int sver = -1; - code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return -1; - } + + TAOS_CHECK_RETURN(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema)); // build keys & multi get from rocks int num_keys = pTSchema->numOfCols; @@ -1652,14 +1689,22 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; for (int i = 0; i < num_keys; ++i) { - SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i], values_list_sizes[i]); + SLastCol *pLastCol = NULL; + code = tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol); + if (code) { + tsdbError("tsdb/cache: vgId:%d, deserialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + } taosThreadMutexLock(&pTsdb->rCache.rMutex); if (NULL != pLastCol && (pLastCol->rowKey.ts <= eKey && pLastCol->rowKey.ts >= sKey)) { rocksdb_writebatch_delete(wb, keys_list[i], klen); } taosMemoryFreeClear(pLastCol); - pLastCol = tsdbCacheDeserialize(values_list[i + num_keys], values_list_sizes[i + num_keys]); + pLastCol = NULL; + code = tsdbCacheDeserialize(values_list[i + num_keys], values_list_sizes[i + num_keys], &pLastCol); + if (code) { + tsdbError("tsdb/cache: vgId:%d, deserialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + } if (NULL != pLastCol && (pLastCol->rowKey.ts <= eKey && pLastCol->rowKey.ts >= sKey)) { rocksdb_writebatch_delete(wb, keys_list[num_keys + i], klen); } @@ -1719,37 +1764,24 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE _exit: taosMemoryFree(pTSchema); - return code; + TAOS_RETURN(code); } int32_t tsdbOpenCache(STsdb *pTsdb) { - int32_t code = 0; + int32_t code = 0, lino = 0; SLRUCache *pCache = NULL; size_t cfgCapacity = pTsdb->pVnode->config.cacheLastSize * 1024 * 1024; pCache = taosLRUCacheInit(cfgCapacity, 0, .5); if (pCache == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err); } - code = tsdbOpenBCache(pTsdb); - if (code != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + TAOS_CHECK_GOTO(tsdbOpenBCache(pTsdb), &lino, _err); - code = tsdbOpenPgCache(pTsdb); - if (code != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + TAOS_CHECK_GOTO(tsdbOpenPgCache(pTsdb), &lino, _err); - code = tsdbOpenRocksCache(pTsdb); - if (code != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + TAOS_CHECK_GOTO(tsdbOpenRocksCache(pTsdb), &lino, _err); taosLRUCacheSetStrictCapacity(pCache, false); @@ -1759,8 +1791,13 @@ int32_t tsdbOpenCache(STsdb *pTsdb) { pTsdb->flushState.flush_count = 0; _err: + if (code) { + tsdbError("tsdb/cache: vgId:%d, open failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); + } + pTsdb->lruCache = pCache; - return code; + + TAOS_RETURN(code); } void tsdbCloseCache(STsdb *pTsdb) { @@ -1818,7 +1855,7 @@ static int32_t getTableDelDataFromDelIdx(SDelFReader *pDelReader, SDelIdx *pDelI code = tsdbReadDelDatav1(pDelReader, pDelIdx, aDelData, INT64_MAX); } - return code; + TAOS_RETURN(code); } static int32_t getTableDelDataFromTbData(STbData *pTbData, SArray *aDelData) { @@ -1895,7 +1932,7 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea code = isFile ? tsdbDataFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block) : tsdbSttFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block); if (code != TSDB_CODE_SUCCESS) { - return code; + TAOS_RETURN(code); } uint64_t uid = uidList[j]; @@ -1959,37 +1996,28 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea tTombBlockDestroy(&block); if (finished) { - return code; + TAOS_RETURN(code); } } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t loadDataTomb(SCacheRowsReader *pReader, SDataFileReader *pFileReader) { - int32_t code = 0; - const TTombBlkArray *pBlkArray = NULL; - code = tsdbDataFileReadTombBlk(pFileReader, &pBlkArray); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - return loadTombFromBlk(pBlkArray, pReader, pFileReader, true); + TAOS_CHECK_RETURN(tsdbDataFileReadTombBlk(pFileReader, &pBlkArray)); + + TAOS_RETURN(loadTombFromBlk(pBlkArray, pReader, pFileReader, true)); } static int32_t loadSttTomb(STsdbReader *pTsdbReader, SSttFileReader *pSttFileReader, SSttBlockLoadInfo *pLoadInfo) { - int32_t code = 0; - - SCacheRowsReader *pReader = (SCacheRowsReader *)pTsdbReader; - + SCacheRowsReader *pReader = (SCacheRowsReader *)pTsdbReader; const TTombBlkArray *pBlkArray = NULL; - code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - return loadTombFromBlk(pBlkArray, pReader, pSttFileReader, false); + TAOS_CHECK_RETURN(tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray)); + + TAOS_RETURN(loadTombFromBlk(pBlkArray, pReader, pSttFileReader, false)); } typedef struct { @@ -2022,14 +2050,11 @@ static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb .pCurRowKey = &pr->rowKey, }; - code = tMergeTreeOpen2(&iter->mergeTree, &conf, NULL); - if (code != TSDB_CODE_SUCCESS) { - return -1; - } + TAOS_CHECK_RETURN(tMergeTreeOpen2(&iter->mergeTree, &conf, NULL)); iter->pMergeTree = &iter->mergeTree; - return code; + TAOS_RETURN(code); } static int32_t lastIterClose(SFSLastIter **iter) { @@ -2042,7 +2067,7 @@ static int32_t lastIterClose(SFSLastIter **iter) { *iter = NULL; - return code; + TAOS_RETURN(code); } static int32_t lastIterNext(SFSLastIter *iter, TSDBROW **ppRow) { @@ -2051,12 +2076,13 @@ static int32_t lastIterNext(SFSLastIter *iter, TSDBROW **ppRow) { bool hasVal = tMergeTreeNext(iter->pMergeTree); if (!hasVal) { *ppRow = NULL; - return code; + + TAOS_RETURN(code); } *ppRow = tMergeTreeGetRow(iter->pMergeTree); - return code; + TAOS_RETURN(code); } typedef enum SFSNEXTROWSTATES { @@ -2107,8 +2133,8 @@ static void clearLastFileSet(SFSNextRowIter *state); static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, int nCols) { + int32_t code = 0, lino = 0; SFSNextRowIter *state = (SFSNextRowIter *)iter; - int32_t code = 0; STsdb *pTsdb = state->pr->pTsdb; if (SFSNEXTROW_FS == state->state) { @@ -2123,7 +2149,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (--state->iFileSet < 0) { *ppRow = NULL; - return code; + + TAOS_RETURN(code); } else { state->pFileSet = TARRAY2_GET(state->aDFileSet, state->iFileSet); } @@ -2153,19 +2180,13 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie filesName[3] = pFileObj[3]->fname; } - code = tsdbDataFileReaderOpen(filesName, &conf, &state->pr->pFileReader); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TAOS_CHECK_GOTO(tsdbDataFileReaderOpen(filesName, &conf, &state->pr->pFileReader), &lino, _err); state->pr->pCurFileSet = state->pFileSet; loadDataTomb(state->pr, state->pr->pFileReader); - int32_t code = tsdbDataFileReadBrinBlk(state->pr->pFileReader, &state->pr->pBlkArray); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TAOS_CHECK_GOTO(tsdbDataFileReadBrinBlk(state->pr->pFileReader, &state->pr->pBlkArray), &lino, _err); } if (!state->pIndexList) { @@ -2202,16 +2223,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie state->pr->pCurFileSet = state->pFileSet; } - code = lastIterOpen(&state->lastIter, state->pFileSet, pTsdb, state->pTSchema, state->suid, state->uid, state->pr, - state->lastTs, aCols, nCols); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TAOS_CHECK_GOTO(lastIterOpen(&state->lastIter, state->pFileSet, pTsdb, state->pTSchema, state->suid, state->uid, + state->pr, state->lastTs, aCols, nCols), + &lino, _err); - code = lastIterNext(&state->lastIter, &state->pLastRow); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TAOS_CHECK_GOTO(lastIterNext(&state->lastIter, &state->pLastRow), &lino, _err); if (!state->pLastRow) { state->lastEmpty = 1; @@ -2228,7 +2244,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie *ppRow = state->pLastRow; state->pLastRow = NULL; - return code; + + TAOS_RETURN(code); } } @@ -2236,10 +2253,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie } if (SFSNEXTROW_NEXTSTTROW == state->state) { - code = lastIterNext(&state->lastIter, &state->pLastRow); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TAOS_CHECK_GOTO(lastIterNext(&state->lastIter, &state->pLastRow), &lino, _err); if (!state->pLastRow) { if (state->pLastIter) { @@ -2252,7 +2266,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie } else { *ppRow = state->pLastRow; state->pLastRow = NULL; - return code; + + TAOS_RETURN(code); } } @@ -2278,10 +2293,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie } else { tBrinBlockClear(&state->brinBlock); } - code = tsdbDataFileReadBrinBlock(state->pr->pFileReader, pBrinBlk, &state->brinBlock); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + + TAOS_CHECK_GOTO(tsdbDataFileReadBrinBlock(state->pr->pFileReader, pBrinBlk, &state->brinBlock), &lino, _err); state->iBrinRecord = state->brinBlock.numOfRecords - 1; state->state = SFSNEXTROW_BRINBLOCK; @@ -2293,10 +2306,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie tBrinBlockClear(&state->brinBlock); goto _next_brinindex; } - code = tBrinBlockGet(&state->brinBlock, state->iBrinRecord, &state->brinRecord); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + + TAOS_CHECK_GOTO(tBrinBlockGet(&state->brinBlock, state->iBrinRecord, &state->brinRecord), &lino, _err); SBrinRecord *pRecord = &state->brinRecord; if (pRecord->uid != state->uid) { @@ -2313,8 +2324,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (!state->pBlockData) { state->pBlockData = &state->blockData; - code = tBlockDataCreate(&state->blockData); - if (code) goto _err; + + TAOS_CHECK_GOTO(tBlockDataCreate(&state->blockData), &lino, _err); } else { tBlockDataReset(state->pBlockData); } @@ -2323,11 +2334,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie --nCols; ++aCols; } - code = tsdbDataFileReadBlockDataByColumn(state->pr->pFileReader, pRecord, state->pBlockData, state->pTSchema, aCols, - nCols); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + + TAOS_CHECK_GOTO(tsdbDataFileReadBlockDataByColumn(state->pr->pFileReader, pRecord, state->pBlockData, + state->pTSchema, aCols, nCols), + &lino, _err); state->nRow = state->blockData.nRow; state->iRow = state->nRow - 1; @@ -2350,10 +2360,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (!state->pLastRow) { // get next row from fslast and process with fs row, --state->Row if select fs row - code = lastIterNext(&state->lastIter, &state->pLastRow); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TAOS_CHECK_GOTO(lastIterNext(&state->lastIter, &state->pLastRow), &lino, _err); } if (!state->pLastRow) { @@ -2372,34 +2379,27 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (lastRowTs > rowTs) { *ppRow = state->pLastRow; state->pLastRow = NULL; - return code; + + TAOS_RETURN(code); } else if (lastRowTs < rowTs) { *ppRow = &state->row; --state->iRow; - return code; + + TAOS_RETURN(code); } else { // TODO: merge rows and *ppRow = mergedRow SRowMerger *pMerger = &state->rowMerger; tsdbRowMergerInit(pMerger, state->pTSchema); - code = tsdbRowMergerAdd(pMerger, &state->row, state->pTSchema); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - code = tsdbRowMergerAdd(pMerger, state->pLastRow, state->pTSchema); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TAOS_CHECK_GOTO(tsdbRowMergerAdd(pMerger, &state->row, state->pTSchema), &lino, _err); + TAOS_CHECK_GOTO(tsdbRowMergerAdd(pMerger, state->pLastRow, state->pTSchema), &lino, _err); if (state->pTSRow) { taosMemoryFree(state->pTSRow); state->pTSRow = NULL; } - code = tsdbRowMergerGetRow(pMerger, &state->pTSRow); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TAOS_CHECK_GOTO(tsdbRowMergerGetRow(pMerger, &state->pTSRow), &lino, _err); state->row = tsdbRowFromTSRow(TSDBROW_VERSION(&state->row), state->pTSRow); *ppRow = &state->row; @@ -2407,7 +2407,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie tsdbRowMergerClear(pMerger); - return code; + TAOS_RETURN(code); } } @@ -2416,7 +2416,12 @@ _err: *ppRow = NULL; - return code; + if (code) { + tsdbError("tsdb/cache: vgId:%d, %s failed at line %d since %s.", TD_VID(pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } + + TAOS_RETURN(code); } typedef enum SMEMNEXTROWSTATES { @@ -2442,7 +2447,8 @@ static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarli if (state->pMem->maxKey <= state->lastTs) { *ppRow = NULL; *pIgnoreEarlierTs = true; - return code; + + TAOS_RETURN(code); } tsdbTbDataIterOpen(state->pMem, NULL, 1, &state->iter); @@ -2451,23 +2457,23 @@ static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarli *ppRow = pMemRow; state->state = SMEMNEXTROW_NEXT; - return code; + TAOS_RETURN(code); } } *ppRow = NULL; - return code; + TAOS_RETURN(code); } case SMEMNEXTROW_NEXT: if (tsdbTbDataIterNext(&state->iter)) { *ppRow = tsdbTbDataIterGet(&state->iter); - return code; + TAOS_RETURN(code); } else { *ppRow = NULL; - return code; + TAOS_RETURN(code); } default: ASSERT(0); @@ -2476,7 +2482,8 @@ static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarli _err: *ppRow = NULL; - return code; + + TAOS_RETURN(code); } static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { @@ -2543,7 +2550,7 @@ int32_t clearNextRowFromFS(void *iter) { SFSNextRowIter *state = (SFSNextRowIter *)iter; if (!state) { - return code; + TAOS_RETURN(code); } if (state->pLastIter) { @@ -2575,7 +2582,7 @@ int32_t clearNextRowFromFS(void *iter) { state->pRowIter->pSkyline = NULL; } - return code; + TAOS_RETURN(code); } static void clearLastFileSet(SFSNextRowIter *state) { @@ -2617,7 +2624,7 @@ static void clearLastFileSet(SFSNextRowIter *state) { static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, SArray *pLDataIterArray, STsdbReadSnap *pReadSnap, int64_t lastTs, SCacheRowsReader *pr) { - int code = 0; + int32_t code = 0, lino = 0; STbData *pMem = NULL; if (pReadSnap->pMem) { @@ -2633,10 +2640,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->pMemDelData = NULL; - code = loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TAOS_CHECK_GOTO(loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer), &lino, _err); pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; @@ -2672,12 +2676,13 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs } pIter->pr = pr; + _err: - return code; + TAOS_RETURN(code); } static int32_t nextRowIterClose(CacheNextRowIter *pIter) { - int code = 0; + int32_t code = 0; for (int i = 0; i < 3; ++i) { if (pIter->input[i].nextRowClearFn) { @@ -2694,19 +2699,20 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) { } _err: - return code; + TAOS_RETURN(code); } // iterate next row non deleted backward ts, version (from high to low) static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, int nCols) { - int code = 0; + int32_t code = 0, lino = 0; + for (;;) { for (int i = 0; i < 3; ++i) { if (pIter->input[i].next && !pIter->input[i].stop) { - code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs, - isLast, aCols, nCols); - if (code) goto _err; + TAOS_CHECK_GOTO(pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, + &pIter->input[i].ignoreEarlierTs, isLast, aCols, nCols), + &lino, _err); if (pIter->input[i].pRow == NULL) { pIter->input[i].stop = true; @@ -2719,7 +2725,8 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI *ppRow = NULL; *pIgnoreEarlierTs = (pIter->input[0].ignoreEarlierTs || pIter->input[1].ignoreEarlierTs || pIter->input[2].ignoreEarlierTs); - return code; + + TAOS_RETURN(code); } // select maxpoint(s) from mem, imem, fs and last @@ -2785,18 +2792,24 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI pIter->input[iMerge[0]].next = true; *ppRow = merge[0]; - return code; + + TAOS_RETURN(code); } } _err: - return code; + + if (code) { + tsdbError("tsdb/cache: %s failed at line %d since %s.", __func__, lino, tstrerror(code)); + } + + TAOS_RETURN(code); } static int32_t initLastColArrayPartial(STSchema *pTSchema, SArray **ppColArray, int16_t *slotIds, int nCols) { SArray *pColArray = taosArrayInit(nCols, sizeof(SLastCol)); if (NULL == pColArray) { - return TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } for (int32_t i = 0; i < nCols; ++i) { @@ -2806,34 +2819,38 @@ static int32_t initLastColArrayPartial(STSchema *pTSchema, SArray **ppColArray, taosArrayPush(pColArray, &col); } *ppColArray = pColArray; - return TSDB_CODE_SUCCESS; + + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t cloneTSchema(STSchema *pSrc, STSchema **ppDst) { int32_t len = sizeof(STSchema) + sizeof(STColumn) * pSrc->numOfCols; *ppDst = taosMemoryMalloc(len); if (NULL == *ppDst) { - return TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } memcpy(*ppDst, pSrc, len); - return TSDB_CODE_SUCCESS; + + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t updateTSchema(int32_t sversion, SCacheRowsReader *pReader, uint64_t uid) { if (NULL == pReader->pCurrSchema && sversion == pReader->pSchema->version) { - return cloneTSchema(pReader->pSchema, &pReader->pCurrSchema); + TAOS_RETURN(cloneTSchema(pReader->pSchema, &pReader->pCurrSchema)); } if (NULL != pReader->pCurrSchema && sversion == pReader->pCurrSchema->version) { - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } taosMemoryFreeClear(pReader->pCurrSchema); - return metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, sversion, &pReader->pCurrSchema); + TAOS_RETURN( + metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, sversion, &pReader->pCurrSchema)); } static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, int nCols, int16_t *slotIds) { + int32_t code = 0, lino = 0; STSchema *pTSchema = pr->pSchema; // metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); int16_t nLastCol = nCols; int16_t noneCol = 0; @@ -2843,15 +2860,13 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC SArray *pColArray = NULL; SColVal *pColVal = &(SColVal){0}; - int32_t code = initLastColArrayPartial(pTSchema, &pColArray, slotIds, nCols); - if (TSDB_CODE_SUCCESS != code) { - return code; - } + TAOS_CHECK_RETURN(initLastColArrayPartial(pTSchema, &pColArray, slotIds, nCols)); + SArray *aColArray = taosArrayInit(nCols, sizeof(int16_t)); if (NULL == aColArray) { taosArrayDestroy(pColArray); - return TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } for (int i = 0; i < nCols; ++i) { @@ -2876,10 +2891,8 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC int32_t sversion = TSDBROW_SVERSION(pRow); if (sversion != -1) { - code = updateTSchema(sversion, pr, uid); - if (TSDB_CODE_SUCCESS != code) { - goto _err; - } + TAOS_CHECK_GOTO(updateTSchema(sversion, pr, uid), &lino, _err); + pTSchema = pr->pCurrSchema; } // int16_t nCol = pTSchema->numOfCols; @@ -2919,9 +2932,7 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC if (pColVal->value.nData > 0) { pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData); if (pCol->colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err); } memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); } else { @@ -2973,9 +2984,7 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC if (pColVal->value.nData > 0) { lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData); if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err); } memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData); } else { @@ -3008,7 +3017,7 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC nextRowIterClose(&iter); taosArrayDestroy(aColArray); - return code; + TAOS_RETURN(code); _err: nextRowIterClose(&iter); @@ -3016,11 +3025,18 @@ _err: *ppLastArray = NULL; taosArrayDestroy(pColArray); taosArrayDestroy(aColArray); - return code; + + if (code) { + tsdbError("tsdb/cache: vgId:%d, %s failed at line %d since %s.", TD_VID(pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } + + TAOS_RETURN(code); } static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, int nCols, int16_t *slotIds) { + int32_t code = 0, lino = 0; STSchema *pTSchema = pr->pSchema; // metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); int16_t nLastCol = nCols; int16_t noneCol = 0; @@ -3030,15 +3046,13 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SArray *pColArray = NULL; SColVal *pColVal = &(SColVal){0}; - int32_t code = initLastColArrayPartial(pTSchema, &pColArray, slotIds, nCols); - if (TSDB_CODE_SUCCESS != code) { - return code; - } + TAOS_CHECK_RETURN(initLastColArrayPartial(pTSchema, &pColArray, slotIds, nCols)); + SArray *aColArray = taosArrayInit(nCols, sizeof(int16_t)); if (NULL == aColArray) { taosArrayDestroy(pColArray); - return TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } for (int i = 0; i < nCols; ++i) { @@ -3061,10 +3075,8 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, int32_t sversion = TSDBROW_SVERSION(pRow); if (sversion != -1) { - code = updateTSchema(sversion, pr, uid); - if (TSDB_CODE_SUCCESS != code) { - goto _err; - } + TAOS_CHECK_GOTO(updateTSchema(sversion, pr, uid), &lino, _err); + pTSchema = pr->pCurrSchema; } // int16_t nCol = pTSchema->numOfCols; @@ -3097,9 +3109,7 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, if (pColVal->value.nData > 0) { pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData); if (pCol->colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err); } if (pColVal->value.nData > 0) { memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); @@ -3131,7 +3141,7 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, nextRowIterClose(&iter); taosArrayDestroy(aColArray); - return code; + TAOS_RETURN(code); _err: nextRowIterClose(&iter); @@ -3139,7 +3149,13 @@ _err: *ppLastArray = NULL; taosArrayDestroy(pColArray); taosArrayDestroy(aColArray); - return code; + + if (code) { + tsdbError("tsdb/cache: vgId:%d, %s failed at line %d since %s.", TD_VID(pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } + + TAOS_RETURN(code); } int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h) { @@ -3194,22 +3210,10 @@ static void getBCacheKey(int32_t fid, int64_t commitID, int64_t blkno, char *key static int32_t tsdbCacheLoadBlockS3(STsdbFD *pFD, uint8_t **ppBlock) { int32_t code = 0; - /* - uint8_t *pBlock = taosMemoryCalloc(1, tsS3BlockSize * pFD->szPage); - if (pBlock == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - */ - int64_t block_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage; - code = s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, 0, ppBlock); - if (code != TSDB_CODE_SUCCESS) { - // taosMemoryFree(pBlock); - // code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } - //*ppBlock = pBlock; + int64_t block_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage; + + TAOS_CHECK_RETURN(s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, 0, ppBlock)); tsdbTrace("block:%p load from s3", *ppBlock); @@ -3247,7 +3251,8 @@ int32_t tsdbCacheGetBlockS3(SLRUCache *pCache, STsdbFD *pFD, LRUHandle **handle) if (code == TSDB_CODE_SUCCESS && !pBlock) { code = TSDB_CODE_OUT_OF_MEMORY; } - return code; + + TAOS_RETURN(code); } size_t charge = tsS3BlockSize * pFD->szPage; @@ -3255,7 +3260,7 @@ int32_t tsdbCacheGetBlockS3(SLRUCache *pCache, STsdbFD *pFD, LRUHandle **handle) LRUStatus status = taosLRUCacheInsert(pCache, key, keyLen, pBlock, charge, deleter, &h, TAOS_LRU_PRIORITY_LOW, NULL); if (status != TAOS_LRU_STATUS_OK) { - code = -1; + // code = -1; } } @@ -3264,7 +3269,7 @@ int32_t tsdbCacheGetBlockS3(SLRUCache *pCache, STsdbFD *pFD, LRUHandle **handle) *handle = h; - return code; + TAOS_RETURN(code); } int32_t tsdbCacheGetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, LRUHandle **handle) { @@ -3304,5 +3309,5 @@ int32_t tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_ tsdbCacheRelease(pFD->pTsdb->pgCache, handle); - return code; + TAOS_RETURN(code); } From d908c1d7108bf6f20025873076e22cc5fca7dfec Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 23 Jul 2024 16:39:52 +0800 Subject: [PATCH 42/52] feat: (errcode) more for ttime.c/geomFunc.c/geosWrapper.c --- include/common/ttime.h | 2 +- source/common/src/ttime.c | 158 +++++++++--------- source/common/test/commonTests.cpp | 3 +- source/libs/geometry/src/geomFunc.c | 217 +++++++++---------------- source/libs/geometry/src/geosWrapper.c | 114 +++++++------ 5 files changed, 213 insertions(+), 281 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index 2d40bd93a6..cec5b15761 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -117,7 +117,7 @@ int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t pr int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, int32_t errMsgLen); -void TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out, int32_t outLen); +int32_t TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out, int32_t outLen); int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const char* tsStr); /// @brief get offset seconds from zero timezone to input timezone diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 4fca564804..d3207a1912 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -96,8 +96,6 @@ char* forwardToTimeStringEnd(char* str) { } int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFraction) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t i = 0; int64_t fraction = 0; @@ -147,8 +145,6 @@ int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFractio } int32_t parseTimezone(char* str, int64_t* tzOffset) { - int32_t code = TSDB_CODE_SUCCESS; - int64_t hour = 0; int32_t i = 0; @@ -224,8 +220,6 @@ int32_t offsetOfTimezone(char* tzStr, int64_t* offset) { * 2013-04-12T15:52:01.123+0800 */ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) { - int32_t code = TSDB_CODE_SUCCESS; - int64_t factor = TSDB_TICK_PER_SECOND(timePrec); int64_t tzOffset = 0; @@ -315,8 +309,6 @@ static FORCE_INLINE bool validateTm(struct tm* pTm) { } int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { - int32_t code = TSDB_CODE_SUCCESS; - *utime = 0; struct tm tm = {0}; @@ -358,8 +350,6 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timeP } int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { - int32_t code = TSDB_CODE_SUCCESS; - *utime = 0; struct tm tm = {0}; tm.tm_isdst = -1; @@ -484,8 +474,6 @@ int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPre // !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = // 1626006833631000064 int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes) { - int32_t code = TSDB_CODE_SUCCESS; - if (fromPrecision != TSDB_TIME_PRECISION_MILLI && fromPrecision != TSDB_TIME_PRECISION_MICRO && fromPrecision != TSDB_TIME_PRECISION_NANO) { TAOS_RETURN(TSDB_CODE_INVALID_PARA); @@ -559,13 +547,14 @@ int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char } int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t charLen = varDataLen(inputData); char* newColData; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) { newColData = taosMemoryCalloc(1, charLen + 1); - memcpy(newColData, varDataVal(inputData), charLen); + if (NULL == newColData) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + (void)memcpy(newColData, varDataVal(inputData), charLen); int32_t ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight); if (ret != TSDB_CODE_SUCCESS) { taosMemoryFree(newColData); @@ -574,6 +563,9 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec taosMemoryFree(newColData); } else if (type == TSDB_DATA_TYPE_NCHAR) { newColData = taosMemoryCalloc(1, charLen + TSDB_NCHAR_SIZE); + if (NULL == newColData) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData); if (len < 0) { taosMemoryFree(newColData); @@ -593,8 +585,6 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec } int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { - int32_t code = TSDB_CODE_SUCCESS; - switch (unit) { case 's': if (val > INT64_MAX / MILLISECOND_PER_SECOND) { @@ -658,8 +648,6 @@ int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecisi */ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) { - int32_t code = TSDB_CODE_SUCCESS; - errno = 0; char* endPtr = NULL; @@ -680,8 +668,6 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow) { - int32_t code = TSDB_CODE_SUCCESS; - errno = 0; /* get the basic numeric value */ @@ -718,7 +704,7 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { struct tm tm; time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); - taosLocalTime(&tt, &tm, NULL); + (void)taosLocalTime(&tt, &tm, NULL); int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)numOfMonth; tm.tm_year = mon / 12; tm.tm_mon = mon % 12; @@ -779,11 +765,11 @@ int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interva struct tm tm; time_t t = (time_t)skey; - taosLocalTime(&t, &tm, NULL); + (void)taosLocalTime(&t, &tm, NULL); int32_t smon = tm.tm_year * 12 + tm.tm_mon; t = (time_t)ekey; - taosLocalTime(&t, &tm, NULL); + (void)taosLocalTime(&t, &tm, NULL); int32_t emon = tm.tm_year * 12 + tm.tm_mon; if (unit == 'y') { @@ -808,7 +794,7 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; time_t tt = (time_t)start; - taosLocalTime(&tt, &tm, NULL); + (void)taosLocalTime(&tt, &tm, NULL); tm.tm_sec = 0; tm.tm_min = 0; tm.tm_hour = 0; @@ -978,8 +964,6 @@ const char* fmtts(int64_t ts) { } int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) { - int32_t code = TSDB_CODE_SUCCESS; - char ts[40] = {0}; struct tm ptm; @@ -1018,7 +1002,7 @@ int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precisio TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - if (taosLocalTime(", &ptm, buf) == NULL) { + if (NULL == taosLocalTime(", &ptm, buf)) { TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); @@ -1032,7 +1016,9 @@ int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precisio int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) { tm->fsec = ts % TICK_PER_SECOND[precision] * (TICK_PER_SECOND[TSDB_TIME_PRECISION_NANO] / TICK_PER_SECOND[precision]); time_t t = ts / TICK_PER_SECOND[precision]; - taosLocalTime(&t, &tm->tm, NULL); + if (NULL == taosLocalTime(&t, &tm->tm, NULL)) { + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + } return TSDB_CODE_SUCCESS; } @@ -1344,7 +1330,7 @@ static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int TSFormatNode* format = taosArrayGet(formats, i); if (format->type != TS_FORMAT_NODE_TYPE_KEYWORD) { if (s - start + format->len + 1 > outLen) break; - strncpy(s, format->c, format->len); + (void)strncpy(s, format->c, format->len); s += format->len; continue; } @@ -1353,37 +1339,37 @@ static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int switch (format->key->id) { case TSFKW_AM: case TSFKW_PM: - sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "PM" : "AM"); + (void)sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "PM" : "AM"); s += 2; break; case TSFKW_A_M: case TSFKW_P_M: - sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "P.M." : "A.M."); + (void)sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "P.M." : "A.M."); s += 4; break; case TSFKW_am: case TSFKW_pm: - sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "pm" : "am"); + (void)sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "pm" : "am"); s += 2; break; case TSFKW_a_m: case TSFKW_p_m: - sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "p.m." : "a.m."); + (void)sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "p.m." : "a.m."); s += 4; break; case TSFKW_DDD: #ifdef WINDOWS return TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED; #endif - sprintf(s, "%03d", tm->tm.tm_yday + 1); + (void)sprintf(s, "%03d", tm->tm.tm_yday + 1); s += strlen(s); break; case TSFKW_DD: - sprintf(s, "%02d", tm->tm.tm_mday); + (void)sprintf(s, "%02d", tm->tm.tm_mday); s += 2; break; case TSFKW_D: - sprintf(s, "%d", tm->tm.tm_wday + 1); + (void)sprintf(s, "%d", tm->tm.tm_wday + 1); s += 1; break; case TSFKW_DAY: { @@ -1391,20 +1377,20 @@ static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int const char* wd = weekDays[tm->tm.tm_wday]; char buf[10] = {0}; for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = toupper(wd[i]); - sprintf(s, "%-9s", buf); + (void)sprintf(s, "%-9s", buf); s += strlen(s); break; } case TSFKW_Day: // Monday, TuesDay... - sprintf(s, "%-9s", weekDays[tm->tm.tm_wday]); + (void)sprintf(s, "%-9s", weekDays[tm->tm.tm_wday]); s += strlen(s); break; case TSFKW_day: { const char* wd = weekDays[tm->tm.tm_wday]; char buf[10] = {0}; for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = tolower(wd[i]); - sprintf(s, "%-9s", buf); + (void)sprintf(s, "%-9s", buf); s += strlen(s); break; } @@ -1413,13 +1399,13 @@ static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int const char* wd = shortWeekDays[tm->tm.tm_wday]; char buf[8] = {0}; for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = toupper(wd[i]); - sprintf(s, "%3s", buf); + (void)sprintf(s, "%3s", buf); s += 3; break; } case TSFKW_Dy: // Mon, Tue - sprintf(s, "%3s", shortWeekDays[tm->tm.tm_wday]); + (void)sprintf(s, "%3s", shortWeekDays[tm->tm.tm_wday]); s += 3; break; case TSFKW_dy: { @@ -1427,33 +1413,33 @@ static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int const char* wd = shortWeekDays[tm->tm.tm_wday]; char buf[8] = {0}; for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = tolower(wd[i]); - sprintf(s, "%3s", buf); + (void)sprintf(s, "%3s", buf); s += 3; break; } case TSFKW_HH24: - sprintf(s, "%02d", tm->tm.tm_hour); + (void)sprintf(s, "%02d", tm->tm.tm_hour); s += 2; break; case TSFKW_HH: case TSFKW_HH12: // 0 or 12 o'clock in 24H coresponds to 12 o'clock (AM/PM) in 12H - sprintf(s, "%02d", tm->tm.tm_hour % 12 == 0 ? 12 : tm->tm.tm_hour % 12); + (void)sprintf(s, "%02d", tm->tm.tm_hour % 12 == 0 ? 12 : tm->tm.tm_hour % 12); s += 2; break; case TSFKW_MI: - sprintf(s, "%02d", tm->tm.tm_min); + (void)sprintf(s, "%02d", tm->tm.tm_min); s += 2; break; case TSFKW_MM: - sprintf(s, "%02d", tm->tm.tm_mon + 1); + (void)sprintf(s, "%02d", tm->tm.tm_mon + 1); s += 2; break; case TSFKW_MONTH: { const char* mon = fullMonths[tm->tm.tm_mon]; char buf[10] = {0}; for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = toupper(mon[i]); - sprintf(s, "%-9s", buf); + (void)sprintf(s, "%-9s", buf); s += strlen(s); break; } @@ -1461,44 +1447,44 @@ static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int const char* mon = months[tm->tm.tm_mon]; char buf[10] = {0}; for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = toupper(mon[i]); - sprintf(s, "%s", buf); + (void)sprintf(s, "%s", buf); s += strlen(s); break; } case TSFKW_Month: - sprintf(s, "%-9s", fullMonths[tm->tm.tm_mon]); + (void)sprintf(s, "%-9s", fullMonths[tm->tm.tm_mon]); s += strlen(s); break; case TSFKW_month: { const char* mon = fullMonths[tm->tm.tm_mon]; char buf[10] = {0}; for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = tolower(mon[i]); - sprintf(s, "%-9s", buf); + (void)sprintf(s, "%-9s", buf); s += strlen(s); break; } case TSFKW_Mon: - sprintf(s, "%s", months[tm->tm.tm_mon]); + (void)sprintf(s, "%s", months[tm->tm.tm_mon]); s += strlen(s); break; case TSFKW_mon: { const char* mon = months[tm->tm.tm_mon]; char buf[10] = {0}; for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = tolower(mon[i]); - sprintf(s, "%s", buf); + (void)sprintf(s, "%s", buf); s += strlen(s); break; } case TSFKW_SS: - sprintf(s, "%02d", tm->tm.tm_sec); + (void)sprintf(s, "%02d", tm->tm.tm_sec); s += 2; break; case TSFKW_MS: - sprintf(s, "%03" PRId64, tm->fsec / 1000000L); + (void)sprintf(s, "%03" PRId64, tm->fsec / 1000000L); s += 3; break; case TSFKW_US: - sprintf(s, "%06" PRId64, tm->fsec / 1000L); + (void)sprintf(s, "%06" PRId64, tm->fsec / 1000L); s += 6; break; case TSFKW_NS: @@ -1506,23 +1492,23 @@ static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int s += 9; break; case TSFKW_TZH: - sprintf(s, "%s%02d", tsTimezone < 0 ? "-" : "+", tsTimezone); + (void)sprintf(s, "%s%02d", tsTimezone < 0 ? "-" : "+", tsTimezone); s += strlen(s); break; case TSFKW_YYYY: - sprintf(s, "%04d", tm->tm.tm_year + 1900); + (void)sprintf(s, "%04d", tm->tm.tm_year + 1900); s += strlen(s); break; case TSFKW_YYY: - sprintf(s, "%03d", (tm->tm.tm_year + 1900) % 1000); + (void)sprintf(s, "%03d", (tm->tm.tm_year + 1900) % 1000); s += strlen(s); break; case TSFKW_YY: - sprintf(s, "%02d", (tm->tm.tm_year + 1900) % 100); + (void)sprintf(s, "%02d", (tm->tm.tm_year + 1900) % 100); s += strlen(s); break; case TSFKW_Y: - sprintf(s, "%01d", (tm->tm.tm_year + 1900) % 10); + (void)sprintf(s, "%01d", (tm->tm.tm_year + 1900) % 10); s += strlen(s); break; default: @@ -1557,7 +1543,7 @@ static const char* tsFormatStr2Int32(int32_t* dest, const char* str, int32_t len s = last; } else { char buf[16] = {0}; - strncpy(buf, s, len); + (void)strncpy(buf, s, len); int32_t copiedLen = strlen(buf); if (copiedLen < len) { if (!needMoreDigit) { @@ -1936,10 +1922,13 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) { if (!*formats) { *formats = taosArrayInit(8, sizeof(TSFormatNode)); + if (!*formats){ + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } parseTsFormat(format, *formats); } struct STm tm; - taosTs2Tm(ts, precision, &tm); + TAOS_CHECK_RETURN(taosTs2Tm(ts, precision, &tm)); return tm2char(*formats, &tm, out, outLen); } @@ -1949,6 +1938,9 @@ int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int int32_t fErrIdx; if (!*formats) { *formats = taosArrayInit(4, sizeof(TSFormatNode)); + if (!*formats) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } parseTsFormat(format, *formats); } int32_t code = char2ts(tsStr, *formats, ts, precision, &sErrPos, &fErrIdx); @@ -1964,16 +1956,24 @@ int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int snprintf(errMsg, errMsgLen, "timestamp format not supported"); code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED; } - return code; + TAOS_RETURN(code); } -void TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out, int32_t outLen) { +int32_t TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out, int32_t outLen) { + int32_t code = TSDB_CODE_SUCCESS; + SArray* formats = taosArrayInit(4, sizeof(TSFormatNode)); + if (!formats) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } parseTsFormat(format, formats); struct STm tm; - taosTs2Tm(ts, precision, &tm); - tm2char(formats, &tm, out, outLen); + TAOS_CHECK_GOTO(taosTs2Tm(ts, precision, &tm), NULL, _exit); + TAOS_CHECK_GOTO(tm2char(formats, &tm, out, outLen), NULL, _exit); + +_exit: taosArrayDestroy(formats); + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const char* tsStr) { @@ -1997,17 +1997,19 @@ static int8_t UNIT_INDEX[26] = {/*a*/ 2, 0, -1, 6, -1, -1, -1, #define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx) - 97] -static int64_t UNIT_MATRIX[10][11] = {/* ns, us, ms, s, min, h, d, w, month, y*/ - /*ns*/ {1, 1000, 0}, - /*us*/ {1000, 1, 1000, 0}, - /*ms*/ {0, 1000, 1, 1000, 0}, - /*s*/ {0, 0, 1000, 1, 60, 0}, - /*min*/ {0, 0, 0, 60, 1, 60, 0}, - /*h*/ {0, 0, 0, 0, 60, 1, 1, 0}, - /*d*/ {0, 0, 0, 0, 0, 24, 1, 7, 1, 0}, - /*w*/ {0, 0, 0, 0, 0, 0, 7, 1, -1, 0}, - /*mon*/ {0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0}, - /*y*/ {0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}}; +// clang-format off +static int64_t UNIT_MATRIX[10][11] = { /* ns, us, ms, s, min, h, d, w, month, y*/ + /*ns*/ { 1, 1000, 0}, + /*us*/ {1000, 1, 1000, 0}, + /*ms*/ { 0, 1000, 1, 1000, 0}, + /*s*/ { 0, 0, 1000, 1, 60, 0}, + /*min*/ { 0, 0, 0, 60, 1, 60, 0}, + /*h*/ { 0, 0, 0, 0, 60, 1, 1, 0}, + /*d*/ { 0, 0, 0, 0, 0, 24, 1, 7, 1, 0}, + /*w*/ { 0, 0, 0, 0, 0, 0, 7, 1, -1, 0}, + /*mon*/ { 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0}, + /*y*/ { 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}}; +// clang-format on static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) { diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 360d1ed31a..b539d6731f 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -465,7 +465,8 @@ TEST(timeTest, timestamp2tm) { void test_ts2char(int64_t ts, const char* format, int32_t precison, const char* expected) { char buf[256] = {0}; - TEST_ts2char(format, ts, precison, buf, 256); + int32_t code = TEST_ts2char(format, ts, precison, buf, 256); + ASSERT_EQ(code, 0); printf("ts: %ld format: %s res: [%s], expected: [%s]\n", ts, format, buf, expected); ASSERT_STREQ(expected, buf); } diff --git a/source/libs/geometry/src/geomFunc.c b/source/libs/geometry/src/geomFunc.c index 601588571e..343e6ec4ce 100644 --- a/source/libs/geometry/src/geomFunc.c +++ b/source/libs/geometry/src/geomFunc.c @@ -21,8 +21,8 @@ #include "sclInt.h" #include "sclvector.h" -typedef int32_t (*_geomDoRelationFunc_t)(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, - bool swapped, char *res); +typedef int32_t (*_geomDoRelationFunc_t)(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, + const GEOSGeometry *geom2, bool swapped, char *res); typedef int32_t (*_geomInitCtxFunc_t)(); typedef int32_t (*_geomExecuteOneParamFunc_t)(SColumnInfoData *pInputData, int32_t i, SColumnInfoData *pOutputData); @@ -35,7 +35,7 @@ int32_t doMakePointFunc(double x, double y, unsigned char **output) { int32_t code = TSDB_CODE_FAILED; unsigned char *outputGeom = NULL; - size_t size = 0; + size_t size = 0; code = doMakePoint(x, y, &outputGeom, &size); if (code != TSDB_CODE_SUCCESS) { goto _exit; @@ -47,7 +47,7 @@ int32_t doMakePointFunc(double x, double y, unsigned char **output) { goto _exit; } - memcpy(varDataVal(*output), outputGeom, size); + (void)memcpy(varDataVal(*output), outputGeom, size); varDataSetLen(*output, size); code = TSDB_CODE_SUCCESS; @@ -62,14 +62,14 @@ _exit: int32_t doGeomFromTextFunc(const char *input, unsigned char **output) { int32_t code = TSDB_CODE_FAILED; - if ((varDataLen(input)) == 0) { //empty value + if ((varDataLen(input)) == 0) { // empty value *output = NULL; return TSDB_CODE_SUCCESS; } - char *inputGeom = NULL; + char *inputGeom = NULL; unsigned char *outputGeom = NULL; - size_t size = 0; + size_t size = 0; // make a zero ending string inputGeom = taosMemoryCalloc(1, varDataLen(input) + 1); @@ -77,12 +77,9 @@ int32_t doGeomFromTextFunc(const char *input, unsigned char **output) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - memcpy(inputGeom, varDataVal(input), varDataLen(input)); + (void)memcpy(inputGeom, varDataVal(input), varDataLen(input)); - code = doGeomFromText(inputGeom, &outputGeom, &size); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } + TAOS_CHECK_GOTO(doGeomFromText(inputGeom, &outputGeom, &size), NULL, _exit); *output = taosMemoryCalloc(1, size + VARSTR_HEADER_SIZE); if (*output == NULL) { @@ -90,7 +87,7 @@ int32_t doGeomFromTextFunc(const char *input, unsigned char **output) { goto _exit; } - memcpy(varDataVal(*output), outputGeom, size); + (void)memcpy(varDataVal(*output), outputGeom, size); varDataSetLen(*output, size); code = TSDB_CODE_SUCCESS; @@ -106,16 +103,13 @@ _exit: int32_t doAsTextFunc(unsigned char *input, char **output) { int32_t code = TSDB_CODE_FAILED; - if ((varDataLen(input)) == 0) { //empty value + if ((varDataLen(input)) == 0) { // empty value *output = NULL; return TSDB_CODE_SUCCESS; } char *outputWKT = NULL; - code = doAsText(varDataVal(input), varDataLen(input), &outputWKT); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } + TAOS_CHECK_GOTO(doAsText(varDataVal(input), varDataLen(input), &outputWKT), NULL, _exit); size_t size = strlen(outputWKT); *output = taosMemoryCalloc(1, size + VARSTR_HEADER_SIZE); @@ -124,7 +118,7 @@ int32_t doAsTextFunc(unsigned char *input, char **output) { goto _exit; } - memcpy(varDataVal(*output), outputWKT, size); + (void)memcpy(varDataVal(*output), outputWKT, size); varDataSetLen(*output, size); code = TSDB_CODE_SUCCESS; @@ -139,28 +133,17 @@ int32_t executeMakePointFunc(SColumnInfoData *pInputData[], int32_t iLeft, int32 int32_t code = TSDB_CODE_FAILED; _getDoubleValue_fn_t getDoubleValueFn[2]; - getDoubleValueFn[0]= getVectorDoubleValueFn(pInputData[0]->info.type); - getDoubleValueFn[1]= getVectorDoubleValueFn(pInputData[1]->info.type); + getDoubleValueFn[0] = getVectorDoubleValueFn(pInputData[0]->info.type); + getDoubleValueFn[1] = getVectorDoubleValueFn(pInputData[1]->info.type); unsigned char *output = NULL; - double leftRes = 0; - double rightRes = 0; - code = getDoubleValueFn[0](pInputData[0]->pData, iLeft, &leftRes); - if (TSDB_CODE_SUCCESS != code) { - goto _exit; - } - code = getDoubleValueFn[1](pInputData[1]->pData, iRight, &rightRes); - if (TSDB_CODE_SUCCESS != code) { - goto _exit; - } - code = doMakePointFunc(leftRes, rightRes, &output); - if (TSDB_CODE_SUCCESS != code) { - goto _exit; - } - code = colDataSetVal(pOutputData, TMAX(iLeft, iRight), output, (output == NULL)); - if (TSDB_CODE_SUCCESS != code) { - goto _exit; - } + double leftRes = 0; + double rightRes = 0; + + TAOS_CHECK_GOTO(getDoubleValueFn[0](pInputData[0]->pData, iLeft, &leftRes), NULL, _exit); + TAOS_CHECK_GOTO(getDoubleValueFn[1](pInputData[1]->pData, iRight, &rightRes), NULL, _exit); + TAOS_CHECK_GOTO(doMakePointFunc(leftRes, rightRes, &output), NULL, _exit); + TAOS_CHECK_GOTO(colDataSetVal(pOutputData, TMAX(iLeft, iRight), output, (output == NULL)), NULL, _exit); _exit: if (output) { @@ -173,14 +156,11 @@ _exit: int32_t executeGeomFromTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnInfoData *pOutputData) { int32_t code = TSDB_CODE_FAILED; - char *input = colDataGetData(pInputData, i); + char *input = colDataGetData(pInputData, i); unsigned char *output = NULL; - code = doGeomFromTextFunc(input, &output); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } - code = colDataSetVal(pOutputData, i, output, (output == NULL)); + TAOS_CHECK_GOTO(doGeomFromTextFunc(input, &output), NULL, _exit); + TAOS_CHECK_GOTO(colDataSetVal(pOutputData, i, output, (output == NULL)), NULL, _exit); _exit: if (output) { @@ -194,13 +174,10 @@ int32_t executeAsTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnInfoDat int32_t code = TSDB_CODE_FAILED; unsigned char *input = colDataGetData(pInputData, i); - char *output = NULL; - code = doAsTextFunc(input, &output); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } + char *output = NULL; - code = colDataSetVal(pOutputData, i, output, (output == NULL)); + TAOS_CHECK_GOTO(doAsTextFunc(input, &output), NULL, _exit); + TAOS_CHECK_GOTO(colDataSetVal(pOutputData, i, output, (output == NULL)), NULL, _exit); _exit: if (output) { @@ -211,36 +188,24 @@ _exit: } int32_t executeRelationFunc(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, - const GEOSGeometry *geom2, int32_t i, - bool swapped, SColumnInfoData *pOutputData, + const GEOSGeometry *geom2, int32_t i, bool swapped, SColumnInfoData *pOutputData, _geomDoRelationFunc_t doRelationFn) { - int32_t code = TSDB_CODE_FAILED; char res = 0; - if (!geom1 || !geom2) { //if empty input value + if (!geom1 || !geom2) { // if empty input value res = -1; - code = TSDB_CODE_SUCCESS; - } - else { - code = doRelationFn(geom1, preparedGeom1, geom2, swapped, &res); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + } else { + TAOS_CHECK_RETURN(doRelationFn(geom1, preparedGeom1, geom2, swapped, &res)); } - code = colDataSetVal(pOutputData, i, &res, (res==-1)); - - return code; + return colDataSetVal(pOutputData, i, &res, (res == -1)); } -int32_t geomOneParamFunction(SScalarParam *pInput, SScalarParam *pOutput, - _geomInitCtxFunc_t initCtxFn, _geomExecuteOneParamFunc_t executeOneParamFn) { +int32_t geomOneParamFunction(SScalarParam *pInput, SScalarParam *pOutput, _geomInitCtxFunc_t initCtxFn, + _geomExecuteOneParamFunc_t executeOneParamFn) { int32_t code = TSDB_CODE_FAILED; - code = initCtxFn(); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TAOS_CHECK_RETURN(initCtxFn()); SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; @@ -249,8 +214,7 @@ int32_t geomOneParamFunction(SScalarParam *pInput, SScalarParam *pOutput, if (IS_NULL_TYPE(GET_PARAM_TYPE(pInput))) { colDataSetNNULL(pOutputData, 0, pInput->numOfRows); code = TSDB_CODE_SUCCESS; - } - else { + } else { for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { colDataSetNULL(pOutputData, i); @@ -258,44 +222,36 @@ int32_t geomOneParamFunction(SScalarParam *pInput, SScalarParam *pOutput, continue; } - code = executeOneParamFn(pInputData, i, pOutputData); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TAOS_CHECK_RETURN(executeOneParamFn(pInputData, i, pOutputData)); } } - return code; + TAOS_RETURN(code); } -int32_t geomTwoParamsFunction(SScalarParam *pInput, SScalarParam *pOutput, - _geomInitCtxFunc_t initCtxFn, _geomExecuteTwoParamsFunc_t executeTwoParamsFn) { +int32_t geomTwoParamsFunction(SScalarParam *pInput, SScalarParam *pOutput, _geomInitCtxFunc_t initCtxFn, + _geomExecuteTwoParamsFunc_t executeTwoParamsFn) { int32_t code = TSDB_CODE_FAILED; - code = initCtxFn(); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TAOS_CHECK_RETURN(initCtxFn()); SColumnInfoData *pInputData[2]; SColumnInfoData *pOutputData = pOutput->columnData; pInputData[0] = pInput[0].columnData; pInputData[1] = pInput[1].columnData; - bool hasNullType = (IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0])) || - IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[1]))); - bool isConstantLeft = (pInput[0].numOfRows == 1); - bool isConstantRight = (pInput[1].numOfRows == 1); + bool hasNullType = (IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0])) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[1]))); + bool isConstantLeft = (pInput[0].numOfRows == 1); + bool isConstantRight = (pInput[1].numOfRows == 1); int32_t numOfRows = TMAX(pInput[0].numOfRows, pInput[1].numOfRows); pOutput->numOfRows = numOfRows; - if (hasNullType || // one of operant is NULL type - (isConstantLeft && colDataIsNull_s(pInputData[0], 0)) || // left operand is constant NULL - (isConstantRight && colDataIsNull_s(pInputData[1], 0))) { // right operand is constant NULL + if (hasNullType || // one of operant is NULL type + (isConstantLeft && colDataIsNull_s(pInputData[0], 0)) || // left operand is constant NULL + (isConstantRight && colDataIsNull_s(pInputData[1], 0))) { // right operand is constant NULL colDataSetNNULL(pOutputData, 0, numOfRows); code = TSDB_CODE_SUCCESS; - } - else { + } else { int32_t iLeft = 0; int32_t iRight = 0; for (int32_t i = 0; i < numOfRows; ++i) { @@ -309,103 +265,78 @@ int32_t geomTwoParamsFunction(SScalarParam *pInput, SScalarParam *pOutput, continue; } - code = executeTwoParamsFn(pInputData, iLeft, iRight, pOutputData); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TAOS_CHECK_RETURN(executeTwoParamsFn(pInputData, iLeft, iRight, pOutputData)); } } - return code; + TAOS_RETURN(code); } -int32_t geomRelationFunction(SScalarParam *pInput, SScalarParam *pOutput, - bool swapAllowed, _geomDoRelationFunc_t doRelationFn) { +int32_t geomRelationFunction(SScalarParam *pInput, SScalarParam *pOutput, bool swapAllowed, + _geomDoRelationFunc_t doRelationFn) { int32_t code = TSDB_CODE_FAILED; - code = initCtxRelationFunc(); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TAOS_CHECK_RETURN(initCtxRelationFunc()); // handle with all NULL output - bool hasNullType = (IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0])) || - IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[1]))); - bool isConstant1 = (pInput[0].numOfRows == 1); - bool isConstant2 = (pInput[1].numOfRows == 1); + bool hasNullType = (IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0])) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[1]))); + bool isConstant1 = (pInput[0].numOfRows == 1); + bool isConstant2 = (pInput[1].numOfRows == 1); int32_t numOfRows = TMAX(pInput[0].numOfRows, pInput[1].numOfRows); pOutput->numOfRows = numOfRows; SColumnInfoData *pOutputData = pOutput->columnData; - if (hasNullType || // at least one of operant is NULL type - (isConstant1 && colDataIsNull_s(pInput[0].columnData, 0)) || // left operand is constant NULL - (isConstant2 && colDataIsNull_s(pInput[1].columnData, 0))) { // right operand is constant NULL + if (hasNullType || // at least one of operant is NULL type + (isConstant1 && colDataIsNull_s(pInput[0].columnData, 0)) || // left operand is constant NULL + (isConstant2 && colDataIsNull_s(pInput[1].columnData, 0))) { // right operand is constant NULL colDataSetNNULL(pOutputData, 0, numOfRows); - code = TSDB_CODE_SUCCESS; - return code; + TAOS_RETURN(TSDB_CODE_SUCCESS); } - bool swapped = false; + bool swapped = false; SColumnInfoData *pInputData[2]; // swap two input data to make sure input data 0 is constant if swapAllowed and only isConstant2 is true - if (swapAllowed && - !isConstant1 && isConstant2) { + if (swapAllowed && !isConstant1 && isConstant2) { pInputData[0] = pInput[1].columnData; pInputData[1] = pInput[0].columnData; isConstant1 = true; isConstant2 = false; swapped = true; - } - else { + } else { pInputData[0] = pInput[0].columnData; pInputData[1] = pInput[1].columnData; } - GEOSGeometry *geom1 = NULL; - GEOSGeometry *geom2 = NULL; + GEOSGeometry *geom1 = NULL; + GEOSGeometry *geom2 = NULL; const GEOSPreparedGeometry *preparedGeom1 = NULL; // if there is constant, make PreparedGeometry from pInputData 0 if (isConstant1) { - code = readGeometry(colDataGetData(pInputData[0], 0), &geom1, &preparedGeom1); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } + TAOS_CHECK_GOTO(readGeometry(colDataGetData(pInputData[0], 0), &geom1, &preparedGeom1), NULL, _exit); } if (isConstant2) { - code = readGeometry(colDataGetData(pInputData[1], 0), &geom2, NULL); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } + TAOS_CHECK_GOTO(readGeometry(colDataGetData(pInputData[1], 0), &geom2, NULL), NULL, _exit); } for (int32_t i = 0; i < numOfRows; ++i) { - if ((!isConstant1 && colDataIsNull_s(pInputData[0], i)) || - (!isConstant2 && colDataIsNull_s(pInputData[1], i))) { + if ((!isConstant1 && colDataIsNull_s(pInputData[0], i)) || (!isConstant2 && colDataIsNull_s(pInputData[1], i))) { colDataSetNULL(pOutputData, i); code = TSDB_CODE_SUCCESS; continue; } if (!isConstant1) { - code = readGeometry(colDataGetData(pInputData[0], i), &geom1, &preparedGeom1); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } + TAOS_CHECK_GOTO(readGeometry(colDataGetData(pInputData[0], i), &geom1, &preparedGeom1), NULL, _exit); } if (!isConstant2) { - code = readGeometry(colDataGetData(pInputData[1], i), &geom2, NULL); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } + TAOS_CHECK_GOTO(readGeometry(colDataGetData(pInputData[1], i), &geom2, NULL), NULL, _exit); } - code = executeRelationFunc(geom1, preparedGeom1, geom2, i, swapped, pOutputData, doRelationFn); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } + TAOS_CHECK_GOTO(executeRelationFunc(geom1, preparedGeom1, geom2, i, swapped, pOutputData, doRelationFn), NULL, + _exit); if (!isConstant1) { destroyGeometry(&geom1, &preparedGeom1); @@ -419,7 +350,7 @@ _exit: destroyGeometry(&geom1, &preparedGeom1); destroyGeometry(&geom2, NULL); - return code; + TAOS_RETURN(code); } int32_t makePointFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index c5250c8481..2142b3d62d 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -18,7 +18,8 @@ #include "types.h" typedef char (*_geosRelationFunc_t)(GEOSContextHandle_t handle, const GEOSGeometry *g1, const GEOSGeometry *g2); -typedef char (*_geosPreparedRelationFunc_t)(GEOSContextHandle_t handle, const GEOSPreparedGeometry *pg1, const GEOSGeometry *g2); +typedef char (*_geosPreparedRelationFunc_t)(GEOSContextHandle_t handle, const GEOSPreparedGeometry *pg1, + const GEOSGeometry *g2); void geosFreeBuffer(void *buffer) { if (buffer) { @@ -27,13 +28,13 @@ void geosFreeBuffer(void *buffer) { } void geosErrMsgeHandler(const char *errMsg, void *userData) { - char* targetErrMsg = userData; + char *targetErrMsg = userData; snprintf(targetErrMsg, 512, "%s", errMsg); } int32_t initCtxMakePoint() { - int32_t code = TSDB_CODE_FAILED; - SGeosContext* geosCtx = getThreadLocalGeosCtx(); + int32_t code = TSDB_CODE_FAILED; + SGeosContext *geosCtx = getThreadLocalGeosCtx(); if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); @@ -41,7 +42,7 @@ int32_t initCtxMakePoint() { return code; } - GEOSContext_setErrorMessageHandler_r(geosCtx->handle, geosErrMsgeHandler, geosCtx->errMsg); + (void)GEOSContext_setErrorMessageHandler_r(geosCtx->handle, geosErrMsgeHandler, geosCtx->errMsg); } if (geosCtx->WKBWriter == NULL) { @@ -57,10 +58,10 @@ int32_t initCtxMakePoint() { // outputWKT is a zero ending string // need to call geosFreeBuffer(*outputGeom) later int32_t doMakePoint(double x, double y, unsigned char **outputGeom, size_t *size) { - int32_t code = TSDB_CODE_FAILED; - SGeosContext* geosCtx = getThreadLocalGeosCtx(); + int32_t code = TSDB_CODE_FAILED; + SGeosContext *geosCtx = getThreadLocalGeosCtx(); - GEOSGeometry *geom = NULL; + GEOSGeometry *geom = NULL; unsigned char *wkb = NULL; geom = GEOSGeom_createPointFromXY_r(geosCtx->handle, x, y); @@ -89,6 +90,10 @@ _exit: static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) { int32_t code = 0; char *wktPatternWithSpace = taosMemoryCalloc(4, 1024); + if (NULL == wktPatternWithSpace) { + return TSDB_CODE_OUT_OF_MEMORY; + } + sprintf( wktPatternWithSpace, "^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " @@ -148,8 +153,8 @@ static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData } int32_t initCtxGeomFromText() { - int32_t code = TSDB_CODE_FAILED; - SGeosContext* geosCtx = getThreadLocalGeosCtx(); + int32_t code = TSDB_CODE_FAILED; + SGeosContext *geosCtx = getThreadLocalGeosCtx(); if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); @@ -157,7 +162,7 @@ int32_t initCtxGeomFromText() { return code; } - GEOSContext_setErrorMessageHandler_r(geosCtx->handle, geosErrMsgeHandler, geosCtx->errMsg); + (void)GEOSContext_setErrorMessageHandler_r(geosCtx->handle, geosErrMsgeHandler, geosCtx->errMsg); } if (geosCtx->WKTReader == NULL) { @@ -184,15 +189,15 @@ int32_t initCtxGeomFromText() { // inputWKT is a zero ending string // need to call geosFreeBuffer(*outputGeom) later int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t *size) { - int32_t code = TSDB_CODE_FAILED; - SGeosContext* geosCtx = getThreadLocalGeosCtx(); + int32_t code = TSDB_CODE_FAILED; + SGeosContext *geosCtx = getThreadLocalGeosCtx(); - GEOSGeometry *geom = NULL; + GEOSGeometry *geom = NULL; unsigned char *wkb = NULL; if (doRegExec(inputWKT, geosCtx->WKTRegex, geosCtx->WKTMatchData) != 0) { - code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE; - goto _exit; + code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + goto _exit; } geom = GEOSWKTReader_read_r(geosCtx->handle, geosCtx->WKTReader, inputWKT); @@ -219,8 +224,8 @@ _exit: } int32_t initCtxAsText() { - int32_t code = TSDB_CODE_FAILED; - SGeosContext* geosCtx = getThreadLocalGeosCtx(); + int32_t code = TSDB_CODE_FAILED; + SGeosContext *geosCtx = getThreadLocalGeosCtx(); if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); @@ -228,7 +233,7 @@ int32_t initCtxAsText() { return code; } - GEOSContext_setErrorMessageHandler_r(geosCtx->handle, geosErrMsgeHandler, geosCtx->errMsg); + (void)GEOSContext_setErrorMessageHandler_r(geosCtx->handle, geosErrMsgeHandler, geosCtx->errMsg); } if (geosCtx->WKBReader == NULL) { @@ -255,10 +260,10 @@ int32_t initCtxAsText() { // outputWKT is a zero ending string // need to call geosFreeBuffer(*outputWKT) later int32_t doAsText(const unsigned char *inputGeom, size_t size, char **outputWKT) { - int32_t code = TSDB_CODE_FAILED; - SGeosContext* geosCtx = getThreadLocalGeosCtx(); + int32_t code = TSDB_CODE_FAILED; + SGeosContext *geosCtx = getThreadLocalGeosCtx(); - GEOSGeometry *geom = NULL; + GEOSGeometry *geom = NULL; unsigned char *wkt = NULL; geom = GEOSWKBReader_read_r(geosCtx->handle, geosCtx->WKBReader, inputGeom, size); @@ -286,8 +291,8 @@ _exit: } int32_t initCtxRelationFunc() { - int32_t code = TSDB_CODE_FAILED; - SGeosContext* geosCtx = getThreadLocalGeosCtx(); + int32_t code = TSDB_CODE_FAILED; + SGeosContext *geosCtx = getThreadLocalGeosCtx(); if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); @@ -295,7 +300,7 @@ int32_t initCtxRelationFunc() { return code; } - GEOSContext_setErrorMessageHandler_r(geosCtx->handle, geosErrMsgeHandler, geosCtx->errMsg); + (void)GEOSContext_setErrorMessageHandler_r(geosCtx->handle, geosErrMsgeHandler, geosCtx->errMsg); } if (geosCtx->WKBReader == NULL) { @@ -309,88 +314,81 @@ int32_t initCtxRelationFunc() { } int32_t doGeosRelation(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, - bool swapped, char *res, - _geosRelationFunc_t relationFn, - _geosRelationFunc_t swappedRelationFn, + bool swapped, char *res, _geosRelationFunc_t relationFn, _geosRelationFunc_t swappedRelationFn, _geosPreparedRelationFunc_t preparedRelationFn, _geosPreparedRelationFunc_t swappedPreparedRelationFn) { - int32_t code = TSDB_CODE_FAILED; - SGeosContext* geosCtx = getThreadLocalGeosCtx(); + SGeosContext *geosCtx = getThreadLocalGeosCtx(); if (!preparedGeom1) { if (!swapped) { ASSERT(relationFn); *res = relationFn(geosCtx->handle, geom1, geom2); - } - else { + } else { ASSERT(swappedRelationFn); *res = swappedRelationFn(geosCtx->handle, geom1, geom2); } - } - else { + } else { if (!swapped) { ASSERT(preparedRelationFn); *res = preparedRelationFn(geosCtx->handle, preparedGeom1, geom2); - } - else { + } else { ASSERT(swappedPreparedRelationFn); *res = swappedPreparedRelationFn(geosCtx->handle, preparedGeom1, geom2); } } - code = TSDB_CODE_SUCCESS; - return code; + return TSDB_CODE_SUCCESS; } int32_t doIntersects(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, bool swapped, char *res) { - return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, - GEOSIntersects_r, GEOSIntersects_r, GEOSPreparedIntersects_r, GEOSPreparedIntersects_r); + return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, GEOSIntersects_r, GEOSIntersects_r, + GEOSPreparedIntersects_r, GEOSPreparedIntersects_r); } int32_t doEquals(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, bool swapped, char *res) { - return doGeosRelation(geom1, NULL, geom2, swapped, res, - GEOSEquals_r, GEOSEquals_r, NULL, NULL); // no prepared version for eguals() + return doGeosRelation(geom1, NULL, geom2, swapped, res, GEOSEquals_r, GEOSEquals_r, NULL, + NULL); // no prepared version for eguals() } int32_t doTouches(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, bool swapped, char *res) { - return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, - GEOSTouches_r, GEOSTouches_r, GEOSPreparedTouches_r, GEOSPreparedTouches_r); + return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, GEOSTouches_r, GEOSTouches_r, GEOSPreparedTouches_r, + GEOSPreparedTouches_r); } int32_t doCovers(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, bool swapped, char *res) { - return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, - GEOSCovers_r, GEOSCoveredBy_r, GEOSPreparedCovers_r, GEOSPreparedCoveredBy_r); + return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, GEOSCovers_r, GEOSCoveredBy_r, GEOSPreparedCovers_r, + GEOSPreparedCoveredBy_r); } int32_t doContains(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, bool swapped, char *res) { - return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, - GEOSContains_r, GEOSWithin_r, GEOSPreparedContains_r, GEOSPreparedWithin_r); + return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, GEOSContains_r, GEOSWithin_r, GEOSPreparedContains_r, + GEOSPreparedWithin_r); } -int32_t doContainsProperly(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, - bool swapped, char *res) { - return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, - NULL, NULL, GEOSPreparedContainsProperly_r, NULL); +int32_t doContainsProperly(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, + const GEOSGeometry *geom2, bool swapped, char *res) { + return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, NULL, NULL, GEOSPreparedContainsProperly_r, NULL); } // input is with VARSTR format // need to call destroyGeometry(outputGeom, outputPreparedGeom) later -int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, const GEOSPreparedGeometry **outputPreparedGeom) { - SGeosContext* geosCtx = getThreadLocalGeosCtx(); +int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, + const GEOSPreparedGeometry **outputPreparedGeom) { + SGeosContext *geosCtx = getThreadLocalGeosCtx(); - ASSERT(outputGeom); //it is not allowed if outputGeom is NULL + ASSERT(outputGeom); // it is not allowed if outputGeom is NULL *outputGeom = NULL; - if (outputPreparedGeom) { //it means not to generate PreparedGeometry if outputPreparedGeom is NULL + if (outputPreparedGeom) { // it means not to generate PreparedGeometry if outputPreparedGeom is NULL *outputPreparedGeom = NULL; } - if (varDataLen(input) == 0) { //empty value + if (varDataLen(input) == 0) { // empty value return TSDB_CODE_SUCCESS; } @@ -410,7 +408,7 @@ int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, cons } void destroyGeometry(GEOSGeometry **geom, const GEOSPreparedGeometry **preparedGeom) { - SGeosContext* geosCtx = getThreadLocalGeosCtx(); + SGeosContext *geosCtx = getThreadLocalGeosCtx(); if (preparedGeom && *preparedGeom) { GEOSPreparedGeom_destroy_r(geosCtx->handle, *preparedGeom); From 12f9116eebee4a94bf8ec71201907003e89b3048 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 24 Jul 2024 19:27:52 +0800 Subject: [PATCH 43/52] fix: (errcode) geom return value --- source/libs/geometry/src/geomFunc.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/source/libs/geometry/src/geomFunc.c b/source/libs/geometry/src/geomFunc.c index 343e6ec4ce..194590c06c 100644 --- a/source/libs/geometry/src/geomFunc.c +++ b/source/libs/geometry/src/geomFunc.c @@ -203,8 +203,6 @@ int32_t executeRelationFunc(const GEOSGeometry *geom1, const GEOSPreparedGeometr int32_t geomOneParamFunction(SScalarParam *pInput, SScalarParam *pOutput, _geomInitCtxFunc_t initCtxFn, _geomExecuteOneParamFunc_t executeOneParamFn) { - int32_t code = TSDB_CODE_FAILED; - TAOS_CHECK_RETURN(initCtxFn()); SColumnInfoData *pInputData = pInput->columnData; @@ -213,12 +211,10 @@ int32_t geomOneParamFunction(SScalarParam *pInput, SScalarParam *pOutput, _geomI if (IS_NULL_TYPE(GET_PARAM_TYPE(pInput))) { colDataSetNNULL(pOutputData, 0, pInput->numOfRows); - code = TSDB_CODE_SUCCESS; } else { for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { colDataSetNULL(pOutputData, i); - code = TSDB_CODE_SUCCESS; continue; } @@ -226,13 +222,11 @@ int32_t geomOneParamFunction(SScalarParam *pInput, SScalarParam *pOutput, _geomI } } - TAOS_RETURN(code); + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t geomTwoParamsFunction(SScalarParam *pInput, SScalarParam *pOutput, _geomInitCtxFunc_t initCtxFn, _geomExecuteTwoParamsFunc_t executeTwoParamsFn) { - int32_t code = TSDB_CODE_FAILED; - TAOS_CHECK_RETURN(initCtxFn()); SColumnInfoData *pInputData[2]; @@ -250,7 +244,6 @@ int32_t geomTwoParamsFunction(SScalarParam *pInput, SScalarParam *pOutput, _geom (isConstantLeft && colDataIsNull_s(pInputData[0], 0)) || // left operand is constant NULL (isConstantRight && colDataIsNull_s(pInputData[1], 0))) { // right operand is constant NULL colDataSetNNULL(pOutputData, 0, numOfRows); - code = TSDB_CODE_SUCCESS; } else { int32_t iLeft = 0; int32_t iRight = 0; @@ -261,7 +254,6 @@ int32_t geomTwoParamsFunction(SScalarParam *pInput, SScalarParam *pOutput, _geom if ((!isConstantLeft && colDataIsNull_s(pInputData[0], iLeft)) || (!isConstantRight && colDataIsNull_s(pInputData[1], iRight))) { colDataSetNULL(pOutputData, i); - code = TSDB_CODE_SUCCESS; continue; } @@ -269,7 +261,7 @@ int32_t geomTwoParamsFunction(SScalarParam *pInput, SScalarParam *pOutput, _geom } } - TAOS_RETURN(code); + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t geomRelationFunction(SScalarParam *pInput, SScalarParam *pOutput, bool swapAllowed, @@ -324,7 +316,6 @@ int32_t geomRelationFunction(SScalarParam *pInput, SScalarParam *pOutput, bool s for (int32_t i = 0; i < numOfRows; ++i) { if ((!isConstant1 && colDataIsNull_s(pInputData[0], i)) || (!isConstant2 && colDataIsNull_s(pInputData[1], i))) { colDataSetNULL(pOutputData, i); - code = TSDB_CODE_SUCCESS; continue; } @@ -346,6 +337,8 @@ int32_t geomRelationFunction(SScalarParam *pInput, SScalarParam *pOutput, bool s } } + code = TSDB_CODE_SUCCESS; + _exit: destroyGeometry(&geom1, &preparedGeom1); destroyGeometry(&geom2, NULL); From 0d5323c587670323b6b56391402420c6ba3d7bca Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 24 Jul 2024 23:46:06 +0800 Subject: [PATCH 44/52] fix: refactor return code --- source/dnode/mnode/impl/src/mndUser.c | 6 +++--- source/dnode/vnode/src/tsdb/tsdbFS2.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index f05baf25bd..5a19b6c762 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1724,7 +1724,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate if (pCreate->numIpRanges == 0) { TAOS_CHECK_RETURN(createDefaultIpWhiteList(&userObj.pIpWhiteList)); } else { - SHashObj *pUniqueTab = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); + SHashObj *pUniqueTab = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); if (pUniqueTab == NULL) { TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } @@ -2082,7 +2082,7 @@ static int32_t mndRemoveTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj int32_t len = strlen(tbFName) + 1; if (taosHashRemove(hash, tbFName, len) != 0) { - TAOS_RETURN(TSDB_CODE_NOT_FOUND); + TAOS_RETURN(0); // not found } int32_t dbKeyLen = strlen(alterReq->objname) + 1; @@ -2093,7 +2093,7 @@ static int32_t mndRemoveTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj if (1 == *currRef) { if (taosHashRemove(useDbHash, alterReq->objname, dbKeyLen) != 0) { - TAOS_RETURN(TSDB_CODE_NOT_FOUND); + TAOS_RETURN(0); // not found } return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 68954e5156..ca99b0eaff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -524,7 +524,7 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { { // clear unreferenced files STfsDir *dir = NULL; - TAOS_CHECK_GOTO(tfsOpendir(fs->tsdb->pVnode->pTfs, fs->tsdb->path, &dir), &lino, _exit); + TAOS_CHECK_GOTO(tfsOpendir(fs->tsdb->pVnode->pTfs, fs->tsdb->path, &dir), &lino, _close_dir); STFileHash fobjHash = {0}; code = tsdbFSCreateFileObjHash(fs, &fobjHash); From eaa36347641311ecd8d67acfc11aa003ba4abf68 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 25 Jul 2024 08:51:19 +0800 Subject: [PATCH 45/52] enh: refactor error code --- include/common/tdatablock.h | 52 +- include/common/tdataformat.h | 10 +- include/common/tmsg.h | 8 +- include/common/ttszip.h | 147 --- include/common/ttypes.h | 2 +- include/libs/function/taosudf.h | 27 +- include/util/tarray2.h | 28 +- include/util/tcoding.h | 24 +- include/util/tdef.h | 10 +- include/util/tencode.h | 14 +- include/util/tutil.h | 4 +- include/util/types.h | 4 +- source/common/src/ttszip.c | 1143 ----------------------- source/libs/executor/inc/executil.h | 12 +- source/libs/executor/inc/executorInt.h | 3 +- source/libs/function/inc/tpercentile.h | 9 +- source/libs/function/src/tfunctionInt.c | 1 - source/libs/index/inc/indexUtil.h | 34 +- source/libs/scalar/inc/filterInt.h | 61 +- source/libs/tdb/src/db/tdbDb.c | 6 +- utils/TSZ/zstd/common/bitstream.h | 2 +- 21 files changed, 156 insertions(+), 1445 deletions(-) delete mode 100644 include/common/ttszip.h delete mode 100644 source/common/src/ttszip.c diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index c09e96d16d..862bbee776 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -32,8 +32,8 @@ typedef struct SBlockOrderInfo { SColumnInfoData* pColData; } SBlockOrderInfo; -#define BLOCK_VERSION_1 1 -#define BLOCK_VERSION_2 2 +#define BLOCK_VERSION_1 1 +#define BLOCK_VERSION_2 2 #define NBIT (3u) #define BitPos(_n) ((_n) & ((1 << NBIT) - 1)) @@ -46,9 +46,9 @@ typedef struct SBlockOrderInfo { BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) -#define colDataSetNull_f_s(c_, r_) \ - do { \ - colDataSetNull_f((c_)->nullbitmap, r_); \ +#define colDataSetNull_f_s(c_, r_) \ + do { \ + colDataSetNull_f((c_)->nullbitmap, r_); \ (void)memset(((char*)(c_)->pData) + (c_)->info.bytes * (r_), 0, (c_)->info.bytes); \ } while (0) @@ -143,7 +143,7 @@ static FORCE_INLINE void colDataSetNNULL(SColumnInfoData* pColumnInfoData, uint3 for (int32_t i = start; i < start + nRows; ++i) { colDataSetNull_f(pColumnInfoData->nullbitmap, i); } - memset(pColumnInfoData->pData + start * pColumnInfoData->info.bytes, 0, pColumnInfoData->info.bytes * nRows); + (void)memset(pColumnInfoData->pData + start * pColumnInfoData->info.bytes, 0, pColumnInfoData->info.bytes * nRows); } pColumnInfoData->hasNull = true; @@ -192,15 +192,17 @@ int32_t getJsonValueLen(const char* data); int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData); -int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue); -void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow, uint32_t numOfRows); -int32_t colDataCopyNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, - uint32_t numOfRows, bool isNull); +int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, + bool trimValue); +void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow, uint32_t numOfRows); +int32_t colDataCopyNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows, + bool isNull); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, int32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo); -int32_t colDataAssignNRows(SColumnInfoData* pDst, int32_t dstIdx, const SColumnInfoData* pSrc, int32_t srcIdx, int32_t numOfRows); +int32_t colDataAssignNRows(SColumnInfoData* pDst, int32_t dstIdx, const SColumnInfoData* pSrc, int32_t srcIdx, + int32_t numOfRows); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); int32_t blockDataUpdatePkRange(SSDataBlock* pDataBlock, int32_t pkColumnIndex, bool asc); @@ -214,7 +216,7 @@ size_t blockDataGetNumOfRows(const SSDataBlock* pBlock); int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); int32_t blockDataMergeNRows(SSDataBlock* pDest, const SSDataBlock* pSrc, int32_t srcIdx, int32_t numOfRows); -void blockDataShrinkNRows(SSDataBlock* pBlock, int32_t numOfRows); +void blockDataShrinkNRows(SSDataBlock* pBlock, int32_t numOfRows); int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize); int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock); @@ -237,12 +239,12 @@ int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo); int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); -void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); -void blockDataCleanup(SSDataBlock* pDataBlock); -void blockDataReset(SSDataBlock* pDataBlock); -void blockDataEmpty(SSDataBlock* pDataBlock); +void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); +void blockDataCleanup(SSDataBlock* pDataBlock); +void blockDataReset(SSDataBlock* pDataBlock); +void blockDataEmpty(SSDataBlock* pDataBlock); int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, - bool clearPayload); + bool clearPayload); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int32_t extraSize); @@ -264,20 +266,20 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColIn SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); -int32_t blockGetEncodeSize(const SSDataBlock* pBlock); -int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); +int32_t blockGetEncodeSize(const SSDataBlock* pBlock); +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); const char* blockDecode(SSDataBlock* pBlock, const char* pData); // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf, const char* taskIdStr); -int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pDataBlocks, const STSchema* pTSchema, int64_t uid, int32_t vgId, - tb_uid_t suid); +int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pDataBlocks, const STSchema* pTSchema, + int64_t uid, int32_t vgId, tb_uid_t suid); -bool alreadyAddGroupId(char* ctbName, int64_t groupId); -bool isAutoTableName(char* ctbName); -void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId); -char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); +bool alreadyAddGroupId(char* ctbName, int64_t groupId); +bool isAutoTableName(char* ctbName); +void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId); +char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf); void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 04e13fbdb3..0acb28fabd 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -315,7 +315,7 @@ struct STag { do { \ VarDataLenT __len = (VarDataLenT)strlen(str); \ *(VarDataLenT *)(x) = __len; \ - memcpy(varDataVal(x), (str), __len); \ + (void)memcpy(varDataVal(x), (str), __len); \ } while (0); #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ @@ -324,10 +324,10 @@ struct STag { varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \ } while (0) -#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ - do { \ - *(VarDataLenT *)(x) = (VarDataLenT)(_size); \ - memcpy(varDataVal(x), (str), (_size)); \ +#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ + do { \ + *(VarDataLenT *)(x) = (VarDataLenT)(_size); \ + (void)memcpy(varDataVal(x), (str), (_size)); \ } while (0); // STSchema ================================ diff --git a/include/common/tmsg.h b/include/common/tmsg.h index e9cde9e99b..b8e642e5d1 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -691,7 +691,7 @@ static FORCE_INLINE SColCmprWrapper* tCloneSColCmprWrapper(const SColCmprWrapper int32_t size = sizeof(SColCmpr) * pDstWrapper->nCols; pDstWrapper->pColCmpr = (SColCmpr*)taosMemoryCalloc(1, size); - memcpy(pDstWrapper->pColCmpr, pSrcWrapper->pColCmpr, size); + (void)memcpy(pDstWrapper->pColCmpr, pSrcWrapper->pColCmpr, size); return pDstWrapper; } @@ -732,7 +732,7 @@ static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* p return NULL; } - memcpy(pSW->pSchema, pSchemaWrapper->pSchema, pSW->nCols * sizeof(SSchema)); + (void)memcpy(pSW->pSchema, pSchemaWrapper->pSchema, pSW->nCols * sizeof(SSchema)); return pSW; } @@ -2837,13 +2837,13 @@ static FORCE_INLINE int32_t tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeR buf = taosDecodeFixedI32(buf, &topicNum); pReq->topicNames = taosArrayInit(topicNum, sizeof(void*)); - if (pReq->topicNames == NULL){ + if (pReq->topicNames == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < topicNum; i++) { char* name = NULL; buf = taosDecodeString(buf, &name); - if (taosArrayPush(pReq->topicNames, &name) == NULL){ + if (taosArrayPush(pReq->topicNames, &name) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } } diff --git a/include/common/ttszip.h b/include/common/ttszip.h deleted file mode 100644 index 8eb99bd45e..0000000000 --- a/include/common/ttszip.h +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_COMMON_TTSZIP_H_ -#define _TD_COMMON_TTSZIP_H_ - -#include "os.h" -#include "tdef.h" -#include "tvariant.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define MEM_BUF_SIZE (1 << 20) -#define TS_COMP_FILE_MAGIC 0x87F5EC4C -#define TS_COMP_FILE_GROUP_MAX 512 - -typedef struct STSList { - char* rawBuf; - int32_t allocSize; - int32_t threshold; - int32_t len; -} STSList; - -typedef struct STSElem { - TSKEY ts; - SVariant* tag; - int32_t id; -} STSElem; - -typedef struct STSCursor { - int32_t vgroupIndex; - int32_t blockIndex; - int32_t tsIndex; - uint32_t order; -} STSCursor; - -typedef struct STSBlock { - SVariant tag; // tag value - int32_t numOfElem; // number of elements - int32_t compLen; // size after compressed - int32_t padding; // 0xFFFFFFFF by default, after the payload - char* payload; // actual data that is compressed -} STSBlock; - -/* - * The size of buffer file should not be greater than 2G, - * and the offset of int32_t type is enough - */ -typedef struct STSGroupBlockInfo { - int32_t id; // group id - int32_t offset; // offset set value in file - int32_t numOfBlocks; // number of total blocks - int32_t compLen; // compressed size -} STSGroupBlockInfo; - -typedef struct STSGroupBlockInfoEx { - STSGroupBlockInfo info; - int32_t len; // length before compress -} STSGroupBlockInfoEx; - -typedef struct STSBuf { - TdFilePtr pFile; - char path[PATH_MAX]; - uint32_t fileSize; - - // todo use array - STSGroupBlockInfoEx* pData; - uint32_t numOfAlloc; - uint32_t numOfGroups; - - char* assistBuf; - int32_t bufSize; - STSBlock block; - STSList tsData; // uncompressed raw ts data - uint64_t numOfTotal; - bool autoDelete; - bool remainOpen; - int32_t tsOrder; // order of timestamp in ts comp buffer - STSCursor cur; -} STSBuf; - -typedef struct STSBufFileHeader { - uint32_t magic; // file magic number - uint32_t numOfGroup; // number of group stored in current file - int32_t tsOrder; // timestamp order in current file -} STSBufFileHeader; - -STSBuf* tsBufCreate(bool autoDelete, int32_t order); -STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete); -STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t id); - -void* tsBufDestroy(STSBuf* pTSBuf); - -void tsBufAppend(STSBuf* pTSBuf, int32_t id, SVariant* tag, const char* pData, int32_t len); -int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf); - -STSBuf* tsBufClone(STSBuf* pTSBuf); - -STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id); - -void tsBufFlush(STSBuf* pTSBuf); -void tsBufResetPos(STSBuf* pTSBuf); -bool tsBufNextPos(STSBuf* pTSBuf); - -STSElem tsBufGetElem(STSBuf* pTSBuf); -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, SVariant* tag); - -STSCursor tsBufGetCursor(STSBuf* pTSBuf); -void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order); - -void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur); - -/** - * display all data in comp block file, for debug purpose only - * @param pTSBuf - */ -void tsBufDisplay(STSBuf* pTSBuf); - -int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf); - -void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id); - -int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t id, void* buf, int32_t* len, int32_t* numOfBlocks); - -STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, SVariant* pTag); - -bool tsBufIsValidElem(STSElem* pElem); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_COMMON_TTSZIP_H_*/ diff --git a/include/common/ttypes.h b/include/common/ttypes.h index f10f419b6f..3934553b1c 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -46,7 +46,7 @@ typedef struct { #pragma pack(pop) #define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) -#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) +#define varDataCopy(dst, v) (void)memcpy((dst), (void *)(v), varDataTLen(v)) #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h index 935f9f9b03..79abbc4e68 100644 --- a/include/libs/function/taosudf.h +++ b/include/libs/function/taosudf.h @@ -105,12 +105,14 @@ typedef uint16_t VarDataLenT; // maxVarDataLen: 65535 #define varDataLen(v) ((VarDataLenT *)(v))[0] #define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE) #define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) -#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) +#define varDataCopy(dst, v) (void)memcpy((dst), (void *)(v), varDataTLen(v)) #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) -#define IS_VAR_DATA_TYPE(t) \ - (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) -#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) +#define IS_VAR_DATA_TYPE(t) \ + (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR) || \ + ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) +#define IS_STR_DATA_TYPE(t) \ + (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) static FORCE_INLINE char *udfColDataGetData(const SUdfColumn *pColumn, int32_t row) { if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { @@ -158,7 +160,7 @@ static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn *pColumn, int32_t ne } data->varLenCol.varOffsets = (int32_t *)tmp; data->varLenCol.varOffsetsLen = sizeof(int32_t) * allocCapacity; - memset(&data->varLenCol.varOffsets[existedRows], 0, sizeof(int32_t) * (allocCapacity - existedRows)); + (void)memset(&data->varLenCol.varOffsets[existedRows], 0, sizeof(int32_t) * (allocCapacity - existedRows)); // for payload, add data in udfColDataAppend } else { char *tmp = (char *)realloc(data->fixLenCol.nullBitmap, BitmapLen(allocCapacity)); @@ -166,11 +168,11 @@ static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn *pColumn, int32_t ne return TSDB_CODE_OUT_OF_MEMORY; } uint32_t extend = BitmapLen(allocCapacity) - BitmapLen(data->rowsAlloc); - memset(tmp + BitmapLen(data->rowsAlloc), 0, extend); + (void)memset(tmp + BitmapLen(data->rowsAlloc), 0, extend); data->fixLenCol.nullBitmap = tmp; data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity); int32_t oldLen = BitmapLen(existedRows); - memset(&data->fixLenCol.nullBitmap[oldLen], 0, BitmapLen(allocCapacity) - oldLen); + (void)memset(&data->fixLenCol.nullBitmap[oldLen], 0, BitmapLen(allocCapacity) - oldLen); if (meta->type == TSDB_DATA_TYPE_NULL) { return TSDB_CODE_SUCCESS; @@ -198,7 +200,8 @@ static FORCE_INLINE void udfColDataSetNull(SUdfColumn *pColumn, int32_t row) { udfColDataSetNull_f(pColumn, row); } pColumn->hasNull = true; - pColumn->colData.numOfRows = ((int32_t)(row + 1) > pColumn->colData.numOfRows) ? (int32_t)(row + 1) : pColumn->colData.numOfRows; + pColumn->colData.numOfRows = + ((int32_t)(row + 1) > pColumn->colData.numOfRows) ? (int32_t)(row + 1) : pColumn->colData.numOfRows; } static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentRow, const char *pData, bool isNull) { @@ -211,7 +214,7 @@ static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentR } else { if (!isVarCol) { udfColDataSetNotNull_f(pColumn, currentRow); - memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes); + (void)memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes); } else { int32_t dataLen = varDataTLen(pData); if (meta->type == TSDB_DATA_TYPE_JSON) { @@ -249,7 +252,7 @@ static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentR uint32_t len = data->varLenCol.payloadLen; data->varLenCol.varOffsets[currentRow] = len; - memcpy(data->varLenCol.payload + len, pData, dataLen); + (void)memcpy(data->varLenCol.payload + len, pData, dataLen); data->varLenCol.payloadLen += dataLen; } } @@ -278,8 +281,8 @@ typedef enum EUdfFuncType { UDF_FUNC_TYPE_SCALAR = 1, UDF_FUNC_TYPE_AGG = 2 } EU typedef struct SScriptUdfInfo { const char *name; - int32_t version; - int64_t createdTime; + int32_t version; + int64_t createdTime; EUdfFuncType funcType; int8_t scriptType; diff --git a/include/util/tarray2.h b/include/util/tarray2.h index 0d1ceded6c..ce24a1d2ce 100644 --- a/include/util/tarray2.h +++ b/include/util/tarray2.h @@ -71,9 +71,9 @@ static FORCE_INLINE int32_t tarray2InsertBatch(void *arr, int32_t idx, const voi } if (ret == 0) { if (idx < a->size) { - memmove(a->data + (idx + numEle) * eleSize, a->data + idx * eleSize, (a->size - idx) * eleSize); + (void)memmove(a->data + (idx + numEle) * eleSize, a->data + idx * eleSize, (a->size - idx) * eleSize); } - memcpy(a->data + idx * eleSize, elePtr, numEle * eleSize); + (void)memcpy(a->data + idx * eleSize, elePtr, numEle * eleSize); a->size += numEle; } return ret; @@ -145,18 +145,18 @@ static FORCE_INLINE int32_t tarray2SortInsert(void *arr, const void *elePtr, int #define TARRAY2_SORT_INSERT(a, e, cmp) tarray2SortInsert(a, &(e), sizeof(((a)->data[0])), (__compar_fn_t)cmp) #define TARRAY2_SORT_INSERT_P(a, ep, cmp) tarray2SortInsert(a, ep, sizeof(((a)->data[0])), (__compar_fn_t)cmp) -#define TARRAY2_REMOVE(a, idx, cb) \ - do { \ - if ((idx) < (a)->size) { \ - if (cb) { \ - TArray2Cb cb_ = (TArray2Cb)(cb); \ - cb_((a)->data + (idx)); \ - } \ - if ((idx) < (a)->size - 1) { \ - memmove((a)->data + (idx), (a)->data + (idx) + 1, sizeof((*(a)->data)) * ((a)->size - (idx)-1)); \ - } \ - (a)->size--; \ - } \ +#define TARRAY2_REMOVE(a, idx, cb) \ + do { \ + if ((idx) < (a)->size) { \ + if (cb) { \ + TArray2Cb cb_ = (TArray2Cb)(cb); \ + cb_((a)->data + (idx)); \ + } \ + if ((idx) < (a)->size - 1) { \ + (void)memmove((a)->data + (idx), (a)->data + (idx) + 1, sizeof((*(a)->data)) * ((a)->size - (idx)-1)); \ + } \ + (a)->size--; \ + } \ } while (0) #define TARRAY2_FOREACH(a, e) for (int32_t __i = 0; __i < (a)->size && ((e) = (a)->data[__i], 1); __i++) diff --git a/include/util/tcoding.h b/include/util/tcoding.h index b1bd09e123..1040adf431 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -79,7 +79,7 @@ static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) { static FORCE_INLINE int32_t taosEncodeFixedU16(void **buf, uint16_t value) { if (buf != NULL) { if (IS_LITTLE_ENDIAN()) { - memcpy(*buf, &value, sizeof(value)); + TAOS_MEMCPY(*buf, &value, sizeof(value)); } else { ((uint8_t *)(*buf))[0] = value & 0xff; ((uint8_t *)(*buf))[1] = (value >> 8) & 0xff; @@ -92,7 +92,7 @@ static FORCE_INLINE int32_t taosEncodeFixedU16(void **buf, uint16_t value) { static FORCE_INLINE void *taosDecodeFixedU16(const void *buf, uint16_t *value) { if (IS_LITTLE_ENDIAN()) { - memcpy(value, buf, sizeof(*value)); + TAOS_MEMCPY(value, buf, sizeof(*value)); } else { ((uint8_t *)value)[1] = ((uint8_t *)buf)[0]; ((uint8_t *)value)[0] = ((uint8_t *)buf)[1]; @@ -117,7 +117,7 @@ static FORCE_INLINE void *taosDecodeFixedI16(const void *buf, int16_t *value) { static FORCE_INLINE int32_t taosEncodeFixedU32(void **buf, uint32_t value) { if (buf != NULL) { if (IS_LITTLE_ENDIAN()) { - memcpy(*buf, &value, sizeof(value)); + TAOS_MEMCPY(*buf, &value, sizeof(value)); } else { ((uint8_t *)(*buf))[0] = value & 0xff; ((uint8_t *)(*buf))[1] = (value >> 8) & 0xff; @@ -132,7 +132,7 @@ static FORCE_INLINE int32_t taosEncodeFixedU32(void **buf, uint32_t value) { static FORCE_INLINE void *taosDecodeFixedU32(const void *buf, uint32_t *value) { if (IS_LITTLE_ENDIAN()) { - memcpy(value, buf, sizeof(*value)); + TAOS_MEMCPY(value, buf, sizeof(*value)); } else { ((uint8_t *)value)[3] = ((uint8_t *)buf)[0]; ((uint8_t *)value)[2] = ((uint8_t *)buf)[1]; @@ -159,7 +159,7 @@ static FORCE_INLINE void *taosDecodeFixedI32(const void *buf, int32_t *value) { static FORCE_INLINE int32_t taosEncodeFixedU64(void **buf, uint64_t value) { if (buf != NULL) { if (IS_LITTLE_ENDIAN()) { - memcpy(*buf, &value, sizeof(value)); + TAOS_MEMCPY(*buf, &value, sizeof(value)); } else { ((uint8_t *)(*buf))[0] = value & 0xff; ((uint8_t *)(*buf))[1] = (value >> 8) & 0xff; @@ -179,7 +179,7 @@ static FORCE_INLINE int32_t taosEncodeFixedU64(void **buf, uint64_t value) { static FORCE_INLINE void *taosDecodeFixedU64(const void *buf, uint64_t *value) { if (IS_LITTLE_ENDIAN()) { - memcpy(value, buf, sizeof(*value)); + TAOS_MEMCPY(value, buf, sizeof(*value)); } else { ((uint8_t *)value)[7] = ((uint8_t *)buf)[0]; ((uint8_t *)value)[6] = ((uint8_t *)buf)[1]; @@ -357,7 +357,7 @@ static FORCE_INLINE int32_t taosEncodeString(void **buf, const char *value) { tlen += taosEncodeVariantU64(buf, size); if (buf != NULL) { - memcpy(*buf, value, size); + TAOS_MEMCPY(*buf, value, size); *buf = POINTER_SHIFT(*buf, size); } tlen += (int32_t)size; @@ -372,7 +372,7 @@ static FORCE_INLINE void *taosDecodeString(const void *buf, char **value) { *value = (char *)taosMemoryMalloc((size_t)size + 1); if (*value == NULL) return NULL; - memcpy(*value, buf, (size_t)size); + TAOS_MEMCPY(*value, buf, (size_t)size); (*value)[size] = '\0'; @@ -383,7 +383,7 @@ static FORCE_INLINE void *taosDecodeStringTo(const void *buf, char *value) { uint64_t size = 0; buf = taosDecodeVariantU64(buf, &size); - memcpy(value, buf, (size_t)size); + TAOS_MEMCPY(value, buf, (size_t)size); value[size] = '\0'; @@ -395,7 +395,7 @@ static FORCE_INLINE int32_t taosEncodeBinary(void **buf, const void *value, int3 int32_t tlen = 0; if (buf != NULL) { - memcpy(*buf, value, valueLen); + TAOS_MEMCPY(*buf, value, valueLen); *buf = POINTER_SHIFT(*buf, valueLen); } tlen += (int32_t)valueLen; @@ -406,13 +406,13 @@ static FORCE_INLINE int32_t taosEncodeBinary(void **buf, const void *value, int3 static FORCE_INLINE void *taosDecodeBinary(const void *buf, void **value, int32_t valueLen) { *value = taosMemoryMalloc((size_t)valueLen); if (*value == NULL) return NULL; - memcpy(*value, buf, (size_t)valueLen); + TAOS_MEMCPY(*value, buf, (size_t)valueLen); return POINTER_SHIFT(buf, valueLen); } static FORCE_INLINE void *taosDecodeBinaryTo(const void *buf, void *value, int32_t valueLen) { - memcpy(value, buf, (size_t)valueLen); + TAOS_MEMCPY(value, buf, (size_t)valueLen); return POINTER_SHIFT(buf, valueLen); } diff --git a/include/util/tdef.h b/include/util/tdef.h index 53bb8a493c..c9677845ac 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -131,10 +131,10 @@ static const int64_t TICK_PER_SECOND[] = { : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1000000LL : 1000000000LL))) #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) -#define T_APPEND_MEMBER(dst, ptr, type, member) \ - do { \ - memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member)); \ - dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member)); \ +#define T_APPEND_MEMBER(dst, ptr, type, member) \ + do { \ + (void)memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member)); \ + dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member)); \ } while (0) #define T_READ_MEMBER(src, type, target) \ do { \ @@ -556,7 +556,7 @@ typedef struct { char name[TSDB_LOG_VAR_LEN]; } SLogVar; -#define TMQ_SEPARATOR ":" +#define TMQ_SEPARATOR ":" #define TMQ_SEPARATOR_CHAR ':' enum { diff --git a/include/util/tencode.h b/include/util/tencode.h index 9b9f8af1b6..b66e79fa60 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -127,7 +127,7 @@ static FORCE_INLINE int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint if (pCoder->pos + size > pCoder->size) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } - memcpy(pCoder->data + pCoder->pos, val, size); + TAOS_MEMCPY(pCoder->data + pCoder->pos, val, size); } pCoder->pos += size; @@ -212,7 +212,7 @@ static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, if (pCoder->pos + len > pCoder->size) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } - memcpy(pCoder->data + pCoder->pos, val, len); + TAOS_MEMCPY(pCoder->data + pCoder->pos, val, len); } pCoder->pos += len; @@ -233,7 +233,7 @@ static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size) { if (pCoder->pos + size > pCoder->size) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } else if (val) { - memcpy(val, pCoder->data + pCoder->pos, size); + TAOS_MEMCPY(val, pCoder->data + pCoder->pos, size); } pCoder->pos += size; return 0; @@ -427,7 +427,7 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { uint32_t len; TAOS_CHECK_RETURN(tDecodeCStrAndLen(pCoder, &pStr, &len)); - memcpy(val, pStr, len + 1); + TAOS_MEMCPY(val, pStr, len + 1); return 0; } @@ -446,7 +446,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uin TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(*val, pCoder->data + pCoder->pos, length); + TAOS_MEMCPY(*val, pCoder->data + pCoder->pos, length); pCoder->pos += length; } else { @@ -468,7 +468,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, u if (*val == NULL) { TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(*val, pCoder->data + pCoder->pos, length); + TAOS_MEMCPY(*val, pCoder->data + pCoder->pos, length); pCoder->pos += length; } else { @@ -757,7 +757,7 @@ static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nDat int n = 0; n += tPutU32v(p ? p + n : p, nData); - if (p) memcpy(p + n, pData, nData); + if (p) TAOS_MEMCPY(p + n, pData, nData); n += nData; return n; diff --git a/include/util/tutil.h b/include/util/tutil.h index 5af79dfc49..c1951dbe79 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -64,7 +64,7 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *tar tMD5Init(&context); tMD5Update(&context, inBuf, (uint32_t)inLen); tMD5Final(&context); - memcpy(target, context.digest, tListLen(context.digest)); + (void)memcpy(target, context.digest, tListLen(context.digest)); } static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *target) { @@ -79,7 +79,7 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12], context.digest[13], context.digest[14], context.digest[15]); - memcpy(target, buf, TSDB_PASSWORD_LEN); + (void)memcpy(target, buf, TSDB_PASSWORD_LEN); } static FORCE_INLINE int32_t taosCreateMD5Hash(char *pBuf, int32_t len) { diff --git a/include/util/types.h b/include/util/types.h index 0aa01a66f5..1797021b55 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -38,7 +38,7 @@ static FORCE_INLINE float taos_align_get_float(const char *pBuf) { assert(sizeof(float) == sizeof(uint32_t)); #endif float fv = 0; - memcpy(&fv, pBuf, sizeof(fv)); // in ARM, return *((const float*)(pBuf)) may cause problem + (void)memcpy(&fv, pBuf, sizeof(fv)); // in ARM, return *((const float*)(pBuf)) may cause problem return fv; } @@ -49,7 +49,7 @@ static FORCE_INLINE double taos_align_get_double(const char *pBuf) { assert(sizeof(double) == sizeof(uint64_t)); #endif double dv = 0; - memcpy(&dv, pBuf, sizeof(dv)); // in ARM, return *((const double*)(pBuf)) may cause problem + (void)memcpy(&dv, pBuf, sizeof(dv)); // in ARM, return *((const double*)(pBuf)) may cause problem return dv; } diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c deleted file mode 100644 index 38659eea44..0000000000 --- a/source/common/src/ttszip.c +++ /dev/null @@ -1,1143 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "ttszip.h" -#include "taoserror.h" -#include "tcompression.h" -#include "tlog.h" - -static int32_t getDataStartOffset(); -static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); -static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); -static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); - -/** - * todo error handling - * support auto closeable tmp file - * @param path - * @return - */ -STSBuf* tsBufCreate(bool autoDelete, int32_t order) { - if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_DISKSPACE; - // tscError("tmp file created failed since %s", terrstr()); - return NULL; - } - - STSBuf* pTSBuf = taosMemoryCalloc(1, sizeof(STSBuf)); - if (pTSBuf == NULL) { - return NULL; - } - - pTSBuf->autoDelete = autoDelete; - - taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path); - // pTSBuf->pFile = fopen(pTSBuf->path, "wb+"); - pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); - if (pTSBuf->pFile == NULL) { - taosMemoryFree(pTSBuf); - return NULL; - } - - if (!autoDelete) { - if (taosRemoveFile(pTSBuf->path) != 0) { - taosMemoryFree(pTSBuf); - return NULL; - } - } - - if (allocResForTSBuf(pTSBuf) == NULL) { - return NULL; - } - - // update the header info - STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = TSDB_ORDER_ASC}; - STSBufUpdateHeader(pTSBuf, &header); - - tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSDB_ORDER_ASC; - - pTSBuf->tsOrder = order; - - return pTSBuf; -} - -STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { - STSBuf* pTSBuf = taosMemoryCalloc(1, sizeof(STSBuf)); - if (pTSBuf == NULL) { - return NULL; - } - - pTSBuf->autoDelete = autoDelete; - - tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - - // pTSBuf->pFile = fopen(pTSBuf->path, "rb+"); - pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ); - if (pTSBuf->pFile == NULL) { - taosMemoryFree(pTSBuf); - return NULL; - } - - if (allocResForTSBuf(pTSBuf) == NULL) { - return NULL; - } - - // validate the file magic number - STSBufFileHeader header = {0}; - int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); - UNUSED(ret); - size_t sz = taosReadFile(pTSBuf->pFile, &header, sizeof(STSBufFileHeader)); - UNUSED(sz); - - // invalid file - if (header.magic != TS_COMP_FILE_MAGIC) { - tsBufDestroy(pTSBuf); - return NULL; - } - - if (header.numOfGroup > pTSBuf->numOfAlloc) { - pTSBuf->numOfAlloc = header.numOfGroup; - STSGroupBlockInfoEx* tmp = taosMemoryRealloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); - if (tmp == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->pData = tmp; - } - - pTSBuf->numOfGroups = header.numOfGroup; - - // check the ts order - pTSBuf->tsOrder = header.tsOrder; - if (pTSBuf->tsOrder != TSDB_ORDER_ASC && pTSBuf->tsOrder != TSDB_ORDER_DESC) { - // tscError("invalid order info in buf:%d", pTSBuf->tsOrder); - tsBufDestroy(pTSBuf); - return NULL; - } - - size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups; - - STSGroupBlockInfo* buf = (STSGroupBlockInfo*)taosMemoryCalloc(1, infoSize); - if (buf == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - // int64_t pos = ftell(pTSBuf->pFile); //pos not used - sz = taosReadFile(pTSBuf->pFile, buf, infoSize); - UNUSED(sz); - - // the length value for each vnode is not kept in file, so does not set the length value - for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) { - STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i]; - memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo)); - } - taosMemoryFree(buf); - - ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END); - UNUSED(ret); - - int64_t file_size; - if (taosFStatFile(pTSBuf->pFile, &file_size, NULL) != 0) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->fileSize = (uint32_t)file_size; - tsBufResetPos(pTSBuf); - - // ascending by default - pTSBuf->cur.order = TSDB_ORDER_ASC; - - // tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, - // fileno(pTSBuf->pFile), - // pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); - - return pTSBuf; -} - -void* tsBufDestroy(STSBuf* pTSBuf) { - if (pTSBuf == NULL) { - return NULL; - } - - taosMemoryFreeClear(pTSBuf->assistBuf); - taosMemoryFreeClear(pTSBuf->tsData.rawBuf); - - taosMemoryFreeClear(pTSBuf->pData); - taosMemoryFreeClear(pTSBuf->block.payload); - - if (!pTSBuf->remainOpen) { - taosCloseFile(&pTSBuf->pFile); - } - - if (pTSBuf->autoDelete) { - // ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); - if (taosRemoveFile(pTSBuf->path) != 0) { - // tscError("tsBuf %p destroyed, failed to remove tmp file:%s", pTSBuf, pTSBuf->path); - } - } else { - // tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); - } - - taosVariantDestroy(&pTSBuf->block.tag); - taosMemoryFree(pTSBuf); - return NULL; -} - -static STSGroupBlockInfoEx* tsBufGetLastGroupInfo(STSBuf* pTSBuf) { - int32_t last = pTSBuf->numOfGroups - 1; - - ASSERT(last >= 0); - return &pTSBuf->pData[last]; -} - -static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { - if (pTSBuf->numOfAlloc <= pTSBuf->numOfGroups) { - uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); - ASSERT((int32_t)newSize > pTSBuf->numOfAlloc); - - STSGroupBlockInfoEx* tmp = - (STSGroupBlockInfoEx*)taosMemoryRealloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); - if (tmp == NULL) { - return NULL; - } - - pTSBuf->pData = tmp; - pTSBuf->numOfAlloc = newSize; - memset(&pTSBuf->pData[pTSBuf->numOfGroups], 0, sizeof(STSGroupBlockInfoEx) * (newSize - pTSBuf->numOfGroups)); - } - - if (pTSBuf->numOfGroups > 0) { - STSGroupBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - - // update prev vnode length info in file - TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pPrevBlockInfoEx->info); - } - - // set initial value for vnode block - STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfGroups].info; - pBlockInfo->id = id; - pBlockInfo->offset = pTSBuf->fileSize; - ASSERT(pBlockInfo->offset >= getDataStartOffset()); - - // update vnode info in file - TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups, pBlockInfo); - - // add one vnode info - pTSBuf->numOfGroups += 1; - - // update the header info - STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - - STSBufUpdateHeader(pTSBuf, &header); - return tsBufGetLastGroupInfo(pTSBuf); -} - -static void shrinkBuffer(STSList* ptsData) { - // shrink tmp buffer size if it consumes too many memory compared to the pre-defined size - if (ptsData->allocSize >= ptsData->threshold * 2) { - char* rawBuf = taosMemoryRealloc(ptsData->rawBuf, MEM_BUF_SIZE); - if (rawBuf) { - ptsData->rawBuf = rawBuf; - ptsData->allocSize = MEM_BUF_SIZE; - } - } -} - -static int32_t getTagAreaLength(SVariant* pa) { - int32_t t = sizeof(pa->nLen) * 2 + sizeof(pa->nType); - if (pa->nType != TSDB_DATA_TYPE_NULL) { - t += pa->nLen; - } - - return t; -} - -static void writeDataToDisk(STSBuf* pTSBuf) { - if (pTSBuf->tsData.len == 0) { - return; - } - - STSBlock* pBlock = &pTSBuf->block; - STSList* pTsData = &pTSBuf->tsData; - - pBlock->numOfElem = pTsData->len / TSDB_KEYSIZE; - pBlock->compLen = tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len / TSDB_KEYSIZE, pBlock->payload, - pTsData->allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - - int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET); - ASSERT(r == 0); - - /* - * format for output data: - * 1. tags, number of ts, size after compressed, payload, size after compressed - * 2. tags, number of ts, size after compressed, payload, size after compressed - * - * both side has the compressed length is used to support load data forwards/backwords. - */ - int32_t metaLen = 0; - metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); - - int32_t trueLen = pBlock->tag.nLen; - if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_VARBINARY || - pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR || pBlock->tag.nType == TSDB_DATA_TYPE_GEOMETRY) { - metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); - metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); - } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { - metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); - float tfloat = (float)pBlock->tag.d; - metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t)pBlock->tag.nLen); - } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); - metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t)pBlock->tag.nLen); - } else { - trueLen = 0; - metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); - } - - taosWriteFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem)); - taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); - taosWriteFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); - taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); - - metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); - ASSERT(metaLen == getTagAreaLength(&pBlock->tag)); - - int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; - pTSBuf->fileSize += blockSize; - - pTSBuf->tsData.len = 0; - - STSGroupBlockInfoEx* pGroupBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - - pGroupBlockInfoEx->info.compLen += blockSize; - pGroupBlockInfoEx->info.numOfBlocks += 1; - - shrinkBuffer(&pTSBuf->tsData); -} - -static void expandBuffer(STSList* ptsData, int32_t inputSize) { - if (ptsData->allocSize - ptsData->len < inputSize) { - int32_t newSize = inputSize + ptsData->len; - char* tmp = taosMemoryRealloc(ptsData->rawBuf, (size_t)newSize); - if (tmp == NULL) { - // todo - } - - ptsData->rawBuf = tmp; - ptsData->allocSize = newSize; - } -} - -STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { - STSBlock* pBlock = &pTSBuf->block; - - // clear the memory buffer - pBlock->compLen = 0; - pBlock->padding = 0; - pBlock->numOfElem = 0; - - int32_t offset = -1; - - if (order == TSDB_ORDER_DESC) { - /* - * set the right position for the reversed traverse, the reversed traverse is started from - * the end of each comp data block - */ - int32_t prev = -(int32_t)(sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); - int32_t ret = taosLSeekFile(pTSBuf->pFile, prev, SEEK_CUR); - size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); - sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); - UNUSED(sz); - - pBlock->compLen = pBlock->padding; - - offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - ret = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); - UNUSED(ret); - } - - int32_t ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); - ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); - - // NOTE: mix types tags are not supported - size_t sz = 0; - if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_VARBINARY || - pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR || pBlock->tag.nType == TSDB_DATA_TYPE_GEOMETRY) { - char* tp = taosMemoryRealloc(pBlock->tag.pz, pBlock->tag.nLen + 1); - ASSERT(tp != NULL); - - memset(tp, 0, pBlock->tag.nLen + 1); - pBlock->tag.pz = tp; - - sz = taosReadFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); - UNUSED(sz); - } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { - float tfloat = 0; - sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t)pBlock->tag.nLen); - pBlock->tag.d = (double)tfloat; - UNUSED(sz); - } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { // TODO check the return value - sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t)pBlock->tag.nLen); - UNUSED(sz); - } - - sz = taosReadFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem)); - UNUSED(sz); - sz = taosReadFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); - UNUSED(sz); - sz = taosReadFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); - - if (decomp) { - pTSBuf->tsData.len = - tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, - pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - } - - // read the comp length at the length of comp block - sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); - ASSERT(pBlock->padding == pBlock->compLen); - - int32_t n = 0; - sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen)); - if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { - ASSERT(n == 0); - } else { - ASSERT(n == pBlock->tag.nLen); - } - - UNUSED(sz); - - // for backwards traverse, set the start position at the end of previous block - if (order == TSDB_ORDER_DESC) { - int32_t r = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); - UNUSED(r); - } - - return pBlock; -} - -// set the order of ts buffer if the ts order has not been set yet -static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { - STSList* ptsData = &pTSBuf->tsData; - - if (pTSBuf->tsOrder == -1) { - if (ptsData->len > 0) { - TSKEY lastKey = *(TSKEY*)(ptsData->rawBuf + ptsData->len - TSDB_KEYSIZE); - - if (lastKey > *(TSKEY*)pData) { - pTSBuf->tsOrder = TSDB_ORDER_DESC; - } else { - pTSBuf->tsOrder = TSDB_ORDER_ASC; - } - } else if (len > TSDB_KEYSIZE) { - // no data in current vnode, more than one ts is added, check the orders - TSKEY k1 = *(TSKEY*)(pData); - TSKEY k2 = *(TSKEY*)(pData + TSDB_KEYSIZE); - - if (k1 < k2) { - pTSBuf->tsOrder = TSDB_ORDER_ASC; - } else if (k1 > k2) { - pTSBuf->tsOrder = TSDB_ORDER_DESC; - } else { - // todo handle error - } - } - } else { - // todo the timestamp order is set, check the asc/desc order of appended data - } - - return TSDB_CODE_SUCCESS; -} - -void tsBufAppend(STSBuf* pTSBuf, int32_t id, SVariant* tag, const char* pData, int32_t len) { - STSGroupBlockInfoEx* pBlockInfo = NULL; - STSList* ptsData = &pTSBuf->tsData; - - if (pTSBuf->numOfGroups == 0 || tsBufGetLastGroupInfo(pTSBuf)->info.id != id) { - writeDataToDisk(pTSBuf); - shrinkBuffer(ptsData); - - pBlockInfo = addOneGroupInfo(pTSBuf, id); - } else { - pBlockInfo = tsBufGetLastGroupInfo(pTSBuf); - } - - ASSERT(pBlockInfo->info.id == id); - - if ((taosVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { - // new arrived data with different tags value, save current value into disk first - writeDataToDisk(pTSBuf); - } else { - expandBuffer(ptsData, len); - } - - taosVariantAssign(&pTSBuf->block.tag, tag); - memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); - - // todo check return value - setCheckTSOrder(pTSBuf, pData, len); - - ptsData->len += len; - pBlockInfo->len += len; - - pTSBuf->numOfTotal += len / TSDB_KEYSIZE; - - // the size of raw data exceeds the size of the default prepared buffer, so - // during getBufBlock, the output buffer needs to be large enough. - if (ptsData->len >= ptsData->threshold) { - writeDataToDisk(pTSBuf); - shrinkBuffer(ptsData); - } - - tsBufResetPos(pTSBuf); -} - -void tsBufFlush(STSBuf* pTSBuf) { - if (pTSBuf->tsData.len <= 0) { - return; - } - - writeDataToDisk(pTSBuf); - shrinkBuffer(&pTSBuf->tsData); - - STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - - // update prev vnode length info in file - TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pBlockInfoEx->info); - - // save the ts order into header - STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - STSBufUpdateHeader(pTSBuf, &header); -} - -static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t numOfGroups, int32_t id) { - int32_t j = -1; - for (int32_t i = 0; i < numOfGroups; ++i) { - if (pGroupInfoEx[i].info.id == id) { - j = i; - break; - } - } - - return j; -} - -// todo opt performance by cache blocks info -static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) { - if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { - return -1; - } - - // sequentially read the compressed data blocks, start from the beginning of the comp data block of this vnode - int32_t i = 0; - bool decomp = false; - - while ((i++) <= blockIndex) { - if (readDataFromDisk(pTSBuf, TSDB_ORDER_ASC, decomp) == NULL) { - return -1; - } - } - - // set the file position to be the end of previous comp block - if (pTSBuf->cur.order == TSDB_ORDER_DESC) { - STSBlock* pBlock = &pTSBuf->block; - int32_t compBlockSize = - pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - int32_t ret = taosLSeekFile(pTSBuf->pFile, -compBlockSize, SEEK_CUR); - UNUSED(ret); - } - - return 0; -} - -static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, SVariant* tag) { - bool decomp = false; - - int64_t offset = 0; - if (pTSBuf->cur.order == TSDB_ORDER_ASC) { - offset = pBlockInfo->offset; - } else { // reversed traverse starts from the end of block - offset = pBlockInfo->offset + pBlockInfo->compLen; - } - - if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { - return -1; - } - - for (int32_t i = 0; i < pBlockInfo->numOfBlocks; ++i) { - if (readDataFromDisk(pTSBuf, pTSBuf->cur.order, decomp) == NULL) { - return -1; - } - - if (taosVariantCompare(&pTSBuf->block.tag, tag) == 0) { - return (pTSBuf->cur.order == TSDB_ORDER_ASC) ? i : (pBlockInfo->numOfBlocks - (i + 1)); - } - } - - return -1; -} - -static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex) { - STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info; - if (pBlockInfo->numOfBlocks <= blockIndex) { - ASSERT(false); - } - - STSCursor* pCur = &pTSBuf->cur; - if (pCur->vgroupIndex == groupIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || - (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) { - int32_t i = 0; - bool decomp = false; - int32_t step = abs(blockIndex - pCur->blockIndex); - - while ((++i) <= step) { - if (readDataFromDisk(pTSBuf, pCur->order, decomp) == NULL) { - return; - } - } - } else { - if (tsBufFindBlock(pTSBuf, pBlockInfo, blockIndex) == -1) { - ASSERT(false); - } - } - - STSBlock* pBlock = &pTSBuf->block; - - size_t s = pBlock->numOfElem * TSDB_KEYSIZE; - - /* - * In order to accommodate all the qualified data, the actual buffer size for one block with identical tags value - * may exceed the maximum allowed size during *tsBufAppend* function by invoking expandBuffer function - */ - if (s > pTSBuf->tsData.allocSize) { - expandBuffer(&pTSBuf->tsData, (int32_t)s); - } - - pTSBuf->tsData.len = - tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, - pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - - ASSERT((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len)); - - pCur->vgroupIndex = groupIndex; - pCur->blockIndex = blockIndex; - - pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1; -} - -static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockInfo* pVInfo) { - if (offset < 0 || offset >= getDataStartOffset()) { - return -1; - } - - if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { - return -1; - } - - taosWriteFile(pTSBuf->pFile, pVInfo, sizeof(STSGroupBlockInfo)); - return 0; -} - -STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) { - int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id); - if (j == -1) { - return NULL; - } - - return &pTSBuf->pData[j].info; -} - -int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { - if ((pTSBuf->pFile == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { - return -1; - } - - if (pHeader->tsOrder != TSDB_ORDER_ASC && pHeader->tsOrder != TSDB_ORDER_DESC) { - return -1; - } - - int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); - if (r != 0) { - // qError("fseek failed, errno:%d", errno); - return -1; - } - - size_t ws = taosWriteFile(pTSBuf->pFile, pHeader, sizeof(STSBufFileHeader)); - if (ws != 1) { - // qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, - // (int32_t)sizeof(STSBufFileHeader)); - return -1; - } - return 0; -} - -bool tsBufNextPos(STSBuf* pTSBuf) { - if (pTSBuf == NULL || pTSBuf->numOfGroups == 0) { - return false; - } - - STSCursor* pCur = &pTSBuf->cur; - - // get the first/last position according to traverse order - if (pCur->vgroupIndex == -1) { - if (pCur->order == TSDB_ORDER_ASC) { - tsBufGetBlock(pTSBuf, 0, 0); - - if (pTSBuf->block.numOfElem == 0) { // the whole list is empty, return - tsBufResetPos(pTSBuf); - return false; - } else { - return true; - } - - } else { // get the last timestamp record in the last block of the last vnode - ASSERT(pTSBuf->numOfGroups > 0); - - int32_t groupIndex = pTSBuf->numOfGroups - 1; - pCur->vgroupIndex = groupIndex; - - int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; - STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); - int32_t blockIndex = pBlockInfo->numOfBlocks - 1; - - tsBufGetBlock(pTSBuf, groupIndex, blockIndex); - - pCur->tsIndex = pTSBuf->block.numOfElem - 1; - if (pTSBuf->block.numOfElem == 0) { - tsBufResetPos(pTSBuf); - return false; - } else { - return true; - } - } - } - - int32_t step = pCur->order == TSDB_ORDER_ASC ? 1 : -1; - - while (1) { - ASSERT(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); - - if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || - (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { - int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; - - STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); - if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) || - (pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { - if ((pCur->vgroupIndex >= pTSBuf->numOfGroups - 1 && pCur->order == TSDB_ORDER_ASC) || - (pCur->vgroupIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { - pCur->vgroupIndex = -1; - return false; - } - - if (pBlockInfo == NULL) { - return false; - } - - int32_t blockIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : (pBlockInfo->numOfBlocks - 1); - tsBufGetBlock(pTSBuf, pCur->vgroupIndex + step, blockIndex); - break; - - } else { - tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex + step); - break; - } - } else { - pCur->tsIndex += step; - break; - } - } - - return true; -} - -void tsBufResetPos(STSBuf* pTSBuf) { - if (pTSBuf == NULL) { - return; - } - - pTSBuf->cur = (STSCursor){.tsIndex = -1, .blockIndex = -1, .vgroupIndex = -1, .order = pTSBuf->cur.order}; -} - -STSElem tsBufGetElem(STSBuf* pTSBuf) { - STSElem elem1 = {.id = -1}; - if (pTSBuf == NULL) { - return elem1; - } - - STSCursor* pCur = &pTSBuf->cur; - if (pCur != NULL && pCur->vgroupIndex < 0) { - return elem1; - } - - STSBlock* pBlock = &pTSBuf->block; - - elem1.id = pTSBuf->pData[pCur->vgroupIndex].info.id; - elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); - elem1.tag = &pBlock->tag; - - return elem1; -} - -/** - * current only support ts comp data from two vnode merge - * @param pDestBuf - * @param pSrcBuf - * @param id - * @return - */ -int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { - if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfGroups <= 0) { - return 0; - } - - if (pDestBuf->numOfGroups + pSrcBuf->numOfGroups > TS_COMP_FILE_GROUP_MAX) { - return -1; - } - - // src can only have one vnode index - ASSERT(pSrcBuf->numOfGroups == 1); - - // there are data in buffer, flush to disk first - tsBufFlush(pDestBuf); - - // compared with the last vnode id - int32_t id = tsBufGetLastGroupInfo((STSBuf*)pSrcBuf)->info.id; - if (id != tsBufGetLastGroupInfo(pDestBuf)->info.id) { - int32_t oldSize = pDestBuf->numOfGroups; - int32_t newSize = oldSize + pSrcBuf->numOfGroups; - - if (pDestBuf->numOfAlloc < newSize) { - pDestBuf->numOfAlloc = newSize; - - STSGroupBlockInfoEx* tmp = taosMemoryRealloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); - if (tmp == NULL) { - return -1; - } - - pDestBuf->pData = tmp; - } - - // directly copy the vnode index information - memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfGroups * sizeof(STSGroupBlockInfoEx)); - - // set the new offset value - for (int32_t i = 0; i < pSrcBuf->numOfGroups; ++i) { - STSGroupBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize]; - pBlockInfoEx->info.offset = (pSrcBuf->pData[i].info.offset - getDataStartOffset()) + pDestBuf->fileSize; - pBlockInfoEx->info.id = id; - } - - pDestBuf->numOfGroups = newSize; - } else { - STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pDestBuf); - - pBlockInfoEx->len += pSrcBuf->pData[0].len; - pBlockInfoEx->info.numOfBlocks += pSrcBuf->pData[0].info.numOfBlocks; - pBlockInfoEx->info.compLen += pSrcBuf->pData[0].info.compLen; - pBlockInfoEx->info.id = id; - } - - int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END); - ASSERT(r == 0); - - int64_t offset = getDataStartOffset(); - int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; - int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size); - - if (written == -1 || written != size) { - return -1; - } - - pDestBuf->numOfTotal += pSrcBuf->numOfTotal; - - int32_t oldSize = pDestBuf->fileSize; - - // file meta data may be cached, close and reopen the file for accurate file size. - taosCloseFile(&pDestBuf->pFile); - // pDestBuf->pFile = fopen(pDestBuf->path, "rb+"); - pDestBuf->pFile = taosOpenFile(pDestBuf->path, TD_FILE_WRITE | TD_FILE_READ); - if (pDestBuf->pFile == NULL) { - return -1; - } - - int64_t file_size; - if (taosFStatFile(pDestBuf->pFile, &file_size, NULL) != 0) { - return -1; - } - pDestBuf->fileSize = (uint32_t)file_size; - - ASSERT(pDestBuf->fileSize == oldSize + size); - - return 0; -} - -STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t id) { - STSBuf* pTSBuf = tsBufCreate(true, order); - - STSGroupBlockInfo* pBlockInfo = &(addOneGroupInfo(pTSBuf, 0)->info); - pBlockInfo->numOfBlocks = numOfBlocks; - pBlockInfo->compLen = len; - pBlockInfo->offset = getDataStartOffset(); - pBlockInfo->id = id; - - // update prev vnode length info in file - TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); - - int32_t ret = taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET); - if (ret == -1) { - // qError("fseek failed, errno:%d", errno); - tsBufDestroy(pTSBuf); - return NULL; - } - size_t sz = taosWriteFile(pTSBuf->pFile, (void*)pData, len); - if (sz != len) { - // qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); - tsBufDestroy(pTSBuf); - return NULL; - } - pTSBuf->fileSize += len; - - pTSBuf->tsOrder = order; - if (order != TSDB_ORDER_ASC && order != TSDB_ORDER_DESC) { - tsBufDestroy(pTSBuf); - return NULL; - } - - STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - if (STSBufUpdateHeader(pTSBuf, &header) < 0) { - tsBufDestroy(pTSBuf); - return NULL; - } - - // TODO taosFsync?? - // if (taosFsync(fileno(pTSBuf->pFile)) == -1) { - //// qError("fsync failed, errno:%d", errno); - // tsBufDestroy(pTSBuf); - // return NULL; - // } - - return pTSBuf; -} - -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, SVariant* tag) { - STSElem elem = {.id = -1}; - - if (pTSBuf == NULL) { - return elem; - } - - int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id); - if (j == -1) { - return elem; - } - - // for debug purpose - // tsBufDisplay(pTSBuf); - - STSCursor* pCur = &pTSBuf->cur; - STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[j].info; - - int32_t blockIndex = tsBufFindBlockByTag(pTSBuf, pBlockInfo, tag); - if (blockIndex < 0) { - return elem; - } - - pCur->vgroupIndex = j; - pCur->blockIndex = blockIndex; - tsBufGetBlock(pTSBuf, j, blockIndex); - - return tsBufGetElem(pTSBuf); -} - -STSCursor tsBufGetCursor(STSBuf* pTSBuf) { - STSCursor c = {.vgroupIndex = -1}; - if (pTSBuf == NULL) { - return c; - } - - return pTSBuf->cur; -} - -void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur) { - if (pTSBuf == NULL || pCur == NULL) { - return; - } - - if (pCur->vgroupIndex != -1) { - tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex); - } - - pTSBuf->cur = *pCur; -} - -void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order) { - if (pTSBuf == NULL) { - return; - } - - pTSBuf->cur.order = order; -} - -STSBuf* tsBufClone(STSBuf* pTSBuf) { - if (pTSBuf == NULL) { - return NULL; - } - - tsBufFlush(pTSBuf); - - return tsBufCreateFromFile(pTSBuf->path, false); -} - -void tsBufDisplay(STSBuf* pTSBuf) { - printf("-------start of ts comp file-------\n"); - printf("number of vnode:%d\n", pTSBuf->numOfGroups); - - int32_t old = pTSBuf->cur.order; - pTSBuf->cur.order = TSDB_ORDER_ASC; - - tsBufResetPos(pTSBuf); - - while (tsBufNextPos(pTSBuf)) { - STSElem elem = tsBufGetElem(pTSBuf); - if (elem.tag->nType == TSDB_DATA_TYPE_BIGINT) { - printf("%d-%" PRId64 "-%" PRId64 "\n", elem.id, elem.tag->i, elem.ts); - } - } - - pTSBuf->cur.order = old; - printf("-------end of ts comp file-------\n"); -} - -static int32_t getDataStartOffset() { - return sizeof(STSBufFileHeader) + TS_COMP_FILE_GROUP_MAX * sizeof(STSGroupBlockInfo); -} - -// update prev vnode length info in file -static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo) { - int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSGroupBlockInfo); - doUpdateGroupInfo(pTSBuf, offset, pBlockInfo); -} - -static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { - const int32_t INITIAL_GROUPINFO_SIZE = 4; - - pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE; - pTSBuf->pData = taosMemoryCalloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); - if (pTSBuf->pData == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->tsData.rawBuf = taosMemoryMalloc(MEM_BUF_SIZE); - if (pTSBuf->tsData.rawBuf == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->bufSize = MEM_BUF_SIZE; - pTSBuf->tsData.threshold = MEM_BUF_SIZE; - pTSBuf->tsData.allocSize = MEM_BUF_SIZE; - - pTSBuf->assistBuf = taosMemoryMalloc(MEM_BUF_SIZE); - if (pTSBuf->assistBuf == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->block.payload = taosMemoryMalloc(MEM_BUF_SIZE); - if (pTSBuf->block.payload == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->fileSize += getDataStartOffset(); - return pTSBuf; -} - -int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf) { - if (pTSBuf == NULL) { - return 0; - } - - return pTSBuf->numOfGroups; -} - -void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) { - int32_t size = tsBufGetNumOfGroup(pTSBuf); - if (num != NULL) { - *num = size; - } - - *id = NULL; - if (size == 0) { - return; - } - - (*id) = taosMemoryMalloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); - - for (int32_t i = 0; i < size; ++i) { - (*id)[i] = pTSBuf->pData[i].info.id; - } -} - -int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) { - ASSERT(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups); - STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info; - - *len = 0; - *numOfBlocks = 0; - - if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { - int32_t code = TAOS_SYSTEM_ERROR(taosGetErrorFile(pTSBuf->pFile)); - // qError("%p: fseek failed: %s", pSql, tstrerror(code)); - return code; - } - - size_t s = taosReadFile(pTSBuf->pFile, buf, pBlockInfo->compLen); - if (s != pBlockInfo->compLen) { - int32_t code = TAOS_SYSTEM_ERROR(taosGetErrorFile(pTSBuf->pFile)); - // tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); - return code; - } - - *len = pBlockInfo->compLen; - *numOfBlocks = pBlockInfo->numOfBlocks; - - return TSDB_CODE_SUCCESS; -} - -STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, SVariant* pTag) { - STSElem el = {.id = -1}; - - for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) { - el = tsBufGetElemStartPos(pTSBuf, pTSBuf->pData[i].info.id, pTag); - if (el.id == pTSBuf->pData[i].info.id) { - return el; - } - } - - return el; -} - -bool tsBufIsValidElem(STSElem* pElem) { return pElem->id >= 0; } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 7b27b41177..fbb61c7c90 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -30,11 +30,11 @@ longjmp((_obj), (_c)); \ } while (0) -#define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ - do { \ - assert(sizeof(_uid) == sizeof(uint64_t)); \ - *(uint64_t*)(_k) = (_uid); \ - memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \ +#define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ + do { \ + assert(sizeof(_uid) == sizeof(uint64_t)); \ + *(uint64_t*)(_k) = (_uid); \ + (void)memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \ } while (0) #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) @@ -175,7 +175,7 @@ int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod int32_t type, SColMatchInfo* pMatchInfo); int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId); -int32_t createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); +int32_t createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 31fdc5b291..eb36141ef0 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -24,7 +24,6 @@ extern "C" { #include "theap.h" #include "tlosertree.h" #include "tsort.h" -#include "ttszip.h" #include "tvariant.h" #include "dataSinkMgt.h" @@ -904,7 +903,7 @@ int32_t appendDataToSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pE uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); void finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, - SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); + SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); bool groupbyTbname(SNodeList* pGroupList); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); diff --git a/source/libs/function/inc/tpercentile.h b/source/libs/function/inc/tpercentile.h index 90fb279259..118571c8aa 100644 --- a/source/libs/function/inc/tpercentile.h +++ b/source/libs/function/inc/tpercentile.h @@ -21,17 +21,16 @@ extern "C" { #endif #include "tpagedbuf.h" -#include "ttszip.h" typedef struct MinMaxEntry { union { - double dMinVal; - //double i64MinVal; + double dMinVal; + // double i64MinVal; uint64_t u64MinVal; }; union { - double dMaxVal; - //double i64MaxVal; + double dMaxVal; + // double i64MaxVal; int64_t u64MaxVal; }; } MinMaxEntry; diff --git a/source/libs/function/src/tfunctionInt.c b/source/libs/function/src/tfunctionInt.c index 80b869b2d2..6a841f6ffa 100644 --- a/source/libs/function/src/tfunctionInt.c +++ b/source/libs/function/src/tfunctionInt.c @@ -25,5 +25,4 @@ #include "tfunctionInt.h" #include "thistogram.h" #include "tpercentile.h" -#include "ttszip.h" #include "tudf.h" diff --git a/source/libs/index/inc/indexUtil.h b/source/libs/index/inc/indexUtil.h index 148a521d5a..308e213ac9 100644 --- a/source/libs/index/inc/indexUtil.h +++ b/source/libs/index/inc/indexUtil.h @@ -21,29 +21,29 @@ extern "C" { #endif -#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ - do { \ - memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ - buf += sizeof(key->mem); \ +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + (void)memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ } while (0) -#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ - do { \ - memcpy((void *)buf, (void *)key->mem, len); \ - buf += len; \ +#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ + do { \ + (void)memcpy((void *)buf, (void *)key->mem, len); \ + buf += len; \ } while (0) -#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ - do { \ - type c = var; \ - memcpy((void *)buf, (void *)&c, sizeof(c)); \ - buf += sizeof(c); \ +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + (void)memcpy((void *)buf, (void *)&c, sizeof(c)); \ + buf += sizeof(c); \ } while (0) -#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ - do { \ - memcpy((void *)buf, (void *)var, len); \ - buf += len; \ +#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ + do { \ + (void)memcpy((void *)buf, (void *)var, len); \ + buf += len; \ } while (0) #define INDEX_MERGE_ADD_DEL(src, dst, tgt) \ diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index a645c43f86..ab04f06b02 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -103,7 +103,7 @@ typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char *, void **); typedef struct SFilterDataInfo { int32_t idx; - void* addr; + void *addr; } SFilterDataInfo; typedef struct SFilterRangeCompare { @@ -227,10 +227,9 @@ typedef struct SFltTreeStat { SFilterInfo *info; } SFltTreeStat; - typedef struct SFltScalarCtx { - SNode *node; - SArray* fltSclRange; + SNode *node; + SArray *fltSclRange; } SFltScalarCtx; typedef struct SFltBuildGroupCtx { @@ -271,9 +270,9 @@ struct SFilterInfo { SFilterPCtx pctx; }; -#define FILTER_NO_MERGE_DATA_TYPE(t) \ - ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_VARBINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON || \ - (t) == TSDB_DATA_TYPE_GEOMETRY) +#define FILTER_NO_MERGE_DATA_TYPE(t) \ + ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_VARBINARY || (t) == TSDB_DATA_TYPE_NCHAR || \ + (t) == TSDB_DATA_TYPE_JSON || (t) == TSDB_DATA_TYPE_GEOMETRY) #define FILTER_NO_MERGE_OPTR(o) ((o) == OP_TYPE_IS_NULL || (o) == OP_TYPE_IS_NOT_NULL || (o) == FILTER_DUMMY_EMPTY_OPTR) #define MR_EMPTY_RES(ctx) (ctx->rs == NULL) @@ -361,29 +360,29 @@ struct SFilterInfo { _r = n; \ } \ } while (0) -#define INSERT_RANGE(ctx, r, ra) \ - do { \ - SFilterRangeNode *n = NULL; \ - FLT_ERR_RET(filterNewRange(ctx, ra, &n)); \ - n->prev = (r)->prev; \ - if ((r)->prev) { \ - (r)->prev->next = n; \ - } else { \ - (ctx)->rs = n; \ - } \ - (r)->prev = n; \ - n->next = r; \ +#define INSERT_RANGE(ctx, r, ra) \ + do { \ + SFilterRangeNode *n = NULL; \ + FLT_ERR_RET(filterNewRange(ctx, ra, &n)); \ + n->prev = (r)->prev; \ + if ((r)->prev) { \ + (r)->prev->next = n; \ + } else { \ + (ctx)->rs = n; \ + } \ + (r)->prev = n; \ + n->next = r; \ } while (0) -#define APPEND_RANGE(ctx, r, ra) \ - do { \ - SFilterRangeNode *n = NULL; \ - FLT_ERR_RET(filterNewRange(ctx, ra, &n)); \ - n->prev = (r); \ - if (r) { \ - (r)->next = n; \ - } else { \ - (ctx)->rs = n; \ - } \ +#define APPEND_RANGE(ctx, r, ra) \ + do { \ + SFilterRangeNode *n = NULL; \ + FLT_ERR_RET(filterNewRange(ctx, ra, &n)); \ + n->prev = (r); \ + if (r) { \ + (r)->next = n; \ + } else { \ + (ctx)->rs = n; \ + } \ } while (0) #define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_UNKNOWN) @@ -454,7 +453,7 @@ struct SFilterInfo { #define FILTER_UNIT_OPTR(u) ((u)->compare.optr) #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) -#define FILTER_UNIT_CLR_F(i) memset((i)->unitFlags, 0, (i)->unitNum * sizeof(*info->unitFlags)) +#define FILTER_UNIT_CLR_F(i) (void)memset((i)->unitFlags, 0, (i)->unitNum * sizeof(*info->unitFlags)) #define FILTER_UNIT_SET_F(i, idx) (i)->unitFlags[idx] = 1 #define FILTER_UNIT_GET_F(i, idx) ((i)->unitFlags[idx]) #define FILTER_UNIT_GET_R(i, idx) ((i)->unitRes[idx]) @@ -482,7 +481,7 @@ struct SFilterInfo { #define FILTER_COPY_IDX(dst, src, n) \ do { \ *(dst) = taosMemoryMalloc(sizeof(uint32_t) * n); \ - memcpy(*(dst), src, sizeof(uint32_t) * n); \ + (void)memcpy(*(dst), src, sizeof(uint32_t) * n); \ } while (0) #define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) \ diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index ca08231d97..47a7b6fe7a 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -31,7 +31,7 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i pPtr = (uint8_t *)tdbOsCalloc(1, zsize); if (pPtr == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pDb = (TDB *)pPtr; @@ -64,7 +64,7 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i tsize = sizeof(SPager *) * pDb->nPgrHash; pDb->pgrHash = tdbOsMalloc(tsize); if (pDb->pgrHash == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } memset(pDb->pgrHash, 0, tsize); @@ -125,7 +125,7 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void ( TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn)); if (!pTxn) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } ret = tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags); diff --git a/utils/TSZ/zstd/common/bitstream.h b/utils/TSZ/zstd/common/bitstream.h index 2f91460c5e..6a154ceb57 100644 --- a/utils/TSZ/zstd/common/bitstream.h +++ b/utils/TSZ/zstd/common/bitstream.h @@ -286,7 +286,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) */ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) { - if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + if (srcSize < 1) { (void)memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } bitD->start = (const char*)srcBuffer; bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); From 7ee18c51784ae92d888ba9f32a7b4a28225bf69f Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 25 Jul 2024 09:31:35 +0800 Subject: [PATCH 46/52] fix: refactor return code --- source/dnode/vnode/src/sma/smaRollup.c | 1 + source/dnode/vnode/src/tsdb/tsdbFS2.c | 2 +- source/libs/tfs/src/tfs.c | 11 +++++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 15c007d393..8c12f80f2b 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -931,6 +931,7 @@ static int32_t tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid, SRSmaInfo **ppR if (ASSERTS(pRSmaInfo->suid == suid, "suid:%" PRIi64 " != %" PRIi64, pRSmaInfo->suid, suid)) { TAOS_RETURN(TSDB_CODE_APP_ERROR); } + *ppRSmaInfo = pRSmaInfo; TAOS_RETURN(code); } taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index ca99b0eaff..68954e5156 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -524,7 +524,7 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { { // clear unreferenced files STfsDir *dir = NULL; - TAOS_CHECK_GOTO(tfsOpendir(fs->tsdb->pVnode->pTfs, fs->tsdb->path, &dir), &lino, _close_dir); + TAOS_CHECK_GOTO(tfsOpendir(fs->tsdb->pVnode->pTfs, fs->tsdb->path, &dir), &lino, _exit); STFileHash fobjHash = {0}; code = tsdbFSCreateFileObjHash(fs, &fobjHash); diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 07c1fab54f..7656e3067a 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -440,7 +440,10 @@ int32_t tfsOpendir(STfs *pTfs, const char *rname, STfsDir **ppDir) { TAOS_CHECK_GOTO(tfsOpendirImpl(pTfs, pDir), NULL, _exit); _exit: - taosMemoryFree(pDir); + if (code != 0) { + taosMemoryFreeClear(pDir); + } + *ppDir = pDir; TAOS_RETURN(code); } @@ -634,9 +637,13 @@ static int32_t tfsGetDiskByName(STfs *pTfs, const char *dir, STfsDisk **ppDisk) static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pTfsDir) { STfsDisk *pDisk = NULL; char adir[TMPNAME_LEN * 2] = "\0"; + int32_t code = 0; if (pTfsDir->pDir != NULL) { - taosCloseDir(&pTfsDir->pDir); + if ((code = taosCloseDir(&pTfsDir->pDir)) != 0) { + fError("%s failed to close dir since %s", __func__, tstrerror(code)); + TAOS_RETURN(code); + } pTfsDir->pDir = NULL; } From 12c44fb8171540307c6fbb6f3c0741583013cc40 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 25 Jul 2024 10:21:06 +0800 Subject: [PATCH 47/52] fix: refactor return code --- source/dnode/mnode/impl/src/mndUser.c | 71 ++++++++++----------------- source/libs/tfs/src/tfs.c | 2 +- 2 files changed, 28 insertions(+), 45 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5a19b6c762..9f37885d6d 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -357,9 +357,9 @@ int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8 memcpy(pNewList->pIpRange, &range, sizeof(SIpV4Range)); pNewList->num = 1; - if (taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *)) != 0) { + if ((code = taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *))) != 0) { taosMemoryFree(pNewList); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(code, &lino, _OVER); } update = true; } else { @@ -375,9 +375,9 @@ int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8 pNewList->num = pList->num + 1; - if (taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *)) != 0) { + if ((code = taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *))) != 0) { taosMemoryFree(pNewList); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(code, &lino, _OVER); } taosMemoryFree(pList); update = true; @@ -405,9 +405,9 @@ int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8 } } pNewList->num = idx; - if (taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *) != 0)) { + if ((code = taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *)) != 0)) { taosMemoryFree(pNewList); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(code, &lino, _OVER); } taosMemoryFree(pList); } @@ -573,11 +573,11 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { sdbCancelFetch(pSdb, pIter); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } - if (taosHashPut(pIpWhiteTab, pUser->user, strlen(pUser->user), &pWhiteList, sizeof(void *)) != 0) { + if ((code = taosHashPut(pIpWhiteTab, pUser->user, strlen(pUser->user), &pWhiteList, sizeof(void *))) != 0) { taosMemoryFree(pWhiteList); sdbRelease(pSdb, pUser); sdbCancelFetch(pSdb, pIter); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(code, &lino, _OVER); } char *name = taosStrdup(pUser->user); @@ -1323,9 +1323,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) - if (taosHashPut(pUser->readTbs, key, keyLen, value, valuelen) != 0) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); - } + TAOS_CHECK_GOTO(taosHashPut(pUser->readTbs, key, keyLen, value, valuelen), &lino, _OVER); } for (int32_t i = 0; i < numOfWriteTbs; ++i) { @@ -1348,9 +1346,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) - if (taosHashPut(pUser->writeTbs, key, keyLen, value, valuelen) != 0) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); - } + TAOS_CHECK_GOTO(taosHashPut(pUser->writeTbs, key, keyLen, value, valuelen), &lino, _OVER); } if (sver >= 6) { @@ -1374,9 +1370,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) - if (taosHashPut(pUser->alterTbs, key, keyLen, value, valuelen) != 0) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); - } + TAOS_CHECK_GOTO(taosHashPut(pUser->alterTbs, key, keyLen, value, valuelen), &lino, _OVER); } for (int32_t i = 0; i < numOfReadViews; ++i) { @@ -1399,9 +1393,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) - if (taosHashPut(pUser->readViews, key, keyLen, value, valuelen) != 0) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); - } + TAOS_CHECK_GOTO(taosHashPut(pUser->readViews, key, keyLen, value, valuelen), &lino, _OVER); } for (int32_t i = 0; i < numOfWriteViews; ++i) { @@ -1424,9 +1416,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) - if (taosHashPut(pUser->writeViews, key, keyLen, value, valuelen) != 0) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); - } + TAOS_CHECK_GOTO(taosHashPut(pUser->writeViews, key, keyLen, value, valuelen), &lino, _OVER); } for (int32_t i = 0; i < numOfAlterViews; ++i) { @@ -1449,9 +1439,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) - if (taosHashPut(pUser->alterViews, key, keyLen, value, valuelen) != 0) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); - } + TAOS_CHECK_GOTO(taosHashPut(pUser->alterViews, key, keyLen, value, valuelen), &lino, _OVER); } } @@ -1469,9 +1457,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t ref = 0; SDB_GET_INT32(pRaw, dataPos, &ref, _OVER); - if (taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref)) != 0) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); - } + TAOS_CHECK_GOTO(taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref)), &lino, _OVER); } } // decoder white list @@ -2135,7 +2121,7 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname); if (pDb == NULL) { mndReleaseDb(pMnode, pDb); - TAOS_CHECK_GOTO(terrno, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); // TODO: refactor the terrno to code } if ((code = taosHashPut(pNewUser->readDbs, pAlterReq->objname, len, pAlterReq->objname, TSDB_DB_FNAME_LEN)) != 0) { @@ -2166,7 +2152,7 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname); if (pDb == NULL) { mndReleaseDb(pMnode, pDb); - TAOS_CHECK_GOTO(terrno, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); // TODO: refactor the terrno to code } if ((code = taosHashPut(pNewUser->writeDbs, pAlterReq->objname, len, pAlterReq->objname, TSDB_DB_FNAME_LEN)) != 0) { @@ -2197,7 +2183,7 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname); if (pDb == NULL) { mndReleaseDb(pMnode, pDb); - TAOS_CHECK_GOTO(terrno, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); // TODO: refactor the terrno to code } taosHashRemove(pNewUser->readDbs, pAlterReq->objname, len); mndReleaseDb(pMnode, pDb); @@ -2213,7 +2199,7 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname); if (pDb == NULL) { mndReleaseDb(pMnode, pDb); - TAOS_CHECK_GOTO(terrno, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); // TODO: refactor the terrno to code } taosHashRemove(pNewUser->writeDbs, pAlterReq->objname, len); mndReleaseDb(pMnode, pDb); @@ -2267,10 +2253,9 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode if (ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) { int32_t len = strlen(pAlterReq->objname) + 1; SMqTopicObj *pTopic = NULL; - int32_t code = mndAcquireTopic(pMnode, pAlterReq->objname, &pTopic); - if (code != 0) { + if ((code = mndAcquireTopic(pMnode, pAlterReq->objname, &pTopic)) != 0) { mndReleaseTopic(pMnode, pTopic); - TAOS_CHECK_GOTO(terrno, &lino, _OVER); + TAOS_CHECK_GOTO(code, &lino, _OVER); } if ((code = taosHashPut(pNewUser->topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN)) != 0) { mndReleaseTopic(pMnode, pTopic); @@ -2282,18 +2267,17 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode if (ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) { int32_t len = strlen(pAlterReq->objname) + 1; SMqTopicObj *pTopic = NULL; - int32_t code = mndAcquireTopic(pMnode, pAlterReq->objname, &pTopic); - if (code != 0) { + if ((code = mndAcquireTopic(pMnode, pAlterReq->objname, &pTopic)) != 0) { mndReleaseTopic(pMnode, pTopic); - TAOS_CHECK_GOTO(terrno, &lino, _OVER); + TAOS_CHECK_GOTO(code, &lino, _OVER); } - taosHashRemove(pNewUser->topics, pAlterReq->objname, len); + (void)taosHashRemove(pNewUser->topics, pAlterReq->objname, len); mndReleaseTopic(pMnode, pTopic); } _OVER: if (code < 0) { - mError("user:%s, failed to alter user privileges at line %d since %s", pAlterReq->user, lino, terrstr()); + mError("user:%s, failed to alter user privileges at line %d since %s", pAlterReq->user, lino, tstrerror(code)); } TAOS_RETURN(code); } @@ -2595,8 +2579,8 @@ static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq) { TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } - contLen =tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp); - if(contLen < 0) { + contLen = tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp); + if (contLen < 0) { TAOS_CHECK_EXIT(contLen); } @@ -3011,7 +2995,6 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); COL_DATA_SET_VAL_GOTO((const char *)condition, false, pUser, _exit); - char notes[2] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 7656e3067a..81c853e546 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -383,7 +383,7 @@ static int32_t tfsRenameAt(STfs *pTfs, SDiskID diskId, const char *orname, const snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname); if (taosRenameFile(oaname, naname) != 0 && errno != ENOENT) { - int32_t code = TAOS_SYSTEM_ERROR(errno); + int32_t code = TAOS_SYSTEM_ERROR(errno); // TODO: use return value of taosRenameFile directly fError("%s failed to rename %s to %s since %s", __func__, oaname, naname, tstrerror(code)); TAOS_RETURN(code); } From ea7eebc8a9be8d7bede8bc008f0fab1f99efab6b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 25 Jul 2024 10:32:56 +0800 Subject: [PATCH 48/52] check memory alloc return values --- source/dnode/vnode/src/tsdb/tsdbCache.c | 182 ++++++++++++++++++++---- 1 file changed, 157 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 459ad39819..b6c389842e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -63,7 +63,6 @@ static void tsdbCloseBCache(STsdb *pTsdb) { static int32_t tsdbOpenPgCache(STsdb *pTsdb) { int32_t code = 0, lino = 0; - // SLRUCache *pCache = taosLRUCacheInit(10 * 1024 * 1024, 0, .5); int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; SLRUCache *pCache = taosLRUCacheInit((int64_t)tsS3PageCacheSize * szPage, 0, .5); @@ -659,11 +658,23 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, int32_t code = 0; // build keys & multi get from rocks - char **keys_list = taosMemoryCalloc(2, sizeof(char *)); - size_t *keys_list_sizes = taosMemoryCalloc(2, sizeof(size_t)); + char **keys_list = taosMemoryCalloc(2, sizeof(char *)); + if (!keys_list) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + size_t *keys_list_sizes = taosMemoryCalloc(2, sizeof(size_t)); + if (!keys_list_sizes) { + taosMemoryFree(keys_list); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } const size_t klen = ROCKS_KEY_LEN; char *keys = taosMemoryCalloc(2, sizeof(SLastKey)); + if (!keys) { + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } ((SLastKey *)keys)[0] = (SLastKey){.lflag = LFLAG_LAST, .uid = uid, .cid = cid}; ((SLastKey *)keys)[1] = (SLastKey){.lflag = LFLAG_LAST_ROW, .uid = uid, .cid = cid}; @@ -672,9 +683,30 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, keys_list_sizes[0] = klen; keys_list_sizes[1] = klen; - char **values_list = taosMemoryCalloc(2, sizeof(char *)); + char **values_list = taosMemoryCalloc(2, sizeof(char *)); + if (!values_list) { + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + taosMemoryFree(keys_list[0]); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } size_t *values_list_sizes = taosMemoryCalloc(2, sizeof(size_t)); - char **errs = taosMemoryCalloc(2, sizeof(char *)); + if (!values_list_sizes) { + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + taosMemoryFree(keys_list[0]); + taosMemoryFree(values_list); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + char **errs = taosMemoryCalloc(2, sizeof(char *)); + if (!errs) { + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + taosMemoryFree(keys_list[0]); + taosMemoryFree(values_list); + taosMemoryFree(values_list_sizes); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } // rocksMayWrite(pTsdb, true, false, false); rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, 2, (const char *const *)keys_list, keys_list_sizes, @@ -960,6 +992,9 @@ static int32_t tsdbCacheUpdateValue(SValue *pOld, SValue *pNew) { if (IS_VAR_DATA_TYPE(pNew->type)) { if (nData < pNew->nData) { pOld->pData = taosMemoryCalloc(1, pNew->nData); + if (!pOld->pData) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } else { pOld->pData = pFree; pFree = NULL; @@ -1044,17 +1079,35 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray num_keys = TARRAY_SIZE(remainCols); } if (remainCols && num_keys > 0) { - char **keys_list = taosMemoryCalloc(num_keys, sizeof(char *)); - size_t *keys_list_sizes = taosMemoryCalloc(num_keys, sizeof(size_t)); + char **keys_list = NULL; + size_t *keys_list_sizes = NULL; + char **values_list = NULL; + size_t *values_list_sizes = NULL; + char **errs = NULL; + keys_list = taosMemoryCalloc(num_keys, sizeof(char *)); + keys_list_sizes = taosMemoryCalloc(num_keys, sizeof(size_t)); + if (!keys_list || !keys_list_sizes) { + taosMemoryFree(keys_list); + taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } for (int i = 0; i < num_keys; ++i) { SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; keys_list[i] = (char *)&idxKey->key; keys_list_sizes[i] = ROCKS_KEY_LEN; } - char **values_list = taosMemoryCalloc(num_keys, sizeof(char *)); - size_t *values_list_sizes = taosMemoryCalloc(num_keys, sizeof(size_t)); - char **errs = taosMemoryCalloc(num_keys, sizeof(char *)); + values_list = taosMemoryCalloc(num_keys, sizeof(char *)); + values_list_sizes = taosMemoryCalloc(num_keys, sizeof(size_t)); + errs = taosMemoryCalloc(num_keys, sizeof(char *)); + if (!values_list || !values_list_sizes || !errs) { + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + taosMemoryFree(values_list); + taosMemoryFree(values_list_sizes); + taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys, (const char *const *)keys_list, keys_list_sizes, values_list, values_list_sizes, errs); for (int i = 0; i < num_keys; ++i) { @@ -1104,6 +1157,16 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray pLastCol = &lastColTmp; SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); + if (!pTmpLastCol) { + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + taosMemoryFree(values_list); + taosMemoryFree(values_list_sizes); + + taosArrayDestroy(remainCols); + taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } *pTmpLastCol = *pLastCol; pLastCol = pTmpLastCol; @@ -1322,18 +1385,30 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr int num_keys = TARRAY_SIZE(remainCols); int16_t *slotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); - int16_t *lastColIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); - int16_t *lastSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); - int16_t *lastrowColIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); - int16_t *lastrowSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); - SArray *lastTmpColArray = NULL; - SArray *lastTmpIndexArray = NULL; - SArray *lastrowTmpColArray = NULL; - SArray *lastrowTmpIndexArray = NULL; + int16_t *lastColIds = NULL; + int16_t *lastSlotIds = NULL; + int16_t *lastrowColIds = NULL; + int16_t *lastrowSlotIds = NULL; + lastColIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); + lastSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); + lastrowColIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); + lastrowSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); + SArray *lastTmpColArray = NULL; + SArray *lastTmpIndexArray = NULL; + SArray *lastrowTmpColArray = NULL; + SArray *lastrowTmpIndexArray = NULL; int lastIndex = 0; int lastrowIndex = 0; + if (!slotIds || !lastColIds || !lastSlotIds || !lastrowColIds || !lastrowSlotIds) { + taosMemoryFree(slotIds); + taosMemoryFree(lastColIds); + taosMemoryFree(lastSlotIds); + taosMemoryFree(lastrowColIds); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + for (int i = 0; i < num_keys; ++i) { SIdxKey *idxKey = taosArrayGet(remainCols, i); slotIds[i] = pr->pSlotIds[idxKey->idx]; @@ -1401,6 +1476,14 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); + if (!pTmpLastCol) { + taosMemoryFree(slotIds); + taosMemoryFree(lastColIds); + taosMemoryFree(lastSlotIds); + taosMemoryFree(lastrowColIds); + taosMemoryFree(lastrowSlotIds); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } *pTmpLastCol = *pLastCol; pLastCol = pTmpLastCol; @@ -1420,7 +1503,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr LRUStatus status = taosLRUCacheInsert(pCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); if (status != TAOS_LRU_STATUS_OK) { - code = -1; + // code = -1; } // store result back to rocks cache @@ -1466,6 +1549,11 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA char **keys_list = taosMemoryMalloc(num_keys * sizeof(char *)); size_t *keys_list_sizes = taosMemoryMalloc(num_keys * sizeof(size_t)); char *key_list = taosMemoryMalloc(num_keys * ROCKS_KEY_LEN); + if (!keys_list || !keys_list_sizes || !key_list) { + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } for (int i = 0; i < num_keys; ++i) { int16_t cid = *(int16_t *)taosArrayGet(remainCols, i); @@ -1477,6 +1565,13 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA char **values_list = taosMemoryCalloc(num_keys, sizeof(char *)); size_t *values_list_sizes = taosMemoryCalloc(num_keys, sizeof(size_t)); char **errs = taosMemoryMalloc(num_keys * sizeof(char *)); + if (!values_list || !values_list_sizes || !errs) { + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + taosMemoryFree(values_list); + taosMemoryFree(values_list_sizes); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys, (const char *const *)keys_list, keys_list_sizes, values_list, values_list_sizes, errs); for (int i = 0; i < num_keys; ++i) { @@ -1486,6 +1581,7 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA rocksdb_free(errs[i]); } } + taosMemoryFree(errs); SLRUCache *pCache = pTsdb->lruCache; for (int i = 0, j = 0; i < num_keys && j < TARRAY_SIZE(remainCols); ++i) { @@ -1498,6 +1594,13 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[j]; if (pLastCol) { SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); + if (!pTmpLastCol) { + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + taosMemoryFree(values_list); + taosMemoryFree(values_list_sizes); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } *pTmpLastCol = *pLastCol; pLastCol = pTmpLastCol; @@ -1535,7 +1638,6 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA } } - taosMemoryFree(errs); taosMemoryFree(key_list); taosMemoryFree(keys_list); taosMemoryFree(keys_list_sizes); @@ -1649,15 +1751,24 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE TAOS_CHECK_RETURN(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema)); // build keys & multi get from rocks - int num_keys = pTSchema->numOfCols; - char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); - size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); + int num_keys = pTSchema->numOfCols; + char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); + size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); + if (!keys_list || !keys_list_sizes) { + taosMemoryFree(keys_list); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } const size_t klen = ROCKS_KEY_LEN; for (int i = 0; i < num_keys; ++i) { int16_t cid = pTSchema->columns[i].colId; char *keys = taosMemoryCalloc(2, sizeof(SLastKey)); + if (!keys) { + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } ((SLastKey *)keys)[0] = (SLastKey){.lflag = LFLAG_LAST, .uid = uid, .cid = cid}; ((SLastKey *)keys)[1] = (SLastKey){.lflag = LFLAG_LAST_ROW, .uid = uid, .cid = cid}; @@ -1669,6 +1780,15 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE char **values_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); size_t *values_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); char **errs = taosMemoryCalloc(num_keys * 2, sizeof(char *)); + if (!values_list_sizes || !values_list) { + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + taosMemoryFree(values_list); + for (int i = 0; i < num_keys; ++i) { + taosMemoryFree(keys_list[i]); + } + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } (void)tsdbCacheCommit(pTsdb); @@ -1885,7 +2005,9 @@ static STableLoadInfo *getTableLoadInfo(SCacheRowsReader *pReader, uint64_t uid) STableLoadInfo **ppInfo = tSimpleHashGet(pReader->pTableMap, &uid, sizeof(uid)); if (!ppInfo) { pInfo = taosMemoryCalloc(1, sizeof(STableLoadInfo)); - tSimpleHashPut(pReader->pTableMap, &uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); + if (pInfo) { + tSimpleHashPut(pReader->pTableMap, &uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); + } return pInfo; } @@ -1898,6 +2020,9 @@ static uint64_t *getUidList(SCacheRowsReader *pReader) { int32_t numOfTables = pReader->numOfTables; pReader->uidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); + if (!pReader->uidList) { + return NULL; + } for (int32_t i = 0; i < numOfTables; ++i) { uint64_t uid = pReader->pTableList[i].uid; @@ -1913,9 +2038,13 @@ static uint64_t *getUidList(SCacheRowsReader *pReader) { static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsReader *pReader, void *pFileReader, bool isFile) { int32_t code = 0; - uint64_t *uidList = getUidList(pReader); int32_t numOfTables = pReader->numOfTables; int64_t suid = pReader->info.suid; + uint64_t *uidList = getUidList(pReader); + + if (!uidList) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } for (int i = 0, j = 0; i < pTombBlkArray->size && j < numOfTables; ++i) { STombBlk *pTombBlk = &pTombBlkArray->data[i]; @@ -3296,6 +3425,9 @@ int32_t tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_ size_t charge = pFD->szPage; _taos_lru_deleter_t deleter = deleteBCache; uint8_t *pPg = taosMemoryMalloc(charge); + if (!pPg) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } memcpy(pPg, pPage, charge); LRUStatus status = From aa921c987097d936db8438fde74498eb5b6d88ed Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 25 Jul 2024 11:13:04 +0800 Subject: [PATCH 49/52] update sync raft log --- source/dnode/vnode/src/tsdb/tsdbCache.c | 7 ++- source/libs/sync/src/syncRaftLog.c | 61 ++++++++++++++++--------- source/util/src/tlrucache.c | 19 ++++---- 3 files changed, 50 insertions(+), 37 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index b6c389842e..1726d8696b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1888,11 +1888,10 @@ _exit: } int32_t tsdbOpenCache(STsdb *pTsdb) { - int32_t code = 0, lino = 0; - SLRUCache *pCache = NULL; - size_t cfgCapacity = pTsdb->pVnode->config.cacheLastSize * 1024 * 1024; + int32_t code = 0, lino = 0; + size_t cfgCapacity = pTsdb->pVnode->config.cacheLastSize * 1024 * 1024; - pCache = taosLRUCacheInit(cfgCapacity, 0, .5); + SLRUCache *pCache = taosLRUCacheInit(cfgCapacity, 0, .5); if (pCache == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err); } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 19c8837d83..57c70d8df1 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -51,7 +51,12 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { taosLRUCacheSetStrictCapacity(pLogStore->pCache, false); pLogStore->data = taosMemoryMalloc(sizeof(SSyncLogStoreData)); - ASSERT(pLogStore->data != NULL); + if (!pLogStore->data) { + taosMemoryFree(pLogStore); + taosLRUCacheCleanup(pLogStore->pCache); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } SSyncLogStoreData* pData = pLogStore->data; pData->pSyncNode = pSyncNode; @@ -60,7 +65,13 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { taosThreadMutexInit(&(pData->mutex), NULL); pData->pWalHandle = walOpenReader(pData->pWal, NULL, 0); - ASSERT(pData->pWalHandle != NULL); + if (!pData->pWalHandle) { + taosMemoryFree(pLogStore); + taosLRUCacheCleanup(pLogStore->pCache); + taosThreadMutexDestroy(&(pData->mutex)); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } pLogStore->syncLogUpdateCommitIndex = raftLogUpdateCommitIndex; pLogStore->syncLogCommitIndex = raftlogCommitIndex; @@ -110,7 +121,7 @@ static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncI SWal* pWal = pData->pWal; int32_t code = walRestoreFromSnapshot(pWal, snapshotIndex); if (code != 0) { - int32_t err = terrno; + int32_t err = code; const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); @@ -118,10 +129,10 @@ static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncI sNError(pData->pSyncNode, "wal restore from snapshot error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", snapshotIndex, err, errStr, sysErr, sysErrStr); - return -1; + TAOS_RETURN(err); } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { @@ -224,7 +235,8 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr sNError(pData->pSyncNode, "wal write error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", pEntry->index, err, errStr, sysErr, sysErrStr); - return -1; + + TAOS_RETURN(err); } code = walFsync(pWal, forceSync); @@ -235,7 +247,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr sNTrace(pData->pSyncNode, "write index:%" PRId64 ", type:%s, origin type:%s, elapsed:%" PRId64, pEntry->index, TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType), tsElapsed); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } // entry found, return 0 @@ -253,10 +265,10 @@ int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncR SWalReader* pWalHandle = pData->pWalHandle; if (pWalHandle == NULL) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; sError("vgId:%d, wal handle is NULL", pData->pSyncNode->vgId); taosThreadMutexUnlock(&(pData->mutex)); - return -1; + + TAOS_RETURN(TSDB_CODE_SYN_INTERNAL_ERROR); } int64_t ts2 = taosGetTimestampNs(); @@ -266,7 +278,7 @@ int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncR // code = walReadVerCached(pWalHandle, index); if (code != 0) { - int32_t err = terrno; + int32_t err = code; const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); @@ -286,7 +298,8 @@ int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncR */ taosThreadMutexUnlock(&(pData->mutex)); - return code; + + TAOS_RETURN(code); } *ppEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); @@ -319,7 +332,7 @@ int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncR ", elapsed-build:%" PRId64, index, tsElapsed, tsElapsedLock, tsElapsedRead, tsElapsedBuild); - return code; + TAOS_RETURN(code); } // truncate semantic @@ -329,7 +342,7 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn int32_t code = walRollback(pWal, fromIndex); if (code != 0) { - int32_t err = terrno; + int32_t err = code; const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); @@ -339,7 +352,8 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn // event log sNTrace(pData->pSyncNode, "log truncate, from-index:%" PRId64, fromIndex); - return code; + + TAOS_RETURN(code); } // entry found, return 0 @@ -352,16 +366,16 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp *ppLastEntry = NULL; if (walIsEmpty(pWal)) { - terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; - return -1; + TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST); } else { SyncIndex lastIndex = raftLogLastIndex(pLogStore); ASSERT(lastIndex >= SYNC_INDEX_BEGIN); int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); - return code; + + TAOS_RETURN(code); } - return -1; + TAOS_RETURN(TSDB_CODE_FAILED); } int32_t raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { @@ -375,20 +389,22 @@ int32_t raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { if (index < snapshotVer || index > wallastVer) { // ignore - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t code = walCommit(pWal, index); if (code != 0) { - int32_t err = terrno; + int32_t err = code; const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); sError("vgId:%d, wal update commit index error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, index, err, errStr, sysErr, sysErrStr); - return -1; + + TAOS_RETURN(code); } - return 0; + + TAOS_RETURN(TSDB_CODE_SUCCESS); } SyncIndex raftlogCommitIndex(SSyncLogStore* pLogStore) { @@ -405,5 +421,6 @@ SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore) { SyncIndex logStoreWalCommitVer(SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; + return walGetCommittedVer(pWal); } diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index 3de159797f..4f8e96c4c7 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -20,6 +20,7 @@ #include "tarray.h" #include "tdef.h" #include "tlog.h" +#include "tutil.h" typedef struct SLRUEntry SLRUEntry; typedef struct SLRUEntryTable SLRUEntryTable; @@ -114,13 +115,13 @@ static int taosLRUEntryTableInit(SLRUEntryTable *table, int maxUpperHashBits) { table->lengthBits = 4; table->list = taosMemoryCalloc(1 << table->lengthBits, sizeof(SLRUEntry *)); if (!table->list) { - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } table->elems = 0; table->maxLengthBits = maxUpperHashBits; - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static void taosLRUEntryTableApply(SLRUEntryTable *table, _taos_lru_table_func_t func, uint32_t begin, uint32_t end) { @@ -349,9 +350,7 @@ static void taosLRUCacheShardSetCapacity(SLRUCacheShard *shard, size_t capacity) static int taosLRUCacheShardInit(SLRUCacheShard *shard, size_t capacity, bool strict, double highPriPoolRatio, int maxUpperHashBits) { - if (taosLRUEntryTableInit(&shard->table, maxUpperHashBits) < 0) { - return -1; - } + TAOS_CHECK_RETURN(taosLRUEntryTableInit(&shard->table, maxUpperHashBits)); taosThreadMutexInit(&shard->mutex, NULL); @@ -372,7 +371,7 @@ static int taosLRUCacheShardInit(SLRUCacheShard *shard, size_t capacity, bool st taosLRUCacheShardSetCapacity(shard, capacity); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static void taosLRUCacheShardCleanup(SLRUCacheShard *shard) { @@ -671,16 +670,13 @@ static int getDefaultCacheShardBits(size_t capacity) { SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoolRatio) { if (numShardBits >= 20) { - terrno = TSDB_CODE_INVALID_PARA; return NULL; } if (highPriPoolRatio < 0.0 || highPriPoolRatio > 1.0) { - terrno = TSDB_CODE_INVALID_PARA; return NULL; } SLRUCache *cache = taosMemoryCalloc(1, sizeof(SLRUCache)); if (!cache) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -692,14 +688,15 @@ SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoo cache->shards = taosMemoryCalloc(numShards, sizeof(SLRUCacheShard)); if (!cache->shards) { taosMemoryFree(cache); - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } bool strictCapacity = 1; size_t perShard = (capacity + (numShards - 1)) / numShards; for (int i = 0; i < numShards; ++i) { - taosLRUCacheShardInit(&cache->shards[i], perShard, strictCapacity, highPriPoolRatio, 32 - numShardBits); + if (TSDB_CODE_SUCCESS != + taosLRUCacheShardInit(&cache->shards[i], perShard, strictCapacity, highPriPoolRatio, 32 - numShardBits)) + return NULL; } cache->numShards = numShards; From a5cd3444d1c09320d7027f6c054d072e3b676efd Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 25 Jul 2024 12:05:38 +0800 Subject: [PATCH 50/52] update sync raft store --- source/libs/sync/src/syncRaftStore.c | 64 ++++++++++++++-------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index c200c6cb4b..7f702c3766 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -25,17 +25,17 @@ static int32_t raftStoreDecode(const SJson *pJson, SRaftStore *pStore) { int32_t code = 0; tjsonGetNumberValue(pJson, "current_term", pStore->currentTerm, code); - if (code < 0) return -1; + if (code < 0) TAOS_RETURN(TSDB_CODE_FAILED); tjsonGetNumberValue(pJson, "vote_for_addr", pStore->voteFor.addr, code); - if (code < 0) return -1; + if (code < 0) TAOS_RETURN(TSDB_CODE_FAILED); tjsonGetInt32ValueFromDouble(pJson, "vote_for_vgid", pStore->voteFor.vgId, code); - if (code < 0) return -1; + if (code < 0) TAOS_RETURN(TSDB_CODE_FAILED); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t raftStoreReadFile(SSyncNode *pNode) { - int32_t code = -1; + int32_t code = -1, lino = 0; TdFilePtr pFile = NULL; char *pData = NULL; SJson *pJson = NULL; @@ -52,41 +52,38 @@ int32_t raftStoreReadFile(SSyncNode *pNode) { pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); sError("vgId:%d, failed to open raft store file:%s since %s", pNode->vgId, file, terrstr()); - goto _OVER; + + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); } int64_t size = 0; if (taosFStatFile(pFile, &size, NULL) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); sError("vgId:%d, failed to fstat raft store file:%s since %s", pNode->vgId, file, terrstr()); - goto _OVER; + + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); } pData = taosMemoryMalloc(size + 1); if (pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); } if (taosReadFile(pFile, pData, size) != size) { - terrno = TAOS_SYSTEM_ERROR(errno); sError("vgId:%d, failed to read raft store file:%s since %s", pNode->vgId, file, terrstr()); - goto _OVER; + + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); } pData[size] = '\0'; pJson = tjsonParse(pData); if (pJson == NULL) { - terrno = TSDB_CODE_INVALID_JSON_FORMAT; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_INVALID_JSON_FORMAT, &lino, _OVER); } if (raftStoreDecode(pJson, pStore) < 0) { - terrno = TSDB_CODE_INVALID_JSON_FORMAT; - goto _OVER; + TAOS_CHECK_GOTO(TSDB_CODE_INVALID_JSON_FORMAT, &lino, _OVER); } code = 0; @@ -100,18 +97,20 @@ _OVER: if (code != 0) { sError("vgId:%d, failed to read raft store file:%s since %s", pNode->vgId, file, terrstr()); } - return code; + + TAOS_RETURN(code); } static int32_t raftStoreEncode(SJson *pJson, SRaftStore *pStore) { - if (tjsonAddIntegerToObject(pJson, "current_term", pStore->currentTerm) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "vote_for_addr", pStore->voteFor.addr) < 0) return -1; - if (tjsonAddDoubleToObject(pJson, "vote_for_vgid", pStore->voteFor.vgId) < 0) return -1; - return 0; + if (tjsonAddIntegerToObject(pJson, "current_term", pStore->currentTerm) < 0) TAOS_RETURN(TSDB_CODE_FAILED); + if (tjsonAddIntegerToObject(pJson, "vote_for_addr", pStore->voteFor.addr) < 0) TAOS_RETURN(TSDB_CODE_FAILED); + if (tjsonAddDoubleToObject(pJson, "vote_for_vgid", pStore->voteFor.vgId) < 0) TAOS_RETURN(TSDB_CODE_FAILED); + + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t raftStoreWriteFile(SSyncNode *pNode) { - int32_t code = -1; + int32_t code = -1, lino = 0; char *buffer = NULL; SJson *pJson = NULL; TdFilePtr pFile = NULL; @@ -120,23 +119,23 @@ int32_t raftStoreWriteFile(SSyncNode *pNode) { char file[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s.bak", realfile); - terrno = TSDB_CODE_OUT_OF_MEMORY; pJson = tjsonCreateObject(); - if (pJson == NULL) goto _OVER; - if (raftStoreEncode(pJson, pStore) != 0) goto _OVER; + if (pJson == NULL) TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + if (raftStoreEncode(pJson, pStore) != 0) TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + buffer = tjsonToString(pJson); - if (buffer == NULL) goto _OVER; - terrno = 0; + if (buffer == NULL) TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); - if (pFile == NULL) goto _OVER; + if (pFile == NULL) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); int32_t len = strlen(buffer); - if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; - if (taosFsyncFile(pFile) < 0) goto _OVER; + if (taosWriteFile(pFile, buffer, len) <= 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); + + if (taosFsyncFile(pFile) < 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); taosCloseFile(&pFile); - if (taosRenameFile(file, realfile) != 0) goto _OVER; + if (taosRenameFile(file, realfile) != 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); code = 0; sInfo("vgId:%d, succeed to write raft store file:%s, term:%" PRId64, pNode->vgId, realfile, pStore->currentTerm); @@ -147,7 +146,6 @@ _OVER: if (pFile != NULL) taosCloseFile(&pFile); if (code != 0) { - if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); sError("vgId:%d, failed to write raft store file:%s since %s", pNode->vgId, realfile, terrstr()); } return code; From 970a5f31a34a30b80e8c98904c732afe709de20f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 25 Jul 2024 12:09:51 +0800 Subject: [PATCH 51/52] update sync replication --- source/libs/sync/src/syncReplication.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 9312719be7..dfe9f51af2 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -52,7 +52,8 @@ int32_t syncNodeReplicateReset(SSyncNode* pNode, SRaftId* pDestId) { SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId); syncLogReplReset(pMgr); taosThreadMutexUnlock(&pBuf->mutex); - return 0; + + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t syncNodeReplicate(SSyncNode* pNode) { @@ -60,13 +61,14 @@ int32_t syncNodeReplicate(SSyncNode* pNode) { taosThreadMutexLock(&pBuf->mutex); int32_t ret = syncNodeReplicateWithoutLock(pNode); taosThreadMutexUnlock(&pBuf->mutex); - return ret; + + TAOS_RETURN(ret); } int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) { if ((pNode->state != TAOS_SYNC_STATE_LEADER && pNode->state != TAOS_SYNC_STATE_ASSIGNED_LEADER) || pNode->raftCfg.cfg.totalReplicaNum == 1) { - return -1; + TAOS_RETURN(TSDB_CODE_FAILED); } for (int32_t i = 0; i < pNode->totalReplicaNum; i++) { if (syncUtilSameId(&pNode->replicasId[i], &pNode->myRaftId)) { @@ -75,14 +77,16 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) { SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; (void)syncLogReplStart(pMgr, pNode); } - return 0; + + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) { SyncAppendEntries* pMsg = pRpcMsg->pCont; pMsg->destId = *destRaftId; syncNodeSendMsgById(destRaftId, pSyncNode, pRpcMsg); - return 0; + + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t syncNodeSendHeartbeat(SSyncNode* pSyncNode, const SRaftId* destId, SRpcMsg* pMsg) { @@ -112,5 +116,5 @@ int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode) { syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg); } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } From 5caa366464402ac0cb8c19e643a105924dffcb42 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 25 Jul 2024 12:15:00 +0800 Subject: [PATCH 52/52] update sync vote & reply --- source/libs/sync/src/syncRequestVote.c | 11 ++++++----- source/libs/sync/src/syncRequestVoteReply.c | 14 +++++++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 77e5498127..0e50cca94c 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -20,7 +20,6 @@ #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" -#include "syncUtil.h" // TLA+ Spec // HandleRequestVoteRequest(i, j, m) == @@ -95,7 +94,8 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &pMsg->srcId)) { syncLogRecvRequestVote(ths, pMsg, -1, "not in my config"); - return -1; + + TAOS_RETURN(TSDB_CODE_FAILED); } bool logOK = syncNodeOnRequestVoteLogOK(ths, pMsg); @@ -122,8 +122,8 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { // send msg SRpcMsg rpcMsg = {0}; - ret = syncBuildRequestVoteReply(&rpcMsg, ths->vgId); - ASSERT(ret == 0); + + TAOS_CHECK_RETURN(syncBuildRequestVoteReply(&rpcMsg, ths->vgId)); SyncRequestVoteReply* pReply = rpcMsg.pCont; pReply->srcId = ths->myRaftId; @@ -138,5 +138,6 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); if (resetElect) syncNodeResetElectTimer(ths); - return 0; + + TAOS_RETURN(TSDB_CODE_SUCCESS); } diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 25c9f813a6..10d9a6c96b 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -45,19 +45,22 @@ int32_t syncNodeOnRequestVoteReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { syncLogRecvRequestVoteReply(ths, pMsg, "not in my config"); - return -1; + + TAOS_RETURN(TSDB_CODE_FAILED); } SyncTerm currentTerm = raftStoreGetTerm(ths); // drop stale response if (pMsg->term < currentTerm) { syncLogRecvRequestVoteReply(ths, pMsg, "drop stale response"); - return -1; + + TAOS_RETURN(TSDB_CODE_FAILED); } if (pMsg->term > currentTerm) { syncLogRecvRequestVoteReply(ths, pMsg, "error term"); syncNodeStepDown(ths, pMsg->term); - return -1; + + TAOS_RETURN(TSDB_CODE_FAILED); } syncLogRecvRequestVoteReply(ths, pMsg, ""); @@ -69,7 +72,8 @@ int32_t syncNodeOnRequestVoteReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { if (ths->pVotesRespond->term != pMsg->term) { sNError(ths, "vote respond error vote-respond-mgr term:%" PRIu64 ", msg term:%" PRIu64 "", ths->pVotesRespond->term, pMsg->term); - return -1; + + TAOS_RETURN(TSDB_CODE_FAILED); } votesRespondAdd(ths->pVotesRespond, pMsg); @@ -93,5 +97,5 @@ int32_t syncNodeOnRequestVoteReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); }