From 120ce576c495da43c376301d9189253845e44ccd Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 5 Sep 2024 09:58:02 +0000 Subject: [PATCH 01/94] fix/TD-31891-remove-void-stb --- source/common/src/tname.c | 6 +- source/dnode/mnode/impl/inc/mndShow.h | 13 ++ source/dnode/mnode/impl/inc/mndStb.h | 6 +- source/dnode/mnode/impl/src/mndStb.c | 190 +++++++++++++++++--------- source/dnode/mnode/impl/src/mndUser.c | 2 +- 5 files changed, 145 insertions(+), 72 deletions(-) diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 8d0f324509..1f0dbad2a7 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -153,7 +153,11 @@ int32_t tNameGetDbName(const SName* name, char* dst) { const char* tNameGetDbNameP(const SName* name) { return &name->dbname[0]; } int32_t tNameGetFullDbName(const SName* name, char* dst) { - return snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); + if (name == NULL || dst == NULL) { + return TSDB_CODE_INVALID_PARA; + } + (void)snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); + return 0; } bool tNameIsEmpty(const SName* name) { return name->type == 0 || name->acctId == 0; } diff --git a/source/dnode/mnode/impl/inc/mndShow.h b/source/dnode/mnode/impl/inc/mndShow.h index 25268c52bb..26d47d6130 100644 --- a/source/dnode/mnode/impl/inc/mndShow.h +++ b/source/dnode/mnode/impl/inc/mndShow.h @@ -32,6 +32,19 @@ extern "C" { } \ } while (0) +#define RETRIEVE_CHECK_GOTO(CMD, pObj, LINO, LABEL) \ + do { \ + code = (CMD); \ + if (code != TSDB_CODE_SUCCESS) { \ + if (LINO) { \ + *((int32_t *)(LINO)) = __LINE__; \ + } \ + if (pObj) sdbRelease(pSdb, (pObj)); \ + if (pObj) sdbCancelFetch(pSdb, (pObj)); \ + goto LABEL; \ + } \ + } while (0) + int32_t mndInitShow(SMnode *pMnode); void mndCleanupShow(SMnode *pMnode); void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp); diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index db960d790f..bc97d4d4f9 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -37,9 +37,9 @@ int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p void mndFreeStb(SStbObj *pStb); int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, void **pCont, int32_t *pLen); -void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst); -void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst); -void mndExtractShortDbNameFromDbFullName(const char *stbFullName, char *dst); +int32_t mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst); +int32_t mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst); +int32_t mndExtractShortDbNameFromDbFullName(const char *stbFullName, char *dst); void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize); const char *mndGetStbStr(const char *src); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 03ff6d425f..124d12fafb 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -482,7 +482,7 @@ SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { if ((terrno = tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) != 0) return NULL; char db[TSDB_TABLE_FNAME_LEN] = {0}; - (void)tNameGetFullDbName(&name, db); + if ((terrno = tNameGetFullDbName(&name, db)) != 0) return NULL; return mndAcquireDb(pMnode, db); } @@ -507,7 +507,9 @@ void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int3 goto _err; } char dbFName[TSDB_DB_FNAME_LEN] = {0}; - (void)tNameGetFullDbName(&name, dbFName); + if ((terrno = tNameGetFullDbName(&name, dbFName)) != 0) { + goto _err; + }; req.name = (char *)tNameGetTableName(&name); req.suid = pStb->uid; @@ -625,8 +627,9 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead)); - (void)tEncodeSVDropStbReq(&encoder, &req); + terrno = tEncodeSVDropStbReq(&encoder, &req); tEncoderClear(&encoder); + if (terrno != 0) return NULL; *pContLen = contLen; return pHead; @@ -1003,7 +1006,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea _OVER: mndTransDrop(pTrans); - (void)mndStbActionDelete(pMnode->pSdb, &stbObj); + if (mndStbActionDelete(pMnode->pSdb, &stbObj) != 0) mError("failed to mndStbActionDelete"); TAOS_RETURN(code); } @@ -1032,6 +1035,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; + int32_t code = 0; SMsgHead *pHead = rpcMallocCont(contLen); if (pHead == NULL) { sdbRelease(pSdb, pVgroup); @@ -1039,12 +1043,16 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { } pHead->contLen = htonl(contLen); pHead->vgId = htonl(pVgroup->vgId); - (void)tSerializeSVDropTtlTableReq((char *)pHead + sizeof(SMsgHead), reqLen, &ttlReq); + if ((code = tSerializeSVDropTtlTableReq((char *)pHead + sizeof(SMsgHead), reqLen, &ttlReq)) < 0) { + mError("vgId:%d, failed to serialize drop ttl table request since %s", pVgroup->vgId, tstrerror(code)); + sdbRelease(pSdb, pVgroup); + continue; + } SRpcMsg rpcMsg = { .msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS, .pCont = pHead, .contLen = contLen, .info = pReq->info}; SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); - int32_t code = tmsgSendReq(&epSet, &rpcMsg); + code = tmsgSendReq(&epSet, &rpcMsg); if (code != 0) { mError("vgId:%d, failed to send drop ttl table request to vnode since 0x%x", pVgroup->vgId, code); } else { @@ -1069,6 +1077,8 @@ static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; + int32_t code = 0; + SMsgHead *pHead = rpcMallocCont(contLen); if (pHead == NULL) { sdbCancelFetch(pSdb, pVgroup); @@ -1077,11 +1087,13 @@ static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq) { } pHead->contLen = htonl(contLen); pHead->vgId = htonl(pVgroup->vgId); - (void)tSerializeSVTrimDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &trimReq); + if ((code = tSerializeSVTrimDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &trimReq)) < 0) { + mError("vgId:%d, failed to serialize trim db request since %s", pVgroup->vgId, tstrerror(code)); + } SRpcMsg rpcMsg = {.msgType = TDMT_VND_TRIM, .pCont = pHead, .contLen = contLen}; SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); - int32_t code = tmsgSendReq(&epSet, &rpcMsg); + code = tmsgSendReq(&epSet, &rpcMsg); if (code != 0) { mError("vgId:%d, timer failed to send vnode-trim request to vnode since 0x%x", pVgroup->vgId, code); } else { @@ -1106,19 +1118,24 @@ static int32_t mndProcessS3MigrateDbTimer(SRpcMsg *pReq) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; + int32_t code = 0; + SMsgHead *pHead = rpcMallocCont(contLen); if (pHead == NULL) { - sdbCancelFetch(pSdb, pVgroup); sdbRelease(pSdb, pVgroup); continue; } pHead->contLen = htonl(contLen); pHead->vgId = htonl(pVgroup->vgId); - (void)tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &s3migrateReq); + if ((code = tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &s3migrateReq)) < 0) { + mError("vgId:%d, failed to serialize s3migrate db request since %s", pVgroup->vgId, tstrerror(code)); + sdbRelease(pSdb, pVgroup); + continue; + } SRpcMsg rpcMsg = {.msgType = TDMT_VND_S3MIGRATE, .pCont = pHead, .contLen = contLen}; SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); - int32_t code = tmsgSendReq(&epSet, &rpcMsg); + code = tmsgSendReq(&epSet, &rpcMsg); if (code != 0) { mError("vgId:%d, timer failed to send vnode-s3migrate request to vnode since 0x%x", pVgroup->vgId, code); } else { @@ -2336,11 +2353,13 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, SStbObj *pObj, void **pCont, i void *cont = taosMemoryMalloc(contLen); tEncoderInit(&ec, cont, contLen); - (void)tEncodeSMAlterStbRsp(&ec, &alterRsp); + code = tEncodeSMAlterStbRsp(&ec, &alterRsp); tEncoderClear(&ec); tFreeSMAlterStbRsp(&alterRsp); + if (code < 0) TAOS_RETURN(code); + *pCont = cont; *pLen = contLen; @@ -2389,7 +2408,7 @@ int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, vo void *cont = taosMemoryMalloc(contLen); tEncoderInit(&ec, cont, contLen); - (void)tEncodeSMCreateStbRsp(&ec, &stbRsp); + TAOS_CHECK_GOTO(tEncodeSMCreateStbRsp(&ec, &stbRsp), NULL, _OVER); tEncoderClear(&ec); tFreeSMCreateStbRsp(&stbRsp); @@ -2628,8 +2647,9 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; SName name = {0}; - // TODO check return value - (void)tNameFromString(&name, alterReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + int32_t ret = 0; + if ((ret = tNameFromString(&name, alterReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) != 0) + mError("stb:%s, failed to tNameFromString since %s", alterReq.name, tstrerror(ret)); auditRecord(pReq, pMnode->clusterId, "alterStb", name.dbname, name.tname, alterReq.sql, alterReq.sqlLen); @@ -2916,8 +2936,9 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; SName name = {0}; - // TODO check return value - (void)tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + int32_t ret = 0; + if ((ret = tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) != 0) + mError("stb:%s, failed to tNameFromString since %s", dropReq.name, tstrerror(ret)); auditRecord(pReq, pMnode->clusterId, "dropStb", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen); @@ -2968,7 +2989,10 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { goto _OVER; } - (void)tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); + if ((rspLen = tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp)) < 0) { + code = rspLen; + goto _OVER; + } pReq->info.rsp = pRsp; pReq->info.rspLen = rspLen; code = 0; @@ -2995,7 +3019,7 @@ static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { TAOS_CHECK_GOTO(tDeserializeSTableCfgReq(pReq->pCont, pReq->contLen, &cfgReq), NULL, _OVER); char dbName[TSDB_DB_NAME_LEN] = {0}; - mndExtractShortDbNameFromDbFullName(cfgReq.dbFName, dbName); + TAOS_CHECK_GOTO(mndExtractShortDbNameFromDbFullName(cfgReq.dbFName, dbName), NULL, _OVER); if (0 == strcmp(dbName, TSDB_INFORMATION_SCHEMA_DB)) { mInfo("information_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); TAOS_CHECK_GOTO(mndBuildInsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp), NULL, _OVER); @@ -3019,7 +3043,10 @@ static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { goto _OVER; } - (void)tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp); + if ((rspLen = tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp)) < 0) { + code = rspLen; + goto _OVER; + } pReq->info.rsp = pRsp; pReq->info.rspLen = rspLen; code = 0; @@ -3140,8 +3167,9 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t TAOS_RETURN(code); } - (void)tSerializeSSTbHbRsp(pRsp, rspLen, &hbRsp); + rspLen = tSerializeSSTbHbRsp(pRsp, rspLen, &hbRsp); tFreeSSTbHbRsp(&hbRsp); + if (rspLen < 0) return rspLen; *ppRsp = pRsp; *pRspLen = rspLen; TAOS_RETURN(code); @@ -3175,25 +3203,31 @@ int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) { TAOS_RETURN(code); } -void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst) { +int32_t mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst) { SName name = {0}; - (void)tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + TAOS_CHECK_RETURN(tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)); - (void)tNameGetFullDbName(&name, dst); + TAOS_CHECK_RETURN(tNameGetFullDbName(&name, dst)); + + return 0; } -void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst) { +int32_t mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst) { SName name = {0}; - (void)tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + TAOS_CHECK_RETURN(tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)); - (void)tNameGetDbName(&name, dst); + TAOS_CHECK_RETURN(tNameGetDbName(&name, dst)); + + return 0; } -void mndExtractShortDbNameFromDbFullName(const char *stbFullName, char *dst) { +int32_t mndExtractShortDbNameFromDbFullName(const char *stbFullName, char *dst) { SName name = {0}; - (void)tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB); + TAOS_CHECK_RETURN(tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB)); - (void)tNameGetDbName(&name, dst); + TAOS_CHECK_RETURN(tNameGetDbName(&name, dst)); + + return 0; } void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize) { @@ -3491,18 +3525,22 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc int32_t numOfRows = 0; SStbObj *pStb = NULL; int32_t cols = 0; + int32_t lino = 0; + int32_t code = 0; SDbObj *pDb = NULL; if (strlen(pShow->db) > 0) { pDb = mndAcquireDb(pMnode, pShow->db); if (pDb == NULL) return terrno; } + int64_t uid = pDb->uid; + mndReleaseDb(pMnode, pDb); while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); if (pShow->pIter == NULL) break; - if (pDb != NULL && pStb->dbUid != pDb->uid) { + if (pStb->dbUid != uid) { sdbRelease(pSdb, pStb); continue; } @@ -3515,42 +3553,44 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc cols = 0; SName name = {0}; + char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN); varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false), pStb, &lino, _ERROR); char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - (void)tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB); - (void)tNameGetDbName(&name, varDataVal(db)); + RETRIEVE_CHECK_GOTO(tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB), pStb, &lino, _ERROR); + RETRIEVE_CHECK_GOTO(tNameGetDbName(&name, varDataVal(db)), pStb, &lino, _ERROR); varDataSetLen(db, strlen(varDataVal(db))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)db, false), pStb, &lino, _ERROR); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)db, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->createdTime, false), pStb, &lino, + _ERROR); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->createdTime, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false), pStb, &lino, + _ERROR); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false), pStb, &lino, _ERROR); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->updateTime, false), pStb, &lino, + _ERROR); // number of tables pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (pStb->commentLen > 0) { char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, pStb->comment); - (void)colDataSetVal(pColInfo, numOfRows, comment, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, comment, false), pStb, &lino, _ERROR); } else if (pStb->commentLen == 0) { char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, ""); - (void)colDataSetVal(pColInfo, numOfRows, comment, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, comment, false), pStb, &lino, _ERROR); } else { colDataSetNULL(pColInfo, numOfRows); } @@ -3560,14 +3600,14 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc varDataSetLen(watermark, strlen(varDataVal(watermark))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)watermark, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)watermark, false), pStb, &lino, _ERROR); char maxDelay[64 + VARSTR_HEADER_SIZE] = {0}; sprintf(varDataVal(maxDelay), "%" PRId64 "a,%" PRId64 "a", pStb->maxdelay[0], pStb->maxdelay[1]); varDataSetLen(maxDelay, strlen(varDataVal(maxDelay))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)maxDelay, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)maxDelay, false), pStb, &lino, _ERROR); char rollup[160 + VARSTR_HEADER_SIZE] = {0}; int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); @@ -3586,16 +3626,19 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc varDataSetLen(rollup, strlen(varDataVal(rollup))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)rollup, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)rollup, false), pStb, &lino, _ERROR); numOfRows++; sdbRelease(pSdb, pStb); } - if (pDb != NULL) { - mndReleaseDb(pMnode, pDb); - } + goto _OVER; +_ERROR: + mError("show:0x%" PRIx64 ", failed to retrieve data at %s:%d since %s", pShow->id, __FUNCTION__, lino, + tstrerror(code)); + +_OVER: pShow->numOfRows += numOfRows; return numOfRows; } @@ -3606,6 +3649,8 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p char dName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t numOfRows = p->info.rows; + int32_t lino = 0; + int32_t code = 0; STR_TO_VARSTR(dName, dbName); STR_TO_VARSTR(typeName, "SYSTEM_TABLE"); @@ -3624,20 +3669,20 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p for (int32_t j = 0; j < pm->colNum; j++) { // table name SColumnInfoData *pColInfoData = taosArrayGet(p->pDataBlock, 0); - (void)colDataSetVal(pColInfoData, numOfRows, tName, false); + TAOS_CHECK_GOTO(colDataSetVal(pColInfoData, numOfRows, tName, false), &lino, _OVER); // database name pColInfoData = taosArrayGet(p->pDataBlock, 1); - (void)colDataSetVal(pColInfoData, numOfRows, dName, false); + TAOS_CHECK_GOTO(colDataSetVal(pColInfoData, numOfRows, dName, false), &lino, _OVER); pColInfoData = taosArrayGet(p->pDataBlock, 2); - (void)colDataSetVal(pColInfoData, numOfRows, typeName, false); + TAOS_CHECK_GOTO(colDataSetVal(pColInfoData, numOfRows, typeName, false), &lino, _OVER); // col name char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(colName, pm->schema[j].name); pColInfoData = taosArrayGet(p->pDataBlock, 3); - (void)colDataSetVal(pColInfoData, numOfRows, colName, false); + TAOS_CHECK_GOTO(colDataSetVal(pColInfoData, numOfRows, colName, false), &lino, _OVER); // col type int8_t colType = pm->schema[j].type; @@ -3652,10 +3697,10 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p (int32_t)((pm->schema[j].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(colTypeStr, colTypeLen); - (void)colDataSetVal(pColInfoData, numOfRows, (char *)colTypeStr, false); + TAOS_CHECK_GOTO(colDataSetVal(pColInfoData, numOfRows, (char *)colTypeStr, false), &lino, _OVER); pColInfoData = taosArrayGet(p->pDataBlock, 5); - (void)colDataSetVal(pColInfoData, numOfRows, (const char *)&pm->schema[j].bytes, false); + TAOS_CHECK_GOTO(colDataSetVal(pColInfoData, numOfRows, (const char *)&pm->schema[j].bytes, false), &lino, _OVER); for (int32_t k = 6; k <= 8; ++k) { pColInfoData = taosArrayGet(p->pDataBlock, k); colDataSetNULL(pColInfoData, numOfRows); @@ -3664,7 +3709,8 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p numOfRows += 1; } } - +_OVER: + mError("failed at %s:%d since %s", __FUNCTION__, lino, tstrerror(code)); return numOfRows; } #define BUILD_COL_FOR_INFO_DB 1 @@ -3712,6 +3758,8 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB SSdb *pSdb = pMnode->pSdb; SStbObj *pStb = NULL; int32_t numOfRows = 0; + int32_t lino = 0; + int32_t code = 0; buildWhichDBs = determineBuildColForWhichDBs(pShow->db); @@ -3771,26 +3819,26 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB mDebug("mndRetrieveStbCol get stable cols, stable name:%s, db:%s", pStb->name, pStb->db); char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - (void)tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB); - (void)tNameGetDbName(&name, varDataVal(db)); + RETRIEVE_CHECK_GOTO(tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB), pStb, &lino, _OVER); + RETRIEVE_CHECK_GOTO(tNameGetDbName(&name, varDataVal(db)), pStb, &lino, _OVER); varDataSetLen(db, strlen(varDataVal(db))); for (int i = 0; i < pStb->numOfColumns; i++) { int32_t cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false), pStb, &lino, _OVER); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)db, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)db, false), pStb, &lino, _OVER); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, typeName, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, typeName, false), pStb, &lino, _OVER); // col name char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(colName, pStb->pColumns[i].name); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, colName, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, colName, false), pStb, &lino, _OVER); // col type int8_t colType = pStb->pColumns[i].type; @@ -3805,10 +3853,11 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB (int32_t)((pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(colTypeStr, colTypeLen); - (void)colDataSetVal(pColInfo, numOfRows, (char *)colTypeStr, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (char *)colTypeStr, false), pStb, &lino, _OVER); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->pColumns[i].bytes, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->pColumns[i].bytes, false), pStb, + &lino, _OVER); while (cols < pShow->numOfColumns) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetNULL(pColInfo, numOfRows); @@ -3824,9 +3873,15 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB } } - pShow->numOfRows += numOfRows; mDebug("mndRetrieveStbCol success, rows:%d, pShow->numOfRows:%d", numOfRows, pShow->numOfRows); + goto _OVER; +_ERROR: + mError("failed to mndRetrieveStbCol, rows:%d, pShow->numOfRows:%d, at %s:%d since %s", numOfRows, pShow->numOfRows, + __FUNCTION__, lino, tstrerror(code)); + +_OVER: + pShow->numOfRows += numOfRows; return numOfRows; } @@ -4111,8 +4166,9 @@ static void *mndBuildVDropTbsReq(SMnode *pMnode, const SVgroupInfo *pVgInfo, con void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead)); - (void)tEncodeSVDropTbBatchReq(&encoder, pReq); + int32_t code = tEncodeSVDropTbBatchReq(&encoder, pReq); tEncoderClear(&encoder); + if (code != 0) return NULL; *len = contLen; return pHead; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 2d979c3327..442b9bb980 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -2800,7 +2800,7 @@ static int32_t mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, i void *key = taosHashGetKey(value, &keyLen); char dbName[TSDB_DB_NAME_LEN] = {0}; - mndExtractShortDbNameFromStbFullName(key, dbName); + (void)mndExtractShortDbNameFromStbFullName(key, dbName); char dbNameContent[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(dbNameContent, dbName, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); From b71e7f402effc87564d05c87992b6ab7f707fcc7 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 5 Sep 2024 10:38:04 +0000 Subject: [PATCH 02/94] fix/TD-31891-remove-void-mndSync --- source/dnode/mnode/impl/src/mndSync.c | 44 ++++++++++++++++++++------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index d669994128..51d27d3c6d 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -238,7 +238,9 @@ static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { pMgmt->transSec = 0; pMgmt->transSeq = 0; pMgmt->errCode = code; - (void)tsem_post(&pMgmt->syncSem); + if (tsem_post(&pMgmt->syncSem) < 0) { + mError("trans:%d, failed to post sem", transId); + } if (pMgmt->errCode != 0) { mError("trans:%d, failed to propose since %s, post sem", transId, tstrerror(pMgmt->errCode)); @@ -310,11 +312,15 @@ void mndRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) { } else { mInfo("vgId:1, sync restore finished"); } - (void)mndRefreshUserIpWhiteList(pMnode); + int32_t code = mndRefreshUserIpWhiteList(pMnode); + if (code != 0) { + mError("vgId:1, failed to refresh user ip white list since %s", tstrerror(code)); + mndSetRestored(pMnode, false); + } SyncIndex fsmIndex = mndSyncAppliedIndex(pFsm); if (commitIdx != fsmIndex) { - mError("vgId:1, sync restore finished, but commitIdx:%" PRId64 " is not equal to appliedIdx:%" PRId64, commitIdx, + mError("vgId:1, failed to sync restore, commitIdx:%" PRId64 " is not equal to appliedIdx:%" PRId64, commitIdx, fsmIndex); mndSetRestored(pMnode, false); } @@ -368,7 +374,9 @@ static void mndBecomeFollower(const SSyncFSM *pFsm) { pMgmt->transSec = 0; pMgmt->transSeq = 0; pMgmt->errCode = TSDB_CODE_SYN_NOT_LEADER; - (void)tsem_post(&pMgmt->syncSem); + if (tsem_post(&pMgmt->syncSem) < 0) { + mError("failed to post sem"); + } } (void)taosThreadMutexUnlock(&pMgmt->lock); @@ -387,7 +395,9 @@ static void mndBecomeLearner(const SSyncFSM *pFsm) { pMgmt->transSec = 0; pMgmt->transSeq = 0; pMgmt->errCode = TSDB_CODE_SYN_NOT_LEADER; - (void)tsem_post(&pMgmt->syncSem); + if (tsem_post(&pMgmt->syncSem) < 0) { + mError("failed to post sem"); + } } (void)taosThreadMutexUnlock(&pMgmt->lock); } @@ -487,13 +497,19 @@ int32_t mndInitSync(SMnode *pMnode) { pNode->nodePort = pMgmt->replicas[i].port; tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); pNode->nodeRole = pMgmt->nodeRoles[i]; - (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + if (tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort) != true) { + mError("failed to open sync, tmsgUpdateDnodeInfo"); + TAOS_RETURN(-1); + } mInfo("vgId:1, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); } int32_t code = 0; - (void)tsem_init(&pMgmt->syncSem, 0, 0); + if ((code = tsem_init(&pMgmt->syncSem, 0, 0)) < 0) { + mError("failed to open sync, tsem_init, since %s", tstrerror(code)); + TAOS_RETURN(code); + } pMgmt->sync = syncOpen(&syncInfo, 1); // always check if (pMgmt->sync <= 0) { if (terrno != 0) code = terrno; @@ -511,7 +527,9 @@ void mndCleanupSync(SMnode *pMnode) { syncStop(pMgmt->sync); mInfo("mnode-sync is stopped, id:%" PRId64, pMgmt->sync); - (void)tsem_destroy(&pMgmt->syncSem); + if (tsem_destroy(&pMgmt->syncSem) < 0) { + mError("failed to destroy sem"); + } (void)taosThreadMutexDestroy(&pMgmt->lock); memset(pMgmt, 0, sizeof(SSyncMgmt)); } @@ -531,7 +549,9 @@ void mndSyncCheckTimeout(SMnode *pMnode) { pMgmt->transSeq = 0; terrno = TSDB_CODE_SYN_TIMEOUT; pMgmt->errCode = TSDB_CODE_SYN_TIMEOUT; - (void)tsem_post(&pMgmt->syncSem); + if (tsem_post(&pMgmt->syncSem) < 0) { + mError("failed to post sem"); + } } else { mDebug("trans:%d, waiting for sync confirm, start:%d cur:%d delta:%d seq:%" PRId64, pMgmt->transId, pMgmt->transSec, curSec, curSec - pMgmt->transSec, pMgmt->transSeq); @@ -572,7 +592,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { mInfo("trans:%d, is proposing and wait sem, seq:%" PRId64, transId, seq); pMgmt->transSeq = seq; (void)taosThreadMutexUnlock(&pMgmt->lock); - (void)tsem_wait(&pMgmt->syncSem); + code = tsem_wait(&pMgmt->syncSem); } else if (code > 0) { mInfo("trans:%d, confirm at once since replica is 1, continue execute", transId); pMgmt->transId = 0; @@ -624,7 +644,9 @@ void mndSyncStop(SMnode *pMnode) { pMgmt->transId = 0; pMgmt->transSec = 0; pMgmt->errCode = TSDB_CODE_APP_IS_STOPPING; - (void)tsem_post(&pMgmt->syncSem); + if (tsem_post(&pMgmt->syncSem) < 0) { + mError("failed to post sem"); + } } (void)taosThreadMutexUnlock(&pMgmt->lock); } From 7a2656636226db04b2dd79d746af5051dfb41b25 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 5 Sep 2024 11:39:49 +0000 Subject: [PATCH 03/94] fix/TD-31891-remove-void-mnode1 --- source/dnode/mnode/impl/src/mndCompact.c | 42 ++++++++++++++++++---- source/dnode/mnode/impl/src/mndDnode.c | 45 +++++++++++++++++------- source/dnode/mnode/impl/src/mndFunc.c | 11 +++--- source/dnode/mnode/impl/src/mndSma.c | 12 ++++--- 4 files changed, 82 insertions(+), 28 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index b8073885fd..76b1deb28b 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -260,7 +260,11 @@ int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompac sdbFreeRaw(pVgRaw); TAOS_RETURN(code); } - (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + + if ((code = sdbSetRawStatus(pVgRaw, SDB_STATUS_READY)) != 0) { + sdbFreeRaw(pVgRaw); + TAOS_RETURN(code); + } rsp->compactId = pCompact->compactId; @@ -349,7 +353,10 @@ static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pC pHead->contLen = htonl(contLen); pHead->vgId = htonl(pVgroup->vgId); - (void)tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req); + if ((contLen = tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req)) <= 0) { + terrno = contLen; + return NULL; + } *pContLen = contLen; return pReq; } @@ -413,7 +420,10 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa mndTransDrop(pTrans); TAOS_RETURN(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)) != 0) { + mndTransDrop(pTrans); + TAOS_RETURN(code); + } void *pIter = NULL; while (1) { @@ -606,7 +616,11 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) { pHead->contLen = htonl(contLen); pHead->vgId = htonl(pDetail->vgId); - (void)tSerializeSQueryCompactProgressReq((char *)pHead + sizeof(SMsgHead), contLen - sizeof(SMsgHead), &req); + if ((contLen = tSerializeSQueryCompactProgressReq((char *)pHead + sizeof(SMsgHead), contLen - sizeof(SMsgHead), + &req)) <= 0) { + sdbRelease(pMnode->pSdb, pDetail); + continue; + } SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS, .contLen = contLen}; @@ -711,7 +725,12 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { mndTransDrop(pTrans); TAOS_RETURN(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)) != 0) { + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDetail); + mndTransDrop(pTrans); + TAOS_RETURN(code); + } } sdbRelease(pMnode->pSdb, pDetail); @@ -774,7 +793,12 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { mndTransDrop(pTrans); TAOS_RETURN(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)) != 0) { + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDetail); + mndTransDrop(pTrans); + TAOS_RETURN(code); + } mInfo("compact:%d, add drop compactdetail action", pDetail->compactDetailId); } @@ -801,7 +825,11 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { mndTransDrop(pTrans); TAOS_RETURN(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)) != 0) { + mError("compact:%d, trans:%d, failed to append commit log since %s", compactId, pTrans->id, terrstr()); + mndTransDrop(pTrans); + TAOS_RETURN(code); + } mInfo("compact:%d, add drop compact action", pCompact->compactId); } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 71952b3bb8..a5a21f6b26 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -182,7 +182,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER); - (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); + TAOS_CHECK_GOTO(sdbSetRawStatus(pRaw, SDB_STATUS_READY), NULL, _OVER); pRaw = NULL; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); @@ -621,7 +621,10 @@ static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) { TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - (void)tSerializeSDnodeInfoReq(pReq, contLen, &infoReq); + if ((contLen = tSerializeSDnodeInfoReq(pReq, contLen, &infoReq)) <= 0) { + code = contLen; + goto _exit; + } SRpcMsg rpcMsg = {.msgType = TDMT_MND_UPDATE_DNODE_INFO, .pCont = pReq, .contLen = contLen}; TAOS_CHECK_EXIT(tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg)); @@ -661,7 +664,7 @@ static int32_t mndProcessUpdateDnodeInfoReq(SRpcMsg *pReq) { mError("trans:%d, failed to append commit log since %s", pTrans->id, tstrerror(code)); TAOS_CHECK_EXIT(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + TAOS_CHECK_EXIT(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); pCommitRaw = NULL; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) { @@ -874,8 +877,12 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { int32_t contLen = tSerializeSStatusRsp(NULL, 0, &statusRsp); void *pHead = rpcMallocCont(contLen); - (void)tSerializeSStatusRsp(pHead, contLen, &statusRsp); + contLen = tSerializeSStatusRsp(pHead, contLen, &statusRsp); taosArrayDestroy(statusRsp.pDnodeEps); + if (contLen < 0) { + code = contLen; + goto _OVER; + } pReq->info.rspLen = contLen; pReq->info.rsp = pHead; @@ -888,8 +895,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { _OVER: mndReleaseDnode(pMnode, pDnode); taosArrayDestroy(statusReq.pVloads); - (void)mndUpdClusterInfo(pReq); - return code; + return mndUpdClusterInfo(pReq); } static int32_t mndProcessNotifyReq(SRpcMsg *pReq) { @@ -955,7 +961,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER); - (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); + TAOS_CHECK_GOTO(sdbSetRawStatus(pRaw, SDB_STATUS_READY), NULL, _OVER); pRaw = NULL; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); @@ -1010,7 +1016,10 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { goto _OVER; } - (void)tSerializeSDnodeListRsp(pRsp, rspLen, &rsp); + if ((rspLen = tSerializeSDnodeListRsp(pRsp, rspLen, &rsp)) <= 0) { + code = rspLen; + goto _OVER; + } pReq->info.rspLen = rspLen; pReq->info.rsp = pRsp; @@ -1151,7 +1160,10 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { goto _OVER; } - (void)tSerializeSShowVariablesRsp(pRsp, rspLen, &rsp); + if ((rspLen = tSerializeSShowVariablesRsp(pRsp, rspLen, &rsp)) <= 0) { + code = rspLen; + goto _OVER; + } pReq->info.rspLen = rspLen; pReq->info.rsp = pRsp; @@ -1247,7 +1259,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRaw), NULL, _OVER); - (void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING); + TAOS_CHECK_GOTO(sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING), NULL, _OVER); pRaw = NULL; pRaw = mndDnodeActionEncode(pDnode); @@ -1257,7 +1269,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER); - (void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); + TAOS_CHECK_GOTO(sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED), NULL, _OVER); pRaw = NULL; if (pMObj != NULL) { @@ -1450,7 +1462,10 @@ static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq void *pBuf = rpcMallocCont(bufLen); if (pBuf != NULL) { - (void)tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq); + if ((bufLen = tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq)) <= 0) { + code = bufLen; + return code; + } mInfo("dnode:%d, send config req to dnode, config:%s value:%s", dnodeId, pDcfgReq->config, pDcfgReq->value); SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; code = tmsgSendReq(&epSet, &rpcMsg); @@ -1590,7 +1605,11 @@ static int32_t mndProcessCreateEncryptKeyReqImpl(SRpcMsg *pReq, int32_t dnodeId, void *pBuf = rpcMallocCont(bufLen); if (pBuf != NULL) { - (void)tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq); + if ((bufLen = tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq)) <= 0) { + code = bufLen; + sdbRelease(pSdb, pDnode); + goto _exit; + } SRpcMsg rpcMsg = {.msgType = TDMT_DND_CREATE_ENCRYPT_KEY, .pCont = pBuf, .contLen = bufLen}; if (0 == tmsgSendReq(&epSet, &rpcMsg)) { (void)atomic_add_fetch_16(&pMnode->encryptMgmt.nEncrypt, 1); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 326f2ffa95..a9f48e1695 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -375,7 +375,7 @@ static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER); - (void)sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + TAOS_CHECK_GOTO(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING), NULL, _OVER); SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); if (pUndoRaw == NULL) { @@ -384,7 +384,7 @@ static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER); - (void)sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + TAOS_CHECK_GOTO(sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY), NULL, _OVER); SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); if (pCommitRaw == NULL) { @@ -393,7 +393,7 @@ static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER); - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + TAOS_CHECK_GOTO(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED), NULL, _OVER); TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); @@ -598,7 +598,10 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { goto RETRIEVE_FUNC_OVER; } - (void)tSerializeSRetrieveFuncRsp(pRsp, contLen, &retrieveRsp); + if ((contLen = tSerializeSRetrieveFuncRsp(pRsp, contLen, &retrieveRsp)) <= 0) { + code = contLen; + goto RETRIEVE_FUNC_OVER; + } pReq->info.rsp = pRsp; pReq->info.rspLen = contLen; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 284d65cd9c..6fd7f180b4 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -368,14 +368,18 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, SEncoder encoder = {0}; int32_t contLen; SName name = {0}; - (void)tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + int32_t ret = 0; + + if ((ret = tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) != 0) { + terrno = ret; + return NULL; + } SVDropTSmaReq req = {0}; req.indexUid = pSma->uid; tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); // get length - int32_t ret = 0; tEncodeSize(tEncodeSVDropTSmaReq, &req, contLen, ret); if (ret < 0) { return NULL; @@ -1669,7 +1673,7 @@ static int32_t mndSetUpdateDbTsmaVersionPrepareLogs(SMnode *pMnode, STrans *pTra TAOS_RETURN(code); } - (void)sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + TAOS_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY)); TAOS_RETURN(code); } @@ -1686,7 +1690,7 @@ static int32_t mndSetUpdateDbTsmaVersionCommitLogs(SMnode *pMnode, STrans *pTran TAOS_RETURN(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + TAOS_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); TAOS_RETURN(code); } From 23507b0eacf2f4e8b6c1b21b04bc4d3a829353dc Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 6 Sep 2024 10:16:15 +0800 Subject: [PATCH 04/94] fix:[TD-31899] remove void(func) --- include/common/tname.h | 2 +- source/client/src/clientHb.c | 5 +- source/client/src/clientMain.c | 9 +- source/client/src/clientMonitor.c | 73 +++++++-- source/client/src/clientRawBlockWrite.c | 38 +++-- source/client/src/clientSml.c | 12 +- source/client/src/clientStmt.c | 5 +- source/client/src/clientTmq.c | 54 ++++-- source/common/src/tname.c | 6 +- source/dnode/mnode/impl/src/mndConsumer.c | 2 +- source/dnode/mnode/impl/src/mndSubscribe.c | 24 +-- source/dnode/mnode/impl/src/mndTopic.c | 21 ++- source/dnode/vnode/src/tq/tq.c | 12 +- source/dnode/vnode/src/tq/tqMeta.c | 53 ++++-- source/dnode/vnode/src/tq/tqSnapshot.c | 5 +- source/libs/catalog/test/catalogTests.cpp | 2 +- source/libs/parser/src/parAuthenticator.c | 7 +- source/libs/parser/src/parTranslater.c | 182 +++++++++++---------- 18 files changed, 329 insertions(+), 183 deletions(-) diff --git a/include/common/tname.h b/include/common/tname.h index f0a68f28c4..426cb2c2af 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -37,7 +37,7 @@ typedef struct SName { char tname[TSDB_TABLE_NAME_LEN]; } SName; -SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName); +void toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName); int32_t tNameExtractFullName(const SName* name, char* dst); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 5d3892d5e0..180c1f63b1 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -1636,7 +1636,10 @@ void hbDeregisterConn(STscObj *pTscObj, SClientHbKey connKey) { SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (pReq) { tFreeClientHbReq(pReq); - (void)taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + if (TSDB_CODE_SUCCESS != code) { + tscError("hbDeregisterConn taosHashRemove error, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code))); + } taosHashRelease(pAppHbMgr->activeInfo, pReq); (void)atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 4a78ce957d..4fc42e0530 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1313,7 +1313,10 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { if (NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d,QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); - (void)refreshMeta(pRequest->pTscObj, pRequest); // ignore return code,try again + code = refreshMeta(pRequest->pTscObj, pRequest); + if (code != 0){ + uInfo("refresh meta error code:%d, msg:%s", code, tstrerror(code)); + } pRequest->prevCode = code; doAsyncQuery(pRequest, true); return; @@ -1492,8 +1495,8 @@ int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - SName tableName; - (void)toName(pTscObj->acctId, db, table, &tableName); + SName tableName = {0}; + toName(pTscObj->acctId, db, table, &tableName); SVgroupInfo vgInfo; code = catalogGetTableHashVgroup(pCtg, &conn, &tableName, &vgInfo); diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 612f57ecdd..a06eb1d09a 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -68,9 +68,14 @@ static void destroyMonitorClient(void* data) { if (pMonitor == NULL) { return; } - (void)taosTmrStopA(&pMonitor->timer); + if (!taosTmrStopA(&pMonitor->timer)) { + tscError("failed to stop timer, pMonitor:%p", pMonitor); + } taosHashCleanup(pMonitor->counters); - (void)taos_collector_registry_destroy(pMonitor->registry); + int ret = taos_collector_registry_destroy(pMonitor->registry); + if (ret){ + tscError("failed to destroy registry, pMonitor:%p ret:%d", pMonitor, ret); + } taosMemoryFree(pMonitor); } @@ -186,7 +191,10 @@ static void generateClusterReport(taos_collector_registry_t* registry, void* pTr } if (strlen(pCont) != 0 && sendReport(pTransporter, epSet, pCont, MONITOR_TYPE_COUNTER, NULL) == 0) { - (void)taos_collector_registry_clear_batch(registry); + int ret = taos_collector_registry_clear_batch(registry); + if (ret){ + tscError("failed to clear registry, ret:%d", ret); + } } taosMemoryFreeClear(pCont); } @@ -207,7 +215,10 @@ static void reportSendProcess(void* param, void* tmrId) { SEpSet ep = getEpSet_s(&pInst->mgmtEp); generateClusterReport(pMonitor->registry, pInst->pTransporter, &ep); - (void)taosTmrReset(reportSendProcess, pInst->monitorParas.tsMonitorInterval * 1000, param, monitorTimer, &tmrId); + bool reset = taosTmrReset(reportSendProcess, pInst->monitorParas.tsMonitorInterval * 1000, param, monitorTimer, &tmrId); + if (!reset){ + tscError("failed to reset timer, pMonitor:%p", pMonitor); + } taosRUnLockLatch(&monitorLock); } @@ -255,7 +266,11 @@ void monitorCreateClient(int64_t clusterId) { goto fail; } - (void)taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector); + int r = taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector); + if (r){ + tscError("failed to register collector, ret:%d", r); + goto fail; + } pMonitor->counters = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pMonitor->counters == NULL) { @@ -304,12 +319,18 @@ void monitorCreateClientCounter(int64_t clusterId, const char* name, const char* MonitorClient* pMonitor = *ppMonitor; if (taos_collector_add_metric(pMonitor->colector, newCounter) != 0){ tscError("failed to add metric to collector"); - (void)taos_counter_destroy(newCounter); + int r = taos_counter_destroy(newCounter); + if (r){ + tscError("failed to destroy counter, code: %d", r); + } goto end; } if (taosHashPut(pMonitor->counters, name, strlen(name), &newCounter, POINTER_BYTES) != 0) { tscError("failed to put counter to monitor"); - (void)taos_counter_destroy(newCounter); + int r = taos_counter_destroy(newCounter); + if (r){ + tscError("failed to destroy counter, code: %d", r); + } goto end; } tscInfo("[monitor] monitorCreateClientCounter %" PRIx64 "(%p):%s : %p.", pMonitor->clusterId, pMonitor, name, @@ -374,7 +395,10 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char* tmpP SlowLogClient* pClient = taosMemoryCalloc(1, sizeof(SlowLogClient)); if (pClient == NULL) { tscError("failed to allocate memory for slow log client"); - (void)taosCloseFile(&pFile); + int32_t ret = taosCloseFile(&pFile); + if (ret != 0){ + tscError("failed to close file:%p ret:%d", pFile, ret); + } return; } pClient->lastCheckTime = taosGetMonoTimestampMs(); @@ -383,7 +407,10 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char* tmpP pClient->pFile = pFile; if (taosHashPut(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES, &pClient, POINTER_BYTES) != 0) { tscError("failed to put clusterId:%" PRId64 " to hash table", slowLogData->clusterId); - (void)taosCloseFile(&pFile); + int32_t ret = taosCloseFile(&pFile); + if (ret != 0){ + tscError("failed to close file:%p ret:%d", pFile, ret); + } taosMemoryFree(pClient); return; } @@ -608,7 +635,11 @@ static void processFileRemoved(SlowLogClient* pClient) { tscError("failed to unlock file:%s since %d", pClient->path, errno); return; } - (void)taosCloseFile(&(pClient->pFile)); + int32_t ret = taosCloseFile(&(pClient->pFile)); + if (ret != 0){ + tscError("failed to close file:%p ret:%d", pClient->pFile, ret); + return; + } TdFilePtr pFile = taosOpenFile(pClient->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_READ | TD_FILE_TRUNC); @@ -697,7 +728,10 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId) { } if (taosLockFile(pFile) < 0) { tscInfo("failed to lock file:%s since %s, maybe used by other process", filename, terrstr()); - (void)taosCloseFile(&pFile); + int32_t ret = taosCloseFile(&pFile); + if (ret != 0){ + tscError("failed to close file:%p ret:%d", pFile, ret); + } continue; } char* tmp = taosStrdup(filename); @@ -705,7 +739,10 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId) { taosMemoryFree(tmp); } - (void)taosCloseDir(&pDir); + int32_t ret = taosCloseDir(&pDir); + if (ret != 0){ + tscError("failed to close dir, ret:%d", ret); + } } static void* monitorThreadFunc(void* param) { @@ -729,7 +766,9 @@ static void* monitorThreadFunc(void* param) { } MonitorSlowLogData* slowLogData = NULL; - (void)taosReadQitem(monitorQueue, (void**)&slowLogData); + if (taosReadQitem(monitorQueue, (void**)&slowLogData) == 0){ + tscDebug("monitorThreadFunc get slow log data from queue null"); + } if (slowLogData != NULL) { if (slowLogData->type == SLOW_LOG_READ_BEGINNIG && quitCnt == 0) { if (slowLogData->pFile != NULL) { @@ -850,7 +889,9 @@ void monitorClose() { taosHashCleanup(monitorSlowLogHash); taosTmrCleanUp(monitorTimer); taosCloseQueue(monitorQueue); - (void)tsem2_destroy(&monitorSem); + if(tsem2_destroy(&monitorSem) != 0) { + tscError("failed to destroy semaphore"); + } taosWUnLockLatch(&monitorLock); } @@ -872,7 +913,9 @@ int32_t monitorPutData2MonitorQueue(MonitorSlowLogData data) { tscDebug("[monitor] write slow log to queue, clusterId:%" PRIx64 " type:%s, data:%s", slowLogData->clusterId, queueTypeStr[slowLogData->type], slowLogData->data); if (taosWriteQitem(monitorQueue, slowLogData) == 0) { - (void)tsem2_post(&monitorSem); + if(tsem2_post(&monitorSem) != 0) { + tscError("failed to post semaphore"); + } } else { if (taosCloseFile(&(slowLogData->pFile)) != 0) { tscError("failed to close file:%p", slowLogData->pFile); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index d5785cce6b..43e17b381b 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -946,7 +946,8 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { pReq.suid); STscObj* pTscObj = pRequest->pTscObj; SName tableName = {0}; - RAW_RETURN_CHECK(tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name)); + toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName); + RAW_RETURN_CHECK(tNameExtractFullName(&tableName, pReq.name)); SCmdMsgInfo pCmdMsg = {0}; pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; @@ -959,6 +960,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { RAW_NULL_CHECK(pCmdMsg.pMsg); if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0) { code = TSDB_CODE_INVALID_PARA; + taosMemoryFree(pCmdMsg.pMsg); goto end; } @@ -970,15 +972,15 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest + taosMemoryFree(pCmdMsg.pMsg); + if (pRequest->code == TSDB_CODE_SUCCESS) { SCatalog* pCatalog = NULL; - // ignore the return value - (void)catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); - (void)catalogRemoveTableMeta(pCatalog, &tableName); + RAW_RETURN_CHECK(catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog)); + RAW_RETURN_CHECK(catalogRemoveTableMeta(pCatalog, &tableName)); } code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); end: uDebug(LOG_ID_TAG " create stable return, msg:%s", LOG_ID_VALUE, tstrerror(code)); @@ -1021,8 +1023,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { .requestObjRefId = pRequest->self, .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)}; SName pName = {0}; - (void)toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, - &pName); // ignore the return value, always return pName + toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); STableMeta* pTableMeta = NULL; code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -1045,7 +1046,8 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { pReq.suid); STscObj* pTscObj = pRequest->pTscObj; SName tableName = {0}; - if (tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name) != 0) { + toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName); + if (tNameExtractFullName(&tableName, pReq.name) != 0) { code = TSDB_CODE_INVALID_PARA; goto end; } @@ -1062,6 +1064,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { RAW_NULL_CHECK(pCmdMsg.pMsg); if (tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0) { code = TSDB_CODE_INVALID_PARA; + taosMemoryFree(pCmdMsg.pMsg); goto end; } @@ -1072,15 +1075,14 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { pQuery.stableQuery = true; (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest - + taosMemoryFree(pCmdMsg.pMsg); if (pRequest->code == TSDB_CODE_SUCCESS) { // ignore the error code - (void)catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); - (void)catalogRemoveTableMeta(pCatalog, &tableName); + RAW_RETURN_CHECK(catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog)); + RAW_RETURN_CHECK(catalogRemoveTableMeta(pCatalog, &tableName)); } code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); end: uDebug(LOG_ID_TAG " drop stable return, msg:%s", LOG_ID_VALUE, tstrerror(code)); @@ -1150,7 +1152,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { SVgroupInfo pInfo = {0}; SName pName = {0}; - (void)toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); + toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -1163,7 +1165,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { SName sName = {0}; tb_uid_t oldSuid = pCreateReq->ctb.suid; // pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb); - (void)toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName); + toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName); code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { code = TSDB_CODE_SUCCESS; @@ -1228,7 +1230,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { (void)launchQueryImpl(pRequest, pQuery, true, NULL); if (pRequest->code == TSDB_CODE_SUCCESS) { - (void)removeMeta(pTscObj, pRequest->tableList, false); + RAW_RETURN_CHECK(removeMeta(pTscObj, pRequest->tableList, false)); } code = pRequest->code; @@ -1307,7 +1309,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { SVgroupInfo pInfo = {0}; SName pName = {0}; - (void)toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); + toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo)); STableMeta* pTableMeta = NULL; @@ -1357,7 +1359,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { (void)launchQueryImpl(pRequest, pQuery, true, NULL); if (pRequest->code == TSDB_CODE_SUCCESS) { - (void)removeMeta(pTscObj, pRequest->tableList, false); + RAW_RETURN_CHECK(removeMeta(pTscObj, pRequest->tableList, false)); } code = pRequest->code; @@ -1451,7 +1453,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { SVgroupInfo pInfo = {0}; SName pName = {0}; - (void)toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName); + toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName); RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo)); pArray = taosArrayInit(1, sizeof(void*)); RAW_NULL_CHECK(pArray); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 05678e1cbf..4caba4cc07 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -116,7 +116,7 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char return TSDB_CODE_SML_INVALID_DATA; } } else { - (void)toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); //ignore + toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); } pAuth.type = type; @@ -1113,7 +1113,10 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.commentLen = -1; pReq.igExists = true; - (void)tNameExtractFullName(pName, pReq.name); + code = tNameExtractFullName(pName, pReq.name); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; @@ -2214,9 +2217,12 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, break; } taosMsleep(100); - (void)refreshMeta(request->pTscObj, request); //ignore return code,try again uInfo("SML:%" PRIx64 " retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code, tstrerror(code)); + code = refreshMeta(request->pTscObj, request); + if (code != 0){ + uInfo("SML:%" PRIx64 " refresh meta error code:%d, msg:%s", info->id, code, tstrerror(code)); + } smlDestroyInfo(info); info = NULL; taos_free_result(request); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 9f7aeabbe4..de9e8bbfa7 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -772,7 +772,10 @@ void* stmtBindThreadFunc(void* param) { continue; } - (void)stmtAsyncOutput(pStmt, asyncParam); + int ret = stmtAsyncOutput(pStmt, asyncParam); + if (ret != 0){ + qError("stmtAsyncOutput failed, reason:%s", tstrerror(ret)); + } } qInfo("stmt bind thread stopped"); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 3927172b61..529392ebfc 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -797,11 +797,16 @@ static void generateTimedTask(int64_t refId, int32_t type) { if (code == TSDB_CODE_SUCCESS) { *pTaskType = type; if (taosWriteQitem(tmq->delayedTask, pTaskType) == 0) { - (void)tsem2_post(&tmq->rspSem); + if (tsem2_post(&tmq->rspSem) != 0){ + tscError("consumer:0x%" PRIx64 " failed to post sem, type:%d", tmq->consumerId, type); + } } } - (void)taosReleaseRef(tmqMgmt.rsetId, refId); + code = taosReleaseRef(tmqMgmt.rsetId, refId); + if (code != 0){ + tscError("failed to release ref:%"PRId64 ", type:%d, code:%d", refId, type, code); + } } void tmqAssignAskEpTask(void* param, void* tmrId) { @@ -814,8 +819,13 @@ void tmqReplayTask(void* param, void* tmrId) { tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) return; - (void)tsem2_post(&tmq->rspSem); - (void)taosReleaseRef(tmqMgmt.rsetId, refId); + if (tsem2_post(&tmq->rspSem) != 0){ + tscError("consumer:0x%" PRIx64 " failed to post sem, replay", tmq->consumerId); + } + int32_t code = taosReleaseRef(tmqMgmt.rsetId, refId); + if (code != 0){ + tscError("failed to release ref:%"PRId64 ", code:%d", refId, code); + } } void tmqAssignDelayedCommitTask(void* param, void* tmrId) { @@ -825,17 +835,17 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) { int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { if (code != 0) { - goto _return; + goto END; } if (pMsg == NULL || param == NULL) { code = TSDB_CODE_INVALID_PARA; - goto _return; + goto END; } SMqHbRsp rsp = {0}; code = tDeserializeSMqHbRsp(pMsg->pData, pMsg->len, &rsp); if (code != 0) { - goto _return; + goto END; } int64_t refId = (int64_t)param; @@ -856,13 +866,15 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { } } taosWUnLockLatch(&tmq->lock); - (void)taosReleaseRef(tmqMgmt.rsetId, refId); + code = taosReleaseRef(tmqMgmt.rsetId, refId); + if (code != 0){ + tscError("failed to release ref:%"PRId64 ", code:%d", refId, code); + } } tDestroySMqHbRsp(&rsp); -_return: - +END: taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); return code; @@ -967,7 +979,10 @@ OVER: if (tmrId != NULL) { (void)taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer); } - (void)taosReleaseRef(tmqMgmt.rsetId, refId); + int32_t ret = taosReleaseRef(tmqMgmt.rsetId, refId); + if (ret != 0){ + tscError("failed to release ref:%"PRId64 ", code:%d", refId, ret); + } } static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) { @@ -1086,7 +1101,9 @@ int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { if (pMsg) { taosMemoryFree(pMsg->pEpSet); } - (void)tsem2_post(&pParam->rspSem); + if (tsem2_post(&pParam->rspSem) != 0){ + tscError("failed to post sem, subscribe cb"); + } return 0; } @@ -1658,7 +1675,10 @@ END: if (tmq) (void)tsem2_post(&tmq->rspSem); if (pMsg) taosMemoryFreeClear(pMsg->pData); if (pMsg) taosMemoryFreeClear(pMsg->pEpSet); - (void)taosReleaseRef(tmqMgmt.rsetId, refId); + ret = taosReleaseRef(tmqMgmt.rsetId, refId); + if (ret != 0){ + tscError("failed to release ref:%"PRId64 ", code:%d", refId, ret); + } return code; } @@ -2903,8 +2923,12 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { } END: - (void)taosReleaseRef(tmqMgmt.rsetId, pParam->refId); - + { + int32_t ret = taosReleaseRef(tmqMgmt.rsetId, pParam->refId); + if (ret != 0){ + tscError("failed to release ref:%"PRId64 ", code:%d", pParam->refId, ret); + } + } FAIL: if (pParam->sync) { SAskEpInfo* pInfo = pParam->pParam; diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 8d0f324509..1d60bab790 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -87,12 +87,14 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in #endif -SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { +void toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { + if (pName == NULL){ + return; + } pName->type = TSDB_TABLE_NAME_T; pName->acctId = acctId; snprintf(pName->dbname, sizeof(pName->dbname), "%s", pDbName); snprintf(pName->tname, sizeof(pName->tname), "%s", pTableName); - return pName; } int32_t tNameExtractFullName(const SName* name, char* dst) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 606b93035f..654dda1113 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -199,7 +199,7 @@ static void storeOffsetRows(SMnode *pMnode, SMqHbReq *req, SMqConsumerObj *pCons taosWLockLatch(&pSub->lock); SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t)); if (pConsumerEp) { - (void)taosArrayDestroy(pConsumerEp->offsetRows); + taosArrayDestroy(pConsumerEp->offsetRows); pConsumerEp->offsetRows = data->offsetRows; data->offsetRows = NULL; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e3bef61bc0..1e957dfee3 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -248,7 +248,7 @@ static int32_t processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash MND_TMQ_RETURN_CHECK(pushVgDataToHash(pConsumerEp->vgs, pHash, *consumerId, pOutput->pSub->key)); } - (void)taosArrayDestroy(pConsumerEp->vgs); + taosArrayDestroy(pConsumerEp->vgs); MND_TMQ_RETURN_CHECK(taosHashRemove(pOutput->pSub->consumerHash, consumerId, sizeof(int64_t))); MND_TMQ_NULL_CHECK(taosArrayPush(pOutput->removedConsumers, consumerId)); actualRemoved++; @@ -682,8 +682,8 @@ END: static void freeRebalanceItem(void *param) { SMqRebInfo *pInfo = param; - (void)taosArrayDestroy(pInfo->newConsumers); - (void)taosArrayDestroy(pInfo->removedConsumers); + taosArrayDestroy(pInfo->newConsumers); + taosArrayDestroy(pInfo->removedConsumers); } // type = 0 remove type = 1 add @@ -738,8 +738,12 @@ static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHash } SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); if (!pVgroup) { - (void)mndGetOrCreateRebSub(rebSubHash, key, NULL); - mInfo("vnode splitted, vgId:%d rebalance will be triggered", pVgEp->vgId); + code = mndGetOrCreateRebSub(rebSubHash, key, NULL); + if (code != 0){ + mError("failed to mndGetOrCreateRebSub vgroup:%d, error:%s", pVgEp->vgId, tstrerror(code)) + }else{ + mInfo("vnode splitted, vgId:%d rebalance will be triggered", pVgEp->vgId); + } } mndReleaseVgroup(pMnode, pVgroup); } @@ -813,10 +817,10 @@ void mndRebCntDec() { } static void clearRebOutput(SMqRebOutputObj *rebOutput) { - (void)taosArrayDestroy(rebOutput->newConsumers); - (void)taosArrayDestroy(rebOutput->modifyConsumers); - (void)taosArrayDestroy(rebOutput->removedConsumers); - (void)taosArrayDestroy(rebOutput->rebVgs); + taosArrayDestroy(rebOutput->newConsumers); + taosArrayDestroy(rebOutput->modifyConsumers); + taosArrayDestroy(rebOutput->removedConsumers); + taosArrayDestroy(rebOutput->rebVgs); tDeleteSubscribeObj(rebOutput->pSub); taosMemoryFree(rebOutput->pSub); } @@ -858,7 +862,7 @@ static int32_t checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) { mError("consumer:0x%" PRIx64 " not exists in sdb for exception", pConsumerEp->consumerId); MND_TMQ_NULL_CHECK(taosArrayAddAll(pSub->unassignedVgs, pConsumerEp->vgs)); - (void)taosArrayDestroy(pConsumerEp->vgs); + taosArrayDestroy(pConsumerEp->vgs); MND_TMQ_RETURN_CHECK(taosHashRemove(pSub->consumerHash, &pConsumerEp->consumerId, sizeof(int64_t))); } END: diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index ed702e7707..78aca0dbee 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -302,7 +302,7 @@ static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) { taosMemoryFreeClear(pTopic->ast); taosMemoryFreeClear(pTopic->physicalPlan); if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema); - (void)taosArrayDestroy(pTopic->ntbColIds); + taosArrayDestroy(pTopic->ntbColIds); return 0; } @@ -467,7 +467,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * MND_TMQ_NULL_CHECK(topicObj.ntbColIds); MND_TMQ_RETURN_CHECK(extractTopicTbInfo(pAst, &topicObj)); if (topicObj.ntbUid == 0) { - (void)taosArrayDestroy(topicObj.ntbColIds); + taosArrayDestroy(topicObj.ntbColIds); topicObj.ntbColIds = NULL; } @@ -505,7 +505,7 @@ END: taosMemoryFreeClear(topicObj.physicalPlan); taosMemoryFreeClear(topicObj.sql); taosMemoryFreeClear(topicObj.ast); - (void)taosArrayDestroy(topicObj.ntbColIds); + taosArrayDestroy(topicObj.ntbColIds); if (topicObj.schema.nCols) { taosMemoryFreeClear(topicObj.schema.pSchema); } @@ -567,9 +567,15 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { { SName dbname = {0}; - (void)tNameFromString(&dbname, createTopicReq.subDbName, T_NAME_ACCT | T_NAME_DB); // ignore error + int32_t ret = tNameFromString(&dbname, createTopicReq.subDbName, T_NAME_ACCT | T_NAME_DB); + if (ret != 0){ + mError("failed to parse db name:%s, ret:%d", createTopicReq.subDbName, ret); + } SName topicName = {0}; - (void)tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); // ignore error + ret = tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (ret != 0){ + mError("failed to parse topic name:%s, ret:%d", createTopicReq.name, ret); + } auditRecord(pReq, pMnode->clusterId, "createTopic", dbname.dbname, topicName.dbname, createTopicReq.sql, strlen(createTopicReq.sql)); } @@ -735,7 +741,10 @@ END: } SName name = {0}; - (void)tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); // ignore error + int32_t ret = tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (ret != 0) { + mError("topic:%s, failed to drop since %s", dropReq.name, tstrerror(ret)); + } auditRecord(pReq, pMnode->clusterId, "dropTopic", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen); tFreeSMDropTopicReq(&dropReq); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 07daab4459..f7c2c4522c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -344,7 +344,10 @@ int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg) { .pCont = pHandle->msg->pCont, .contLen = pHandle->msg->contLen, .info = pHandle->msg->info}; - (void)tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg); + int32_t ret = tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg); + if (ret != 0){ + tqError("vgId:%d tmsgPutToQueue failed, consumer:0x%" PRIx64, vgId, pHandle->consumerId); + } taosMemoryFree(pHandle->msg); pHandle->msg = NULL; } @@ -643,7 +646,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tDecoderInit(&dc, (uint8_t*)msg, msgLen); ret = tDecodeSMqRebVgReq(&dc, &req); - // decode req if (ret < 0) { goto end; } @@ -653,7 +655,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg taosRLockLatch(&pTq->lock); STqHandle* pHandle = NULL; - (void)tqMetaGetHandle(pTq, req.subKey, &pHandle); // ignore return code + int32_t code = tqMetaGetHandle(pTq, req.subKey, &pHandle); + if (code != 0){ + tqInfo("vgId:%d, tq process sub req:%s, no such handle, create new one", pTq->pVnode->config.vgId, req.subKey); + } taosRUnLockLatch(&pTq->lock); if (pHandle == NULL) { if (req.oldConsumerId != -1) { @@ -662,6 +667,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } if (req.newConsumerId == -1) { tqError("vgId:%d, tq invalid rebalance request, new consumerId %" PRId64 "", req.vgId, req.newConsumerId); + ret = TSDB_CODE_INVALID_PARA; goto end; } STqHandle handle = {0}; diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index db7766a7bb..7b56ded81b 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -374,7 +374,10 @@ static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pOld, TTB* pNew) { END: tdbFree(pKey); tdbFree(pVal); - (void)tdbTbcClose(pCur); + int32_t ret = tdbTbcClose(pCur); + if (ret != 0) { + tqError("failed to close tbc, ret:%d", ret); + } return code; } @@ -446,7 +449,10 @@ static int32_t tqMetaRestoreCheckInfo(STQ* pTq) { END: tdbFree(pKey); tdbFree(pVal); - (void)tdbTbcClose(pCur); + int32_t ret = tdbTbcClose(pCur); + if (ret != 0) { + tqError("failed to close tbc, ret:%d", ret); + } tDeleteSTqCheckInfo(&info); return code; } @@ -461,13 +467,13 @@ int32_t tqMetaOpen(STQ* pTq) { TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq)); } else { TQ_ERR_GO_TO_END(tqMetaTransform(pTq)); - (void)taosRemoveFile(maindb); + TQ_ERR_GO_TO_END(taosRemoveFile(maindb)); } TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME)); if(taosCheckExistFile(offsetNew)){ TQ_ERR_GO_TO_END(tqOffsetRestoreFromFile(pTq, offsetNew)); - (void)taosRemoveFile(offsetNew); + TQ_ERR_GO_TO_END(taosRemoveFile(offsetNew)); } TQ_ERR_GO_TO_END(tqMetaRestoreCheckInfo(pTq)); @@ -503,7 +509,7 @@ int32_t tqMetaTransform(STQ* pTq) { if (taosCopyFile(offset, offsetNew) < 0) { tqError("copy offset file error"); } else { - (void)taosRemoveFile(offset); + TQ_ERR_GO_TO_END(taosRemoveFile(offset)); } } @@ -511,23 +517,44 @@ END: taosMemoryFree(offset); taosMemoryFree(offsetNew); - // return 0 always, so ignore - (void)tdbTbClose(pExecStore); - (void)tdbTbClose(pCheckStore); - (void)tdbClose(pMetaDB); + int32_t ret = tdbTbClose(pExecStore); + if (ret != 0) { + tqError("failed to close tb, ret:%d", ret); + } + ret = tdbTbClose(pCheckStore); + if (ret != 0) { + tqError("failed to close tb, ret:%d", ret); + } + ret = tdbClose(pMetaDB); + if (ret != 0) { + tqError("failed to close tdb, ret:%d", ret); + } return code; } void tqMetaClose(STQ* pTq) { + int32_t ret = 0; if (pTq->pExecStore) { - (void)tdbTbClose(pTq->pExecStore); + ret = tdbTbClose(pTq->pExecStore); + if (ret != 0) { + tqError("failed to close tb, ret:%d", ret); + } } if (pTq->pCheckStore) { - (void)tdbTbClose(pTq->pCheckStore); + ret = tdbTbClose(pTq->pCheckStore); + if (ret != 0) { + tqError("failed to close tb, ret:%d", ret); + } } if (pTq->pOffsetStore) { - (void)tdbTbClose(pTq->pOffsetStore); + ret = tdbTbClose(pTq->pOffsetStore); + if (ret != 0) { + tqError("failed to close tb, ret:%d", ret); + } + } + ret = tdbClose(pTq->pMetaDB); + if (ret != 0) { + tqError("failed to close tdb, ret:%d", ret); } - (void)tdbClose(pTq->pMetaDB); } diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index 1abc94f653..12b1e5ec19 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -77,7 +77,10 @@ _err: } void tqSnapReaderClose(STqSnapReader** ppReader) { - (void)tdbTbcClose((*ppReader)->pCur); + int32_t ret = tdbTbcClose((*ppReader)->pCur); + if (ret != 0){ + tqError("vgId:%d, vnode snapshot tq reader close failed since %s", TD_VID((*ppReader)->pTq->pVnode), tstrerror(ret)); + } taosMemoryFree(*ppReader); *ppReader = NULL; } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 534070c540..25c82b8452 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -2889,7 +2889,7 @@ TEST(apiTest, catalogChkAuth_test) { SUserAuthInfo authInfo = {0}; SUserAuthRes authRes = {0}; TAOS_STRCPY(authInfo.user, ctgTestUsername); - (void)toName(1, ctgTestDbname, ctgTestSTablename, &authInfo.tbName); + toName(1, ctgTestDbname, ctgTestSTablename, &authInfo.tbName); authInfo.type = AUTH_TYPE_READ; bool exists = false; code = catalogChkAuthFromCache(pCtg, &authInfo, &authRes, &exists); diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 60cc1024bf..65e26f10e0 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -46,7 +46,7 @@ static int32_t setUserAuthInfo(SParseContext* pCxt, const char* pDbName, const c int32_t code = tNameSetDbName(&pAuth->tbName, pCxt->acctId, pDbName, strlen(pDbName)); if (TSDB_CODE_SUCCESS != code) return code; } else { - (void)toName(pCxt->acctId, pDbName, pTabName, &pAuth->tbName); + toName(pCxt->acctId, pDbName, pTabName, &pAuth->tbName); } pAuth->type = type; pAuth->isView = isView; @@ -169,10 +169,11 @@ static EDealRes authSelectImpl(SNode* pNode, void* pContext) { SNode* pTagCond = NULL; STableNode* pTable = (STableNode*)pNode; #ifdef TD_ENTERPRISE - SName name; + SName name = {0}; + toName(pAuthCxt->pParseCxt->acctId, pTable->dbName, pTable->tableName, &name); STableMeta* pTableMeta = NULL; int32_t code = getTargetMetaImpl( - pAuthCxt->pParseCxt, pAuthCxt->pMetaCache, toName(pAuthCxt->pParseCxt->acctId, pTable->dbName, pTable->tableName, &name), &pTableMeta, true); + pAuthCxt->pParseCxt, pAuthCxt->pMetaCache, &name, &pTableMeta, true); if (TSDB_CODE_SUCCESS == code && TSDB_VIEW_TABLE == pTableMeta->tableType) { isView = true; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6c86a6c12f..d1206e27b2 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -525,8 +525,9 @@ static int32_t getTargetMeta(STranslateContext* pCxt, const SName* pName, STable } static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName, STableMeta** pMeta) { - SName name; - return getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pMeta, false); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); + return getTargetMeta(pCxt, &name, pMeta, false); } static int32_t getTableCfg(STranslateContext* pCxt, const SName* pName, STableCfg** pCfg) { @@ -556,8 +557,8 @@ static int32_t getTableCfg(STranslateContext* pCxt, const SName* pName, STableCf static int32_t refreshGetTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName, STableMeta** pMeta) { SParseContext* pParCxt = pCxt->pParseCxt; - SName name; - (void)toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); int32_t code = TSDB_CODE_SUCCESS; if (pParCxt->async) { code = getTableMetaFromCache(pCxt->pMetaCache, &name, pMeta); @@ -634,8 +635,9 @@ static int32_t getTableHashVgroupImpl(STranslateContext* pCxt, const SName* pNam static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName, const char* pTableName, SVgroupInfo* pInfo) { - SName name; - return getTableHashVgroupImpl(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pInfo); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); + return getTableHashVgroupImpl(pCxt, &name, pInfo); } static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, @@ -4015,7 +4017,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo SName tsmaTargetTbName = {0}; SVgroupInfo vgInfo = {0}; bool exists = false; - (void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); int32_t len = snprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); @@ -4676,10 +4678,9 @@ int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinPare pRealTable->ratio = (NULL != pCxt->pExplainOpt ? pCxt->pExplainOpt->ratio : 1.0); // The SRealTableNode created through ROLLUP already has STableMeta. if (NULL == pRealTable->pMeta) { - SName name; - code = getTargetMeta( - pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), - &(pRealTable->pMeta), true); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name); + code = getTargetMeta(pCxt, &name, &(pRealTable->pMeta), true); if (TSDB_CODE_SUCCESS != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); } @@ -6228,9 +6229,9 @@ static void findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTbnames } for (int j = 0; j < nTbls; ++j) { - SName snameTb; + SName snameTb = {0}; char* tbName = taosArrayGetP(aTbnames, j); - (void)toName(pCxt->pParseCxt->acctId, dbName, tbName, &snameTb); + toName(pCxt->pParseCxt->acctId, dbName, tbName, &snameTb); SVgroupInfo vgInfo = {0}; bool bExists; int32_t code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &snameTb, &vgInfo, &bExists); @@ -6255,11 +6256,11 @@ static void findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTbnames } static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo) { - SName snameTb; + SName snameTb = {0}; int32_t code = 0; SRealTableNode* pRealTable = pInfo->pRealTable; char* tbName = taosArrayGetP(pInfo->aTbnames, 0); - (void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, tbName, &snameTb); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, tbName, &snameTb); STableMeta* pMeta = NULL; TAOS_CHECK_RETURN(catalogGetCachedTableMeta(pCxt->pParseCxt->pCatalog, &snameTb, &pMeta)); @@ -6280,7 +6281,7 @@ static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTa for (int32_t i = 0; i < pRealTable->pTsmas->size; ++i) { STableTSMAInfo* pTsma = taosArrayGetP(pRealTable->pTsmas, i); SName tsmaTargetTbName = {0}; - (void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); int32_t len = snprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); @@ -8778,7 +8779,7 @@ static int32_t buildRollupFuncs(SNodeList* pFuncs, SArray** pArray) { static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) { int32_t code = TSDB_CODE_SUCCESS; - SName tableName; + SName tableName = {0}; pReq->igExists = pStmt->ignoreExists; pReq->delay1 = pStmt->pOptions->maxDelay1; pReq->delay2 = pStmt->pOptions->maxDelay2; @@ -8810,7 +8811,8 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm } if (TSDB_CODE_SUCCESS == code) { pReq->numOfFuncs = taosArrayGetSize(pReq->pFuncs); - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName); + code = tNameExtractFullName(&tableName, pReq->name); } if (TSDB_CODE_SUCCESS == code) code = collectUseTable(&tableName, pCxt->pTables); @@ -8852,21 +8854,22 @@ static int32_t doTranslateDropSuperTable(STranslateContext* pCxt, const SName* p static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt) { SDropTableClause* pClause = (SDropTableClause*)nodesListGetNode(pStmt->pTables, 0); - SName tableName; + SName tableName = {0}; if (pStmt->withTsma) return TSDB_CODE_SUCCESS; - return doTranslateDropSuperTable( - pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), pClause->ignoreNotExists); + toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName); + return doTranslateDropSuperTable(pCxt, &tableName, pClause->ignoreNotExists); } static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableStmt* pStmt) { - SName tableName; - return doTranslateDropSuperTable(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), - pStmt->ignoreNotExists); + SName tableName = {0}; + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName); + return doTranslateDropSuperTable(pCxt, &tableName, pStmt->ignoreNotExists); } static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SMAlterStbReq* pAlterReq) { - SName tableName; - int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pAlterReq->name); + SName tableName = {0}; + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName); + int32_t code = tNameExtractFullName(&tableName, pAlterReq->name); if (TSDB_CODE_SUCCESS != code) return code; pAlterReq->alterType = pStmt->alterType; @@ -9381,11 +9384,13 @@ static int32_t getSmaIndexAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, } static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SMCreateSmaReq* pReq) { - SName name; - int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), pReq->name); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name); + int32_t code = tNameExtractFullName(&name, pReq->name); if (TSDB_CODE_SUCCESS == code) { memset(&name, 0, sizeof(SName)); - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), pReq->stb); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + code = tNameExtractFullName(&name, pReq->stb); } if (TSDB_CODE_SUCCESS == code) { pReq->igExists = pStmt->ignoreExists; @@ -9532,11 +9537,13 @@ static int32_t buildCreateFullTextReq(STranslateContext* pCxt, SCreateIndexStmt* } static int32_t buildCreateTagIndexReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SCreateTagIndexReq* pReq) { - SName name; - int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), pReq->idxName); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name); + int32_t code = tNameExtractFullName(&name, pReq->idxName); if (TSDB_CODE_SUCCESS == code) { memset(&name, 0, sizeof(SName)); - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), pReq->stbName); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + code = tNameExtractFullName(&name, pReq->stbName); } if (TSDB_CODE_SUCCESS == code) { memset(&name, 0, sizeof(SName)); @@ -9570,10 +9577,11 @@ static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateInde static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { int32_t code = 0; - SName name; + SName name = {0}; STableMeta* pMeta = NULL; - code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), &pMeta, false); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + code = getTargetMeta(pCxt, &name, &pMeta, false); if (code) { taosMemoryFree(pMeta); return code; @@ -9614,8 +9622,9 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { SMDropSmaReq dropSmaReq = {0}; - SName name; - int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), dropSmaReq.name); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name); + int32_t code = tNameExtractFullName(&name, dropSmaReq.name); if (TSDB_CODE_SUCCESS != code) return code; dropSmaReq.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_DROP_SMA, (FSerializeFunc)tSerializeSMDropSmaReq, &dropSmaReq); @@ -9687,10 +9696,10 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS } int32_t code = TSDB_CODE_SUCCESS; - SName name; + SName name = {0}; if ('\0' != pStmt->subSTbName[0]) { pReq->subType = TOPIC_SUB_TYPE__TABLE; - (void)toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); (void)tNameGetFullDbName(&name, pReq->subDbName); if (TSDB_CODE_SUCCESS == code) { (void)tNameExtractFullName(&name, pReq->subStbName); @@ -9815,10 +9824,10 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt .requestId = pParCxt->requestId, .requestObjRefId = pParCxt->requestRid, .mgmtEps = pParCxt->mgmtEpSet}; - SName name; + SName name = {0}; STableMeta* pMeta = NULL; - int32_t code = - getTargetMeta(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta, false); + toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); + int32_t code = getTargetMeta(pCxt, &name, &pMeta, false); if (code) { taosMemoryFree(pMeta); return code; @@ -9939,8 +9948,8 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) #ifdef TD_ENTERPRISE if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { int32_t origCode = code; - SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); SViewMeta* pMeta = NULL; code = getViewMetaFromMetaCache(pCxt, &name, &pMeta); if (TSDB_CODE_SUCCESS != code) { @@ -10057,12 +10066,11 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt #ifdef TD_ENTERPRISE SRealTableNode* pRealTable = (SRealTableNode*)((SSelectStmt*)pStmt->pQuery)->pFromTable; - SName name; + SName name = {0}; STableMeta* pMeta = NULL; int8_t tableType = 0; - int32_t code = - getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), - &pMeta, true); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name); + int32_t code = getTargetMeta(pCxt, &name, &pMeta, true); if (NULL != pMeta) { tableType = pMeta->tableType; taosMemoryFree(pMeta); @@ -11298,9 +11306,9 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt #endif SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA}; - SName name; + SName name = {0}; char dbFName[TSDB_DB_FNAME_LEN]; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); (void)tNameGetFullDbName(&name, dbFName); int32_t code = validateCreateView(pCxt, pStmt); @@ -11340,7 +11348,7 @@ static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) #endif SCMDropViewReq dropReq = {0}; - SName name; + SName name = {0}; int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); if (TSDB_CODE_SUCCESS == code) { (void)tNameGetFullDbName(&name, dropReq.dbFName); @@ -11351,7 +11359,7 @@ static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) return TSDB_CODE_OUT_OF_MEMORY; } dropReq.igNotExists = pStmt->ignoreNotExists; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); code = collectUseTable(&name, pCxt->pTargetTables); } @@ -11460,9 +11468,9 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, int32_t code = createRealTableForGrantTable(pStmt, &pTable); if (TSDB_CODE_SUCCESS == code) { - SName name; - code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name), - &(pTable->pMeta), false); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name); + code = getTargetMeta(pCxt, &name, &(pTable->pMeta), false); if (code) { nodesDestroyNode((SNode*)pTable); return code; @@ -11502,10 +11510,10 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { req.privileges = pStmt->privileges; #ifdef TD_ENTERPRISE if (0 != pStmt->tabName[0]) { - SName name; + SName name = {0}; STableMeta* pTableMeta = NULL; - code = - getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true); + toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name); + code = getTargetMeta(pCxt, &name, &pTableMeta, true); if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); @@ -11538,10 +11546,10 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { #ifdef TD_ENTERPRISE if (0 != pStmt->tabName[0]) { - SName name; + SName name = {0}; STableMeta* pTableMeta = NULL; - code = - getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true); + toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name); + code = getTargetMeta(pCxt, &name, &pTableMeta, true); if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); @@ -11654,8 +11662,8 @@ static int32_t translateShowCreateTable(STranslateContext* pCxt, SShowCreateTabl } int32_t code = getDBCfg(pCxt, pStmt->dbName, (SDbCfgInfo*)pStmt->pDbCfg); if (TSDB_CODE_SUCCESS == code) { - SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); code = getTableCfg(pCxt, &name, (STableCfg**)&pStmt->pTableCfg); } return code; @@ -11665,8 +11673,8 @@ static int32_t translateShowCreateView(STranslateContext* pCxt, SShowCreateViewS #ifndef TD_ENTERPRISE return TSDB_CODE_OPS_NOT_SUPPORT; #else - SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); return getViewMetaFromMetaCache(pCxt, &name, (SViewMeta**)&pStmt->pViewMeta); #endif } @@ -11934,13 +11942,14 @@ static int32_t rewriteTSMAFuncs(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, SMCreateSmaReq* pReq, SName* useTbName) { - SName name; + SName name = {0}; SDbCfgInfo pDbInfo = {0}; int32_t code = TSDB_CODE_SUCCESS; - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name), pReq->name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name); + code = tNameExtractFullName(&name, pReq->name); if (TSDB_CODE_SUCCESS == code) { memset(&name, 0, sizeof(SName)); - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, useTbName); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, useTbName); code = tNameExtractFullName(useTbName, pReq->stb); } if (TSDB_CODE_SUCCESS == code) { @@ -12013,7 +12022,8 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm if (TSDB_CODE_SUCCESS == code) { memset(useTbName, 0, sizeof(SName)); memcpy(pStmt->originalTbName, pRecursiveTsma->tb, TSDB_TABLE_NAME_LEN); - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pRecursiveTsma->tb, useTbName), pReq->stb); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pRecursiveTsma->tb, useTbName); + code = tNameExtractFullName(useTbName, pReq->stb); } if (TSDB_CODE_SUCCESS == code) { numOfCols = pRecursiveTsma->pUsedCols->size; @@ -12134,7 +12144,7 @@ int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSData if (TSDB_CODE_SUCCESS == code) { SName name = {0}; - (void)toName(pParseCxt->acctId, pStmt->dbName, pStmt->originalTbName, &name); + toName(pParseCxt->acctId, pStmt->dbName, pStmt->originalTbName, &name); code = collectUseTable(&name, cxt.pTargetTables); } @@ -12151,15 +12161,16 @@ int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSData static int32_t translateDropTSMA(STranslateContext* pCxt, SDropTSMAStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; SMDropSmaReq dropReq = {0}; - SName name; + SName name = {0}; STableTSMAInfo* pTsma = NULL; - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name), dropReq.name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name); + code = tNameExtractFullName(&name, dropReq.name); if (TSDB_CODE_SUCCESS == code) { dropReq.igNotExists = pStmt->ignoreNotExists; code = getTsma(pCxt, &name, &pTsma); } if (code == TSDB_CODE_SUCCESS) { - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pTsma->tb, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pTsma->tb, &name); code = collectUseTable(&name, pCxt->pTargetTables); } if (TSDB_CODE_SUCCESS == code) @@ -13057,12 +13068,11 @@ static int32_t rewriteShowVgroups(STranslateContext* pCxt, SQuery* pQuery) { static int32_t checkShowTags(STranslateContext* pCxt, const SShowStmt* pShow) { int32_t code = 0; - SName name; + SName name = {0}; STableMeta* pTableMeta = NULL; - code = getTargetMeta(pCxt, - toName(pCxt->pParseCxt->acctId, ((SValueNode*)pShow->pDbName)->literal, - ((SValueNode*)pShow->pTbName)->literal, &name), - &pTableMeta, true); + toName(pCxt->pParseCxt->acctId, ((SValueNode*)pShow->pDbName)->literal, + ((SValueNode*)pShow->pTbName)->literal, &name); + code = getTargetMeta(pCxt, &name, &pTableMeta, true); if (TSDB_CODE_SUCCESS != code) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); goto _exit; @@ -13420,8 +13430,8 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { int32_t code = checkCreateTable(pCxt, pStmt, false); SVgroupInfo info = {0}; - SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); if (TSDB_CODE_SUCCESS == code) { code = getTableHashVgroupImpl(pCxt, &name, &info); } @@ -13668,8 +13678,8 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta); } if (TSDB_CODE_SUCCESS == code) { - SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); code = collectUseTable(&name, pCxt->pTargetTables); } @@ -14434,8 +14444,8 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch); FOREACH(pNode, pStmt->pTables) { SDropTableClause* pClause = (SDropTableClause*)pNode; - SName name; - (void)toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &name); + SName name = {0}; + toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &name); int32_t code = buildDropTableVgroupHashmap(pCxt, pClause, &name, &tableType, pVgroupHashmap); if (TSDB_CODE_SUCCESS != code) { taosHashCleanup(pVgroupHashmap); @@ -14511,7 +14521,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS SArray* pTsmas = NULL; int32_t code = TSDB_CODE_SUCCESS; if (pCxt->pMetaCache) { - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); code = getTableTsmasFromCache(pCxt->pMetaCache, &tbName, &pTsmas); if (code != TSDB_CODE_SUCCESS) return code; if (pTsmas && pTsmas->size > 0) return TSDB_CODE_TSMA_MUST_BE_DROPPED; @@ -14693,9 +14703,9 @@ static int32_t buildRenameColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt } if (TSDB_NORMAL_TABLE == pTableMeta->tableType) { SArray* pTsmas = NULL; - SName tbName; + SName tbName = {0}; int32_t code = 0; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); if (pCxt->pMetaCache) code = getTableTsmasFromCache(pCxt->pMetaCache, &tbName, &pTsmas); if (TSDB_CODE_SUCCESS != code) return code; if (pTsmas && pTsmas->size > 0) { From 883d2bfb7ab248ae8b932ad2d79e45809f0d5a74 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 10 Sep 2024 14:54:00 +0800 Subject: [PATCH 05/94] fix: oom with large submit --- source/libs/sync/src/syncPipeline.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index a6e9c7de32..4cdc6c3d83 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -846,7 +846,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm } // recycle - SyncIndex until = pBuf->commitIndex - TSDB_SYNC_LOG_BUFFER_RETENTION; + SyncIndex until = pBuf->commitIndex; // - TSDB_SYNC_LOG_BUFFER_RETENTION; for (SyncIndex index = pBuf->startIndex; index < until; index++) { SSyncRaftEntry* pEntry = pBuf->entries[(index + pBuf->size) % pBuf->size].pItem; if (pEntry == NULL) return TSDB_CODE_SYN_INTERNAL_ERROR; From a322de01144d4ba1fa17bbeffecba1297356e4fd Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Sep 2024 09:36:19 +0800 Subject: [PATCH 06/94] fix:[TD-31899] remove void(func) --- source/client/src/clientTmq.c | 222 +++++++++++++++++++++------------- 1 file changed, 138 insertions(+), 84 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 529392ebfc..8d053defdb 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -800,6 +800,8 @@ static void generateTimedTask(int64_t refId, int32_t type) { if (tsem2_post(&tmq->rspSem) != 0){ tscError("consumer:0x%" PRIx64 " failed to post sem, type:%d", tmq->consumerId, type); } + }else{ + taosFreeQitem(pTaskType); } } @@ -977,7 +979,10 @@ void tmqSendHbReq(void* param, void* tmrId) { OVER: tDestroySMqHbReq(&req); if (tmrId != NULL) { - (void)taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer); + int32_t ret = taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer); + if (!ret){ + tscError("failed to reset timer fo tmq hb"); + } } int32_t ret = taosReleaseRef(tmqMgmt.rsetId, refId); if (ret != 0){ @@ -1001,9 +1006,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { return; } - (void)taosReadAllQitems(pTmq->delayedTask, qall); - - int32_t numOfItems = taosQallItemSize(qall); + int32_t numOfItems = taosReadAllQitems(pTmq->delayedTask, qall); if (numOfItems == 0) { taosFreeQall(qall); return; @@ -1011,9 +1014,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, numOfItems); int8_t* pTaskType = NULL; - (void)taosGetQitem(qall, (void**)&pTaskType); - - while (pTaskType != NULL) { + while (taosGetQitem(qall, (void**)&pTaskType) != 0) { if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) { code = askEp(pTmq, NULL, false, false); if (code != 0) { @@ -1021,21 +1022,26 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { continue; } tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); - (void)taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, + code = taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->epTimer); + if (!code){ + tscError("failed to reset timer fo tmq ask ep"); + } } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn; asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, pTmq->autoCommitInterval / 1000.0); - (void)taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, + code = taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->commitTimer); + if (!code){ + tscError("failed to reset timer fo commit"); + } } else { tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType); } taosFreeQitem(pTaskType); - (void)taosGetQitem(qall, (void**)&pTaskType); } taosFreeQall(qall); @@ -1067,26 +1073,18 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { void tmqClearUnhandleMsg(tmq_t* tmq) { SMqRspWrapper* rspWrapper = NULL; - while (1) { - (void)taosGetQitem(tmq->qall, (void**)&rspWrapper); - if (rspWrapper) { + while (taosGetQitem(tmq->qall, (void**)&rspWrapper) != 0) { tmqFreeRspWrapper(rspWrapper); taosFreeQitem(rspWrapper); - } else { - break; - } } rspWrapper = NULL; - (void)taosReadAllQitems(tmq->mqueue, tmq->qall); - while (1) { - (void)taosGetQitem(tmq->qall, (void**)&rspWrapper); - if (rspWrapper) { - tmqFreeRspWrapper(rspWrapper); - taosFreeQitem(rspWrapper); - } else { - break; - } + if (taosReadAllQitems(tmq->mqueue, tmq->qall) == 0){ + return; + } + while (taosGetQitem(tmq->qall, (void**)&rspWrapper) != 0) { + tmqFreeRspWrapper(rspWrapper); + taosFreeQitem(rspWrapper); } } @@ -1162,19 +1160,27 @@ void tmqFreeImpl(void* handle) { } taosFreeQall(tmq->qall); - (void)tsem2_destroy(&tmq->rspSem); + if(tsem2_destroy(&tmq->rspSem) != 0) { + tscError("failed to destroy sem in free tmq"); + } taosArrayDestroyEx(tmq->clientTopics, freeClientTopic); taos_close_internal(tmq->pTscObj); if (tmq->commitTimer) { - (void)taosTmrStopA(&tmq->commitTimer); + if (!taosTmrStopA(&tmq->commitTimer)) { + tscError("failed to stop commit timer"); + } } if (tmq->epTimer) { - (void)taosTmrStopA(&tmq->epTimer); + if (!taosTmrStopA(&tmq->epTimer)) { + tscError("failed to stop ep timer"); + } } if (tmq->hbLiveTimer) { - (void)taosTmrStopA(&tmq->hbLiveTimer); + if (!taosTmrStopA(&tmq->hbLiveTimer)) { + tscError("failed to stop hb timer"); + } } taosMemoryFree(tmq); @@ -1320,7 +1326,6 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (code) { terrno = code; tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); - (void)tsem2_destroy(&pTmq->rspSem); SET_ERROR_MSG_TMQ("init tscObj failed") goto _failed; } @@ -1427,7 +1432,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } void* abuf = buf; - (void)tSerializeSCMSubscribeReq(&abuf, &req); + tlen = tSerializeSCMSubscribeReq(&abuf, &req); sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { @@ -1459,8 +1464,12 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } - (void)tsem2_wait(¶m.rspSem); - (void)tsem2_destroy(¶m.rspSem); + if (tsem2_wait(¶m.rspSem) != 0){ + tscError("consumer:0x%" PRIx64 ", failed to wait semaphore in subscribe", tmq->consumerId); + } + if(tsem2_destroy(¶m.rspSem) != 0) { + tscError("consumer:0x%" PRIx64 ", failed to destroy semaphore in subscribe", tmq->consumerId); + } if (param.rspErr != 0) { code = param.rspErr; @@ -1665,6 +1674,8 @@ END: (void)strcpy(pRspWrapper->topicName, pParam->topicName); code = taosWriteQitem(tmq->mqueue, pRspWrapper); if (code != 0) { + tmqFreeRspWrapper((SMqRspWrapper*)pRspWrapper); + taosFreeQitem(pRspWrapper); tscError("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code); } } @@ -1672,7 +1683,11 @@ END: tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d,QID:0x%" PRIx64, tmq ? tmq->consumerId : 0, rspType, vgId, total, requestId); - if (tmq) (void)tsem2_post(&tmq->rspSem); + if (tmq) { + if (tsem2_post(&tmq->rspSem) != 0){ + tscError("failed to post rsp sem, consumer:0x%" PRIx64, tmq->consumerId); + } + } if (pMsg) taosMemoryFreeClear(pMsg->pData); if (pMsg) taosMemoryFreeClear(pMsg->pEpSet); ret = taosReleaseRef(tmqMgmt.rsetId, refId); @@ -1775,7 +1790,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) SHashObj* pVgOffsetHashMap = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); if (pVgOffsetHashMap == NULL) { - (void)taosArrayDestroy(newTopics); + taosArrayDestroy(newTopics); return false; } @@ -1860,10 +1875,10 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->enableBatchMeta = tmq->enableBatchMeta; } -int32_t tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqMetaRspObj** ppRspObj) { +void tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqMetaRspObj** ppRspObj) { SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj)); if (pRspObj == NULL) { - return terrno; + return; } pRspObj->resType = RES_TYPE__TMQ_META; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); @@ -1872,13 +1887,12 @@ int32_t tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqMetaRspObj** (void)memcpy(&pRspObj->metaRsp, &pWrapper->metaRsp, sizeof(SMqMetaRsp)); *ppRspObj = pRspObj; - return 0; } -int32_t tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqBatchMetaRspObj** ppRspObj) { +void tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqBatchMetaRspObj** ppRspObj) { SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj)); if (pRspObj == NULL) { - return terrno; + return; } pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META; tstrncpy(pRspObj->common.topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); @@ -1888,7 +1902,6 @@ int32_t tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqBatchMet (void)memcpy(&pRspObj->rsp, &pWrapper->batchMetaRsp, sizeof(SMqBatchMetaRsp)); tscDebug("build batchmeta Rsp from wrapper"); *ppRspObj = pRspObj; - return 0; } void changeByteEndian(char* pData) { @@ -1985,31 +1998,29 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg } } -int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, +void tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj** ppRspObj) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); if (pRspObj == NULL) { - return terrno; + return; } pRspObj->common.resType = RES_TYPE__TMQ; (void)memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); *ppRspObj = pRspObj; - return 0; } -int32_t tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, +void tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqTaosxRspObj** ppRspObj) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); if (pRspObj == NULL) { - return terrno; + return; } pRspObj->common.resType = RES_TYPE__TMQ_METADATA; (void)memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); *ppRspObj = pRspObj; - return 0; } static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* pVg, int64_t timeout) { @@ -2168,12 +2179,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { while (1) { SMqRspWrapper* pRspWrapper = NULL; - (void)taosGetQitem(tmq->qall, (void**)&pRspWrapper); - - if (pRspWrapper == NULL) { - (void)taosReadAllQitems(tmq->mqueue, tmq->qall); - (void)taosGetQitem(tmq->qall, (void**)&pRspWrapper); - if (pRspWrapper == NULL) { + if (taosGetQitem(tmq->qall, (void**)&pRspWrapper) == 0) { + if (taosReadAllQitems(tmq->mqueue, tmq->qall) == 0){ + return NULL; + } + if (taosGetQitem(tmq->qall, (void**)&pRspWrapper) == 0) { return NULL; } } @@ -2251,7 +2261,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } else { // build rsp int64_t numOfRows = 0; SMqRspObj* pRsp = NULL; - (void)tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp); + tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp); tmq->totalRows += numOfRows; pVg->emptyBlockReceiveTs = 0; if (pRsp && tmq->replayEnable) { @@ -2305,7 +2315,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId, true); // build rsp SMqMetaRspObj* pRsp = NULL; - (void)tmqBuildMetaRspFromWrapper(pollRspWrapper, &pRsp); + tmqBuildMetaRspFromWrapper(pollRspWrapper, &pRsp); taosMemoryFreeClear(pollRspWrapper->pEpset); taosFreeQitem(pRspWrapper); taosWUnLockLatch(&tmq->lock); @@ -2343,7 +2353,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever, tmq->consumerId, true); SMqBatchMetaRspObj* pRsp = NULL; - (void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp); + tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp); taosMemoryFreeClear(pollRspWrapper->pEpset); taosFreeQitem(pRspWrapper); taosWUnLockLatch(&tmq->lock); @@ -2393,9 +2403,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { // build rsp int64_t numOfRows = 0; SMqTaosxRspObj* pRsp = NULL; - if (tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp) != 0) { - tscError("consumer:0x%" PRIx64 " build taosx rsp failed, vgId:%d", tmq->consumerId, pVg->vgId); - } + tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp); tmq->totalRows += numOfRows; char buf[TSDB_OFFSET_LEN] = {0}; @@ -2563,7 +2571,10 @@ int32_t tmq_consumer_close(tmq_t* tmq) { } if (code == 0) { - (void)taosRemoveRef(tmqMgmt.rsetId, tmq->refId); + code = taosRemoveRef(tmqMgmt.rsetId, tmq->refId); + if (code != 0){ + tscError("tmq close failed to remove ref:%" PRId64 ", code:%d", tmq->refId, code); + } } return code; } @@ -2721,7 +2732,9 @@ void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* static void commitCallBackFn(tmq_t* UNUSED_PARAM(tmq), int32_t code, void* param) { SSyncCommitInfo* pInfo = (SSyncCommitInfo*)param; pInfo->code = code; - (void)tsem2_post(&pInfo->sem); + if (tsem2_post(&pInfo->sem) != 0){ + tscError("failed to post rsp sem in commit cb"); + } } int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { @@ -2750,10 +2763,14 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { asyncCommitFromResult(tmq, pRes, commitCallBackFn, pInfo); } - (void)tsem2_wait(&pInfo->sem); + if (tsem2_wait(&pInfo->sem) != 0){ + tscError("failed to wait sem for sync commit"); + } code = pInfo->code; - (void)tsem2_destroy(&pInfo->sem); + if(tsem2_destroy(&pInfo->sem) != 0) { + tscError("failed to destroy sem for sync commit"); + } taosMemoryFree(pInfo); tscInfo("consumer:0x%" PRIx64 " sync res commit done, code:%s", tmq->consumerId, tstrerror(code)); @@ -2818,12 +2835,16 @@ int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, commitCallBackFn, pInfo); if (code == 0) { - (void)tsem2_wait(&pInfo->sem); + if (tsem2_wait(&pInfo->sem) != 0){ + tscError("failed to wait sem for sync commit offset"); + } code = pInfo->code; } if (code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; - (void)tsem2_destroy(&pInfo->sem); + if(tsem2_destroy(&pInfo->sem) != 0) { + tscError("failed to destroy sem for sync commit offset"); + } taosMemoryFree(pInfo); tscInfo("consumer:0x%" PRIx64 " sync send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, @@ -2876,11 +2897,11 @@ end: } int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { - if (param == NULL) { + SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; + if (pParam == NULL) { goto FAIL; } - SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); if (tmq == NULL) { code = TSDB_CODE_TMQ_CONSUMER_CLOSED; @@ -2918,7 +2939,12 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); taosFreeQitem(pWrapper); } else { - (void)taosWriteQitem(tmq->mqueue, pWrapper); + code = taosWriteQitem(tmq->mqueue, pWrapper); + if (code != 0) { + tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); + taosFreeQitem(pWrapper); + tscError("consumer:0x%" PRIx64 " put ep res into mqueue failed, code:%d", tmq->consumerId, code); + } } } @@ -2930,11 +2956,13 @@ END: } } FAIL: - if (pParam->sync) { + if (pParam && pParam->sync) { SAskEpInfo* pInfo = pParam->pParam; if (pInfo) { pInfo->code = code; - (void)tsem2_post(&pInfo->sem); + if (tsem2_post(&pInfo->sem) != 0){ + tscError("failed to post rsp sem askep cb"); + } } } @@ -2956,11 +2984,15 @@ int32_t syncAskEp(tmq_t* pTmq) { int32_t code = askEp(pTmq, pInfo, true, false); if (code == 0) { - (void)tsem2_wait(&pInfo->sem); + if (tsem2_wait(&pInfo->sem) != 0){ + tscError("consumer:0x%" PRIx64 ", failed to wait for sem", pTmq->consumerId); + } code = pInfo->code; } - (void)tsem2_destroy(&pInfo->sem); + if(tsem2_destroy(&pInfo->sem) != 0) { + tscError("failed to destroy sem sync ask ep"); + } taosMemoryFree(pInfo); return code; } @@ -3131,7 +3163,9 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { END: pCommon->code = code; if (total == pParam->totalReq) { - (void)tsem2_post(&pCommon->rsp); + if (tsem2_post(&pCommon->rsp) != 0) { + tscError("failed to post semaphore in get wal cb"); + } } if (pMsg) { @@ -3146,8 +3180,10 @@ static void destroyCommonInfo(SMqVgCommon* pCommon) { if (pCommon == NULL) { return; } - (void)taosArrayDestroy(pCommon->pList); - (void)tsem2_destroy(&pCommon->rsp); + taosArrayDestroy(pCommon->pList); + if(tsem2_destroy(&pCommon->rsp) != 0) { + tscError("failed to destroy semaphore for topic:%s", pCommon->pTopicName); + } (void)taosThreadMutexDestroy(&pCommon->mutex); taosMemoryFree(pCommon->pTopicName); taosMemoryFree(pCommon); @@ -3183,7 +3219,9 @@ end: taosMemoryFree(pMsg->pEpSet); } pParam->code = code; - (void)tsem2_post(&pParam->sem); + if (tsem2_post(&pParam->sem) != 0){ + tscError("failed to post semaphore in tmCommittedCb"); + } return code; } @@ -3248,12 +3286,16 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep int64_t transporterId = 0; code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, sendInfo); if (code != 0) { - (void)tsem2_destroy(&pParam->sem); + if(tsem2_destroy(&pParam->sem) != 0) { + tscError("failed to destroy semaphore in get committed from server1"); + } taosMemoryFree(pParam); return code; } - (void)tsem2_wait(&pParam->sem); + if (tsem2_wait(&pParam->sem) != 0){ + tscError("failed to wait semaphore in get committed from server"); + } code = pParam->code; if (code == TSDB_CODE_SUCCESS) { if (pParam->vgOffset.offset.val.type == TMQ_OFFSET__LOG) { @@ -3263,7 +3305,9 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; } } - (void)tsem2_destroy(&pParam->sem); + if(tsem2_destroy(&pParam->sem) != 0) { + tscError("failed to destroy semaphore in get committed from server2"); + } taosMemoryFree(pParam); return code; @@ -3534,7 +3578,9 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } } - (void)tsem2_wait(&pCommon->rsp); + if (tsem2_wait(&pCommon->rsp) != 0){ + tscError("consumer:0x%" PRIx64 " failed to wait sem in get assignment", tmq->consumerId); + } code = pCommon->code; if (code != TSDB_CODE_SUCCESS) { @@ -3597,7 +3643,9 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { } SMqSeekParam* pParam = param; pParam->code = code; - (void)tsem2_post(&pParam->sem); + if (tsem2_post(&pParam->sem) != 0){ + tscError("failed to post sem in tmqSeekCb"); + } return 0; } @@ -3695,14 +3743,20 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ int64_t transporterId = 0; code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); if (code != 0) { - (void)tsem2_destroy(&pParam->sem); + if(tsem2_destroy(&pParam->sem) != 0) { + tscError("consumer:0x%" PRIx64 "destroy rsp sem failed in seek offset", tmq->consumerId); + } taosMemoryFree(pParam); return code; } - (void)tsem2_wait(&pParam->sem); + if (tsem2_wait(&pParam->sem) != 0){ + tscError("consumer:0x%" PRIx64 "wait rsp sem failed in seek offset", tmq->consumerId); + } code = pParam->code; - (void)tsem2_destroy(&pParam->sem); + if(tsem2_destroy(&pParam->sem) != 0) { + tscError("consumer:0x%" PRIx64 "destroy rsp sem failed in seek offset", tmq->consumerId); + } taosMemoryFree(pParam); tscInfo("consumer:0x%" PRIx64 "send seek to vgId:%d, return code:%s", tmq->consumerId, vgId, tstrerror(code)); From 118eeec480c1a0b8e4c1f4efb4b76b7a10252279 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Sep 2024 10:06:58 +0800 Subject: [PATCH 07/94] fix:[TD-31899] remove void(func) --- source/client/src/clientMonitor.c | 5 +---- source/client/src/clientTmq.c | 10 +++++----- source/dnode/vnode/src/tq/tq.c | 3 +-- source/dnode/vnode/src/tq/tqStreamTask.c | 4 ++-- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index fbf266b517..98a4dcf5d0 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -767,10 +767,7 @@ static void* monitorThreadFunc(void* param) { } MonitorSlowLogData* slowLogData = NULL; - if (taosReadQitem(monitorQueue, (void**)&slowLogData) == 0){ - tscDebug("monitorThreadFunc get slow log data from queue null"); - } - if (slowLogData != NULL) { + if (taosReadQitem(monitorQueue, (void**)&slowLogData) != 0) { if (slowLogData->type == SLOW_LOG_READ_BEGINNIG && quitCnt == 0) { if (slowLogData->pFile != NULL) { monitorSendSlowLogAtBeginning(slowLogData->clusterId, &(slowLogData->fileName), slowLogData->pFile, diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 72af228ea9..f4aeda78ad 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -978,7 +978,7 @@ void tmqSendHbReq(void* param, void* tmrId) { OVER: tDestroySMqHbReq(&req); if (tmrId != NULL) { - int32_t ret = taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer); + bool ret = taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer); if (!ret){ tscError("failed to reset timer fo tmq hb"); } @@ -1021,9 +1021,9 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { continue; } tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); - code = taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, + bool ret = taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->epTimer); - if (!code){ + if (!ret){ tscError("failed to reset timer fo tmq ask ep"); } } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { @@ -1031,9 +1031,9 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, pTmq->autoCommitInterval / 1000.0); - code = taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, + bool ret = taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->commitTimer); - if (!code){ + if (!ret){ tscError("failed to reset timer fo commit"); } } else { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 70a6efcf2e..3911822068 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -344,8 +344,7 @@ int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg) { .pCont = pHandle->msg->pCont, .contLen = pHandle->msg->contLen, .info = pHandle->msg->info}; - int32_t ret = tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg); - if (ret != 0){ + if (tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg) != 0){ tqError("vgId:%d tmsgPutToQueue failed, consumer:0x%" PRIx64, vgId, pHandle->consumerId); } taosMemoryFree(pHandle->msg); diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 6463a59dfb..6558012551 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -112,8 +112,8 @@ int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) { if (pMeta->scanInfo.scanTimer == NULL) { pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, pTimer); } else { - code = taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer); - if (code) { + bool ret = taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer); + if (!ret) { tqError("vgId:%d failed to start scan wal in:%dms", vgId, idleDuration); } } From ca2e761a86b78e6e7595ce15966f20847903ce42 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 11 Sep 2024 10:24:04 +0800 Subject: [PATCH 08/94] Add mutex while read wal --- include/util/tutil.h | 22 +++++++++++++++------- source/libs/wal/src/walRead.c | 8 +++++--- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/include/util/tutil.h b/include/util/tutil.h index 6321d2011a..5fa06b7e61 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -73,14 +73,14 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar char buf[TSDB_PASSWORD_LEN + 1]; buf[TSDB_PASSWORD_LEN] = 0; - (void)sprintf(buf, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], - context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], - context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], - context.digest[12], context.digest[13], context.digest[14], context.digest[15]); + (void)sprintf(buf, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], + context.digest[6], context.digest[7], context.digest[8], context.digest[9], context.digest[10], + context.digest[11], context.digest[12], context.digest[13], context.digest[14], context.digest[15]); (void)memcpy(target, buf, TSDB_PASSWORD_LEN); } -static FORCE_INLINE int32_t taosHashBinary(char* pBuf, int32_t len) { +static FORCE_INLINE int32_t taosHashBinary(char *pBuf, int32_t len) { uint64_t hashVal = MurmurHash3_64(pBuf, len); return sprintf(pBuf, "%" PRIu64, hashVal); } @@ -161,8 +161,7 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, #define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member))) -#define TAOS_GET_TERRNO(code) \ - (terrno == 0 ? code : terrno) +#define TAOS_GET_TERRNO(code) (terrno == 0 ? code : terrno) #define TAOS_RETURN(CODE) \ do { \ @@ -177,6 +176,15 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } \ } while (0) +#define TAOS_CHECK_RETURN_WITH_MUTEX(CMD, MUTEX) \ + do { \ + int32_t __c = (CMD); \ + if (__c != TSDB_CODE_SUCCESS) { \ + taosThreadMutexUnlock(MUTEX); \ + TAOS_RETURN(__c); \ + } \ + } while (0) + #define TAOS_CHECK_GOTO(CMD, LINO, LABEL) \ do { \ code = (CMD); \ diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index deb5a07672..5568270efe 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -201,21 +201,23 @@ static int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { // bsearch in fileSet SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; + taosThreadMutexLock(&pWal->mutex); SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); if (pRet == NULL) { wError("failed to find WAL log file with ver:%" PRId64, ver); - + taosThreadMutexUnlock(&pWal->mutex); TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER); } if (pReader->curFileFirstVer != pRet->firstVer) { // error code was set inner - TAOS_CHECK_RETURN(walReadChangeFile(pReader, pRet->firstVer)); + TAOS_CHECK_RETURN_WITH_MUTEX(walReadChangeFile(pReader, pRet->firstVer), &pWal->mutex); } // error code was set inner - TAOS_CHECK_RETURN(walReadSeekFilePos(pReader, pRet->firstVer, ver)); + TAOS_CHECK_RETURN_WITH_MUTEX(walReadSeekFilePos(pReader, pRet->firstVer, ver), &pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, pReader->curVersion, ver); pReader->curVersion = ver; From 1850f3fb5227f62a34591b7e2c91efeab08be5c1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Sep 2024 11:51:02 +0800 Subject: [PATCH 09/94] fix:[TD-31962]memory leak by crash_gen --- include/util/taoserror.h | 1 + source/client/src/clientEnv.c | 2 +- source/client/src/clientTmq.c | 5 ++++- source/common/src/tmsg.c | 4 ++++ source/dnode/mnode/impl/src/mndConsumer.c | 11 ++++++++++- source/libs/executor/src/scanoperator.c | 12 ++++-------- source/util/src/terror.c | 1 + 7 files changed, 25 insertions(+), 11 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4591c7fbcc..16027730f7 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -962,6 +962,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP TAOS_DEF_ERROR_CODE(0, 0x4013) #define TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x4014) #define TSDB_CODE_TMQ_NO_TABLE_QUALIFIED TAOS_DEF_ERROR_CODE(0, 0x4015) +#define TSDB_CODE_TMQ_NO_NEED_REBALANCE TAOS_DEF_ERROR_CODE(0, 0x4016) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index fec1060042..101bd9341c 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -941,7 +941,7 @@ void taos_init_imp(void) { tscInitRes = TSDB_CODE_OUT_OF_MEMORY; return; } - taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst); +// taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst); avoid heap use after free deltaToUtcInitOnce(); char logDirName[64] = {0}; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 783815d97f..0395869367 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -2896,7 +2896,10 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); taosFreeQitem(pWrapper); } else { - (void)taosWriteQitem(tmq->mqueue, pWrapper); + if (taosWriteQitem(tmq->mqueue, pWrapper) != 0){ + tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); + taosFreeQitem(pWrapper); + } } } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 758a4aeec3..4aed628a92 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -10346,7 +10346,11 @@ int32_t tDecodeSMCreateStbRsp(SDecoder *pDecoder, SMCreateStbRsp *pRsp) { } tEndDecode(pDecoder); + return code; + _exit: + tFreeSTableMetaRsp(pRsp->pMeta); + taosMemoryFreeClear(pRsp->pMeta); return code; } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 606b93035f..403a5b89a8 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -511,6 +511,11 @@ static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerOb } } } + // no topics need to be rebalanced + if (taosArrayGetSize(pConsumerNew->rebNewTopics) == 0 && taosArrayGetSize(pConsumerNew->rebRemovedTopics) == 0) { + code = TSDB_CODE_TMQ_NO_NEED_REBALANCE; + } + END: return code; } @@ -581,6 +586,10 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { if(ubSubscribe){ SMqConsumerObj *pConsumerTmp = NULL; MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, subscribe.consumerId, &pConsumerTmp)); + if (taosArrayGetSize(pConsumerTmp->assignedTopics) == 0){ + mndReleaseConsumer(pMnode, pConsumerTmp); + goto END; + } mndReleaseConsumer(pMnode, pConsumerTmp); } MND_TMQ_RETURN_CHECK(checkAndSortTopic(pMnode, subscribe.topicNames)); @@ -599,7 +608,7 @@ END: mndTransDrop(pTrans); tDeleteSMqConsumerObj(pConsumerNew); taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree); - return code; + return (code == TSDB_CODE_TMQ_NO_NEED_REBALANCE || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) ? 0 : code; } SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 42e7e4ac3b..717b8793f2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -668,8 +668,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int val = *pVal; } else { pCache->cacheHit += 1; - STableCachedVal* pVal = taosLRUCacheValue(pCache->pTableMetaEntryCache, h); - val = *pVal; + STableCachedVal* pValTmp = taosLRUCacheValue(pCache->pTableMetaEntryCache, h); + val = *pValTmp; bool bRes = taosLRUCacheRelease(pCache->pTableMetaEntryCache, h, false); qTrace("release LRU cache, res %d", bRes); @@ -721,12 +721,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int if (IS_VAR_DATA_TYPE(((const STagVal*)p)->type)) { taosMemoryFree(data); } - if (code) { - if (freeReader) { - pHandle->api.metaReaderFn.clearReader(&mr); - } - return code; - } + QUERY_CHECK_CODE(code, lino, _end); } else { // todo opt for json tag for (int32_t i = 0; i < pBlock->info.rows; ++i) { code = colDataSetVal(pColInfoData, i, data, false); @@ -746,6 +741,7 @@ _end: sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW, NULL); if (insertRet != TAOS_LRU_STATUS_OK) { qWarn("failed to put meta into lru cache, code:%d, %s", insertRet, idStr); + freeTableCachedVal(pVal); } } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 58dde5cd23..f85c76f157 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -806,6 +806,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_SAME_COMMITTED_VALUE, "Same committed valu TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP, "Replay need only one vgroup if subscribe super table") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT, "Replay is disabled if subscribe db or stable") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NO_TABLE_QUALIFIED, "No table qualified for query") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NO_NEED_REBALANCE, "No need rebalance") // stream TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") From 96c59cc7462f0c57313cd08ac074ff3e67b27d54 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 11 Sep 2024 13:47:43 +0800 Subject: [PATCH 10/94] reduce mutex lock usage time --- include/util/tutil.h | 14 +++++++------- source/libs/wal/src/walRead.c | 20 +++++++++++++------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/include/util/tutil.h b/include/util/tutil.h index 5fa06b7e61..6a8f58e360 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -176,13 +176,13 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } \ } while (0) -#define TAOS_CHECK_RETURN_WITH_MUTEX(CMD, MUTEX) \ - do { \ - int32_t __c = (CMD); \ - if (__c != TSDB_CODE_SUCCESS) { \ - taosThreadMutexUnlock(MUTEX); \ - TAOS_RETURN(__c); \ - } \ +#define TAOS_CHECK_RETURN_WITH_FREE(CMD, PTR) \ + do { \ + int32_t __c = (CMD); \ + if (__c != TSDB_CODE_SUCCESS) { \ + taosMemoryFree(PTR); \ + TAOS_RETURN(__c); \ + } \ } while (0) #define TAOS_CHECK_GOTO(CMD, LINO, LABEL) \ diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 5568270efe..1e9fcc70aa 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -202,22 +202,28 @@ static int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; taosThreadMutexLock(&pWal->mutex); - SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - if (pRet == NULL) { + SWalFileInfo *gloablPRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + if (gloablPRet == NULL) { wError("failed to find WAL log file with ver:%" PRId64, ver); taosThreadMutexUnlock(&pWal->mutex); TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER); } - + SWalFileInfo *pRet = taosMemoryMalloc(sizeof(SWalFileInfo)); + if (pRet == NULL) { + wError("failed to allocate memory for localRet"); + taosThreadMutexUnlock(&pWal->mutex); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + TAOS_MEMCPY(pRet, gloablPRet, sizeof(SWalFileInfo)); + taosThreadMutexUnlock(&pWal->mutex); if (pReader->curFileFirstVer != pRet->firstVer) { // error code was set inner - TAOS_CHECK_RETURN_WITH_MUTEX(walReadChangeFile(pReader, pRet->firstVer), &pWal->mutex); + TAOS_CHECK_RETURN_WITH_FREE(walReadChangeFile(pReader, pRet->firstVer), pRet); } // error code was set inner - TAOS_CHECK_RETURN_WITH_MUTEX(walReadSeekFilePos(pReader, pRet->firstVer, ver), &pWal->mutex); - - taosThreadMutexUnlock(&pWal->mutex); + TAOS_CHECK_RETURN_WITH_FREE(walReadSeekFilePos(pReader, pRet->firstVer, ver), pRet); + taosMemoryFree(pRet); wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, pReader->curVersion, ver); pReader->curVersion = ver; From 2a13aa08e15c4962b154cc0b9675932c55d074d7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 11 Sep 2024 14:23:47 +0800 Subject: [PATCH 11/94] fix: oom with large msg --- include/common/tglobal.h | 1 + include/util/tdef.h | 1 + source/common/src/tglobal.c | 17 ++++++++++ source/libs/sync/inc/syncInt.h | 2 +- source/libs/sync/inc/syncPipeline.h | 1 + source/libs/sync/src/syncPipeline.c | 50 ++++++++++++++++++++++++----- 6 files changed, 63 insertions(+), 9 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 2d4d437649..bece14c17d 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -94,6 +94,7 @@ extern int32_t tsElectInterval; extern int32_t tsHeartbeatInterval; extern int32_t tsHeartbeatTimeout; extern int32_t tsSnapReplMaxWaitN; +extern int64_t tsLogBufferMemoryAllowed; // maximum allowed log buffer size in bytes for each dnode // arbitrator extern int32_t tsArbHeartBeatIntervalSec; diff --git a/include/util/tdef.h b/include/util/tdef.h index 46a0d01457..46c84ab26a 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -332,6 +332,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_LEARNER_REPLICA 10 #define TSDB_SYNC_LOG_BUFFER_SIZE 4096 #define TSDB_SYNC_LOG_BUFFER_RETENTION 256 +#define TSDB_SYNC_LOG_BUFFER_THRESHOLD (1024 * 1024 * 5) #define TSDB_SYNC_APPLYQ_SIZE_LIMIT 512 #define TSDB_SYNC_NEGOTIATION_WIN 512 diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5b67e1267b..2519ead06e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -81,6 +81,7 @@ int32_t tsElectInterval = 25 * 1000; int32_t tsHeartbeatInterval = 1000; int32_t tsHeartbeatTimeout = 20 * 1000; int32_t tsSnapReplMaxWaitN = 128; +int64_t tsLogBufferMemoryAllowed = 0; // bytes // mnode int64_t tsMndSdbWriteDelta = 200; @@ -702,6 +703,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; tsQueueMemoryAllowed = TRANGE(tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); + tsLogBufferMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; + tsLogBufferMemoryAllowed = TRANGE(tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); + // clang-format off TAOS_CHECK_RETURN(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); @@ -736,6 +740,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "syncLogBufferMemoryAllowed", tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbCheckSyncIntervalSec", tsArbCheckSyncIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); @@ -970,6 +975,14 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } + pItem = cfgGetItem(tsCfg, "syncLogBufferMemoryAllowed"); + if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { + tsLogBufferMemoryAllowed = totalMemoryKB * 1024 * 0.1; + tsLogBufferMemoryAllowed = TRANGE(tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); + pItem->i64 = tsLogBufferMemoryAllowed; + pItem->stype = stype; + } + TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -1520,6 +1533,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "syncSnapReplMaxWaitN"); tsSnapReplMaxWaitN = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "syncLogBufferMemoryAllowed"); + tsLogBufferMemoryAllowed = pItem->i64; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "arbHeartBeatIntervalSec"); tsArbHeartBeatIntervalSec = pItem->i32; @@ -1954,6 +1970,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"randErrorChance", &tsRandErrChance}, {"randErrorDivisor", &tsRandErrDivisor}, {"randErrorScope", &tsRandErrScope}, + {"syncLogBufferMemoryAllowed", &tsLogBufferMemoryAllowed}, {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold}, {"checkpointInterval", &tsStreamCheckpointInterval}, diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 009854b45b..0b653ddbe9 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "sync.h" -#include "taosdef.h" +#include "tglobal.h" #include "trpc.h" #include "ttimer.h" diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index ea85b796d5..427a3690f2 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -54,6 +54,7 @@ typedef struct SSyncLogBuffer { int64_t matchIndex; int64_t endIndex; int64_t size; + int64_t bytes; TdThreadMutex mutex; TdThreadMutexAttr attr; int64_t totalIndex; diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 4cdc6c3d83..5d20f9290b 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -28,6 +28,8 @@ #include "syncUtil.h" #include "syncVoteMgr.h" +static int64_t sSyncLogBufferBytes = 0; // total bytes of vnode log buffer + static bool syncIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM); @@ -101,6 +103,10 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = pMatch->index, .prevLogTerm = pMatch->term}; pBuf->entries[index % pBuf->size] = tmp; pBuf->endIndex = index + 1; + if (pNode->vgId > 1) { + pBuf->bytes += pEntry->bytes; + atomic_add_fetch_64(&sSyncLogBufferBytes, (int64_t)pEntry->bytes); + } (void)taosThreadMutexUnlock(&pBuf->mutex); TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); @@ -330,6 +336,7 @@ int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode) { (void)memset(&pBuf->entries[(index + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); } pBuf->startIndex = pBuf->commitIndex = pBuf->matchIndex = pBuf->endIndex = 0; + pBuf->bytes = 0; int32_t code = syncLogBufferInitWithoutLock(pBuf, pNode); if (code < 0) { sError("vgId:%d, failed to re-initialize sync log buffer since %s.", pNode->vgId, tstrerror(code)); @@ -470,8 +477,12 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt goto _out; } SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = prevIndex, .prevLogTerm = prevTerm}; - pEntry = NULL; pBuf->entries[index % pBuf->size] = tmp; + if (pNode->vgId > 1) { + pBuf->bytes += pEntry->bytes; + atomic_add_fetch_64(&sSyncLogBufferBytes, (int64_t)pEntry->bytes); + } + pEntry = NULL; // update end index pBuf->endIndex = TMAX(index + 1, pBuf->endIndex); @@ -846,14 +857,36 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm } // recycle - SyncIndex until = pBuf->commitIndex; // - TSDB_SYNC_LOG_BUFFER_RETENTION; - for (SyncIndex index = pBuf->startIndex; index < until; index++) { - SSyncRaftEntry* pEntry = pBuf->entries[(index + pBuf->size) % pBuf->size].pItem; - if (pEntry == NULL) return TSDB_CODE_SYN_INTERNAL_ERROR; + bool isVnode = pNode->vgId > 1; + SyncIndex until = pBuf->commitIndex - TSDB_SYNC_LOG_BUFFER_RETENTION; + do { + if (pBuf->startIndex >= pBuf->commitIndex) { + break; + } + SSyncRaftEntry* pEntry = pBuf->entries[(pBuf->startIndex + pBuf->size) % pBuf->size].pItem; + if (pEntry == NULL) { + sError("vgId:%d, invalid log entry to recycle. index:%" PRId64 ", startIndex:%" PRId64 ", until:%" PRId64 + ", commitIndex:%" PRId64 ", endIndex:%" PRId64 ", term:%" PRId64, + pNode->vgId, pEntry->index, pBuf->startIndex, until, pBuf->commitIndex, pBuf->endIndex, pEntry->term); + return TSDB_CODE_SYN_INTERNAL_ERROR; + } + if (!((pBuf->startIndex < until) || (isVnode && pBuf->bytes >= TSDB_SYNC_LOG_BUFFER_THRESHOLD && + atomic_load_64(&sSyncLogBufferBytes) >= tsLogBufferMemoryAllowed))) { + break; + } + if (isVnode) { + pBuf->bytes -= pEntry->bytes; + atomic_sub_fetch_64(&sSyncLogBufferBytes, (int64_t)pEntry->bytes); + } + sDebug("vgId:%d, recycle log entry. index:%" PRId64 ", startIndex:%" PRId64 ", until:%" PRId64 + ", commitIndex:%" PRId64 ", endIndex:%" PRId64 ", term:%" PRId64 ", entry bytes:%u, buf bytes:%" PRId64 + ", total bytes:%" PRId64, + pNode->vgId, pEntry->index, pBuf->startIndex, until, pBuf->commitIndex, pBuf->endIndex, pEntry->term, + pEntry->bytes, pBuf->bytes, atomic_load_64(&sSyncLogBufferBytes)); syncEntryDestroy(pEntry); - (void)memset(&pBuf->entries[(index + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); - pBuf->startIndex = index + 1; - } + memset(&pBuf->entries[(pBuf->startIndex + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); + ++pBuf->startIndex; + } while (true); code = 0; _out: @@ -1324,6 +1357,7 @@ void syncLogBufferClear(SSyncLogBuffer* pBuf) { (void)memset(&pBuf->entries[(index + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); } pBuf->startIndex = pBuf->commitIndex = pBuf->matchIndex = pBuf->endIndex = 0; + pBuf->bytes = 0; (void)taosThreadMutexUnlock(&pBuf->mutex); } From e92a58043ab8923b2f0b2301185c092db190bf1f Mon Sep 17 00:00:00 2001 From: Kaili Xu Date: Wed, 11 Sep 2024 14:44:29 +0800 Subject: [PATCH 12/94] enh: code optimization --- source/libs/sync/src/syncPipeline.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 5d20f9290b..1ba9de8fa7 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -860,7 +860,9 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm bool isVnode = pNode->vgId > 1; SyncIndex until = pBuf->commitIndex - TSDB_SYNC_LOG_BUFFER_RETENTION; do { - if (pBuf->startIndex >= pBuf->commitIndex) { + if ((pBuf->startIndex >= pBuf->commitIndex) || + !((pBuf->startIndex < until) || (isVnode && pBuf->bytes >= TSDB_SYNC_LOG_BUFFER_THRESHOLD && + atomic_load_64(&sSyncLogBufferBytes) >= tsLogBufferMemoryAllowed))) { break; } SSyncRaftEntry* pEntry = pBuf->entries[(pBuf->startIndex + pBuf->size) % pBuf->size].pItem; @@ -870,10 +872,6 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm pNode->vgId, pEntry->index, pBuf->startIndex, until, pBuf->commitIndex, pBuf->endIndex, pEntry->term); return TSDB_CODE_SYN_INTERNAL_ERROR; } - if (!((pBuf->startIndex < until) || (isVnode && pBuf->bytes >= TSDB_SYNC_LOG_BUFFER_THRESHOLD && - atomic_load_64(&sSyncLogBufferBytes) >= tsLogBufferMemoryAllowed))) { - break; - } if (isVnode) { pBuf->bytes -= pEntry->bytes; atomic_sub_fetch_64(&sSyncLogBufferBytes, (int64_t)pEntry->bytes); From ffaa1092c027577eb28e616c8feebb6e9ccfe977 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 11 Sep 2024 14:51:52 +0800 Subject: [PATCH 13/94] fix: oom with large msg --- source/libs/sync/src/syncPipeline.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 1ba9de8fa7..d7c55fcd8a 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -28,7 +28,7 @@ #include "syncUtil.h" #include "syncVoteMgr.h" -static int64_t sSyncLogBufferBytes = 0; // total bytes of vnode log buffer +int64_t tsLogBufferMemoryUsed = 0; // total bytes of vnode log buffer static bool syncIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || @@ -105,7 +105,7 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt pBuf->endIndex = index + 1; if (pNode->vgId > 1) { pBuf->bytes += pEntry->bytes; - atomic_add_fetch_64(&sSyncLogBufferBytes, (int64_t)pEntry->bytes); + atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); } (void)taosThreadMutexUnlock(&pBuf->mutex); @@ -480,7 +480,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt pBuf->entries[index % pBuf->size] = tmp; if (pNode->vgId > 1) { pBuf->bytes += pEntry->bytes; - atomic_add_fetch_64(&sSyncLogBufferBytes, (int64_t)pEntry->bytes); + atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); } pEntry = NULL; @@ -862,7 +862,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm do { if ((pBuf->startIndex >= pBuf->commitIndex) || !((pBuf->startIndex < until) || (isVnode && pBuf->bytes >= TSDB_SYNC_LOG_BUFFER_THRESHOLD && - atomic_load_64(&sSyncLogBufferBytes) >= tsLogBufferMemoryAllowed))) { + atomic_load_64(&tsLogBufferMemoryUsed) >= tsLogBufferMemoryAllowed))) { break; } SSyncRaftEntry* pEntry = pBuf->entries[(pBuf->startIndex + pBuf->size) % pBuf->size].pItem; @@ -874,13 +874,13 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm } if (isVnode) { pBuf->bytes -= pEntry->bytes; - atomic_sub_fetch_64(&sSyncLogBufferBytes, (int64_t)pEntry->bytes); + atomic_sub_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); } sDebug("vgId:%d, recycle log entry. index:%" PRId64 ", startIndex:%" PRId64 ", until:%" PRId64 ", commitIndex:%" PRId64 ", endIndex:%" PRId64 ", term:%" PRId64 ", entry bytes:%u, buf bytes:%" PRId64 ", total bytes:%" PRId64, pNode->vgId, pEntry->index, pBuf->startIndex, until, pBuf->commitIndex, pBuf->endIndex, pEntry->term, - pEntry->bytes, pBuf->bytes, atomic_load_64(&sSyncLogBufferBytes)); + pEntry->bytes, pBuf->bytes, atomic_load_64(&tsLogBufferMemoryUsed)); syncEntryDestroy(pEntry); memset(&pBuf->entries[(pBuf->startIndex + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); ++pBuf->startIndex; From 8b24bae5334a8fe188b32fae3d784eddb77af612 Mon Sep 17 00:00:00 2001 From: Kaili Xu Date: Wed, 11 Sep 2024 17:08:34 +0800 Subject: [PATCH 14/94] enh: return value of memset --- source/libs/sync/src/syncPipeline.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index d7c55fcd8a..ae0d5b7ddb 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -882,7 +882,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm pNode->vgId, pEntry->index, pBuf->startIndex, until, pBuf->commitIndex, pBuf->endIndex, pEntry->term, pEntry->bytes, pBuf->bytes, atomic_load_64(&tsLogBufferMemoryUsed)); syncEntryDestroy(pEntry); - memset(&pBuf->entries[(pBuf->startIndex + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); + (void)memset(&pBuf->entries[(pBuf->startIndex + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); ++pBuf->startIndex; } while (true); From b49d9f369199fb69ce2028b6717b96e940bd2e4b Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 11 Sep 2024 19:48:50 +0800 Subject: [PATCH 15/94] change mutex lock in SWal to rwlock --- include/libs/wal/wal.h | 4 +-- source/libs/sync/src/syncMain.c | 12 +++---- source/libs/wal/src/walMeta.c | 4 +-- source/libs/wal/src/walMgmt.c | 24 +++++++------- source/libs/wal/src/walRead.c | 12 +++---- source/libs/wal/src/walRef.c | 14 ++++----- source/libs/wal/src/walWrite.c | 56 ++++++++++++++++----------------- 7 files changed, 63 insertions(+), 63 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index a5d5316d23..5a2cf3a3a0 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -112,8 +112,8 @@ typedef struct SWal { int64_t totSize; int64_t lastRollSeq; // ctl - int64_t refId; - TdThreadMutex mutex; + int64_t refId; + TdThreadRwlock mutex; // ref SHashObj *pRefHash; // refId -> SWalRef // path diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 8e709f8809..8081de60c9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -3294,11 +3294,11 @@ _out:; // proceed match index, with replicating on needed SyncIndex matchIndex = syncLogBufferProceed(ths->pLogBuf, ths, NULL, "Append"); - if(pEntry != NULL) + if (pEntry != NULL) sTrace("vgId:%d, append raft entry. index:%" PRId64 ", term:%" PRId64 " pBuf: [%" PRId64 " %" PRId64 " %" PRId64 - ", %" PRId64 ")", - ths->vgId, pEntry->index, pEntry->term, ths->pLogBuf->startIndex, ths->pLogBuf->commitIndex, - ths->pLogBuf->matchIndex, ths->pLogBuf->endIndex); + ", %" PRId64 ")", + ths->vgId, pEntry->index, pEntry->term, ths->pLogBuf->startIndex, ths->pLogBuf->commitIndex, + ths->pLogBuf->matchIndex, ths->pLogBuf->endIndex); if (code == 0 && ths->state == TAOS_SYNC_STATE_ASSIGNED_LEADER) { (void)syncNodeUpdateAssignedCommitIndex(ths, matchIndex); @@ -3449,8 +3449,8 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pMsgReply->startTime = ths->startTime; pMsgReply->timeStamp = tsMs; - sGTrace("vgId:%d, process sync-heartbeat msg from dnode:%d, cluster:%d, Msgterm:%" PRId64 " currentTerm:%" PRId64, ths->vgId, - DID(&(pMsg->srcId)), CID(&(pMsg->srcId)), pMsg->term, currentTerm); + sGTrace("vgId:%d, process sync-heartbeat msg from dnode:%d, cluster:%d, Msgterm:%" PRId64 " currentTerm:%" PRId64, + ths->vgId, DID(&(pMsg->srcId)), CID(&(pMsg->srcId)), pMsg->term, currentTerm); if (pMsg->term > currentTerm && ths->state == TAOS_SYNC_STATE_LEARNER) { raftStoreSetTerm(ths, pMsg->term); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 0eb011113a..925524affb 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -684,7 +684,7 @@ _err: int64_t walGetVerRetention(SWal* pWal, int64_t bytes) { int64_t ver = -1; int64_t totSize = 0; - (void)taosThreadMutexLock(&pWal->mutex); + (void)taosThreadRwlockRdlock(&pWal->mutex); int32_t fileIdx = taosArrayGetSize(pWal->fileInfoSet); while (--fileIdx) { SWalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); @@ -694,7 +694,7 @@ int64_t walGetVerRetention(SWal* pWal, int64_t bytes) { } totSize += pInfo->fileSize; } - (void)taosThreadMutexUnlock(&pWal->mutex); + (void)taosThreadRwlockUnlock(&pWal->mutex); return ver + 1; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index e1d31ce113..31085e7919 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -21,11 +21,11 @@ #include "walInt.h" typedef struct { - int8_t stop; - int8_t inited; - uint32_t seq; - int32_t refSetId; - TdThread thread; + int8_t stop; + int8_t inited; + uint32_t seq; + int32_t refSetId; + TdThread thread; stopDnodeFn stopDnode; } SWalMgmt; @@ -88,7 +88,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } - if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { + if (taosThreadRwlockInit(&pWal->mutex, NULL) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosMemoryFree(pWal); return NULL; @@ -179,7 +179,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { _err: taosArrayDestroy(pWal->fileInfoSet); taosHashCleanup(pWal->pRefHash); - TAOS_UNUSED(taosThreadMutexDestroy(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockDestroy(&pWal->mutex)); taosMemoryFreeClear(pWal); return NULL; @@ -215,15 +215,15 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { int32_t walPersist(SWal *pWal) { int32_t code = 0; - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); code = walSaveMeta(pWal); - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(code); } void walClose(SWal *pWal) { - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); (void)walSaveMeta(pWal); TAOS_UNUSED(taosCloseFile(&pWal->pLogFile)); pWal->pLogFile = NULL; @@ -243,7 +243,7 @@ void walClose(SWal *pWal) { } taosHashCleanup(pWal->pRefHash); pWal->pRefHash = NULL; - (void)taosThreadMutexUnlock(&pWal->mutex); + (void)taosThreadRwlockUnlock(&pWal->mutex); if (pWal->cfg.level == TAOS_WAL_SKIP) { wInfo("vgId:%d, remove all wals, path:%s", pWal->cfg.vgId, pWal->path); @@ -258,7 +258,7 @@ static void walFreeObj(void *wal) { SWal *pWal = wal; wDebug("vgId:%d, wal:%p is freed", pWal->cfg.vgId, pWal); - (void)taosThreadMutexDestroy(&pWal->mutex); + (void)taosThreadRwlockDestroy(&pWal->mutex); taosMemoryFreeClear(pWal); } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1e9fcc70aa..e44e7038d0 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -115,9 +115,9 @@ void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *eve void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal *pOffset) { // if offset version is small than first version , let's seek to first version - TAOS_UNUSED(taosThreadMutexLock(&pWalReader->pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockRdlock(&pWalReader->pWal->mutex)); int64_t firstVer = walGetFirstVer((pWalReader)->pWal); - TAOS_UNUSED(taosThreadMutexUnlock(&pWalReader->pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWalReader->pWal->mutex)); if (pOffset->version < firstVer) { pOffset->version = firstVer; @@ -201,21 +201,21 @@ static int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { // bsearch in fileSet SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; - taosThreadMutexLock(&pWal->mutex); + TAOS_UNUSED(taosThreadRwlockRdlock(&pWal->mutex)); SWalFileInfo *gloablPRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); if (gloablPRet == NULL) { wError("failed to find WAL log file with ver:%" PRId64, ver); - taosThreadMutexUnlock(&pWal->mutex); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER); } SWalFileInfo *pRet = taosMemoryMalloc(sizeof(SWalFileInfo)); if (pRet == NULL) { wError("failed to allocate memory for localRet"); - taosThreadMutexUnlock(&pWal->mutex); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } TAOS_MEMCPY(pRet, gloablPRet, sizeof(SWalFileInfo)); - taosThreadMutexUnlock(&pWal->mutex); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); if (pReader->curFileFirstVer != pRet->firstVer) { // error code was set inner TAOS_CHECK_RETURN_WITH_FREE(walReadChangeFile(pReader, pRet->firstVer), pRet); diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index 579921a7e0..bf24ed89fb 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -61,34 +61,34 @@ int32_t walSetRefVer(SWalRef *pRef, int64_t ver) { SWal *pWal = pRef->pWal; wDebug("vgId:%d, wal ref version %" PRId64 ", refId %" PRId64, pWal->cfg.vgId, ver, pRef->refId); if (pRef->refVer != ver) { - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); if (ver < pWal->vers.firstVer || ver > pWal->vers.lastVer) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER); } pRef->refVer = ver; - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); } TAOS_RETURN(TSDB_CODE_SUCCESS); } void walRefFirstVer(SWal *pWal, SWalRef *pRef) { - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockRdlock(&pWal->mutex)); pRef->refVer = pWal->vers.firstVer; - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, pRef->refVer); } void walRefLastVer(SWal *pWal, SWalRef *pRef) { - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockRdlock(&pWal->mutex)); pRef->refVer = pWal->vers.lastVer; - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, pRef->refVer); } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 52af3e8528..b3c5f30ba3 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -23,7 +23,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { int32_t code = 0; - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); wInfo("vgId:%d, restore from snapshot, version %" PRId64, pWal->cfg.vgId, ver); @@ -34,7 +34,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { SWalRef *pRef = *(SWalRef **)pIter; if (pRef->refVer != -1 && pRef->refVer <= ver) { taosHashCancelIterate(pWal->pRefHash, pIter); - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_FAILED); } @@ -51,7 +51,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); if (taosRemoveFile(fnameStr) < 0) { wError("vgId:%d, restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } @@ -60,7 +60,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); if (taosRemoveFile(fnameStr) < 0) { wError("vgId:%d, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } @@ -81,7 +81,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { pWal->vers.snapshotVer = ver; pWal->vers.verInSnapshotting = -1; - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -160,12 +160,12 @@ static int64_t walChangeWrite(SWal *pWal, int64_t ver) { } int32_t walRollback(SWal *pWal, int64_t ver) { - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); wInfo("vgId:%d, wal rollback for version %" PRId64, pWal->cfg.vgId, ver); int64_t code; char fnameStr[WAL_FILE_LEN]; if (ver > pWal->vers.lastVer || ver <= pWal->vers.commitVer || ver <= pWal->vers.snapshotVer) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER); } @@ -175,7 +175,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // change current files code = walChangeWrite(pWal, ver); if (code < 0) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(code); } @@ -198,21 +198,21 @@ int32_t walRollback(SWal *pWal, int64_t ver) { TAOS_UNUSED(taosCloseFile(&pWal->pIdxFile)); TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); if (pIdxFile == NULL) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = taosLSeekFile(pIdxFile, idxOff, SEEK_SET); if (code < 0) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } // read idx file and get log file pos SWalIdxEntry entry; if (taosReadFile(pIdxFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } @@ -223,14 +223,14 @@ int32_t walRollback(SWal *pWal, int64_t ver) { wDebug("vgId:%d, wal truncate file %s", pWal->cfg.vgId, fnameStr); if (pLogFile == NULL) { // TODO - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } code = taosLSeekFile(pLogFile, entry.offset, SEEK_SET); if (code < 0) { // TODO - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } @@ -238,19 +238,19 @@ int32_t walRollback(SWal *pWal, int64_t ver) { SWalCkHead head; int64_t size = taosReadFile(pLogFile, &head, sizeof(SWalCkHead)); if (size != sizeof(SWalCkHead)) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } code = walValidHeadCksum(&head); if (code != 0) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED); } if (head.head.version != ver) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED); } @@ -258,13 +258,13 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // truncate old files code = taosFtruncateFile(pLogFile, entry.offset); if (code < 0) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } code = taosFtruncateFile(pIdxFile, idxOff); if (code < 0) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } @@ -278,13 +278,13 @@ int32_t walRollback(SWal *pWal, int64_t ver) { code = walSaveMeta(pWal); if (code < 0) { wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(code); } // unlock - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -375,7 +375,7 @@ int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) { TAOS_RETURN(TSDB_CODE_FAILED); } - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); pWal->vers.verInSnapshotting = ver; pWal->vers.logRetention = logRetention; @@ -391,7 +391,7 @@ int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) { } _exit: - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(code); } @@ -399,7 +399,7 @@ _exit: int32_t walEndSnapshot(SWal *pWal) { int32_t code = 0, lino = 0; - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); int64_t ver = pWal->vers.verInSnapshotting; wDebug("vgId:%d, wal end snapshot for version %" PRId64 ", log retention %" PRId64 " first ver %" PRId64 @@ -508,7 +508,7 @@ int32_t walEndSnapshot(SWal *pWal) { taosArrayClear(pWal->toDeleteFiles); _exit: - taosThreadMutexUnlock(&pWal->mutex); + taosThreadRwlockWrlock(&pWal->mutex); if (code) { wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code)); @@ -719,7 +719,7 @@ int32_t walAppendLog(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syn int32_t bodyLen) { int32_t code = 0, lino = 0; - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); if (index != pWal->vers.lastVer + 1) { TAOS_CHECK_GOTO(TSDB_CODE_WAL_INVALID_VER, &lino, _exit); @@ -738,7 +738,7 @@ _exit: wError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); return code; } @@ -749,7 +749,7 @@ int32_t walFsync(SWal *pWal, bool forceFsync) { return code; } - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (taosFsyncFile(pWal->pLogFile) < 0) { @@ -758,7 +758,7 @@ int32_t walFsync(SWal *pWal, bool forceFsync) { code = TAOS_SYSTEM_ERROR(errno); } } - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); return code; } From e127a29e64870330a78497180dd8e3ac1f0cbb91 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 12 Sep 2024 05:55:22 +0800 Subject: [PATCH 16/94] fix: oom with large msg --- source/libs/sync/src/syncPipeline.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index d7c55fcd8a..72d586f47e 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -266,6 +266,10 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = -1, .prevLogTerm = -1}; pBuf->entries[index % pBuf->size] = tmp; taken = true; + if (pNode->vgId > 1) { + pBuf->bytes += pEntry->bytes; + atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); + } } if (index < toIndex) { @@ -292,6 +296,10 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { } SSyncLogBufEntry tmp = {.pItem = pDummy, .prevLogIndex = commitIndex - 1, .prevLogTerm = commitTerm}; pBuf->entries[(commitIndex + pBuf->size) % pBuf->size] = tmp; + if (pNode->vgId > 1) { + pBuf->bytes += pDummy->bytes; + atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pDummy->bytes); + } if (index < toIndex) { pBuf->entries[(index + 1) % pBuf->size].prevLogIndex = commitIndex; @@ -878,9 +886,9 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm } sDebug("vgId:%d, recycle log entry. index:%" PRId64 ", startIndex:%" PRId64 ", until:%" PRId64 ", commitIndex:%" PRId64 ", endIndex:%" PRId64 ", term:%" PRId64 ", entry bytes:%u, buf bytes:%" PRId64 - ", total bytes:%" PRId64, + ", used:%" PRId64 ", allowed:%" PRId64, pNode->vgId, pEntry->index, pBuf->startIndex, until, pBuf->commitIndex, pBuf->endIndex, pEntry->term, - pEntry->bytes, pBuf->bytes, atomic_load_64(&tsLogBufferMemoryUsed)); + pEntry->bytes, pBuf->bytes, atomic_load_64(&tsLogBufferMemoryUsed), tsLogBufferMemoryAllowed); syncEntryDestroy(pEntry); memset(&pBuf->entries[(pBuf->startIndex + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); ++pBuf->startIndex; From e3121c24ebae09ba5e09f1be26406256046e358e Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 12 Sep 2024 06:38:15 +0800 Subject: [PATCH 17/94] fix: compile problem --- source/libs/sync/inc/syncInt.h | 2 +- source/libs/sync/src/syncPipeline.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 0b653ddbe9..2ff890da56 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "sync.h" -#include "tglobal.h" +#include "tdef.h" #include "trpc.h" #include "ttimer.h" diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 5eac01b884..279583340d 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -27,6 +27,7 @@ #include "syncSnapshot.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "tglobal.h" int64_t tsLogBufferMemoryUsed = 0; // total bytes of vnode log buffer From 15c92cea3fda0e8204fac0af9044ab4fd57b5dfa Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 12 Sep 2024 07:11:26 +0800 Subject: [PATCH 18/94] fix: compile problem --- source/libs/sync/inc/syncInt.h | 2 +- source/libs/sync/src/syncPipeline.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 2ff890da56..0b653ddbe9 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "sync.h" -#include "tdef.h" +#include "tglobal.h" #include "trpc.h" #include "ttimer.h" diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 279583340d..5eac01b884 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -27,7 +27,6 @@ #include "syncSnapshot.h" #include "syncUtil.h" #include "syncVoteMgr.h" -#include "tglobal.h" int64_t tsLogBufferMemoryUsed = 0; // total bytes of vnode log buffer From d9a25b31fe6346edc709bace7cc7698d1e40744f Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 12 Sep 2024 08:32:59 +0800 Subject: [PATCH 19/94] fix: compile problem --- source/libs/sync/inc/syncInt.h | 2 +- source/libs/sync/src/syncPipeline.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 0b653ddbe9..2ff890da56 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "sync.h" -#include "tglobal.h" +#include "tdef.h" #include "trpc.h" #include "ttimer.h" diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 5eac01b884..c891db07c6 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -27,8 +27,9 @@ #include "syncSnapshot.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "tglobal.h" -int64_t tsLogBufferMemoryUsed = 0; // total bytes of vnode log buffer +static int64_t tsLogBufferMemoryUsed = 0; // total bytes of vnode log buffer static bool syncIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || From 84e675c24862bd44c20aee73d742271bb9b0e720 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 11 Sep 2024 17:39:24 +0800 Subject: [PATCH 20/94] fix: memleak in taos (create table) --- source/client/src/clientImpl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d77b8dcbb7..ac5866401e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1163,6 +1163,9 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { (void)atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertRows, pResult->numOfRows); } } + if (TSDB_CODE_SUCCESS == code) { + schedulerFreeJob(&pRequest->body.queryJob, 0); + } } taosMemoryFree(pResult); From 8af5e89d4e0a77414e15b6afb774582d3157e638 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 11 Sep 2024 17:46:53 +0800 Subject: [PATCH 21/94] enh: create from csv rm tbname hash --- source/libs/parser/src/parTranslater.c | 33 ++++---------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 854bf83a1f..81adfff9a1 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5993,7 +5993,7 @@ static int32_t isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pO *pRet = false; return TSDB_CODE_SUCCESS; } - + SFunctionNode* pTbnameFunc = NULL; SValueNode* pValueNode = NULL; if (nodeType(pOperator->pLeft) == QUERY_NODE_FUNCTION && @@ -6275,7 +6275,7 @@ static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTa if (NULL == pMeta || TSDB_CHILD_TABLE != pMeta->tableType || pMeta->suid != pRealTable->pMeta->suid) { goto _return; } - + pRealTable->pMeta->uid = pMeta->uid; pRealTable->pMeta->vgId = pMeta->vgId; pRealTable->pMeta->tableType = pMeta->tableType; @@ -6386,11 +6386,11 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* } qDebug("before ctbname optimize, code:%d, aTableNum:%d, nTbls:%d, stableQuery:%d", code, aTableNum, nTbls, stableQuery); - + if (TSDB_CODE_SUCCESS == code && 1 == aTableNum && 1 == nTbls && stableQuery && NULL == pInfo->pRealTable->pTsmas) { code = replaceToChildTableQuery(pCxt, pInfo); } - + return code; } @@ -6797,7 +6797,7 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { if (pCxt->pParseCxt && pCxt->pParseCxt->setQueryFp) { (*pCxt->pParseCxt->setQueryFp)(pCxt->pParseCxt->requestRid); } - + if (NULL == pSelect->pFromTable) { return translateSelectWithoutFrom(pCxt, pSelect); } else { @@ -13801,7 +13801,6 @@ _OUT: } typedef struct SParseFileContext { - SHashObj* pTbNameHash; SArray* aTagNames; bool tagNameFilled; STableMeta* pStbMeta; @@ -13936,18 +13935,6 @@ static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFi code = parseOneStbRow(pMsgBuf, pParFileCxt); - if (TSDB_CODE_SUCCESS == code) { - if (taosHashGet(pParFileCxt->pTbNameHash, pParFileCxt->ctbName.tname, strlen(pParFileCxt->ctbName.tname) + 1) != - NULL) { - taosMemoryFreeClear(pParFileCxt->pTag); - code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TBNAME_DUPLICATED, pParFileCxt->ctbName.tname); - break; - } - - code = taosHashPut(pParFileCxt->pTbNameHash, pParFileCxt->ctbName.tname, strlen(pParFileCxt->ctbName.tname) + 1, - NULL, 0); - } - if (TSDB_CODE_SUCCESS == code) { code = fillVgroupInfo(pParseCxt, &pParFileCxt->ctbName, &pParFileCxt->vg); } @@ -13987,7 +13974,6 @@ static void destructParseFileContext(SParseFileContext** ppParFileCxt) { SParseFileContext* pParFileCxt = *ppParFileCxt; - taosHashCleanup(pParFileCxt->pTbNameHash); taosArrayDestroy(pParFileCxt->aTagNames); taosMemoryFreeClear(pParFileCxt->pStbMeta); taosArrayDestroy(pParFileCxt->aTagIndexs); @@ -14012,15 +13998,6 @@ static int32_t constructParseFileContext(SCreateSubTableFromFileClause* pStmt, S pParFileCxt->ctbName.acctId = acctId; strcpy(pParFileCxt->ctbName.dbname, pStmt->useDbName); - if (NULL == pParFileCxt->pTbNameHash) { - pParFileCxt->pTbNameHash = - taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); - if (!pParFileCxt->pTbNameHash) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _ERR; - } - } - if (NULL == pParFileCxt->aTagNames) { pParFileCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); if (NULL == pParFileCxt->aTagNames) { From d1cb30c47396a2cba83d9148ae4b5f653e36ebaf Mon Sep 17 00:00:00 2001 From: Kaili Xu Date: Thu, 12 Sep 2024 10:47:49 +0800 Subject: [PATCH 22/94] chore: restore header file --- source/libs/sync/inc/syncInt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 2ff890da56..009854b45b 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "sync.h" -#include "tdef.h" +#include "taosdef.h" #include "trpc.h" #include "ttimer.h" From 36a3a035b033ba7f55cd1dcdc9da8080b26a467b Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 12 Sep 2024 11:02:27 +0800 Subject: [PATCH 23/94] fix: compile problem --- source/libs/sync/src/syncPipeline.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index c891db07c6..a7c22ffd55 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -106,7 +106,7 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt pBuf->endIndex = index + 1; if (pNode->vgId > 1) { pBuf->bytes += pEntry->bytes; - atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); + (void)atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); } (void)taosThreadMutexUnlock(&pBuf->mutex); @@ -269,7 +269,7 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { taken = true; if (pNode->vgId > 1) { pBuf->bytes += pEntry->bytes; - atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); + (void)atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); } } @@ -299,7 +299,7 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { pBuf->entries[(commitIndex + pBuf->size) % pBuf->size] = tmp; if (pNode->vgId > 1) { pBuf->bytes += pDummy->bytes; - atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pDummy->bytes); + (void)atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pDummy->bytes); } if (index < toIndex) { @@ -489,7 +489,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt pBuf->entries[index % pBuf->size] = tmp; if (pNode->vgId > 1) { pBuf->bytes += pEntry->bytes; - atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); + (void)atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); } pEntry = NULL; @@ -883,7 +883,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm } if (isVnode) { pBuf->bytes -= pEntry->bytes; - atomic_sub_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); + (void)atomic_sub_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); } sDebug("vgId:%d, recycle log entry. index:%" PRId64 ", startIndex:%" PRId64 ", until:%" PRId64 ", commitIndex:%" PRId64 ", endIndex:%" PRId64 ", term:%" PRId64 ", entry bytes:%u, buf bytes:%" PRId64 From c2d28a208eabb537d813fa9e07bc018c51c44b0b Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 12 Sep 2024 11:18:10 +0800 Subject: [PATCH 24/94] chore: code optimization --- source/libs/sync/inc/syncInt.h | 2 +- source/libs/sync/src/syncPipeline.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 009854b45b..0b653ddbe9 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "sync.h" -#include "taosdef.h" +#include "tglobal.h" #include "trpc.h" #include "ttimer.h" diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index a7c22ffd55..44f5a293a9 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -27,7 +27,6 @@ #include "syncSnapshot.h" #include "syncUtil.h" #include "syncVoteMgr.h" -#include "tglobal.h" static int64_t tsLogBufferMemoryUsed = 0; // total bytes of vnode log buffer From db7b54168a69aef7bc2fdc1d7ba5cd41a80834ac Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 12 Sep 2024 06:24:03 +0000 Subject: [PATCH 25/94] fix/TD-31891-remove-void-stb-fix-case --- source/dnode/mnode/impl/src/mndStb.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 124d12fafb..37502a69ec 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -3533,14 +3533,12 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc pDb = mndAcquireDb(pMnode, pShow->db); if (pDb == NULL) return terrno; } - int64_t uid = pDb->uid; - mndReleaseDb(pMnode, pDb); while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); if (pShow->pIter == NULL) break; - if (pStb->dbUid != uid) { + if (pDb != NULL && pStb->dbUid != pDb->uid) { sdbRelease(pSdb, pStb); continue; } @@ -3632,6 +3630,10 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc sdbRelease(pSdb, pStb); } + if (pDb != NULL) { + mndReleaseDb(pMnode, pDb); + } + goto _OVER; _ERROR: From 5c5de6e9d126c780a80c7160290debfed1232c53 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 12 Sep 2024 07:04:47 +0000 Subject: [PATCH 26/94] fix/TD-31891-remove-void-mnd-fix-case --- source/dnode/mnode/impl/src/mndSync.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 51d27d3c6d..517553faac 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -498,8 +498,7 @@ int32_t mndInitSync(SMnode *pMnode) { tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); pNode->nodeRole = pMgmt->nodeRoles[i]; if (tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort) != true) { - mError("failed to open sync, tmsgUpdateDnodeInfo"); - TAOS_RETURN(-1); + mError("failed to open sync, tmsgUpdateDnodeInfo is false"); } mInfo("vgId:1, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); From 8393565d52382b5eae3f1e8d754548af2d3252f5 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 12 Sep 2024 08:08:08 +0000 Subject: [PATCH 27/94] fix/TD-31891-remove-void-mnode1-fix-case --- source/dnode/mnode/impl/src/mndCompact.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 76b1deb28b..d0cea30bf3 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -353,7 +353,7 @@ static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pC pHead->contLen = htonl(contLen); pHead->vgId = htonl(pVgroup->vgId); - if ((contLen = tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req)) <= 0) { + if ((contLen = tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req)) < 0) { terrno = contLen; return NULL; } @@ -616,8 +616,7 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) { pHead->contLen = htonl(contLen); pHead->vgId = htonl(pDetail->vgId); - if ((contLen = tSerializeSQueryCompactProgressReq((char *)pHead + sizeof(SMsgHead), contLen - sizeof(SMsgHead), - &req)) <= 0) { + if (tSerializeSQueryCompactProgressReq((char *)pHead + sizeof(SMsgHead), contLen - sizeof(SMsgHead), &req) <= 0) { sdbRelease(pMnode->pSdb, pDetail); continue; } From 295a34fc55654a9954f6cf6e3f5459ec8922172c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 12 Sep 2024 16:09:09 +0800 Subject: [PATCH 28/94] fix:[TD-31962]memory leak by crash_gen --- source/client/src/clientEnv.c | 2 +- source/client/src/clientImpl.c | 5 ++++- source/client/src/clientMain.c | 5 ++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 101bd9341c..fec1060042 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -941,7 +941,7 @@ void taos_init_imp(void) { tscInitRes = TSDB_CODE_OUT_OF_MEMORY; return; } -// taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst); avoid heap use after free + taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst); deltaToUtcInitOnce(); char logDirName[64] = {0}; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d77b8dcbb7..80ab094eaf 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -77,7 +77,10 @@ bool chkRequestKilled(void* param) { return killed; } -void cleanupAppInfo() { taosHashCleanup(appInfo.pInstMap); } +void cleanupAppInfo() { + taosHashCleanup(appInfo.pInstMap); + taosHashCleanup(appInfo.pInstMapByClusterId); +} static int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1c1fff9b7b..829c696c7c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -73,6 +73,8 @@ void taos_cleanup(void) { tscWarn("failed to cleanup task queue"); } + tmqMgmtClose(); + int32_t id = clientReqRefPool; clientReqRefPool = -1; taosCloseRef(id); @@ -87,9 +89,6 @@ void taos_cleanup(void) { tscDebug("rpc cleanup"); taosConvDestroy(); - - tmqMgmtClose(); - DestroyRegexCache(); tscInfo("all local resources released"); From d0731d2ece7ce289e27d0c261c9bd133a4bc9897 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 12 Sep 2024 08:26:48 +0000 Subject: [PATCH 29/94] fix/TD-31891-remove-void-mnode1-merge-code --- source/dnode/mnode/impl/src/mndSma.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 01afb789fd..f91f7d49cc 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -379,6 +379,7 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); // get length + int32_t ret = 0; tEncodeSize(tEncodeSVDropTSmaReq, &req, contLen, ret); if (ret < 0) { return NULL; From 6347d16121f6cd2e42ece17daaa4274b2407c477 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 12 Sep 2024 17:32:42 +0800 Subject: [PATCH 30/94] fix:[TD-31899] remove void(func) --- source/client/src/clientMsgHandler.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index c36162f102..aef3cef1c5 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -386,9 +386,11 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) { SMCreateStbRsp createRsp = {0}; SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - code = tDecodeSMCreateStbRsp(&coder, &createRsp); // pMsg->len == 0 - if (code != TSDB_CODE_SUCCESS) { - setErrno(pRequest, code); + if (pMsg->len > 0){ + code = tDecodeSMCreateStbRsp(&coder, &createRsp); // pMsg->len == 0 + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + } } tDecoderClear(&coder); @@ -477,9 +479,11 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) { SMAlterStbRsp alterRsp = {0}; SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - code = tDecodeSMAlterStbRsp(&coder, &alterRsp); // pMsg->len = 0 - if (code != TSDB_CODE_SUCCESS) { - setErrno(pRequest, code); + if (pMsg->len > 0){ + code = tDecodeSMAlterStbRsp(&coder, &alterRsp); // pMsg->len == 0 + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + } } tDecoderClear(&coder); From c8ee66eee2cceb4b4e917f3f83fa282c3bec34e1 Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Thu, 12 Sep 2024 20:57:36 +0800 Subject: [PATCH 31/94] modify error code passing --- source/common/src/cos_cp.c | 2 +- source/common/src/tdatablock.c | 29 ++++++++++----------- source/common/src/tdataformat.c | 18 ++++++------- source/common/src/tmisce.c | 4 +-- source/common/src/tmsg.c | 8 +++--- source/common/src/trow.c | 2 +- source/common/src/ttime.c | 12 ++++----- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 8 +++--- source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 8 +++--- source/dnode/mgmt/mgmt_snode/src/smWorker.c | 6 ++--- source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 8 +++--- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 3 +-- source/dnode/mgmt/node_util/src/dmEps.c | 12 ++++----- source/dnode/mgmt/node_util/src/dmFile.c | 10 +++---- 16 files changed, 66 insertions(+), 68 deletions(-) diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index e9af9c0306..1db417f6d9 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -170,7 +170,7 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) { cp_body = taosMemoryMalloc(size + 1); if (!cp_body) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } int64_t n = taosReadFile(fd, cp_body, size); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index ea3e88919b..757813dfbc 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -124,7 +124,7 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pColumnInfoData->pData = buf; @@ -170,7 +170,7 @@ static int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) if (pColumnInfoData->varmeta.allocLen < newSize) { char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pColumnInfoData->pData = buf; @@ -388,7 +388,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int if (finalNumOfRows > (*capacity)) { char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } *capacity = finalNumOfRows; @@ -409,7 +409,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int if (pColumnInfoData->varmeta.allocLen < len + oldLen) { char* tmp = taosMemoryRealloc(pColumnInfoData->pData, len + oldLen); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pColumnInfoData->pData = tmp; @@ -425,14 +425,14 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int // all data may be null, when the pColumnInfoData->info.type == 0, bytes == 0; char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pColumnInfoData->pData = tmp; if (BitmapLen(numOfRow1) < BitmapLen(finalNumOfRows)) { char* btmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(finalNumOfRows)); if (btmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } uint32_t extend = BitmapLen(finalNumOfRows) - BitmapLen(numOfRow1); memset(btmp + BitmapLen(numOfRow1), 0, extend); @@ -469,7 +469,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p if (pColumnInfoData->varmeta.allocLen < newLen) { char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newLen); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pColumnInfoData->pData = tmp; @@ -545,7 +545,7 @@ int32_t colDataAssignNRows(SColumnInfoData* pDst, int32_t dstIdx, const SColumnI if (pDst->varmeta.allocLen < pDst->varmeta.length + allLen) { char* tmp = taosMemoryRealloc(pDst->pData, pDst->varmeta.length + allLen); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pDst->pData = tmp; @@ -995,7 +995,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { size_t metaSize = pBlock->info.rows * sizeof(int32_t); char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize); // preview calloc is too small if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pCol->varmeta.offset = (int32_t*)tmp; @@ -1013,7 +1013,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { if (pCol->varmeta.allocLen < colLength) { char* tmp = taosMemoryRealloc(pCol->pData, colLength); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pCol->pData = tmp; @@ -1094,7 +1094,7 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) if (pCol->varmeta.allocLen < colLength) { char* tmp = taosMemoryRealloc(pCol->pData, colLength); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pCol->pData = tmp; @@ -1417,7 +1417,6 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { int32_t* index = createTupleIndex(rows); if (index == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -1540,7 +1539,7 @@ int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockI if (IS_VAR_DATA_TYPE(pColumn->info.type)) { char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pColumn->varmeta.offset = (int32_t*)tmp; @@ -1549,7 +1548,7 @@ int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockI // prepare for the null bitmap char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows)); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int32_t oldLen = BitmapLen(existedRows); @@ -1563,7 +1562,7 @@ int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockI // to MALLOC_ALIGN_BYTES tmp = taosMemoryMallocAlign(MALLOC_ALIGN_BYTES, numOfRows * pColumn->info.bytes); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } // memset(tmp, 0, numOfRows * pColumn->info.bytes); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 24e669577a..4b44e4af43 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -489,7 +489,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, colVal = COL_VAL_VALUE(infos[iInfo].columnId, value); } if (taosArrayPush(colValArray, &colVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -500,7 +500,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, } if ((taosArrayPush(rowArray, &row)) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -702,7 +702,7 @@ static int32_t tRowMergeImpl(SArray *aRowP, STSchema *pTSchema, int32_t iStart, if (pColVal) { if (taosArrayPush(aColVal, pColVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -1756,7 +1756,7 @@ int32_t tTagToValArray(const STag *pTag, SArray **ppArray) { } (void)tGetTagVal(p + offset, &tv, pTag->flags & TD_TAG_JSON); if (taosArrayPush(*ppArray, &tv) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _err; } } @@ -3251,13 +3251,13 @@ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorte } if ((bufArray = taosArrayInit(numOfInfos, sizeof(uint8_t *))) == NULL) { taosArrayDestroy(colValArray); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int i = 0; i < numOfInfos; ++i) { if (!taosArrayPush(bufArray, &infos[i].bind->buffer)) { taosArrayDestroy(colValArray); taosArrayDestroy(bufArray); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -3294,7 +3294,7 @@ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorte colVal = COL_VAL_VALUE(infos[iInfo].columnId, value); } if (taosArrayPush(colValArray, &colVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -3305,7 +3305,7 @@ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorte } if ((taosArrayPush(rowArray, &row)) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -3502,7 +3502,7 @@ static int32_t tColDataMerge(SArray **colArr) { dst = taosArrayInit(taosArrayGetSize(src), sizeof(SColData)); if (dst == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int32_t i = 0; i < taosArrayGetSize(src); i++) { diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index f3f4b29617..338de6242e 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -179,7 +179,7 @@ int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) { int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) { int32_t code = 0; SJson* pJson = tjsonCreateObject(); - if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pJson == NULL) return terrno; char tmp[4096] = {0}; @@ -242,7 +242,7 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t char* pCont = tjsonToString(pJson); if (pCont == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_CHECK_GOTO(code, NULL, _exit); goto _exit; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 758a4aeec3..1f2268242f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -12026,19 +12026,19 @@ int32_t tCloneTbTSMAInfo(STableTSMAInfo *pInfo, STableTSMAInfo **pRes) { *pRet = *pInfo; if (pInfo->pFuncs) { pRet->pFuncs = taosArrayDup(pInfo->pFuncs, NULL); - if (!pRet->pFuncs) code = TSDB_CODE_OUT_OF_MEMORY; + if (!pRet->pFuncs) code = terrno; } if (pInfo->pTags && code == TSDB_CODE_SUCCESS) { pRet->pTags = taosArrayDup(pInfo->pTags, NULL); - if (!pRet->pTags) code = TSDB_CODE_OUT_OF_MEMORY; + if (!pRet->pTags) code = terrno; } if (pInfo->pUsedCols && code == TSDB_CODE_SUCCESS) { pRet->pUsedCols = taosArrayDup(pInfo->pUsedCols, NULL); - if (!pRet->pUsedCols) code = TSDB_CODE_OUT_OF_MEMORY; + if (!pRet->pUsedCols) code = terrno; } if (pInfo->ast && code == TSDB_CODE_SUCCESS) { pRet->ast = taosStrdup(pInfo->ast); - if (!pRet->ast) code = TSDB_CODE_OUT_OF_MEMORY; + if (!pRet->ast) code = terrno; } if (code) { tFreeAndClearTableTSMAInfo(pRet); diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 760b86dee1..40b4f863cb 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -482,7 +482,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r if (isAlloc) { taosMemoryFreeClear(*ppRow); } - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index d771830d3f..7e8749ef8b 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -1247,7 +1247,7 @@ static int32_t parseTsFormat(const char* formatStr, SArray* formats) { const TSFormatKeyWord* key = keywordSearch(formatStr); if (key) { TSFormatNode format = {.key = key, .type = TS_FORMAT_NODE_TYPE_KEYWORD}; - if (NULL == taosArrayPush(formats, &format)) TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + if (NULL == taosArrayPush(formats, &format)) TAOS_RETURN(terrno); formatStr += key->len; lastOtherFormat = NULL; } else { @@ -1274,7 +1274,7 @@ static int32_t parseTsFormat(const char* formatStr, SArray* formats) { TSFormatNode format = {.type = TS_FORMAT_NODE_TYPE_CHAR, .key = NULL}; format.c = formatStr; format.len = 1; - if (NULL == taosArrayPush(formats, &format)) TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + if (NULL == taosArrayPush(formats, &format)) TAOS_RETURN(terrno); formatStr++; last = taosArrayGetLast(formats); } @@ -1301,7 +1301,7 @@ static int32_t parseTsFormat(const char* formatStr, SArray* formats) { .key = NULL}; format.c = formatStr; format.len = 1; - if (NULL == taosArrayPush(formats, &format)) TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + if (NULL == taosArrayPush(formats, &format)) TAOS_RETURN(terrno); formatStr++; if (format.type == TS_FORMAT_NODE_TYPE_CHAR) lastOtherFormat = taosArrayGetLast(formats); } @@ -1910,7 +1910,7 @@ int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t pr if (!*formats) { *formats = taosArrayInit(8, sizeof(TSFormatNode)); if (!*formats) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } TAOS_CHECK_RETURN(parseTsFormat(format, *formats)); } @@ -1926,7 +1926,7 @@ int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int if (!*formats) { *formats = taosArrayInit(4, sizeof(TSFormatNode)); if (!*formats) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } TAOS_CHECK_RETURN(parseTsFormat(format, *formats)); } @@ -1951,7 +1951,7 @@ int32_t TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* ou SArray* formats = taosArrayInit(4, sizeof(TSFormatNode)); if (!formats) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } TAOS_CHECK_RETURN(parseTsFormat(format, formats)); struct STm tm; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 9c22a11674..f9ae94c53f 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -345,7 +345,7 @@ int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; // rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; // return rspMsg.code; } @@ -383,7 +383,7 @@ int32_t dmBuildVariablesBlock(SSDataBlock **ppBlock) { pBlock->pDataBlock = taosArrayInit(pMeta[index].colNum, sizeof(SColumnInfoData)); if (pBlock->pDataBlock == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -393,7 +393,7 @@ int32_t dmBuildVariablesBlock(SSDataBlock **ppBlock) { colInfoData.info.type = pMeta[index].schema[i].type; colInfoData.info.bytes = pMeta[index].schema[i].bytes; if (taosArrayPush(pBlock->pDataBlock, &colInfoData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -457,7 +457,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; dError("failed to retrieve data since %s", tstrerror(code)); blockDataDestroy(pBlock); return code; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index d7170398fb..1561ab0a6b 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -54,7 +54,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { int32_t code = 0; SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt)); if (pMgmt == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pMgmt->pData = pInput->pData; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 563d123b83..f02970e4b2 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -91,7 +91,7 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) { pData = taosMemoryMalloc(size + 1); if (pData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -134,14 +134,14 @@ static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) { SJson *replicas = tjsonCreateArray(); if (replicas == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if ((code = tjsonAddItemToObject(pJson, "replicas", replicas)) < 0) return code; for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { SJson *replica = tjsonCreateObject(); if (replica == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } const SReplica *pReplica = pOption->replicas + i; @@ -183,7 +183,7 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { // terrno = TSDB_CODE_OUT_OF_MEMORY; pJson = tjsonCreateObject(); if (pJson == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index ad602313e0..e8594130d6 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -71,14 +71,14 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) { int32_t code = 0; pMgmt->writeWroker = taosArrayInit(0, sizeof(SMultiWorker *)); if (pMgmt->writeWroker == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; return code; } for (int32_t i = 0; i < tsNumOfSnodeWriteThreads; i++) { SMultiWorker *pWriteWorker = taosMemoryMalloc(sizeof(SMultiWorker)); if (pWriteWorker == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; return code; } @@ -93,7 +93,7 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) { return code; } if (taosArrayPush(pMgmt->writeWroker, &pWriteWorker) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; return code; } } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index 8513d31695..1e61d450b9 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -128,7 +128,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t pData = taosMemoryMalloc(size + 1); if (pData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -169,7 +169,7 @@ static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t num int32_t code = 0; SJson *vnodes = tjsonCreateArray(); if (vnodes == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if ((code = tjsonAddItemToObject(pJson, "vnodes", vnodes)) < 0) { tjsonDelete(vnodes); @@ -181,7 +181,7 @@ static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t num if (pVnode == NULL) continue; SJson *vnode = tjsonCreateObject(); - if (vnode == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (vnode == NULL) return terrno; if ((code = tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId)) < 0) return code; if ((code = tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped)) < 0) return code; if ((code = tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion)) < 0) return code; @@ -221,7 +221,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { // terrno = TSDB_CODE_OUT_OF_MEMORY; pJson = tjsonCreateObject(); if (pJson == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } if ((code = vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes)) != 0) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 2abf292e73..f020ba2f5e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -921,7 +921,7 @@ int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = terrno; goto _OVER; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index fdce9fd4c9..ba0a40e048 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -161,8 +161,7 @@ int32_t dmInitVars(SDnode *pDnode) { pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if (pData->dnodeHash == NULL) { dError("failed to init dnode hash"); - code = TSDB_CODE_OUT_OF_MEMORY; - return terrno = code; + return terrno; } if ((code = dmReadEps(pData)) != 0) { diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 315c4d7430..7cd794500c 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -217,7 +217,7 @@ int32_t dmReadEps(SDnodeData *pData) { content = taosMemoryMalloc(size + 1); if (content == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -321,7 +321,7 @@ int32_t dmWriteEps(SDnodeData *pData) { TAOS_CHECK_GOTO(dmInitDndInfo(pData), NULL, _OVER); pJson = tjsonCreateObject(); - if (pJson == NULL) TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); + if (pJson == NULL) TAOS_CHECK_GOTO(terrno, NULL, _OVER); pData->engineVer = tsVersion; @@ -329,7 +329,7 @@ int32_t dmWriteEps(SDnodeData *pData) { buffer = tjsonToString(pJson); if (buffer == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); + TAOS_CHECK_GOTO(terrno, NULL, _OVER); } pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); @@ -575,7 +575,7 @@ static int32_t dmDecodeEpPairs(SJson *pJson, SDnodeData *pData) { tjsonGetUInt16ValueFromDouble(dnode, "new_port", pair.newPort, code); if (code < 0) return TSDB_CODE_INVALID_CFG_VALUE; - if (taosArrayPush(pData->oldDnodeEps, &pair) == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (taosArrayPush(pData->oldDnodeEps, &pair) == NULL) return terrno; } return code; @@ -621,7 +621,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) { content = taosMemoryMalloc(size + 1); if (content == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -641,7 +641,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) { pData->oldDnodeEps = taosArrayInit(1, sizeof(SDnodeEpPair)); if (pData->oldDnodeEps == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; dError("failed to calloc dnodeEp array since %s", strerror(errno)); goto _OVER; } diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index c8b24d5469..ae66999bcf 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -70,7 +70,7 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) { content = taosMemoryMalloc(size + 1); if (content == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -134,7 +134,7 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) { pJson = tjsonCreateObject(); if (pJson == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -230,7 +230,7 @@ static int32_t dmWriteCheckCodeFile(char *file, char *realfile, char *key, bool int32_t len = ENCRYPTED_LEN(sizeof(DM_KEY_INDICATOR)); result = taosMemoryMalloc(len); if (result == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } SCryptOpts opts; @@ -338,7 +338,7 @@ static int32_t dmCompareEncryptKey(char *file, char *key, bool toLogFile) { content = taosMemoryMalloc(size); if (content == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -479,7 +479,7 @@ static int32_t dmReadEncryptCodeFile(char *file, char **output) { content = taosMemoryMalloc(size + 1); if (content == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } From 0335799e1cc9b813aff10d065cf71b1e522123db Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 12 Sep 2024 21:48:54 +0800 Subject: [PATCH 32/94] refactor: remove void. --- source/dnode/mnode/impl/src/mndStream.c | 12 ++++---- source/dnode/mnode/impl/src/mndStreamHb.c | 11 +++++--- source/dnode/mnode/impl/src/mndStreamUtil.c | 31 +++++++++++++++------ 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index cbe631912c..e0b8caa938 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2221,7 +2221,11 @@ static int32_t refreshNodeListFromExistedStreams(SMnode *pMnode, SArray *pNodeLi } char buf[256] = {0}; - (void) epsetToStr(&pEntry->epset, buf, tListLen(buf)); // ignore this error since it is only for log file + int32_t ret = epsetToStr(&pEntry->epset, buf, tListLen(buf)); // ignore this error since it is only for log file + if (ret != 0) { // print error and continue + mError("failed to convert epset to str, code:%s", tstrerror(ret)); + } + mDebug("extract nodeInfo from stream obj, nodeId:%d, %s", pEntry->nodeId, buf); } @@ -2231,7 +2235,7 @@ static int32_t refreshNodeListFromExistedStreams(SMnode *pMnode, SArray *pNodeLi return code; } -static int32_t addAllDbsIntoHashmap(SHashObj *pDBMap, SSdb *pSdb) { +static void addAllDbsIntoHashmap(SHashObj *pDBMap, SSdb *pSdb) { void *pIter = NULL; int32_t code = 0; while (1) { @@ -2249,8 +2253,6 @@ static int32_t addAllDbsIntoHashmap(SHashObj *pDBMap, SSdb *pSdb) { mDebug("add Db:%s into Dbs list (total:%d) for kill checkpoint trans", pVgroup->dbName, size); } } - - return code; } // this function runs by only one thread, so it is not multi-thread safe @@ -2311,7 +2313,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { mInfo("rollback all stream due to mnode leader/follower switch by using nodeUpdate trans"); updateAllVgroups = true; execInfo.switchFromFollower = false; // reset the flag - (void) addAllDbsIntoHashmap(changeInfo.pDBMap, pMnode->pSdb); + addAllDbsIntoHashmap(changeInfo.pDBMap, pMnode->pSdb); } } diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 43f9d8d055..941956ae2b 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -194,10 +194,13 @@ int32_t mndSendDropOrphanTasksMsg(SMnode *pMnode, SArray *pList) { return terrno; } - (void)tSerializeDropOrphanTaskMsg(pReq, contLen, &msg); + int32_t code = tSerializeDropOrphanTaskMsg(pReq, contLen, &msg); + if (code <= 0) { + mError("failed to serialize the drop orphan task msg, code:%s", tstrerror(code)); + } SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_DROP_ORPHANTASKS, .pCont = pReq, .contLen = contLen}; - int32_t code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); if (code) { mError("failed to put drop-orphan task msg into write queue, code:%s", tstrerror(code)); } else { @@ -216,7 +219,7 @@ int32_t mndProcessResetStatusReq(SRpcMsg *pReq) { mndKillTransImpl(pMnode, pMsg->transId, ""); streamMutexLock(&execInfo.lock); - (void) mndResetChkptReportInfo(execInfo.pChkptStreams, pMsg->streamId); + code = mndResetChkptReportInfo(execInfo.pChkptStreams, pMsg->streamId); // do thing if failed streamMutexUnlock(&execInfo.lock); code = mndGetStreamObj(pMnode, pMsg->streamId, &pStream); @@ -393,7 +396,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes); if (numOfUpdated > 0) { mDebug("%d stream node(s) need updated from hbMsg(vgId:%d)", numOfUpdated, req.vgId); - (void) setNodeEpsetExpiredFlag(req.pUpdateNodes); + int32_t unused = setNodeEpsetExpiredFlag(req.pUpdateNodes); } bool snodeChanged = false; diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index b2e35827af..bad44a8687 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -165,7 +165,10 @@ int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray **pList) { } char buf[256] = {0}; - (void)epsetToStr(&entry.epset, buf, tListLen(buf)); + code = epsetToStr(&entry.epset, buf, tListLen(buf)); + if (code != 0) { // print error and continue + mError("failed to convert epset to str, code:%s", tstrerror(code)); + } void *p = taosArrayPush(pVgroupList, &entry); if (p == NULL) { @@ -198,7 +201,10 @@ int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray **pList) { } char buf[256] = {0}; - (void)epsetToStr(&entry.epset, buf, tListLen(buf)); + code = epsetToStr(&entry.epset, buf, tListLen(buf)); + if (code != 0) { // print error and continue + mError("failed to convert epset to str, code:%s", tstrerror(code)); + } void *p = taosArrayPush(pVgroupList, &entry); if (p == NULL) { @@ -424,9 +430,12 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa } char buf[256] = {0}; - (void) epsetToStr(&epset, buf, tListLen(buf)); - mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); + code = epsetToStr(&epset, buf, tListLen(buf)); + if (code != 0) { // print error and continue + mError("failed to convert epset to str, code:%s", tstrerror(code)); + } + mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != 0) { taosMemoryFree(pReq); @@ -639,8 +648,7 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) { void *pBuf = NULL; int32_t len = 0; - (void)streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); - + bool unusedRet = streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); int32_t code = doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id); if (code) { return code; @@ -914,8 +922,15 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { } // 2. remove stream entry in consensus hash table and checkpoint-report hash table - (void) mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid); - (void) mndClearChkptReportInfo(execInfo.pChkptStreams, pStream->uid); + code = mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid); + if (code) { + mError("failed to clear consensus checkpointId, code:%s", tstrerror(code)); + } + + code = mndClearChkptReportInfo(execInfo.pChkptStreams, pStream->uid); + if (code) { + mError("failed to clear the checkpoint report info, code:%s", tstrerror(code)); + } streamMutexUnlock(&pExecNode->lock); destroyStreamTaskIter(pIter); From 3c8ae1fc9f30ba1f19a8916462c4ee4fe8344170 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 13 Sep 2024 09:27:11 +0800 Subject: [PATCH 33/94] fix:[TD-31962]memory leak by crash_gen --- source/libs/executor/src/scanoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 717b8793f2..b641c66e45 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -741,7 +741,6 @@ _end: sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW, NULL); if (insertRet != TAOS_LRU_STATUS_OK) { qWarn("failed to put meta into lru cache, code:%d, %s", insertRet, idStr); - freeTableCachedVal(pVal); } } From 49e2f1f0a4137f0962b56847a4ffd333dff79a1c Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Thu, 12 Sep 2024 14:57:03 +0800 Subject: [PATCH 34/94] avoid runtime error for double to int64_t conversion --- source/util/src/tunit.c | 28 ++++++++++++++++++++-------- tests/system-test/0-others/show.py | 14 ++++++++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/source/util/src/tunit.c b/source/util/src/tunit.c index 57226878dd..e73045cc89 100644 --- a/source/util/src/tunit.c +++ b/source/util/src/tunit.c @@ -23,14 +23,16 @@ #define UNIT_ONE_PEBIBYTE (UNIT_ONE_TEBIBYTE * UNIT_SIZE_CONVERT_FACTOR) #define UNIT_ONE_EXBIBYTE (UNIT_ONE_PEBIBYTE * UNIT_SIZE_CONVERT_FACTOR) -static int32_t parseCfgIntWithUnit(const char* str, double* res) { - double val, temp = (double)INT64_MAX; +static int32_t parseCfgIntWithUnit(const char* str, int64_t* res) { + double val = 0, temp = (double)INT64_MAX; char* endPtr; + bool useDouble = false; errno = 0; - val = taosStr2Int64(str, &endPtr, 0); + int64_t int64Val = taosStr2Int64(str, &endPtr, 0); if (*endPtr == '.' || errno == ERANGE) { errno = 0; val = taosStr2Double(str, &endPtr); + useDouble = true; } if (endPtr == str || errno == ERANGE || isnan(val)) { return terrno = TSDB_CODE_INVALID_CFG_VALUE; @@ -67,23 +69,33 @@ static int32_t parseCfgIntWithUnit(const char* str, double* res) { default: return terrno = TSDB_CODE_INVALID_CFG_VALUE; } + endPtr++; + if ((val > 0 && val > temp) || (val < 0 && val < -temp)) { return terrno = TSDB_CODE_OUT_OF_RANGE; } - endPtr++; val *= factor; + int64Val *= factor; } while (isspace((unsigned char)*endPtr)) endPtr++; if (*endPtr) { return terrno = TSDB_CODE_INVALID_CFG_VALUE; } - val = rint(val); - *res = val; + if (useDouble) { + val = rint(val); + if ((val > 0 && val >= (double)INT64_MAX) || (val < 0 && val <= (double)INT64_MIN)) { + return terrno = TSDB_CODE_OUT_OF_RANGE; + } else { + *res = (int64_t)val; + } + } else { + *res = int64Val; + } return TSDB_CODE_SUCCESS; } int32_t taosStrHumanToInt64(const char* str, int64_t* out) { - double res; + int64_t res; int32_t code = parseCfgIntWithUnit(str, &res); if (code == TSDB_CODE_SUCCESS) *out = (int64_t)res; return code; @@ -109,7 +121,7 @@ void taosInt64ToHumanStr(int64_t val, char* outStr) { #endif int32_t taosStrHumanToInt32(const char* str, int32_t* out) { - double res; + int64_t res; int32_t code = parseCfgIntWithUnit(str, &res); if (code == TSDB_CODE_SUCCESS) { if (res < INT32_MIN || res > INT32_MAX) { diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py index 64696c5e6d..032e5eebe1 100644 --- a/tests/system-test/0-others/show.py +++ b/tests/system-test/0-others/show.py @@ -284,6 +284,20 @@ class TDTestCase: for error_val in error_vals: tdSql.error(f'ALTER DNODE 1 "{var}" "{error_val}"') + var = 'randErrorDivisor' + vals = ['9223372036854775807', '9223372036854775807.1', '9223372036854775806', '9223372036854775808', '9223372036854775808.1', '9223372036854775807.0', '9223372036854775806.1'] + expected_vals = ['9223372036854775807', 'err', '9223372036854775806', 'err', 'err', 'err', 'err'] + for val_str, expected_val in zip(vals, expected_vals): + sql = f'ALTER dnode 1 "{var}" "{val_str}"' + if expected_val == 'err': + tdSql.error(sql) + else: + tdSql.execute(sql, queryTimes=1) + actual_val = self.get_variable(var, False) + if expected_val != actual_val: + tdLog.exit(f"failed to set local {var} to {expected_val} actually {actual_val}") + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 8547b8c942dbb01e043ba654b34e55f38a80a99f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Sep 2024 09:54:26 +0800 Subject: [PATCH 35/94] fix(tsdb/cache/read): comment lastTs out to load stt data --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index b24aa6fb1d..72f2052867 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -625,7 +625,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } double cost = (taosGetTimestampUs() - st) / 1000.0; if (cost > tsCacheLazyLoadThreshold) { - pr->lastTs = totalLastTs; + // pr->lastTs = totalLastTs; } } } From 1eb3124cbd7e7039940dce7866998397cd365a3b Mon Sep 17 00:00:00 2001 From: wangmeng Date: Fri, 13 Sep 2024 10:31:10 +0800 Subject: [PATCH 36/94] add a test case to verify TS-5349 --- .../query/sys/tb_perf_queries_exist_test.py | 26 +++++++++++++++++++ tests/parallel_test/cases.task | 1 + 2 files changed, 27 insertions(+) create mode 100644 tests/army/query/sys/tb_perf_queries_exist_test.py diff --git a/tests/army/query/sys/tb_perf_queries_exist_test.py b/tests/army/query/sys/tb_perf_queries_exist_test.py new file mode 100644 index 0000000000..e6afc0bec6 --- /dev/null +++ b/tests/army/query/sys/tb_perf_queries_exist_test.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.autogen import * + +''' + TS-5349: https://jira.taosdata.com:18080/browse/TS-5349 + 查询 performance_schema.perf_queries 后, 再查询 information_schema.perf_queries, + 正常情况下在 information_schema 中不存在表 perf_queries +''' + +class TDTestCase(TBase): + + def run(self): + tdSql.query("select * from performance_schema.perf_queries;") + tdLog.info("Table [perf_queries] exist in schema [performance_schema]") + + tdSql.error("select * from information_schema.perf_queries;") + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3c2adfb46f..92f6f32943 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -42,6 +42,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_compare_asc_desc.py ,,y,army,./pytest.sh python3 ./test.py -f query/last/test_last.py ,,y,army,./pytest.sh python3 ./test.py -f query/window/base.py +,,y,army,./pytest.sh python3 ./test.py -f query/sys/tb_perf_queries_exist_test.py -N 3 # # system test From 9cf662f1642be01a76895ee67c48ef2a562b5e6b Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Fri, 13 Sep 2024 10:58:47 +0800 Subject: [PATCH 37/94] fix: disable dynamic adjustment of 'keepaliveidle' --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5b67e1267b..0937a56b3c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -613,7 +613,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); tsNumOfTaskQueueThreads = tsNumOfCores * 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 16); From 0cb4c927ec38d9a02e0511b34c5a919ecc5160db Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Wed, 11 Sep 2024 15:49:44 +0800 Subject: [PATCH 38/94] enh(query)[TD-31903]. Handle return values for function calls --- source/common/src/tdatablock.c | 52 ++++++++++++++---------- source/libs/executor/src/mergeoperator.c | 2 +- source/libs/executor/src/tlinearhash.c | 27 ++++++++---- source/libs/executor/src/tsort.c | 3 +- source/os/src/osFile.c | 3 +- source/os/src/osString.c | 3 +- source/util/src/tpagedbuf.c | 11 +++-- 7 files changed, 63 insertions(+), 38 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 08f6842c4b..22c4be5cc9 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2517,24 +2517,25 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf pDataBlock->info.id.groupId, pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName); if (len >= size - 1) { - return code; + goto _exit; } for (int32_t j = 0; j < rows; j++) { len += snprintf(dumpBuf + len, size - len, "%s|", flag); if (len >= size - 1) { - return code; + goto _exit; } for (int32_t k = 0; k < colNum; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); if (pColInfoData == NULL) { - return terrno; + code = terrno; + goto _exit; } if (colDataIsNull(pColInfoData, rows, j, NULL) || !pColInfoData->pData) { len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL"); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; continue; } @@ -2542,53 +2543,53 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: memset(pBuf, 0, sizeof(pBuf)); - (void) formatTimestamp(pBuf, *(uint64_t*)var, pColInfoData->info.precision); + (void)formatTimestamp(pBuf, *(uint64_t*)var, pColInfoData->info.precision); len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_TINYINT: len += snprintf(dumpBuf + len, size - len, " %15d |", *(int8_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_UTINYINT: len += snprintf(dumpBuf + len, size - len, " %15d |", *(uint8_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_SMALLINT: len += snprintf(dumpBuf + len, size - len, " %15d |", *(int16_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_USMALLINT: len += snprintf(dumpBuf + len, size - len, " %15d |", *(uint16_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_INT: len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_UINT: len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_BIGINT: len += snprintf(dumpBuf + len, size - len, " %15" PRId64 " |", *(int64_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_UBIGINT: len += snprintf(dumpBuf + len, size - len, " %15" PRIu64 " |", *(uint64_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_FLOAT: len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_DOUBLE: len += snprintf(dumpBuf + len, size - len, " %15f |", *(double*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_BOOL: len += snprintf(dumpBuf + len, size - len, " %15d |", *(bool*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: @@ -2599,24 +2600,33 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf dataSize = TMIN(dataSize, 50); memcpy(pBuf, varDataVal(pData), dataSize); len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; } break; case TSDB_DATA_TYPE_NCHAR: { char* pData = colDataGetVarData(pColInfoData, j); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); memset(pBuf, 0, sizeof(pBuf)); - (void)taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); + code = taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); + if (code < 0) { + uError("func %s failed to convert to ucs charset since %s", __func__, tstrerror(code)); + goto _exit; + } len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; } break; } } len += snprintf(dumpBuf + len, size - len, "%d\n", j); - if (len >= size - 1) return code; + if (len >= size - 1) goto _exit; } len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag); *pDataBuf = dumpBuf; + dumpBuf = NULL; +_exit: + if (dumpBuf) { + taosMemoryFree(dumpBuf); + } return code; } diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index c94a330dbc..49973ac373 100644 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -229,7 +229,7 @@ int32_t doSortMerge(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { resetLimitInfoForNextGroup(&pInfo->limitInfo); } - (void)applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo); + bool limitReached = applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo); if (p->info.rows > 0) { break; diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 763fc6a412..69ce50e150 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -150,10 +150,11 @@ static void doRemoveFromBucket(SFilePage* pPage, SLHashNode* pNode, SLHashBucket pBucket->size -= 1; } -static void doTrimBucketPages(SLHashObj* pHashObj, SLHashBucket* pBucket) { +static int32_t doTrimBucketPages(SLHashObj* pHashObj, SLHashBucket* pBucket) { + int32_t code = 0; size_t numOfPages = taosArrayGetSize(pBucket->pPageIdList); if (numOfPages <= 1) { - return; + return code; } int32_t* firstPage = taosArrayGet(pBucket->pPageIdList, 0); @@ -164,11 +165,14 @@ static void doTrimBucketPages(SLHashObj* pHashObj, SLHashBucket* pBucket) { if (pLast->num <= sizeof(SFilePage)) { // this is empty - // TODO check ret - (void)dBufSetBufPageRecycled(pHashObj->pBuf, pLast); + code = dBufSetBufPageRecycled(pHashObj->pBuf, pLast); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed to recycle buf page since %s", __func__, tstrerror(code)); + return code; + } releaseBufPage(pHashObj->pBuf, pFirst); taosArrayRemove(pBucket->pPageIdList, numOfPages - 1); - return; + return code; } char* pStart = pLast->data; @@ -191,8 +195,11 @@ static void doTrimBucketPages(SLHashObj* pHashObj, SLHashBucket* pBucket) { pStart += nodeSize; if (pLast->num <= sizeof(SFilePage)) { // this is empty - // TODO check ret - (void)dBufSetBufPageRecycled(pHashObj->pBuf, pLast); + code = dBufSetBufPageRecycled(pHashObj->pBuf, pLast); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed to recycle buf page since %s", __func__, tstrerror(code)); + return code; + } releaseBufPage(pHashObj->pBuf, pFirst); taosArrayRemove(pBucket->pPageIdList, numOfPages - 1); break; @@ -210,6 +217,7 @@ static void doTrimBucketPages(SLHashObj* pHashObj, SLHashBucket* pBucket) { break; } } + return code; } static int32_t doAddNewBucket(SLHashObj* pHashObj) { @@ -403,7 +411,10 @@ int32_t tHashPut(SLHashObj* pHashObj, const void* key, size_t keyLen, void* data releaseBufPage(pHashObj->pBuf, p); } - doTrimBucketPages(pHashObj, pBucket); + code = doTrimBucketPages(pHashObj, pBucket); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } return code; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 796ebbeb84..19b825b0ca 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -1308,8 +1308,7 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i return terrno; } - // todo - (void)taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET); + TAOS_CHECK_RETURN(taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET)); int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize); int32_t ret = taosReadFromCFile(pRegion->buf, readBytes, 1, pMemFile->pTdFile); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 0e5b6b71a1..40f48af266 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -1535,8 +1535,9 @@ int taosSeekCFile(FILE *file, int64_t offset, int whence) { int code = fseeko(file, offset, whence); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); + code = terrno; } - return terrno; + return code; #endif } diff --git a/source/os/src/osString.c b/source/os/src/osString.c index ffc64f3493..b1732a2ae1 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -351,7 +351,8 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { int32_t code = 0; iconv_t conv = taosAcquireConv(&idx, C2M); if ((iconv_t)-1 == conv || (iconv_t)0 == conv) { - return TSDB_CODE_APP_ERROR; + code = TAOS_SYSTEM_ERROR(errno);; + return code; } size_t ucs4_input_len = ucs4_max_len; diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index f22233a757..e8303b563e 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -316,7 +316,7 @@ static char* evictBufPage(SDiskbasedBuf* pBuf) { } terrno = 0; - (void)tdListPopNode(pBuf->lruList, pn); + pn = tdListPopNode(pBuf->lruList, pn); SPageInfo* d = *(SPageInfo**)pn->data; @@ -337,7 +337,7 @@ static int32_t lruListPushFront(SList* pList, SPageInfo* pi) { } static void lruListMoveToFront(SList* pList, SPageInfo* pi) { - (void)tdListPopNode(pList, pi->pn); + pi->pn = tdListPopNode(pList, pi->pn); tdListPrependNode(pList, pi->pn); } @@ -474,8 +474,11 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { pBuf->totalBufSize += pBuf->pageSize; } else { taosMemoryFree(availablePage); - (void)taosArrayPop(pBuf->pIdList); - (void)tSimpleHashRemove(pBuf->all, pageId, sizeof(int32_t)); + SPageInfo **pLast = taosArrayPop(pBuf->pIdList); + int32_t ret = tSimpleHashRemove(pBuf->all, pageId, sizeof(int32_t)); + if (ret != TSDB_CODE_SUCCESS) { + uError("%s failed to clear pageId %d from buf hash-set since %s", __func__, *pageId, tstrerror(ret)); + } taosMemoryFree(pi); terrno = code; return NULL; From 987e086918c8a3cdfad79cdc378331a6f1e35b15 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Sep 2024 12:32:25 +0800 Subject: [PATCH 39/94] enh(stmt2/gettbname): ignore tbname not set error --- source/client/src/clientStmt2.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 01a7c4be4c..ad8fc52b95 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1888,7 +1888,12 @@ int stmtGetParamTbName(TAOS_STMT2* stmt, int* nums) { STMT_ERR_RET(stmtParseSql(pStmt)); } - *nums = STMT_TYPE_MULTI_INSERT == pStmt->sql.type ? 1 : 0; + if (TSDB_CODE_TSC_STMT_TBNAME_ERROR == pStmt->errCode) { + *nums = 1; + pStmt->errCode = TSDB_CODE_SUCCESS; + } else { + *nums = STMT_TYPE_MULTI_INSERT == pStmt->sql.type ? 1 : 0; + } return TSDB_CODE_SUCCESS; } From b69112bf416af52c762dbd647b2a8b7c5f429d21 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 13 Sep 2024 02:20:07 +0000 Subject: [PATCH 40/94] fix/TS-5404-disable-mnode-sync --- source/dnode/mnode/impl/src/mndSync.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 2b32dc7781..e73cc1b5db 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -543,14 +543,14 @@ void mndSyncCheckTimeout(SMnode *pMnode) { if (delta > MNODE_TIMEOUT_SEC) { mError("trans:%d, failed to propose since timeout, start:%d cur:%d delta:%d seq:%" PRId64, pMgmt->transId, pMgmt->transSec, curSec, delta, pMgmt->transSeq); - pMgmt->transId = 0; - pMgmt->transSec = 0; - pMgmt->transSeq = 0; - terrno = TSDB_CODE_SYN_TIMEOUT; - pMgmt->errCode = TSDB_CODE_SYN_TIMEOUT; - if (tsem_post(&pMgmt->syncSem) < 0) { - mError("failed to post sem"); - } + // pMgmt->transId = 0; + // pMgmt->transSec = 0; + // pMgmt->transSeq = 0; + // terrno = TSDB_CODE_SYN_TIMEOUT; + // pMgmt->errCode = TSDB_CODE_SYN_TIMEOUT; + //if (tsem_post(&pMgmt->syncSem) < 0) { + // mError("failed to post sem"); + //} } else { mDebug("trans:%d, waiting for sync confirm, start:%d cur:%d delta:%d seq:%" PRId64, pMgmt->transId, pMgmt->transSec, curSec, curSec - pMgmt->transSec, pMgmt->transSeq); From 19580c89eff7f42a4bdc7d9bb10ce5573383c710 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 13 Sep 2024 13:25:36 +0800 Subject: [PATCH 41/94] postfix conn/app cache check time --- source/dnode/mnode/impl/src/mndProfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index f4c5e9f2a0..5d91b0b0d8 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -91,7 +91,7 @@ int32_t mndInitProfile(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; // in ms - int32_t checkTime = tsShellActivityTimer * 2 * 1000; + int32_t checkTime = CACHE_OBJ_KEEP_TIME * 1000; pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, false, (__cache_free_fn_t)mndFreeConn, "conn"); if (pMgmt->connCache == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; From 54eeefba7e5aec3cbd3efc5beb4cdbde1b81770d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 13 Sep 2024 15:14:12 +0800 Subject: [PATCH 42/94] fix:[TD-32010] sml return success if modify meta error --- source/client/src/clientImpl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 484d9bb7e5..8efbf81258 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1276,9 +1276,9 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue } if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type) && NULL == pRequest->body.resInfo.execRes.res) { - code = removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)); - if (TSDB_CODE_SUCCESS != code) { - tscError("0x%" PRIx64 " remove meta failed,QID:0x%" PRIx64, pRequest->self, pRequest->requestId); + int ret = removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)); + if (TSDB_CODE_SUCCESS != ret) { + tscError("0x%" PRIx64 " remove meta failed,code:%d,QID:0x%" PRIx64, pRequest->self, ret, pRequest->requestId); } } From 92171e080ed7c44d972cc220111fa043459040cf Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 13 Sep 2024 15:20:23 +0800 Subject: [PATCH 43/94] ostime and ossocket --- include/os/osTimezone.h | 2 +- include/util/taoserror.h | 1 + source/common/src/tglobal.c | 5 +- source/os/src/osEnv.c | 5 +- source/os/src/osRand.c | 2 +- source/os/src/osTimezone.c | 96 ++++++++++++++++++------------------- 6 files changed, 59 insertions(+), 52 deletions(-) diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index fab42e8e11..2a8d5a442d 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -54,7 +54,7 @@ enum TdTimezone { TdEastZone12 }; -void taosGetSystemTimezone(char *outTimezone, enum TdTimezone *tsTimezone); +int32_t taosGetSystemTimezone(char *outTimezone, enum TdTimezone *tsTimezone); int32_t taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight, enum TdTimezone *tsTimezone); #ifdef __cplusplus diff --git a/include/util/taoserror.h b/include/util/taoserror.h index eb329192a2..17bb6d373c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -157,6 +157,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0138) #define TSDB_CODE_SOCKET_ERROR TAOS_DEF_ERROR_CODE(0, 0x0139) #define TSDB_CODE_UNSUPPORT_OS TAOS_DEF_ERROR_CODE(0, 0x013A) +#define TSDB_CODE_TIME_ERROR TAOS_DEF_ERROR_CODE(0, 0x013B) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5b67e1267b..b852a096f8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1696,7 +1696,10 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * SArray *pArgs) { int32_t code = TSDB_CODE_SUCCESS; - if (tsCfg == NULL) osDefaultInit(); + if (tsCfg == NULL) code = osDefaultInit(); + if (code != 0) { + (void)printf("failed to init os since %s\n", tstrerror(code)); + } SConfig *pCfg = NULL; TAOS_CHECK_RETURN(cfgInit(&pCfg)); diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index c0273e4a6f..f2c90e778d 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -50,7 +50,10 @@ int32_t osDefaultInit() { taosSeedRand(taosSafeRand()); taosGetSystemLocale(tsLocale, tsCharset); - taosGetSystemTimezone(tsTimezoneStr, &tsTimezone); + code = taosGetSystemTimezone(tsTimezoneStr, &tsTimezone); + if(code != 0) { + return code; + } if (strlen(tsTimezoneStr) > 0) { // ignore empty timezone if ((code = taosSetSystemTimezone(tsTimezoneStr, tsTimezoneStr, &tsDaylight, &tsTimezone)) != TSDB_CODE_SUCCESS) return code; diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index da4e8dfb9d..b99017782b 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -69,7 +69,7 @@ uint32_t taosSafeRand(void) { if (len < 0) { seed = (int)taosGetTimestampSec(); } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); } return (uint32_t)seed; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 8fdd296655..1d8a552862 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -811,7 +811,11 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i #elif defined(_TD_DARWIN_64) - setenv("TZ", buf, 1); + code = setenv("TZ", buf, 1); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } tzset(); int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); *tsTimezone = tz; @@ -839,13 +843,17 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i return code; } -void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { +int32_t taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { + int32_t code = 0; #ifdef WINDOWS char value[100]; char keyPath[100]; DWORD bufferSize = sizeof(value); - RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", + LONG result = RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + if (result != ERROR_SUCCESS) { + return TAOS_SYSTEM_WINAPI_ERROR(result); + } strcpy(outTimezoneStr, "not configured"); *tsTimezone = 0; if (bufferSize > 0) { @@ -854,7 +862,10 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { strcpy(outTimezoneStr, win_tz[i][1]); bufferSize = sizeof(value); sprintf(keyPath, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s", value); - RegGetValue(HKEY_LOCAL_MACHINE, keyPath, "Display", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + result = RegGetValue(HKEY_LOCAL_MACHINE, keyPath, "Display", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + if (result != ERROR_SUCCESS) { + return TAOS_SYSTEM_WINAPI_ERROR(result); + } if (bufferSize > 0) { // value[4] = (value[4] == '+' ? '-' : '+'); sprintf(outTimezoneStr, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, value[4], value[5], value[6], value[8], @@ -865,6 +876,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { } } } + return 0; #elif defined(_TD_DARWIN_64) char buf[4096] = {0}; char *tz = NULL; @@ -883,21 +895,11 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { } tz = zi + strlen("zoneinfo") + 1; - // for (int i = n - 1; i >= 0; --i) { - // if (buf[i] == '/') { - // if (tz) { - // tz = buf + i + 1; - // break; - // } - // tz = buf + i + 1; - // } - // } - // if (!tz || 0 == strchr(tz, '/')) { - // printf("parsing /etc/localtime failed\n"); - // return; - // } - - setenv("TZ", tz, 1); + code = setenv("TZ", tz, 1); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } tzset(); } @@ -908,7 +910,9 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - taosLocalTime(&tx1, &tm1, NULL); + if (taosLocalTime(&tx1, &tm1, NULL) == NULL) { + return TSDB_CODE_TIME_ERROR; + } daylight = tm1.tm_isdst; isdst_now = tm1.tm_isdst; @@ -920,6 +924,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone / 3600); + return 0; #else char buf[4096] = {0}; @@ -927,8 +932,6 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { { int n = readlink("/etc/localtime", buf, sizeof(buf)-1); if (n < 0) { - (void)printf("read /etc/localtime error, reason:%s\n", strerror(errno)); - if (taosCheckExistFile("/etc/timezone")) { /* * NOTE: do not remove it. @@ -937,7 +940,9 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - (void)taosLocalTime(&tx1, &tm1, NULL); + if(taosLocalTime(&tx1, &tm1, NULL) == NULL) { + return TSDB_CODE_TIME_ERROR; + } /* load time zone string from /etc/timezone */ // FILE *f = fopen("/etc/timezone", "r"); errno = 0; @@ -946,12 +951,11 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { if (pFile != NULL) { int len = taosReadFile(pFile, buf, 64); if (len < 0) { - (void)taosCloseFile(&pFile); - (void)printf("read /etc/timezone error, reason:%s\n", strerror(errno)); - return; + TAOS_UNUSED(taosCloseFile(&pFile)); + return TSDB_CODE_TIME_ERROR; } - (void)taosCloseFile(&pFile); + TAOS_UNUSED(taosCloseFile(&pFile)); buf[sizeof(buf) - 1] = 0; char *lineEnd = strstr(buf, "\n"); @@ -961,7 +965,11 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables if (strlen(buf) > 0) { - (void)setenv("TZ", buf, 1); + code = setenv("TZ", buf, 1); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } } } // get and set default timezone @@ -986,34 +994,23 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); } else { - (void)printf("There is not /etc/timezone.\n"); + return TSDB_CODE_TIME_ERROR; } - return; + return 0; } buf[n] = '\0'; char *zi = strstr(buf, "zoneinfo"); if (!zi) { - (void)printf("parsing /etc/localtime failed\n"); - return; + return TSDB_CODE_TIME_ERROR; } tz = zi + strlen("zoneinfo") + 1; - // for (int i = n - 1; i >= 0; --i) { - // if (buf[i] == '/') { - // if (tz) { - // tz = buf + i + 1; - // break; - // } - // tz = buf + i + 1; - // } - // } - // if (!tz || 0 == strchr(tz, '/')) { - // printf("parsing /etc/localtime failed"); - // return; - // } - - (void)setenv("TZ", tz, 1); + code = setenv("TZ", tz, 1); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } tzset(); } @@ -1024,7 +1021,9 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - (void)taosLocalTime(&tx1, &tm1, NULL); + if(taosLocalTime(&tx1, &tm1, NULL) == NULL) { + return TSDB_CODE_TIME_ERROR; + } isdst_now = tm1.tm_isdst; /* @@ -1035,5 +1034,6 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone / 3600); + return 0; #endif } From 44e29059830f2db1c945ffacfdc3ed344429925d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 13 Sep 2024 15:25:01 +0800 Subject: [PATCH 44/94] fix:conflicts from 3.0 --- source/client/src/clientMonitor.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index f87de30dff..aeaa3bef8b 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -767,7 +767,8 @@ static void* monitorThreadFunc(void* param) { } MonitorSlowLogData* slowLogData = NULL; - if (taosReadQitem(monitorQueue, (void**)&slowLogData) != 0) { + taosReadQitem(monitorQueue, (void**)&slowLogData); + if (slowLogData != NULL) { if (slowLogData->type == SLOW_LOG_READ_BEGINNIG && quitCnt == 0) { if (slowLogData->pFile != NULL) { monitorSendSlowLogAtBeginning(slowLogData->clusterId, &(slowLogData->fileName), slowLogData->pFile, From 661a170a9b988a86b4606abb01286add287b67bc Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 13 Sep 2024 15:26:14 +0800 Subject: [PATCH 45/94] osTimer --- source/os/src/osSocket.c | 29 +++++++++++++---------------- source/os/src/osTimer.c | 2 +- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 4d650715f9..851615fb7f 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -120,7 +120,12 @@ int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t int32_t taosCloseSocketNoCheck1(SocketFd fd) { #ifdef WINDOWS - return closesocket(fd); + int ret = closesocket(fd); + if (ret == SOCKET_ERROR) { + int errorCode = WSAGetLastError(); + return terrno = TAOS_SYSTEM_WINSOCKET_ERROR(errorCode); + } + return 0; #else int32_t code = close(fd); if (-1 == code) { @@ -770,10 +775,7 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { - code = terrno; - (void)taosCloseSocketNoCheck1(fd); - terrno = code; - + TAOS_SKIP_ERROR(taosCloseSocketNoCheck1(fd)); return false; } pSocket->refId = 0; @@ -782,22 +784,18 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { /* set REUSEADDR option, so the portnumber can be re-used */ reuse = 1; if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - code = terrno; - (void)taosCloseSocket(&pSocket); - terrno = code; - + TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); return false; } /* bind socket to server address */ if (-1 == bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd))) { - code = TAOS_SYSTEM_ERROR(errno); - (void)taosCloseSocket(&pSocket); - terrno = code; + terrno = TAOS_SYSTEM_ERROR(errno); + TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); return false; } - (void)taosCloseSocket(&pSocket); + TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); return true; } @@ -1160,9 +1158,8 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) { #else // Linux like systems uint32_t conn_timeout_ms = timeout; if (-1 == setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { - int32_t code = TAOS_SYSTEM_ERROR(errno); - (void)taosCloseSocketNoCheck1(fd); - terrno = code; + terrno = TAOS_SYSTEM_ERROR(errno); + TAOS_SKIP_ERROR(taosCloseSocketNoCheck1(fd)); return -1; } #endif diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index 6e5d9844a9..fe3a0dcf95 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -80,7 +80,7 @@ void taos_block_sigalrm(void) { static void taosDeleteTimer(void *tharg) { timer_t *pTimer = tharg; - (void)timer_delete(*pTimer); + TAOS_SKIP_ERROR(timer_delete(*pTimer)); } static TdThread timerThread; From a91085eef6afc78cba8bdae69dc10a92919404b9 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 13 Sep 2024 16:14:40 +0800 Subject: [PATCH 46/94] fix: void --- source/libs/function/src/tudf.c | 5 ++++- source/os/src/osDir.c | 6 +++--- source/os/src/osSleep.c | 15 ++++++++++----- source/os/src/osSysinfo.c | 27 ++++++++++++++------------- source/util/src/tlog.c | 5 ++++- 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index d0b3b191ad..0510b6cff5 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -127,7 +127,10 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pData->dnodeId); float numCpuCores = 4; - taosGetCpuCores(&numCpuCores, false); + int32_t code = taosGetCpuCores(&numCpuCores, false); + if(code != 0) { + fnError("failed to get cpu cores, code:%d", code); + } numCpuCores = TMAX(numCpuCores, 2); snprintf(thrdPoolSizeEnvItem, 32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores * 2); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index e06c1a6aac..04747cacd4 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -109,8 +109,8 @@ void taosRemoveDir(const char *dirname) { } } - (void)taosCloseDir(&pDir); - (void)rmdir(dirname); + TAOS_UNUSED(taosCloseDir(&pDir)); + TAOS_UNUSED(rmdir(dirname)); // printf("dir:%s is removed\n", dirname); return; @@ -374,7 +374,7 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { bool taosIsDir(const char *dirname) { TdDirPtr pDir = taosOpenDir(dirname); if (pDir != NULL) { - (void)taosCloseDir(&pDir); + TAOS_SKIP_ERROR(taosCloseDir(&pDir)); return true; } return false; diff --git a/source/os/src/osSleep.c b/source/os/src/osSleep.c index f72805b15d..67a621e8b7 100644 --- a/source/os/src/osSleep.c +++ b/source/os/src/osSleep.c @@ -37,7 +37,7 @@ void taosMsleep(int32_t ms) { #ifdef WINDOWS Sleep(ms); #else - (void)usleep(ms * 1000); + TAOS_SKIP_ERROR(usleep(ms * 1000)); #endif } @@ -49,10 +49,15 @@ void taosUsleep(int32_t us) { interval.QuadPart = (10 * us); timer = CreateWaitableTimer(NULL, TRUE, NULL); - SetWaitableTimer(timer, &interval, 0, NULL, NULL, 0); - WaitForSingleObject(timer, INFINITE); - CloseHandle(timer); + if (timer == NULL) { + return; + } + if (!SetWaitableTimer(timer, &interval, 0, NULL, NULL, 0)) { + return; + } + TAOS_SKIP_ERROR(WaitForSingleObject(timer, INFINITE)); + TAOS_SKIP_ERROR(CloseHandle(timer)); #else - (void)usleep(us); + TAOS_SKIP_ERROR(usleep(us)); #endif } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 36b1d7b60d..fc6296bc04 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -191,10 +191,11 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { &cpuInfo->si, &cpuInfo->st, &cpuInfo->guest, &cpuInfo->guest_nice); if (EOF == code) { terrno = TAOS_SYSTEM_ERROR(errno); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); return terrno; } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); #endif return 0; @@ -263,9 +264,9 @@ bool taosCheckSystemIsLittleEnd() { void taosGetSystemInfo() { #ifdef WINDOWS - taosGetCpuCores(&tsNumOfCores, false); - taosGetTotalMemory(&tsTotalMemoryKB); - taosGetCpuUsage(NULL, NULL); + TAOS_SKIP_ERROR(taosGetCpuCores(&tsNumOfCores, false)); + TAOS_SKIP_ERROR(taosGetTotalMemory(&tsTotalMemoryKB)); + TAOS_SKIP_ERROR(taosGetCpuUsage(NULL, NULL)); #elif defined(_TD_DARWIN_64) long physical_pages = sysconf(_SC_PHYS_PAGES); long page_size = sysconf(_SC_PAGESIZE); @@ -274,10 +275,10 @@ void taosGetSystemInfo() { tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); #else taosGetProcIOnfos(); - (void)taosGetCpuCores(&tsNumOfCores, false); - (void)taosGetTotalMemory(&tsTotalMemoryKB); - (void)taosGetCpuUsage(NULL, NULL); - (void)taosGetCpuInstructions(&tsSSE42Supported, &tsAVXSupported, &tsAVX2Supported, &tsFMASupported, &tsAVX512Supported); + TAOS_SKIP_ERROR(taosGetCpuCores(&tsNumOfCores, false)); + TAOS_SKIP_ERROR(taosGetTotalMemory(&tsTotalMemoryKB)); + TAOS_SKIP_ERROR(taosGetCpuUsage(NULL, NULL)); + TAOS_SKIP_ERROR(taosGetCpuInstructions(&tsSSE42Supported, &tsAVXSupported, &tsAVX2Supported, &tsFMASupported, &tsAVX512Supported)); #endif } @@ -313,11 +314,11 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { if (taosReadFile(pFile, (void *)email, maxLen) < 0) { int32_t code = terrno; - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); return code; } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); return 0; #endif @@ -748,7 +749,7 @@ int32_t taosGetProcMemory(int64_t *usedKB) { char tmp[10]; (void)sscanf(line, "%s %" PRId64, tmp, usedKB); - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); return 0; #endif @@ -1045,7 +1046,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { return terrno; } else { len = taosReadFile(pFile, uid, uidlen); - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); if (len < 0) { return len; } @@ -1087,7 +1088,7 @@ char *taosGetCmdlineByPID(int pid) { cmdline[n] = 0; - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); } else { cmdline[0] = 0; } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index bd22a4e42b..7d905e843d 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -215,7 +215,10 @@ int32_t taosInitSlowLog() { int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; - TAOS_CHECK_RETURN(osUpdate()); + int32_t code = osUpdate(); + if (code != 0) { + uError("failed to update os info, reason:%s", tstrerror(code)); + } TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles)); if (tsc){ From 8bf6b5f6ec37ba455aabf6a21a5ff255df40b21c Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Fri, 13 Sep 2024 16:17:12 +0800 Subject: [PATCH 47/94] Update 22-meta.md --- docs/zh/14-reference/03-taos-sql/22-meta.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/14-reference/03-taos-sql/22-meta.md b/docs/zh/14-reference/03-taos-sql/22-meta.md index 35b71feb2c..397b3d3362 100644 --- a/docs/zh/14-reference/03-taos-sql/22-meta.md +++ b/docs/zh/14-reference/03-taos-sql/22-meta.md @@ -93,8 +93,8 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 | 4 | vgroups | INT | 数据库中有多少个 vgroup。需要注意，`vgroups` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 6 | replica | INT | 副本数。需要注意，`replica` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 7 | strict | VARCHAR(4) | 废弃参数 | -| 8 | duration | VARCHAR(10) | 单文件存储数据的时间跨度。需要注意，`duration` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 9 | keep | VARCHAR(32) | 数据保留时长。需要注意，`keep` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 8 | duration | VARCHAR(10) | 单文件存储数据的时间跨度。需要注意，`duration` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。内部存储单位为分钟，查询时有可能转换为天或小时展示 | +| 9 | keep | VARCHAR(32) | 数据保留时长。需要注意，`keep` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 内部存储单位为分钟，查询时有可能转换为天或小时展示 | | 10 | buffer | INT | 每个 vnode 写缓存的内存块大小，单位 MB。需要注意，`buffer` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 11 | pagesize | INT | 每个 VNODE 中元数据存储引擎的页大小，单位为 KB。需要注意，`pagesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 12 | pages | INT | 每个 vnode 元数据存储引擎的缓存页个数。需要注意，`pages` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | From b836235b67d3514bc6f06af2b4750edd1d042885 Mon Sep 17 00:00:00 2001 From: qevolg <2227465945@qq.com> Date: Fri, 13 Sep 2024 13:14:10 +0000 Subject: [PATCH 48/94] [TS-4893]: add CI test cases test_function --- tests/army/query/function/ans/rand.csv | 21 +++++ tests/army/query/function/in/rand.in | 3 + tests/army/query/function/test_function.py | 101 +++++++++++++++++---- 3 files changed, 108 insertions(+), 17 deletions(-) create mode 100644 tests/army/query/function/ans/rand.csv create mode 100644 tests/army/query/function/in/rand.in diff --git a/tests/army/query/function/ans/rand.csv b/tests/army/query/function/ans/rand.csv new file mode 100644 index 0000000000..40e20c5ba4 --- /dev/null +++ b/tests/army/query/function/ans/rand.csv @@ -0,0 +1,21 @@ +0.663936012733698 +0.840187717154710 +0.840187717154710 +0.700976369297587 +0.561380175203728 +0.916457875592847 +0.274745596235034 +0.135438768721856 +0.486904139391568 +0.352760728612896 +0.206965447965528 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 \ No newline at end of file diff --git a/tests/army/query/function/in/rand.in b/tests/army/query/function/in/rand.in new file mode 100644 index 0000000000..185a76d6f9 --- /dev/null +++ b/tests/army/query/function/in/rand.in @@ -0,0 +1,3 @@ +select RAND(1245); +select RAND(id) from ts_4893.d0 limit 10; +select RAND(id) from ts_4893.d0 order by id desc limit 10; \ No newline at end of file diff --git a/tests/army/query/function/test_function.py b/tests/army/query/function/test_function.py index 4981e93563..18a0d46711 100644 --- a/tests/army/query/function/test_function.py +++ b/tests/army/query/function/test_function.py @@ -509,31 +509,98 @@ class TDTestCase(TBase): tdSql.error( "select * from (select to_iso8601(ts, timezone()), timezone() from meters order by ts desc) limit 1000;", expectErrInfo="Not supported timzone format") # TS-5340 + + def test_rand(self): + self.test_normal_query("rand") + + tdSql.query("select rand();") + tdSql.checkRows(1) + tdSql.checkCols(1) + self.check_result_in_range(0, 0) + + tdSql.query("select rand(null);") + tdSql.checkRows(1) + tdSql.checkCols(1) + self.check_result_in_range(0, 0) + + tdSql.query("select rand() from (select 1) t limit 1;") + tdSql.checkRows(1) + tdSql.checkCols(1) + self.check_result_in_range(0, 0) + + tdSql.query("select rand(id) from ts_4893.d0 limit 100;") + tdSql.checkRows(100) + tdSql.checkCols(1) + for i in range(len(tdSql.res)): + self.check_result_in_range(i, 0) + + tdSql.query("select rand(id) from ts_4893.meters limit 100;") + tdSql.checkRows(100) + tdSql.checkCols(1) + for i in range(len(tdSql.res)): + self.check_result_in_range(i, 0) + + tdSql.query("select rand(123), rand(123);") + tdSql.checkRows(1) + tdSql.checkCols(2) + if tdSql.res[0][0] != tdSql.res[0][1]: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, tdSql.sql, tdSql.res[0][0], tdSql.res[0][1]) + tdLog.exit("%s(%d) failed: sql:%s data1:%s ne data2:%s" % args) + + def check_result_in_range(self, row, col): + res = tdSql.res[row][col] + if res < 0 or res >= 1: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, tdSql.sql, row, col, res) + tdLog.exit("%s(%d) failed: sql:%s row:%s col:%s data:%s lt 0 or ge 1" % args) + + def test_max(self): + self.test_normal_query("max") + + tdSql.query("select max(null) from ts_4893.meters;") + tdSql.checkRows(1) + tdSql.checkCols(1) + tdSql.checkData(0, 0, 'None') + + tdSql.query("select max(id) from ts_4893.meters;") + tdSql.checkRows(1) + + tdSql.query("select max(name) from ts_4893.meters;") + tdSql.checkRows(1) + + tdSql.query("select max(current) from ts_4893.meters;") + tdSql.checkRows(1) + + tdSql.query("select max(nch1) from ts_4893.meters;") + tdSql.checkRows(1) + + tdSql.query("select max(var1) from ts_4893.meters;") + tdSql.checkRows(1) + def test_min(self): self.test_normal_query("min") - tdSql.query("select min(var1), min(id) from ts_4893.d0;") + tdSql.query("select min(null) from ts_4893.meters;") tdSql.checkRows(1) - tdSql.checkData(0, 0, 'abc一二三abc一二三abc') - tdSql.checkData(0, 1, 0) - def test_max(self): - self.test_normal_query("max") - tdSql.query("select max(var1), max(id) from ts_4893.d0;") - tdSql.checkRows(1) - tdSql.checkData(0, 0, '一二三四五六七八九十') - tdSql.checkData(0, 1, 9999) - def test_rand(self): - tdSql.query("select rand();") + tdSql.checkCols(1) + tdSql.checkData(0, 0, 'None') + + tdSql.query("select min(id) from ts_4893.meters;") tdSql.checkRows(1) - tdSql.query("select rand(1);") + tdSql.query("select min(name) from ts_4893.meters;") tdSql.checkRows(1) - tdSql.query("select rand(1) from ts_4893.meters limit 10;") - tdSql.checkRows(10) + tdSql.query("select min(current) from ts_4893.meters;") + tdSql.checkRows(1) + + tdSql.query("select min(nch1) from ts_4893.meters;") + tdSql.checkRows(1) + + tdSql.query("select min(var1) from ts_4893.meters;") + tdSql.checkRows(1) - tdSql.query("select rand(id) from ts_4893.d0 limit 10;") - tdSql.checkRows(10) # run def run(self): tdLog.debug(f"start to excute {__file__}") @@ -576,8 +643,8 @@ class TDTestCase(TBase): self.test_varpop() # select function - self.test_min() self.test_max() + self.test_min() # error function self.test_error() From 3acac77f9f503ac5abef39dd53022ed16d4b21c6 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 13 Sep 2024 16:41:13 +0800 Subject: [PATCH 49/94] build on mac --- source/os/src/osTimezone.c | 6 ++---- source/util/src/terror.c | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 1d8a552862..89ced69f97 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -883,15 +883,13 @@ int32_t taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { int n = readlink("/etc/localtime", buf, sizeof(buf)); if (n < 0) { - printf("read /etc/localtime error, reason:%s\n", strerror(errno)); - return; + return TSDB_CODE_TIME_ERROR; } buf[n] = '\0'; char *zi = strstr(buf, "zoneinfo"); if (!zi) { - printf("parsing /etc/localtime failed\n"); - return; + return TSDB_CODE_TIME_ERROR; } tz = zi + strlen("zoneinfo") + 1; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 5ceec33831..31ede562bd 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -112,6 +112,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FAILED_TO_CONNECT_S3, "Failed to connect to TAOS_DEFINE_ERROR(TSDB_CODE_MSG_PREPROCESSED, "Message has been processed in preprocess") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_BUFFER, "Out of buffer") TAOS_DEFINE_ERROR(TSDB_CODE_INTERNAL_ERROR, "Internal error") +TAOS_DEFINE_ERROR(TSDB_CODE_TIME_ERROR, "Internal error in time") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") From a13b385a16c639975d1447538f3e5e635012a46f Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 13 Sep 2024 16:49:03 +0800 Subject: [PATCH 50/94] docs: add description of rpcQueueMemoryAllowed/syncLogBufferMemoryAllowed --- docs/zh/14-reference/01-components/01-taosd.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 004116b478..163e95d228 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -160,6 +160,12 @@ charset 的有效值是 UTF-8。 | :----------------: | :---------------------------------------------: | | numOfCommitThreads | 写入线程的最大数量，取值范围 0-1024，缺省值为 4 | +### 内存相关 +| 参数名称 | 参数说明 | +| :----------------: | :---------------------------------------------: | +| rpcQueueMemoryAllowed | 一个 dnode 允许的 rpc 消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10 | +| syncLogBufferMemoryAllowed | 一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10, 3.1.3.2/3.3.2.13 版本开始生效 | + ### 日志相关 | 参数名称 | 参数说明 | From 719a5f4a8df7d2a46267e3647509126fdad20d23 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 13 Sep 2024 16:50:52 +0800 Subject: [PATCH 51/94] docs: add description of rpcQueueMemoryAllowed/syncLogBufferMemoryAllowed --- docs/zh/14-reference/01-components/01-taosd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 163e95d228..a3e6d7693d 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -164,7 +164,7 @@ charset 的有效值是 UTF-8。 | 参数名称 | 参数说明 | | :----------------: | :---------------------------------------------: | | rpcQueueMemoryAllowed | 一个 dnode 允许的 rpc 消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10 | -| syncLogBufferMemoryAllowed | 一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10, 3.1.3.2/3.3.2.13 版本开始生效 | +| syncLogBufferMemoryAllowed | 一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 | ### 日志相关 From 36a038529a35f2dc0bf9de5aa1f282cbdba8b359 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 13 Sep 2024 16:51:57 +0800 Subject: [PATCH 52/94] docs: add description of rpcQueueMemoryAllowed/syncLogBufferMemoryAllowed --- docs/zh/14-reference/01-components/01-taosd.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index a3e6d7693d..ff6b27092d 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -154,18 +154,18 @@ charset 的有效值是 UTF-8。 | :-----------: | :-------------------------------------------------------------------------: | | supportVnodes | dnode 支持的最大 vnode 数目，取值范围：0-4096，缺省值： CPU 核数的 2 倍 + 5 | -### 性能调优 - -| 参数名称 | 参数说明 | -| :----------------: | :---------------------------------------------: | -| numOfCommitThreads | 写入线程的最大数量，取值范围 0-1024，缺省值为 4 | - ### 内存相关 | 参数名称 | 参数说明 | | :----------------: | :---------------------------------------------: | | rpcQueueMemoryAllowed | 一个 dnode 允许的 rpc 消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10 | | syncLogBufferMemoryAllowed | 一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 | +### 性能调优 + +| 参数名称 | 参数说明 | +| :----------------: | :---------------------------------------------: | +| numOfCommitThreads | 写入线程的最大数量，取值范围 0-1024，缺省值为 4 | + ### 日志相关 | 参数名称 | 参数说明 | From 98413626aa8a762e5d654905f7fea4eb4eabb348 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 13 Sep 2024 18:22:42 +0800 Subject: [PATCH 53/94] fix:[TD-31965] code rewrite if error happen --- source/libs/scalar/src/filter.c | 39 +++++++++++++-------------------- 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 8ffc588d41..588c252669 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -2912,6 +2912,8 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum uint32_t uidx = 0; uint32_t code = TSDB_CODE_SUCCESS; SArray *group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); + SFilterGroup ng = {0}; + if (group == NULL) { FLT_ERR_JRET(terrno); } @@ -2927,11 +2929,8 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum for (int32_t i = 0; i < gResNum; ++i) { res = gRes[i]; - optr = (res->colNum > 1) ? LOGIC_COND_TYPE_AND : LOGIC_COND_TYPE_OR; - SFilterGroup ng = {0}; - for (uint32_t m = 0; m < res->colNum; ++m) { colInfo = &res->colInfo[res->colIdx[m]]; if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { @@ -2944,28 +2943,16 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum for (int32_t n = 0; n < usize; ++n) { SFilterUnit *u = (SFilterUnit *)taosArrayGetP((SArray *)colInfo->info, n); if (NULL == u) { - code = TSDB_CODE_OUT_OF_RANGE; - break; + FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); } code = filterAddUnitFromUnit(info, &oinfo, u, &uidx); - if (TSDB_CODE_SUCCESS != code) { - break; - } + FLT_ERR_JRET(code); code = filterAddUnitToGroup(&ng, uidx); - if (TSDB_CODE_SUCCESS != code) { - break; - } - } - if (TSDB_CODE_SUCCESS != code) { - break; + FLT_ERR_JRET(code); } continue; } - if (TSDB_CODE_SUCCESS != code) { - filterFreeGroup((void*)&ng); - FLT_ERR_JRET(code); - } if (colInfo->type != RANGE_TYPE_MR_CTX) { fltError("filterRewrite get invalid col type : %d", colInfo->type); @@ -2973,24 +2960,28 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum } code = filterAddGroupUnitFromCtx(info, &oinfo, colInfo->info, res->colIdx[m], &ng, optr, group); - if (TSDB_CODE_SUCCESS != code) { - filterFreeGroup((void*)&ng); - FLT_ERR_JRET(code); - } + FLT_ERR_JRET(code); } if (ng.unitNum > 0) { if (NULL == taosArrayPush(group, &ng)) { - filterFreeGroup((void*)&ng); FLT_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + ng = (SFilterGroup){0}; } } FLT_ERR_JRET(filterConvertGroupFromArray(info, group)); -_return: taosArrayDestroy(group); + filterFreeInfo(&oinfo); + return 0; + +_return: + filterFreeGroup((void*)&ng); + + taosArrayDestroyEx(group, filterFreeGroup); + filterFreeInfo(&oinfo); FLT_RET(code); From 4ef4dddc7bb7f7ed2c85f590af729897f676ad66 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Fri, 13 Sep 2024 18:59:54 +0800 Subject: [PATCH 54/94] fix: schedulerFreeJob reset jobId only on the last reference --- include/util/tref.h | 1 + source/client/src/clientImpl.c | 4 +--- source/libs/scheduler/inc/schInt.h | 1 + source/libs/scheduler/src/schUtil.c | 13 +++++++++++-- source/libs/scheduler/src/scheduler.c | 9 ++++++--- source/util/src/tref.c | 13 +++++++++---- 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/include/util/tref.h b/include/util/tref.h index 1520ced14e..f17fb73f3d 100644 --- a/include/util/tref.h +++ b/include/util/tref.h @@ -46,6 +46,7 @@ void *taosAcquireRef(int32_t rsetId, int64_t rid); // release ref, rid is the reference ID returned by taosAddRef // return 0 if success. On error, -1 is returned, and terrno is set appropriately int32_t taosReleaseRef(int32_t rsetId, int64_t rid); +int32_t taosReleaseRefEx(int32_t rsetId, int64_t rid, int32_t* isReleased); // return the first reference if rid is 0, otherwise return the next after current reference. // if return value is NULL, it means list is over(if terrno is set, it means error happens) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ac5866401e..5afd8a0cbe 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1163,9 +1163,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { (void)atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertRows, pResult->numOfRows); } } - if (TSDB_CODE_SUCCESS == code) { - schedulerFreeJob(&pRequest->body.queryJob, 0); - } + schedulerFreeJob(&pRequest->body.queryJob, 0); } taosMemoryFree(pResult); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 3a25f37895..8d77ba1c98 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -570,6 +570,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType, void *param); int32_t schAcquireJob(int64_t refId, SSchJob **ppJob); int32_t schReleaseJob(int64_t refId); +int32_t schReleaseJobEx(int64_t refId, int32_t* released); void schFreeFlowCtrl(SSchJob *pJob); int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 3f610ed387..bcca820dff 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -41,6 +41,15 @@ FORCE_INLINE int32_t schReleaseJob(int64_t refId) { return taosReleaseRef(schMgmt.jobRef, refId); } +FORCE_INLINE int32_t schReleaseJobEx(int64_t refId, int32_t* released) { + if (0 == refId) { + return TSDB_CODE_SUCCESS; + } + + qDebug("sch release ex jobId:0x%" PRIx64, refId); + return taosReleaseRefEx(schMgmt.jobRef, refId, released); +} + int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes) { *ppRes = NULL; if (NULL == pEpSet) { @@ -189,7 +198,7 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); return; } - + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; @@ -334,7 +343,7 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) { *pTask = NULL; - + int32_t s = taosHashGetSize(pTaskList); if (s <= 0) { return; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 3ce5cd5714..7a69691573 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -119,7 +119,7 @@ int32_t schedulerGetTasksStatus(int64_t jobId, SArray *pSub) { qError("failed to get task %d, total: %d", m, pLevel->taskNum); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SQuerySubDesc subDesc = {0}; subDesc.tid = pTask->taskId; TAOS_STRCPY(subDesc.status, jobTaskStatusStr(pTask->status)); @@ -179,8 +179,11 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) { SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode)); (void)schHandleJobDrop(pJob, errCode); // ignore any error - (void)schReleaseJob(*jobId); // ignore error - *jobId = 0; + int32_t released = false; + (void)schReleaseJobEx(*jobId, &released); // ignore error + if (released) { + *jobId = 0; + } } void schedulerDestroy(void) { diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 0eac7b4427..d387156e2c 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -55,7 +55,7 @@ static void taosLockList(int64_t *lockedBy); static void taosUnlockList(int64_t *lockedBy); static void taosIncRsetCount(SRefSet *pSet); static void taosDecRsetCount(SRefSet *pSet); -static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove); +static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove, int32_t* isReleased); int32_t taosOpenRef(int32_t max, RefFp fp) { SRefNode **nodeList; @@ -181,7 +181,7 @@ int64_t taosAddRef(int32_t rsetId, void *p) { return rid; } -int32_t taosRemoveRef(int32_t rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 1); } +int32_t taosRemoveRef(int32_t rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 1, NULL); } // if rid is 0, return the first p in hash list, otherwise, return the next after current rid void *taosAcquireRef(int32_t rsetId, int64_t rid) { @@ -245,7 +245,8 @@ void *taosAcquireRef(int32_t rsetId, int64_t rid) { return p; } -int32_t taosReleaseRef(int32_t rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 0); } +int32_t taosReleaseRef(int32_t rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 0, NULL); } +int32_t taosReleaseRefEx(int32_t rsetId, int64_t rid, int32_t* isReleased) { return taosDecRefCount(rsetId, rid, 0, isReleased); } // if rid is 0, return the first p in hash list, otherwise, return the next after current rid void *taosIterateRef(int32_t rsetId, int64_t rid) { @@ -372,7 +373,7 @@ int32_t taosListRef() { return num; } -static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { +static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove, int32_t* isReleased) { int32_t hash; SRefSet *pSet; SRefNode *pNode; @@ -440,6 +441,10 @@ static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { taosDecRsetCount(pSet); } + if (isReleased) { + *isReleased = released; + } + return code; } From 83d531284b53d83668c71a19591a636273872a9f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Sep 2024 19:02:56 +0800 Subject: [PATCH 55/94] other: merge 3.0 --- include/libs/stream/tstream.h | 2 +- source/libs/stream/src/streamExec.c | 70 ++++++++++++++++++----------- 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e71a6c4dce..b77c8535f1 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -673,7 +673,7 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask); int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt); int32_t streamExecTask(SStreamTask* pTask); -void streamResumeTask(SStreamTask* pTask); +int32_t streamResumeTask(SStreamTask* pTask); int32_t streamTrySchedExec(SStreamTask* pTask); int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType); void streamTaskResumeInFuture(SStreamTask* pTask); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index f3279a0f01..4fc00a6f59 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -24,7 +24,7 @@ #define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec static int32_t streamTransferStateDoPrepare(SStreamTask* pTask); -static void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); +static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); bool streamTaskShouldStop(const SStreamTask* pTask) { SStreamTaskState pState = streamTaskGetStatus(pTask); @@ -95,7 +95,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* return code; } -void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) { +int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) { int32_t code = TSDB_CODE_SUCCESS; void* pExecutor = pTask->exec.pExecutor; int32_t size = 0; @@ -112,7 +112,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to if (streamTaskShouldStop(pTask) || (pRes == NULL)) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - return; + return code; } SSDataBlock* output = NULL; @@ -122,8 +122,13 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to resetTaskInfo(pExecutor); } - stError("unexpected stream execution, s-task:%s since %s", pTask->id.idStr, tstrerror(code)); - continue; + if (code == TSDB_CODE_OUT_OF_MEMORY || code == TSDB_CODE_INVALID_PARA || code == TSDB_CODE_FILE_CORRUPTED) { + stFatal("s-task:%s failed to continue execute since %s", pTask->id.idStr, tstrerror(code)); + return code; + } else { + qResetTaskCode(pExecutor); + continue; + } } if (output == NULL) { @@ -194,7 +199,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks); // todo: here we need continue retry to put it into output buffer if (code != TSDB_CODE_SUCCESS) { - return; + return code; } pRes = NULL; @@ -208,6 +213,8 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to } else { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); } + + return code; } // todo contiuous try to create result blocks @@ -627,7 +634,7 @@ static void doRecordThroughput(STaskExecStatisInfo* pInfo, int64_t totalBlocks, } } -static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, int32_t num) { +static int32_t doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, int32_t num) { const char* id = pTask->id.idStr; int32_t blockSize = 0; int64_t st = taosGetTimestampMs(); @@ -635,23 +642,28 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i int64_t ver = pInfo->processedVer; int64_t totalSize = 0; int32_t totalBlocks = 0; + int32_t code = 0; stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type)); - int32_t code = doSetStreamInputBlock(pTask, pBlock, &ver, id); + code = doSetStreamInputBlock(pTask, pBlock, &ver, id); if (code) { stError("s-task:%s failed to set input block, not exec for these blocks", id); - return; + return code; + } + + code = streamTaskExecImpl(pTask, pBlock, &totalSize, &totalBlocks); + if (code) { + return code; } - streamTaskExecImpl(pTask, pBlock, &totalSize, &totalBlocks); doRecordThroughput(&pTask->execInfo, totalBlocks, totalSize, blockSize, st, pTask->id.idStr); // update the currentVer if processing the submit blocks. if (!(pInfo->checkpointVer <= pInfo->nextProcessVer && ver >= pInfo->checkpointVer)) { stError("s-task:%s invalid info, checkpointVer:%" PRId64 ", nextProcessVer:%" PRId64 " currentVer:%" PRId64, id, pInfo->checkpointVer, pInfo->nextProcessVer, ver); - return; + return code; } if (ver != pInfo->processedVer) { @@ -660,6 +672,8 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i id, pInfo->processedVer, ver, pInfo->nextProcessVer, pInfo->checkpointVer); pInfo->processedVer = ver; } + + return code; } void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock) { @@ -712,6 +726,7 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB */ static int32_t doStreamExecTask(SStreamTask* pTask) { const char* id = pTask->id.idStr; + int32_t code = 0; // merge multiple input data if possible in the input queue. stDebug("s-task:%s start to extract data block from inputQ", id); @@ -784,9 +799,9 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { if (type == STREAM_INPUT__DATA_BLOCK) { pTask->execInfo.sink.dataSize += blockSize; stDebug("s-task:%s sink task start to sink %d blocks, size:%.2fKiB", id, numOfBlocks, SIZE_IN_KiB(blockSize)); - int32_t code = doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput); + code = doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput); if (code != TSDB_CODE_SUCCESS) { - // todo handle error. + return code; } double el = (taosGetTimestampMs() - st) / 1000.0; @@ -801,17 +816,19 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { } if (type != STREAM_INPUT__CHECKPOINT) { - doStreamTaskExecImpl(pTask, pInput, numOfBlocks); + code = doStreamTaskExecImpl(pTask, pInput, numOfBlocks); streamFreeQitem(pInput); + if (code) { + return code; + } } else { // todo other thread may change the status // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. streamMutexLock(&pTask->lock); SStreamTaskState pState = streamTaskGetStatus(pTask); if (pState.state == TASK_STATUS__CK) { stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); - int32_t code = streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue + code = streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue } else { // todo refactor - int32_t code = 0; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { code = streamTaskSendCheckpointSourceRsp(pTask); } else { @@ -827,7 +844,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { streamMutexUnlock(&pTask->lock); streamFreeQitem(pInput); - return 0; + return code; } } } @@ -858,21 +875,21 @@ bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) { } } -void streamResumeTask(SStreamTask* pTask) { +int32_t streamResumeTask(SStreamTask* pTask) { const char* id = pTask->id.idStr; int32_t code = 0; if (pTask->status.schedStatus != TASK_SCHED_STATUS__ACTIVE) { - stError("s-task:%s invalid sched status:%d, not resume task", id, pTask->status.schedStatus); - return; + stError("s-task:%s invalid sched status:%d, not resume task", pTask->id.idStr, pTask->status.schedStatus); + return code; } while (1) { code = doStreamExecTask(pTask); if (code) { stError("s-task:%s failed to exec stream task, code:%s", id, tstrerror(code)); + return code; } - // check if continue streamMutexLock(&pTask->lock); @@ -888,7 +905,7 @@ void streamResumeTask(SStreamTask* pTask) { stDebug("s-task:%s exec completed, status:%s, sched-status:%d, lastExecTs:%" PRId64, id, p, pTask->status.schedStatus, pTask->status.lastExecTs); - return; + return code; } else { // check if this task needs to be idle for a while if (pTask->status.schedIdleTime > 0) { @@ -896,28 +913,31 @@ void streamResumeTask(SStreamTask* pTask) { streamMutexUnlock(&pTask->lock); setLastExecTs(pTask, taosGetTimestampMs()); - return; + return code; } } streamMutexUnlock(&pTask->lock); } + + return code; } int32_t streamExecTask(SStreamTask* pTask) { // this function may be executed by multi-threads, so status check is required. const char* id = pTask->id.idStr; + int32_t code = 0; int8_t schedStatus = streamTaskSetSchedStatusActive(pTask); if (schedStatus == TASK_SCHED_STATUS__WAITING) { - streamResumeTask(pTask); + code = streamResumeTask(pTask); } else { char* p = streamTaskGetStatus(pTask).name; stDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", id, p, pTask->status.schedStatus); } - return 0; + return code; } int32_t streamTaskReleaseState(SStreamTask* pTask) { From b9c1ab16e138f021b149bee3415f7cf4e2f87b36 Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Fri, 13 Sep 2024 19:06:28 +0800 Subject: [PATCH 56/94] fix(query)[TD-32059]. Enable HAVING clause to work with FILL clause --- source/libs/planner/src/planLogicCreater.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 6886260d0a..6ad30eccb8 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1312,6 +1312,10 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect code = createColumnByRewriteExpr(pFill->pWStartTs, &pFill->node.pTargets); } + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { + code = nodesCloneNode(pSelect->pHaving, &pFill->node.pConditions); + } + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pFill; } else { From a33015e7120660022239758fa2bcc433fbb2a83a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Sep 2024 19:35:02 +0800 Subject: [PATCH 57/94] refactor: pass error code to invoker. --- include/util/taoserror.h | 1 + source/dnode/vnode/src/tq/tq.c | 6 +++- source/dnode/vnode/src/tq/tqRead.c | 5 +--- source/dnode/vnode/src/tq/tqStreamTask.c | 32 +++++++++++++++------- source/dnode/vnode/src/tqCommon/tqCommon.c | 6 ++-- source/libs/stream/src/streamQueue.c | 4 +-- source/util/src/terror.c | 1 + 7 files changed, 35 insertions(+), 20 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ba0930955c..9ea1b53564 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -976,6 +976,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_STREAM_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x4105) #define TSDB_CODE_STREAM_CONFLICT_EVENT TAOS_DEF_ERROR_CODE(0, 0x4106) #define TSDB_CODE_STREAM_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x4107) +#define TSDB_CODE_STREAM_INPUTQ_FULL TAOS_DEF_ERROR_CODE(0, 0x4108) // TDLite #define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x5100) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3911822068..2d1d3ed357 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1007,9 +1007,13 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); + if (code) { + tqError("vgId:%d failed to create task run req, code:%s", TD_VID(pTq->pVnode), tstrerror(code)); + return code; + } // let's continue scan data in the wal files - if (code == 0 && (pReq->reqType >= 0 || pReq->reqType == STREAM_EXEC_T_RESUME_TASK)) { + if (pReq->reqType >= 0 || pReq->reqType == STREAM_EXEC_T_RESUME_TASK) { code = tqScanWalAsync(pTq, false); // it's ok to failed if (code) { tqError("vgId:%d failed to start scan wal file, code:%s", pTq->pStreamMeta->vgId, tstrerror(code)); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index e85c3a5f3a..eb5d6aafe7 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -351,11 +351,8 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con if (data == NULL) { // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then // retry - code = TSDB_CODE_OUT_OF_MEMORY; - terrno = code; - tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", 0); - return code; + return terrno; } (void)memcpy(data, pBody, len); diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 6558012551..8f575b540f 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -23,7 +23,7 @@ static int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle); static int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId); static bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver); static bool taskReadyForDataFromWal(SStreamTask* pTask); -static bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems); +static int32_t doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems, bool* pSucc); static int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration); // extract data blocks(submit/delete) from WAL, and add them into the input queue for all the sources tasks. @@ -40,7 +40,7 @@ int32_t tqScanWal(STQ* pTq) { int32_t code = doScanWalForAllTasks(pMeta, &shouldIdle); if (code) { - tqError("vgId:%d failed to start all tasks, try next time", vgId); + tqError("vgId:%d failed to start all tasks, try next time, code:%s", vgId, tstrerror(code)); return code; } @@ -293,9 +293,11 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) { return true; } -bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems) { +int32_t doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems, bool* pSucc) { const char* id = pTask->id.idStr; int32_t numOfNewItems = 0; + int32_t code = 0; + *pSucc = false; while (1) { if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) { @@ -304,7 +306,7 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems } SStreamQueueItem* pItem = NULL; - int32_t code = extractMsgFromWal(pTask->exec.pWalReader, (void**)&pItem, maxVer, id); + code = extractMsgFromWal(pTask->exec.pWalReader, (void**)&pItem, maxVer, id); if (code != TSDB_CODE_SUCCESS || pItem == NULL) { // failed, continue int64_t currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader); bool itemInFillhistory = handleFillhistoryScanComplete(pTask, currentVer); @@ -327,10 +329,17 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems break; } } else { - tqTrace("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer); - code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer); - if (code) { - tqError("s-task:%s failed to seek ver to:%"PRId64 " in wal", id, pTask->chkInfo.nextProcessVer); + if (code == TSDB_CODE_OUT_OF_MEMORY) { + tqError("s-task:%s failed to put data into inputQ, since out of memory"); + } else { + tqTrace("s-task:%s append input queue failed, code:inputQ is full, ver:%" PRId64, id, + pTask->chkInfo.nextProcessVer); + code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer); + if (code) { + tqError("s-task:%s failed to seek ver to:%" PRId64 " in wal", id, pTask->chkInfo.nextProcessVer); + } + + code = 0; // reset the error code } break; @@ -339,7 +348,8 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems } *numOfItems += numOfNewItems; - return numOfNewItems > 0; + *pSucc = (numOfNewItems > 0); + return code; } int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { @@ -358,6 +368,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { pTaskList = taosArrayDup(pStreamMeta->pTaskList, NULL); streamMetaWUnLock(pStreamMeta); if (pTaskList == NULL) { + tqError("vgId:%d failed to create task list dup, code:%s", vgId, tstrerror(terrno)); return terrno; } @@ -405,7 +416,8 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { continue; } - bool hasNewData = doPutDataIntoInputQ(pTask, maxVer, &numOfItems); + bool hasNewData = false; + code = doPutDataIntoInputQ(pTask, maxVer, &numOfItems, &hasNewData); streamMutexUnlock(&pTask->lock); if ((numOfItems > 0) || hasNewData) { diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 83cbb4d3b9..c31b9b598c 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -78,7 +78,7 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) { pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId, pTask->id.taskId); if (pTask->exec.pExecutor == NULL) { tqError("s-task:%s failed to create exec taskInfo, failed to expand task", pTask->id.idStr); - return -1; + return terrno; } qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); } @@ -840,7 +840,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t idle = taosGetTimestampMs() - execTs; tqDebug("s-task:%s task resume to run after idle for:%dms from:%" PRId64, pTask->id.idStr, idle, execTs); - streamResumeTask(pTask); + code = streamResumeTask(pTask); } else { int8_t status = streamTaskSetSchedStatusInactive(pTask); tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, @@ -849,7 +849,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead streamMetaReleaseTask(pMeta, pTask); } - return 0; + return code; } SStreamTask* pTask = NULL; diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index cfa49fd92e..6c3685bb0f 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -287,7 +287,7 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) "s-task:%s inputQ is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data", pTask->id.idStr, STREAM_TASK_QUEUE_CAPACITY, STREAM_TASK_QUEUE_CAPACITY_IN_SIZE, total, size); streamDataSubmitDestroy(px); - return TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_STREAM_INPUTQ_FULL; } int32_t msgLen = px->submit.msgLen; @@ -312,7 +312,7 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) stTrace("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", pTask->id.idStr, STREAM_TASK_QUEUE_CAPACITY, STREAM_TASK_QUEUE_CAPACITY_IN_SIZE, total, size); streamFreeQitem(pItem); - return TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_STREAM_INPUTQ_FULL; } int32_t code = taosWriteQitem(pQueue, pItem); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ccf30438bd..2c9de4526e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -816,6 +816,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_IVLD_STATUS, "Invalid task status TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_CONFLICT_EVENT, "Stream conflict event") TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INTERNAL_ERROR, "Stream internal error") TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_NOT_LEADER, "Stream task not on leader vnode") +TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INPUTQ_FULL, "Task input queue is full") // TDLite TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS, "Invalid TDLite open flags") From 99dbb78992f3be1477f0e0b91aade9d6977f45a8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Sep 2024 22:43:55 +0800 Subject: [PATCH 58/94] refactor: check return value for stream. --- source/dnode/vnode/src/tq/tq.c | 16 +++-- source/libs/stream/src/streamCheckStatus.c | 34 ++++++----- source/libs/stream/src/streamMeta.c | 70 +++++++--------------- source/libs/stream/src/streamTask.c | 3 +- 4 files changed, 54 insertions(+), 69 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2d1d3ed357..a2c088de68 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -102,7 +102,6 @@ int32_t tqOpen(const char* path, SVnode* pVnode) { int32_t tqInitialize(STQ* pTq) { int32_t vgId = TD_VID(pTq->pVnode); - int32_t code = streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1, tqStartTaskCompleteCallback, &pTq->pStreamMeta); if (code != TSDB_CODE_SUCCESS) { @@ -110,7 +109,6 @@ int32_t tqInitialize(STQ* pTq) { } streamMetaLoadAllTasks(pTq->pStreamMeta); - return tqMetaOpen(pTq); } @@ -713,8 +711,7 @@ end: static void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); } int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessVer) { - STQ* pTq = (STQ*)pTqObj; - + STQ* pTq = (STQ*)pTqObj; int32_t vgId = TD_VID(pTq->pVnode); tqDebug("s-task:0x%x start to build task", pTask->id.taskId); @@ -744,16 +741,25 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV SSchemaWrapper* pschemaWrapper = pOutputInfo->tbSink.pSchemaWrapper; pOutputInfo->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1); if (pOutputInfo->tbSink.pTSchema == NULL) { - return -1; + return terrno; } pOutputInfo->tbSink.pTblInfo = tSimpleHashInit(10240, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + if (pOutputInfo->tbSink.pTblInfo == NULL) { + tqError("vgId:%d failed init sink tableInfo, code:%s", vgId, tstrerror(terrno)); + return terrno; + } + tSimpleHashSetFreeFp(pOutputInfo->tbSink.pTblInfo, freePtr); } if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { SWalFilterCond cond = {.deleteMsg = 1}; // delete msg also extract from wal files pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId); + if (pTask->exec.pWalReader == NULL) { + tqError("vgId:%d failed init wal reader, code:%s", vgId, tstrerror(terrno)); + return terrno; + } } streamTaskResetUpstreamStageInfo(pTask); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 540199bb90..2bfab82805 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -21,7 +21,7 @@ #define CHECK_NOT_RSP_DURATION 10 * 1000 // 10 sec static void processDownstreamReadyRsp(SStreamTask* pTask); -static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId); +static int32_t addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId); static void rspMonitorFn(void* param, void* tmrId); static void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs); static int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id); @@ -226,13 +226,13 @@ int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* stError("s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%" PRId64 ", current stage:%" PRId64 ", not check wait for downstream task nodeUpdate, and all tasks restart", id, pRsp->upstreamNodeId, pRsp->oldStage, pTask->pMeta->stage); - addIntoNodeUpdateList(pTask, pRsp->upstreamNodeId); + code = addIntoNodeUpdateList(pTask, pRsp->upstreamNodeId); } else { stError( "s-task:%s downstream taskId:0x%x (vgId:%d) not leader, self dispatch epset needs to be updated, not check " "downstream again, nodeUpdate needed", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); - addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); + code = addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); } streamMetaAddFailedTaskSelf(pTask, now); @@ -371,12 +371,14 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { } } -void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { +int32_t addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { int32_t vgId = pTask->pMeta->vgId; + int32_t code = 0;; + bool existed = false; streamMutexLock(&pTask->lock); + int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList); - bool existed = false; for (int i = 0; i < num; ++i) { SDownstreamTaskEpset* p = taosArrayGet(pTask->outputInfo.pNodeEpsetUpdateList, i); if (p == NULL) { @@ -391,15 +393,19 @@ void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { if (!existed) { SDownstreamTaskEpset t = {.nodeId = nodeId}; - void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t); + + void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t); if (p == NULL) { - // todo let's retry + code = terrno; + stError("s-task:%s vgId:%d failed to update epset, code:%s", pTask->id.idStr, tstrerror(code)); + } else { + stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, + vgId, t.nodeId, (num + 1)); } - stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, vgId, - t.nodeId, (num + 1)); } streamMutexUnlock(&pTask->lock); + return code; } void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs) { @@ -629,6 +635,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { const char* id = pTask->id.idStr; int32_t vgId = pTask->pMeta->vgId; int32_t numOfTimeout = taosArrayGetSize(pTimeoutList); + int32_t code = 0; pInfo->timeoutStartTs = taosGetTimestampMs(); for (int32_t i = 0; i < numOfTimeout; ++i) { @@ -640,14 +647,13 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { int32_t taskId = *px; SDownstreamStatusInfo* p = NULL; findCheckRspStatus(pInfo, taskId, &p); + if (p != NULL) { if (p->status != -1 || p->rspTs != 0) { - stError("s-task:%s invalid rsp record entry, index:%d, status:%d, rspTs:%" PRId64, pTask->id.idStr, i, - p->status, p->rspTs); + stError("s-task:%s invalid rsp record entry, index:%d, status:%d, rspTs:%" PRId64, id, i, p->status, p->rspTs); continue; } - - int32_t code = doSendCheckMsg(pTask, p); + code = doSendCheckMsg(pTask, p); } } @@ -666,7 +672,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { SDownstreamStatusInfo* p = NULL; findCheckRspStatus(pInfo, *pTaskId, &p); if (p != NULL) { - addIntoNodeUpdateList(pTask, p->vgId); + code = addIntoNodeUpdateList(pTask, p->vgId); stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 100sec, add into nodeUpate list", id, vgId, p->taskId, p->vgId); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 0417fb2182..514e25c689 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -368,8 +368,9 @@ void streamMetaRemoveDB(void* arg, char* key) { int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, int32_t vgId, int64_t stage, startComplete_fn_t fn, SStreamMeta** p) { - int32_t code = 0; QRY_PARAM_CHECK(p); + int32_t code = 0; + int32_t lino = 0; SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { @@ -379,23 +380,18 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, int32_t len = strlen(path) + 64; char* tpath = taosMemoryCalloc(1, len); - if (tpath == NULL) { - code = terrno; - goto _err; - } + TSDB_CHECK_NULL(tpath, code, lino, _err, terrno); sprintf(tpath, "%s%s%s", path, TD_DIRSEP, "stream"); pMeta->path = tpath; code = streamMetaOpenTdb(pMeta); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); if ((code = streamMetaMayCvtDbFormat(pMeta)) < 0) { stError("vgId:%d convert sub info format failed, open stream meta failed, reason: %s", pMeta->vgId, tstrerror(terrno)); - goto _err; + TSDB_CHECK_CODE(code, lino, _err); } if ((code = streamMetaBegin(pMeta) < 0)) { @@ -405,28 +401,17 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, _hash_fn_t fp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); pMeta->pTasksMap = taosHashInit(64, fp, true, HASH_NO_LOCK); - if (pMeta->pTasksMap == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + TSDB_CHECK_NULL(pMeta->pTasksMap, code, lino, _err, terrno); pMeta->updateInfo.pTasks = taosHashInit(64, fp, false, HASH_NO_LOCK); - if (pMeta->updateInfo.pTasks == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + TSDB_CHECK_NULL(pMeta->updateInfo.pTasks, code, lino, _err, terrno); code = streamMetaInitStartInfo(&pMeta->startInfo); - if (code) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); // task list pMeta->pTaskList = taosArrayInit(4, sizeof(SStreamTaskId)); - if (pMeta->pTaskList == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + TSDB_CHECK_NULL(pMeta->pTaskList, code, lino, _err, terrno); pMeta->scanInfo.scanCounter = 0; pMeta->vgId = vgId; @@ -440,59 +425,47 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, pMeta->startInfo.completeFn = fn; pMeta->pTaskDbUnique = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + TSDB_CHECK_NULL(pMeta->pTaskDbUnique, code, lino, _err, terrno); pMeta->numOfPausedTasks = 0; pMeta->numOfStreamTasks = 0; pMeta->closeFlag = false; stInfo("vgId:%d open stream meta succ, latest checkpoint:%" PRId64 ", stage:%" PRId64, vgId, pMeta->chkpId, stage); - pMeta->rid = taosAddRef(streamMetaId, pMeta); // set the attribute when running on Linux OS TdThreadRwlockAttr attr; code = taosThreadRwlockAttrInit(&attr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); #ifdef LINUX code = pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); #endif code = taosThreadRwlockInit(&pMeta->lock, &attr); - if (code) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); code = taosThreadRwlockAttrDestroy(&attr); - if (code) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); code = metaRefMgtAdd(pMeta->vgId, pRid); - if (code) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); code = createMetaHbInfo(pRid, &pMeta->pHbInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL); + TSDB_CHECK_NULL(pMeta->qHandle, code, lino, _err, terrno); code = bkdMgtCreate(tpath, (SBkdMgt**)&pMeta->bkdChkptMgt); - if (code != 0) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); code = taosThreadMutexInit(&pMeta->backendMutex, NULL); + TSDB_CHECK_CODE(code, lino, _err); *p = pMeta; return code; @@ -526,9 +499,10 @@ _err: if (pMeta->startInfo.pReadyTaskSet) taosHashCleanup(pMeta->startInfo.pReadyTaskSet); if (pMeta->startInfo.pFailedTaskSet) taosHashCleanup(pMeta->startInfo.pFailedTaskSet); if (pMeta->bkdChkptMgt) bkdMgtDestroy(pMeta->bkdChkptMgt); + taosMemoryFree(pMeta); - stError("failed to open stream meta, reason:%s", tstrerror(terrno)); + stError("vgId:%d failed to open stream meta, at line:%d reason:%s", vgId, lino, tstrerror(code)); return code; } @@ -1274,7 +1248,7 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) { void streamMetaStartHb(SStreamMeta* pMeta) { int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); if (pRid == NULL) { - stError("vgId:%d failed to prepare the metaHb to mnode, hbMsg will not started, code: out of memory", pMeta->vgId); + stFatal("vgId:%d failed to prepare the metaHb to mnode, hbMsg will not started, code: out of memory", pMeta->vgId); return; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index fb2456e1cd..9a324084ff 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -487,8 +487,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i STaskOutputInfo* pOutputInfo = &pTask->outputInfo; pOutputInfo->pTokenBucket = taosMemoryCalloc(1, sizeof(STokenBucket)); if (pOutputInfo->pTokenBucket == NULL) { - stError("s-task:%s failed to prepare the tokenBucket, code:%s", pTask->id.idStr, - tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + stError("s-task:%s failed to prepare the tokenBucket, code:%s", pTask->id.idStr, tstrerror(terrno)); return terrno; } From 1a90e9612db9748ffa2ec10675ac6f6da6ceb735 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Sep 2024 23:04:41 +0800 Subject: [PATCH 59/94] refactor: check return value. --- include/libs/executor/executor.h | 4 +-- include/libs/executor/storageapi.h | 2 +- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 5 +-- source/dnode/vnode/src/tq/tqUtil.c | 5 +-- source/dnode/vnode/src/tqCommon/tqCommon.c | 20 +++++++---- source/dnode/vnode/src/tsdb/tsdbRead2.c | 9 +++-- source/dnode/vnode/src/vnd/vnodeInitApi.c | 2 +- source/libs/executor/src/executor.c | 40 ++++++++++++---------- source/libs/stream/src/streamState.c | 4 ++- 10 files changed, 56 insertions(+), 37 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index ed56b7e6b2..ae26d5f2ae 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -74,7 +74,7 @@ typedef enum { * @param vgId * @return */ -qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t vgId, int32_t taskId); +int32_t qCreateStreamExecTaskInfo(qTaskInfo_t* pInfo, void* msg, SReadHandle* readers, int32_t vgId, int32_t taskId); /** * Create the exec task for queue mode @@ -93,7 +93,7 @@ int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList * @param taskId * @param queryId */ -void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); +int32_t qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 61ae034450..4fc7e5eac5 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -173,7 +173,7 @@ typedef struct TsdReader { int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables); void (*tsdReaderClose)(); - void (*tsdSetReaderTaskId)(void *pReader, const char *pId); + int32_t (*tsdSetReaderTaskId)(void *pReader, const char *pId); int32_t (*tsdSetQueryTableList)(); int32_t (*tsdNextDataBlock)(); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2f56aac7d6..827f7e2044 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -164,7 +164,7 @@ typedef struct STsdbReader STsdbReader; int32_t tsdbReaderOpen2(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader, const char *idstr, SHashObj **pIgnoreTables); int32_t tsdbSetTableList2(STsdbReader *pReader, const void *pTableList, int32_t num); -void tsdbReaderSetId2(STsdbReader *pReader, const char *idstr); +int32_t tsdbReaderSetId(void *pReader, const char *idstr); void tsdbReaderClose2(STsdbReader *pReader); int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext); int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 19c5b5d481..69819c87dc 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -314,8 +314,9 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat SReadHandle handle = {.vnode = pVnode, .initTqReader = 1, .skipRollup = 1, .pStateBackend = pStreamState}; initStorageAPI(&handle.api); - pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode), 0); - if (!pRSmaInfo->taskInfo[idx]) { + + code = qCreateStreamExecTaskInfo(&pRSmaInfo->taskInfo[idx], param->qmsg[idx], &handle, TD_VID(pVnode), 0); + if (!pRSmaInfo->taskInfo[idx] || (code != 0)) { TAOS_RETURN(TSDB_CODE_RSMA_QTASKINFO_CREATE); } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index ff9f1e524e..495fcd771a 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -145,12 +145,13 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, terrno = 0; SMqDataRsp dataRsp = {0}; - int code = tqInitDataRsp(&dataRsp.common, *pOffset); + + int code = tqInitDataRsp(&dataRsp.common, *pOffset); if (code != 0) { goto end; } - qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); + code = qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); code = tqScanData(pTq, pHandle, &dataRsp, pOffset, pRequest); if (code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { goto end; diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index c31b9b598c..3f4329f22b 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -36,6 +36,7 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) { int64_t st = taosGetTimestampMs(); int64_t streamId = 0; int32_t taskId = 0; + int32_t code = 0; tqDebug("s-task:%s vgId:%d start to expand stream task", pTask->id.idStr, vgId); @@ -52,7 +53,7 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) { pTask->pState = streamStateOpen(pMeta->path, pTask, streamId, taskId); if (pTask->pState == NULL) { tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId); - return -1; + return terrno; } else { tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } @@ -75,18 +76,23 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) { initStorageAPI(&handle.api); if (pTask->info.taskLevel == TASK_LEVEL__SOURCE || pTask->info.taskLevel == TASK_LEVEL__AGG) { - pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId, pTask->id.taskId); - if (pTask->exec.pExecutor == NULL) { - tqError("s-task:%s failed to create exec taskInfo, failed to expand task", pTask->id.idStr); - return terrno; + code = qCreateStreamExecTaskInfo(&pTask->exec.pExecutor, pTask->exec.qmsg, &handle, vgId, pTask->id.taskId); + if (code) { + tqError("s-task:%s failed to expand task, code:%s", pTask->id.idStr, tstrerror(code)); + return code; + } + + code = qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); + if (code) { + + return code; } - qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); } double el = (taosGetTimestampMs() - st) / 1000.0; tqDebug("s-task:%s vgId:%d expand stream task completed, elapsed time:%.2fsec", pTask->id.idStr, vgId, el); - return TSDB_CODE_SUCCESS; + return code; } void tqSetRestoreVersionInfo(SStreamTask* pTask) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index e39ac2e45d..731b733b52 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -5986,13 +5986,18 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact } // if failed, do nothing -void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { +int32_t tsdbReaderSetId(void* p, const char* idstr) { + STsdbReader* pReader = (STsdbReader*) p; taosMemoryFreeClear(pReader->idStr); + pReader->idStr = taosStrdup(idstr); if (pReader->idStr == NULL) { - // no need to do anything + tsdbError("%s failed to build reader id, code:%s", idstr, tstrerror(terrno)); + return terrno; } + pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr; + return 0; } void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 9be84b99f4..59a129cac8 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -59,7 +59,7 @@ void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable2; // todo this function should be moved away pReader->tsdSetQueryTableList = tsdbSetTableList2; - pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId2; + pReader->tsdSetReaderTaskId = tsdbReaderSetId; pReader->tsdSetFilesetDelimited = (void (*)(void*))tsdbSetFilesetDelimited; pReader->tsdSetSetNotifyCb = (void (*)(void*, TsdReaderNotifyCbFn, void*))tsdbReaderSetNotifyCb; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index ebec9f9373..0033e14a2d 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -204,28 +204,34 @@ _end: return code; } -void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI* pAPI) { +int32_t doSetTaskId(SOperatorInfo* pOperator, SStorageAPI* pAPI) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pStreamScanInfo = pOperator->info; if (pStreamScanInfo->pTableScanOp != NULL) { STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info; if (pScanInfo->base.dataReader != NULL) { - pAPI->tsdReader.tsdSetReaderTaskId(pScanInfo->base.dataReader, pTaskInfo->id.str); + int32_t code = pAPI->tsdReader.tsdSetReaderTaskId(pScanInfo->base.dataReader, pTaskInfo->id.str); + if (code) { + qError("failed to set reader id for executor, code:%s", tstrerror(code)); + return code; + } } } } else { - doSetTaskId(pOperator->pDownstream[0], pAPI); + return doSetTaskId(pOperator->pDownstream[0], pAPI); } + + return 0; } -void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { +int32_t qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { SExecTaskInfo* pTaskInfo = tinfo; pTaskInfo->id.queryId = queryId; buildTaskId(taskId, queryId, pTaskInfo->id.str); // set the idstr for tsdbReader - doSetTaskId(pTaskInfo->pRoot, &pTaskInfo->storageAPI); + return doSetTaskId(pTaskInfo->pRoot, &pTaskInfo->storageAPI); } int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { @@ -337,33 +343,31 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 return pTaskInfo; } -qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t vgId, int32_t taskId) { +int32_t qCreateStreamExecTaskInfo(qTaskInfo_t* pTaskInfo, void* msg, SReadHandle* readers, int32_t vgId, int32_t taskId) { if (msg == NULL) { - return NULL; + return TSDB_CODE_INVALID_PARA; } + *pTaskInfo = NULL; + SSubplan* pPlan = NULL; int32_t code = qStringToSubplan(msg, &pPlan); if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; + return code; } - qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(readers, vgId, taskId, pPlan, &pTaskInfo, NULL, 0, NULL, OPTR_EXEC_MODEL_STREAM); + code = qCreateExecTask(readers, vgId, taskId, pPlan, pTaskInfo, NULL, 0, NULL, OPTR_EXEC_MODEL_STREAM); if (code != TSDB_CODE_SUCCESS) { - qDestroyTask(pTaskInfo); - terrno = code; - return NULL; + qDestroyTask(*pTaskInfo); + return code; } code = qStreamInfoResetTimewindowFilter(pTaskInfo); if (code != TSDB_CODE_SUCCESS) { - qDestroyTask(pTaskInfo); - terrno = code; - return NULL; + qDestroyTask(*pTaskInfo); } - return pTaskInfo; + + return code; } static int32_t filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr, diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index e6754d7bfd..0e2d31cc8f 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -101,8 +101,10 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; + SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); stDebug("open stream state %p, %s", pState, path); + if (pState == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; QUERY_CHECK_CODE(code, lino, _end); @@ -138,7 +140,7 @@ SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, i _end: if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + qError("0x%x %s failed at line %d since %s", taskId, __func__, lino, tstrerror(code)); } return NULL; } From 29d9e9acfcf8a0eb88e48e998f57e0d2d59c15e4 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Wed, 11 Sep 2024 08:05:54 +0800 Subject: [PATCH 60/94] docs:[TS-4893] Add doc for new function. --- .../14-reference/03-taos-sql/10-function.md | 898 +++++++++++++++++- 1 file changed, 873 insertions(+), 25 deletions(-) diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index ee71abbdec..007e1dd64a 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -169,14 +169,43 @@ POW(expr1, expr2) #### ROUND - ```sql -ROUND(expr) +ROUND(expr[, digits]) ``` **功能说明**：获得指定字段的四舍五入的结果。 - 其他使用说明参见 CEIL 函数描述。 +**返回结果类型**：与指定字段的原始数据类型一致。 + +**适用数据类型**： +- `expr`：数值类型。 +- `digits`：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `expr` 或 `digits` 为 NULL，返回 NULL。 +- 若指定了`digits`，则会保留 `digits` 位小数，默认为 0。 +- 若输入值是 INTEGER 类型, 无论 `digits` 值为多少，都只会返回 INTEGER 类型，不会保留小数。 +- `digits` 大于零表示对小数位进行操作，四舍五入到 `digits` 位小数。若小数位数小于 `digits` 位，不进行四舍五入操作，直接返回。 +- `digits` 小于零表示丢掉小数位，并将数字四舍五入到小数点左侧 `digits` 位。若小数点左侧的位数小于 `digits`位，返回 0。 +- 由于暂未支持 DECIMAL 类型，所以该函数会用 DOUBLE 和 FLOAT 来表示包含小数的结果，但是 DOUBLE 和 FLOAT 是有精度上限的，当位数太多时使用该函数可能没有意义。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select round(8888.88); + round(8888.88) | +============================ + 8889.000000000000000 | + +taos> select round(8888.88,-1); + round(8888.88,-1) | +============================ + 8890.000000000000000 | +``` #### SIN @@ -232,6 +261,297 @@ TAN(expr) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 +#### PI +```sql +PI() +``` + +**功能说明**：返回圆周率 π 的值。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：无。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- π ≈ 3.141592653589793。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select pi(); + pi() | +============================ + 3.141592653589793 | +``` + +##### TRUNCATE +```sql +TRUNCATE(expr, digits) +``` + +**功能说明**：获得指定字段按照指定位数截断的值。 + +**返回结果类型**：与 `expr` 字段的原始数据类型一致。 + +**适用数据类型**： +- `expr`：数值类型。 +- `digits`：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `expr` 或 `digits` 为 NULL，返回 NULL。 +- 截取指按保留位数直接进行截取，没有四舍五入。 +- `digits` 大于零表示对小数位进行操作，截取到 `digits` 位小数，若小数位数小于 `digits` 位，不进行截取操作，直接返回。 +- `digits` 等于零表示丢掉小数位。 +- `digits` 小于零表示丢掉小数位，并将小数点左侧 `digits` 位置 `0`。若小数点左侧的位数小于 `digits`位，返回 0。 +- 由于暂未支持 DECIMAL 类型，所以该函数会用 DOUBLE 和 FLOAT 来表示包含小数的结果，但是 DOUBLE 和 FLOAT 是有精度上限的，当位数太多时使用该函数可能没有意义。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select truncate(8888.88, 0); + truncate(8888.88, 0) | +============================ + 8888.000000000000000 | + +taos> select truncate(8888.88, -1); + truncate(8888.88, -1) | +============================ + 8880.000000000000000 | +``` + +#### EXP +```sql +EXP(expr) +``` +**功能说明**：返回 e（自然对数的底）的指定乘方后的值。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select exp(2); + exp(2) | +============================ + 7.389056098930650 | +``` + +#### LN +```sql +LN(expr) +``` + +**功能说明**：返回指定参数的自然对数。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，返回 NULL。 +- 如果 `epxr` 小于等于 0，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select ln(10); + ln(10) | +============================ + 2.302585092994046 | +``` + +#### MOD +```sql +MOD(expr1, expr2) +``` + +**功能说明**：计算 expr1 % expr2 的结果。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr2` 为 0 则返回 NULL。 +- 如果 `expr1` 或 `expr2` 为 NULL，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +``` sql +taos> select mod(10,3); + mod(10,3) | +============================ + 1.000000000000000 | + +taos> select mod(1,0); + mod(1,0) | +============================ + NULL | +``` + +#### RAND +```sql +RAND([seed]) +``` + +**功能说明**：返回一个从0到1均匀分布的随机数。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**： +- `seed`：INTEGER。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果指定了 `seed` 值，那么将会用指定的 `seed` 作为随机种子，确保生成的随机数序列具有确定性。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +``` sql +taos> select rand(); + rand() | +============================ + 0.202092426923147 | + +taos> select rand(); + rand() | +============================ + 0.131537788143166 | + +taos> select rand(1); + rand(1) | +============================ + 0.000007826369259 | + +taos> select rand(1); + rand(1) | +============================ + 0.000007826369259 | +``` + +#### SIGN +```sql +SIGN(expr) +``` + +**功能说明**：返回指定参数的符号。 + +**返回结果类型**：与指定字段的原始数据类型一致。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr` 为负，返回 -1 ， +- 如果 `expr` 为正，返回 1 ， +- 如果 `expr` 为 0 ，返回 0 。 +- 如果 `expr` 为 NULL ，返回 NULL 。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select sign(-1); + sign(-1) | +======================== + -1 | + +taos> select sign(1); + sign(1) | +======================== + 1 | + +taos> select sign(0); + sign(0) | +======================== + 0 | +``` + +#### DEGREES +```sql +DEGREES(expr) +``` + +**功能说明**：计算指定参数由弧度值转为角度后的值。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，则返回 NULL。 +- degree = radian * 180 / π。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select degrees(PI()); + degrees(pi()) | +============================ + 180.000000000000000 | +``` + +#### RADIANS +```sql +RADIANS(expr) +``` + +**功能说明**：计算指定参数由角度值转为弧度后的值。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，则返回 NULL。 +- radian = degree * π / 180。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select radians(180); + radians(180) | +============================ + 3.141592653589793 | +``` ### 字符串函数 字符串函数的输入参数为字符串类型，返回结果为数值类型或字符串类型。 @@ -252,6 +572,27 @@ CHAR_LENGTH(expr) **适用于**: 表和超级表。 +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 与 `LENGTH()` 函数不同在于，对于多字节字符，比如中文字符， `CHAR_LENGTH()` 函数会将其算做一个字符，长度为 1，而 `LENGTH()` 会计算其字节数，长度为 3。比如 `CHAR_LENGTH('你好') = 2`， `LENGTH('你好') = 6`。 +- 如果 `expr` 为 NULL，返回 NULL。 + +**举例**： +```sql +taos> select char_length('Hello world'); + char_length('Hello world') | +============================= + 11 | + +taos> select char_length('你好 世界'); + char_length('你好 世界') | +=============================== + 5 | +``` + #### CONCAT ```sql @@ -353,23 +694,150 @@ RTRIM(expr) **适用于**: 表和超级表。 - -#### SUBSTR - +#### TRIM ```sql -SUBSTR(expr, pos [,len]) +TRIM([{LEADING | TRAILING | BOTH} [remstr] FROM] expr) +TRIM([remstr FROM] expr) ``` -**功能说明**：从源字符串 str 中的指定位置 pos 开始取一个长度为 len 的子串并返回。如果输入参数 len 被忽略，返回的子串包含从 pos 开始的整个字串。 +**功能说明**：返回去掉了所有 remstr 前缀或后缀的字符串 epxr 。 -**返回结果类型**：与输入字段的原始类型相同。 +**返回结果类型**：与输入字段 epxr 的原始类型相同。 -**适用数据类型**：VARCHAR, NCHAR。输入参数 pos 可以为正数，也可以为负数。如果 pos 是正数，表示开始位置从字符串开头正数计算。如果 pos 为负数，表示开始位置从字符串结尾倒数计算。 +**适用数据类型**： +- remstr：VARCHAR,NCHAR。 +- epxr：VARCHAR,NCHAR。 **嵌套子查询支持**：适用于内层查询和外层查询。 **适用于**: 表和超级表。 +**使用说明**： +- 第一个可选变量[LEADING | BOTH | TRAILING]指定要剪裁字符串的哪一侧： + - LEADING 将移除字符串开头的指定字符。 + - TRAILING 将移除字符串末尾的指定字符。 + - BOTH（默认值）将移除字符串开头和末尾的指定字符。 +- 第二个可选变量[remstr]指定要裁剪掉的字符串： + - 如果不指定 remstr ，默认裁剪空格。 + - remstr 可以指定多个字符，如trim('ab' from 'abacd') ，此时会将 'ab' 看做一个整体来裁剪，得到裁剪结果 'acd'。 +- 若 expr 为 NULL, 返回 NULL。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select trim(' a '); + trim(' a ') | +============================= + a | + +taos> select trim(leading from ' a '); + trim(leading from ' a ') | +========================================== + a | + + +taos> select trim(leading 'b' from 'bbbbbbbba '); + trim(leading 'b' from 'bbbbbbbba ') | +============================================== + a | + +taos> select trim(both 'b' from 'bbbbbabbbbbb'); + trim(both 'b' from 'bbbbbabbbbbb') | +===================================== + a | +``` + +#### SUBSTRING/SUBSTR +```sql +SUBSTRING/SUBSTR(expr, pos [, len]) +SUBSTRING/SUBSTR(expr FROM pos [FOR len]) +``` +**功能说明**：返回字符串 `expr` 在 `pos` 位置开始的子串，若指定了 `len` ，则返回在 `pos` 位置开始，长度为 `len` 的子串。 + +**返回结果类型**：与输入字段 `expr` 的原始类型相同。 + +**适用数据类型**： +- `expr`：VARCHAR,NCHAR。 +- `pos`：整数类型。 +- `len`：整数类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `pos` 为正数，则返回的结果为 `expr` 从左到右开始数 `pos` 位置开始的右侧的子串。 +- 若 `pos` 为负数，则返回的结果为 `expr` 从右到左开始数 `pos` 位置开始的右侧的子串。 +- 任意参数为 NULL，返回 NULL。 +- 该函数是多字节安全的。 +- 若 `len` 小于 1，返回空串。 +- `pos` 是 1-base 的，若 `pos` 为 0，返回空串。 +- 若 `pos` + `len` 大于 `len(expr)`，返回从 `pos` 开始到字符串结尾的子串，等同于执行 `substring(expr, pos)`。 + +**举例**： +```sql +taos> select substring('tdengine', 0); + substring('tdengine', 0) | +=========================== + | + +taos> select substring('tdengine', 3); + substring('tdengine', 3) | +=========================== + engine | + +taos> select substring('tdengine', 3,3); + substring('tdengine', 3,3) | +============================= + eng | + +taos> select substring('tdengine', -3,3); + substring('tdengine', -3,3) | +============================== + ine | + +taos> select substring('tdengine', -3,-3); + substring('tdengine', -3,-3) | +=============================== + | +``` + +#### SUBSTRING_INDEX +```sql +SUBSTRING_INDEX(expr, delim, count) +``` + +**功能说明**：返回字符串 `expr` 在出现指定次数分隔符的位置截取的子串。 + +**返回结果类型**：与输入字段 `expr` 的原始类型相同。 + +**适用数据类型**： +- `expr`：VARCHAR,NCHAR。 +- `delim`：VARCHAR, NCHAR。 +- `count`：INTEGER。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `count` 为正数，则返回的结果为 `expr` 从左到右开始数第 `count` 次 出现 `delim` 的位置左侧的字符串。 +- 若 `count` 为负数，则返回的结果为 `expr` 从右到左开始数第 `count` 的绝对值次 出现 `delim` 的位置右侧的字符串。 +- 任意参数为 NULL，返回 NULL。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select substring_index('www.taosdata.com','.',2); + substring_index('www.taosdata.com','.',2) | +============================================ + www.taosdata | + +taos> select substring_index('www.taosdata.com','.',-2); + substring_index('www.taosdata.com','.',-2) | +============================================= + taosdata.com | +``` #### UPPER @@ -387,7 +855,182 @@ UPPER(expr) **适用于**: 表和超级表。 +#### CHAR +```sql +CHAR(expr1 [, expr2] [, epxr3] ...) +``` +**功能说明**：将输入参数当作整数，并返回这些整数在 ASCII 编码中对应的字符。 + +**返回结果类型**：VARCHAR。 + +**适用数据类型**：整数类型,VARCHAR,NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 输入的值超过 255 会被转化成多字节的结果，如 `CHAR(256)` 等同于 `CHAR(1,0)`, `CHAR(256 * 256)` 等同于 `CHAR(1,0,0)`。 +- 输入参数的 NULL 值会被跳过。 +- 输入参数若为字符串类型，会将其转换为数值类型处理。 +- 若输入的参数对应的字符为不可打印字符，返回值中仍有该参数对应的字符，但是可能无法显示出来。 + +**举例**： +```sql +taos> select char(77); + char(77) | +=========== + M | + +taos> select char(77,77); + char(77,77) | +============== + MM | + +taos> select char(77 * 256 + 77); + char(77 * 256 + 77) | +====================== + MM | + +taos> select char(77,NULL,77); + char(77,null,77) | +=================== + MM | +``` + +#### ASCII +```sql +ASCII(expr) +``` + +**功能说明**：返回字符串第一个字符的 ASCII 码。 + +**返回结果数据类型**：BIGINT。 + +**适用数据类型**：VARCHAR, NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，返回 NULL。 +- 如果 `expr` 的第一个字符为多字节字符，只会返回该字符第一个字节的值对应的 ASCII 码。 + +**举例**： +```sql +taos> select ascii('testascii'); + ascii('testascii') | +===================== + 116 | +``` + +#### POSITION +```sql +POSITION(expr1 IN expr2) +``` + +**功能说明**：计算字符串 `expr1` 在字符串 `expr2` 中的位置。 + +**返回结果类型**：BIGINT。 + +**适用数据类型**： +- `expr1`：VARCHAR, NCHAR。 +- `expr2`：VARCHAR, NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `expr1` 或 `expr2` 为 NULL，返回 NULL。 +- 若 `expr1` 在 `expr2` 中不存在，返回 0。 +- 若 `expr1` 为空串，认为 `expr1` 在 `expr2` 中总能匹配成功，返回 1。 +- 返回的位置是 1-base 的。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select position('a' in 'cba'); + position('a' in 'cba') | +========================= + 3 | + + +taos> select position('' in 'cba'); + position('' in 'cba') | +======================== + 1 | + +taos> select position('d' in 'cba'); + position('d' in 'cba') | +========================= + 0 | +``` + +#### REPLACE +```sql +REPLACE(expr, from_str, to_str) +``` +**功能说明**：将字符串中的 `from_str` 全部替换为 `to_str`。 + +**返回结果类型**：与输入字段 `expr` 的原始类型相同。 + +**适用数据类型**： +- `expr`：VARCHAR, NCHAR。 +- `from_str`：VARCHAR, NCHAR。 +- `to_str`：VARCHAR, NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 该函数是大小写敏感的。 +- 任意参数为 NULL，返回 NULL。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select replace('aabbccAABBCC', 'AA', 'DD'); + replace('aabbccAABBCC', 'AA', 'DD') | +====================================== + aabbccDDBBCC | +``` + +#### REPEAT +```sql +REPEAT(expr, count) +``` +**功能说明**：返回将字符串重复指定次数得到的字符串。 + +**返回结果类型**：与输入字段 `expr` 的原始类型相同。 + +**适用数据类型**： +- `expr`： VARCHAR,NCHAR。 +- `count`：INTEGER。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `count < 1`，返回空串。 +- 若 `expr` 或 `count` 为 NULL，返回 NULL。 + +**举例**： +```sql +taos> select repeat('abc',5); + repeat('abc',5) | +============================ + abcabcabcabcabc | + +taos> select repeat('abc',-1); + repeat('abc',-1) | +=================== + | +``` ### 转换函数 转换函数将值从一种数据类型转换为另一种数据类型。 @@ -609,22 +1252,39 @@ NOW() TIMEDIFF(expr1, expr2 [, time_unit]) ``` -**功能说明**：计算两个时间戳之间的差值，并近似到时间单位 time_unit 指定的精度。 +**功能说明**：返回时间戳 `expr1` - `expr2` 的结果，结果可能为负，并近似到时间单位 `time_unit` 指定的精度。 -**返回结果数据类型**：BIGINT。 +**返回结果类型**：BIGINT。 -**应用字段**：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 - -**适用于**：表和超级表。 +**适用数据类型**： +- `expr1`：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 +- `expr2`：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 +- `time_unit`：见使用说明。 **嵌套子查询支持**：适用于内层查询和外层查询。 -**使用说明**： -- 支持的时间单位 time_unit 如下： - 1b(纳秒), 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天), 1w(周)。 -- 如果时间单位 time_unit 未指定， 返回的时间差值精度与当前 DATABASE 设置的时间精度一致。 -- 输入包含不符合时间日期格式的字符串则返回 NULL。 +**适用于**: 表和超级表。 +**使用说明**： +- 支持的时间单位 `time_unit` 如下： 1b(纳秒), 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天), 1w(周)。 +- 如果时间单位 `time_unit` 未指定， 返回的时间差值精度与当前 DATABASE 设置的时间精度一致。 +- 输入包含不符合时间日期格式的字符串则返回 NULL。 +- `expr1` 或 `expr2` 为 NULL，返回 NULL。 +- `time_unit` 为 NULL，等同于未指定时间单位。 +- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. + +**举例**： +```sql +taos> select timediff('2022-01-01 08:00:00', '2022-01-01 08:00:01',1s); + timediff('2022-01-01 08:00:00', '2022-01-01 08:00:01',1s) | +============================================================ + -1 | + +taos> select timediff('2022-01-01 08:00:01', '2022-01-01 08:00:00',1s); + timediff('2022-01-01 08:00:01', '2022-01-01 08:00:00',1s) | +============================================================ + 1 | +``` #### TIMETRUNCATE @@ -695,6 +1355,148 @@ TODAY() b(纳秒)，u(微秒)，a(毫秒)，s(秒)，m(分)，h(小时)，d(天)，w(周)。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 +#### WEEK +```sql +WEEK(expr [, mode]) +``` +**功能说明**：返回输入日期的周数。 + +**返回结果类型**：BIGINT。 + +**适用数据类型**： +- `expr`：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 +- `mode`：0 - 7 之间的整数。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `expr` 为 NULL，返回 NULL。 +- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. +- `mode` 用来指定一周是从周日开始还是周一开始，以及指定返回值范围是 1 - 53 还是 0 - 53。下表详细描述不同的 mode 对应的计算方法： + +| Mode | 每周的第一天 | 返回值范围 | 第 1 周的计算方法 | +| ---- | ------ | ------ | ------------------ | +| 0 | 周日 | 0 - 53 | 第一个包含周日的周为第 1 周 | +| 1 | 周一 | 0 - 53 | 第一个包含四天及以上的周为第 1 周 | +| 2 | 周日 | 1 - 53 | 第一个包含周日的周为第 1 周 | +| 3 | 周一 | 1 - 53 | 第一个包含四天及以上的周为第 1 周 | +| 4 | 周日 | 0 - 53 | 第一个包含四天及以上的周为第 1 周 | +| 5 | 周一 | 0 - 53 | 第一个包含周一的周为第 1 周 | +| 6 | 周日 | 1 - 53 | 第一个包含四天及以上的周为第 1 周 | +| 7 | 周一 | 1 - 53 | 第一个包含周一的周为第 1 周 | +- 当返回值范围为0 - 53时，第 1 周之前的日期为第 0 周。 +- 当返回值范围为1 - 53时，第 1 周之前的日期为上一年的最后一周。 +- 以`2000-01-01`为例， + - 在 `mode=0`时返回值为`0`，因为该年第一个周日为`2000-01-02`，从`2000-01-02`起才是第 1 周，所以 `2000-01-01`为第 0 周，返回 0。 + - 在 `mode=1`时返回值为`0`，因为`2000-01-01`所在的周只有两天，分别是 `2000-01-01(周六)`和`2000-01-02(周日)`，所以`2000-01-03`起才是第一周，`2000-01-01`为第 0 周，返回 0。 + - 在 `mode=2`时返回值为`52`，因为从`2000-01-02`起才是第 1 周，并且返回值范围为 1-53，所以`2000-01-01`算做上一年的最后一周，即 1999 年的第 52 周，返回 52。 + +**举例**： +```sql +taos> select week('2000-01-01',0); + week('2000-01-01',0) | +======================== + 0 | + +taos> select week('2000-01-01',1); + week('2000-01-01',1) | +======================== + 0 | + +taos> select week('2000-01-01',2); + week('2000-01-01',2) | +======================== + 52 | + +taos> select week('2000-01-01',3); + week('2000-01-01',3) | +======================== + 52 | +``` + +#### WEEKOFYEAR +```sql +WEEKOFYEAR(expr) +``` +**功能说明**：返回输入日期的周数。 + +**返回结果类型**：BIGINT。 + +**适用数据类型**：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 等同于`WEEK(expr, 3)`，即在每周第一天是周一，返回值范围为 1 - 53，第一个包含四天及以上的周为第 1 周的条件下判断输入日期的周数。 +- 若 `expr` 为 NULL，返回 NULL。 +- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. + +**举例**： +```sql +taos> select weekofyear('2000-01-01'); + weekofyear('2000-01-01') | +=========================== + 52 | +``` + +#### WEEKDAY +```sql +WEEKDAY(expr) +``` +**功能说明**：返回输入日期是周几。 + +**返回结果类型**：BIGINT。 + +**适用数据类型**：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 返回值 0 代表周日，1 代表周一 ... 6 代表周六。 +- 若 `expr` 为 NULL，返回 NULL。 +- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. + +**举例**： +```sql +taos> select weekday('2000-01-01'); + weekday('2000-01-01') | +======================== + 5 | +``` + +#### DAYOFWEEK +```sql +DAYOFWEEK(expr) +``` +**功能说明**：返回输入日期是周几。 + +**返回结果类型**：BIGINT。 + +**适用数据类型**：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 返回值 1 代表周日，2 代表周一 ... 7 代表周六 +- 若 `expr` 为 NULL，返回 NULL。 +- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. + +**举例**： +```sql +taos> select dayofweek('2000-01-01'); + dayofweek('2000-01-01') | +========================== + 7 | +``` + ## 聚合函数 @@ -817,13 +1619,13 @@ SPREAD(expr) **适用于**：表和超级表。 -### STDDEV +### STDDEV/STDDEV_POP ```sql -STDDEV(expr) +STDDEV/STDDEV_POP(expr) ``` -**功能说明**：统计表中某列的均方差。 +**功能说明**：统计表中某列的总体标准差。 **返回数据类型**：DOUBLE。 @@ -831,7 +1633,51 @@ STDDEV(expr) **适用于**：表和超级表。 +**举例**： +```sql +taos> select id from test_stddev; + id | +============== + 1 | + 2 | + 3 | + 4 | + 5 | +taos> select stddev_pop(id) from test_stddev; + stddev_pop(id) | +============================ + 1.414213562373095 | +``` +### VAR_POP +```sql +VAR_POP(expr) +``` + +**功能说明**：统计表中某列的总体方差。 + +**返回数据类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**适用于**：表和超级表。 + +**举例**: +```sql +taos> select id from test_var; + id | +============== + 3 | + 1 | + 2 | + 4 | + 5 | + +taos> select var_pop(id) from test_var; + var_pop(id) | +============================ + 2.000000000000000 | +``` ### SUM ```sql @@ -1045,10 +1891,11 @@ MAX(expr) **返回数据类型**：同应用的字段。 -**适用数据类型**：数值类型。 +**适用数据类型**：数值类型, VARCHAR，NCHAR。 **适用于**：表和超级表。 +**使用说明**：max 函数可以接受字符串作为输入参数，当输入参数为字符串类型时，返回最大的字符串值。 ### MIN @@ -1060,10 +1907,11 @@ MIN(expr) **返回数据类型**：同应用的字段。 -**适用数据类型**：数值类型。 +**适用数据类型**：数值类型, VARCHAR，NCHAR。 **适用于**：表和超级表。 +**使用说明**：min 函数可以接受字符串作为输入参数，当输入参数为字符串类型时，返回最大的字符串值。 ### MODE From 379a837b8c0d0af7c3a6c0ed52b46d78d385ee9a Mon Sep 17 00:00:00 2001 From: charles Date: Sat, 14 Sep 2024 08:46:13 +0800 Subject: [PATCH 61/94] Add ci test case to cover ts-5400 by charles --- tests/army/query/accuracy/test_ts5400.py | 51 ++++++++++++++++++++++++ tests/parallel_test/cases.task | 1 + 2 files changed, 52 insertions(+) create mode 100644 tests/army/query/accuracy/test_ts5400.py diff --git a/tests/army/query/accuracy/test_ts5400.py b/tests/army/query/accuracy/test_ts5400.py new file mode 100644 index 0000000000..5263df8014 --- /dev/null +++ b/tests/army/query/accuracy/test_ts5400.py @@ -0,0 +1,51 @@ +import taos +import socket +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.eos import * +from frame.server.dnodes import * + +class TDTestCase(TBase): + """Add test case to cover TS-5400 + """ + updatecfgDict = { + "timezone": "UTC" + } + + def init(self, conn, logSql, replicaVar=1): + host = socket.gethostname() + con = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath(), timezone='UTC') + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(con.cursor()) + + def prepare_data(self): + # prepare data for TS-5400 + tdSql.execute("create database db_ts5400 BUFFER 512 CACHESIZE 1024 CACHEMODEL 'both' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 KEEP 365000d;") + tdSql.execute("use db_ts5400;") + #tdSql.execute("create stable st(ts TIMESTAMP ENCODE 'delta-i' COMPRESS 'lz4' LEVEL 'medium', `uk` VARCHAR(64) ENCODE 'disabled' COMPRESS 'lz4' LEVEL 'medium' PRIMARY KEY ) tags(ta int,tb int,tc int);") + tdSql.execute("create stable st(ts TIMESTAMP, `uk` VARCHAR(64)) tags(ta int,tb int,tc int);") + tdSql.execute("create table t1 using st tags(1,1,1);") + tdSql.execute("insert into t1 values ('1970-01-29 05:04:53.000','22:: ');") + + def test_ts5400(self): + self.prepare_data() + tdSql.execute("use db_ts5400;") + tdSql.query("select _wstart, count(*) from st interval(1y);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '1970-01-01 00:00:00.000') + tdSql.checkData(0, 1, 1) + + def run(self): + self.test_ts5400() + + def stop(self): + tdSql.execute("drop database db_ts5400;") + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 04efaf92c4..a1b23bb282 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -27,6 +27,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f cluster/clusterBasic.py -N 5 ,,y,army,./pytest.sh python3 ./test.py -f query/query_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/accuracy/test_query_accuracy.py +,,y,army,./pytest.sh python3 ./test.py -f query/accuracy/test_ts5400.py ,,y,army,./pytest.sh python3 ./test.py -f insert/insert_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/splitVgroupByLearner.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f authorith/authBasic.py -N 3 From 3b9ca7b7be3a24bb888cf1446dd655a8e48bd57f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 14 Sep 2024 09:21:15 +0800 Subject: [PATCH 62/94] fix(stream): fix syntax error. --- source/libs/stream/src/streamCheckStatus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 2bfab82805..6353904b07 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -397,7 +397,7 @@ int32_t addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t); if (p == NULL) { code = terrno; - stError("s-task:%s vgId:%d failed to update epset, code:%s", pTask->id.idStr, tstrerror(code)); + stError("s-task:%s vgId:%d failed to update epset, code:%s", pTask->id.idStr, vgId, tstrerror(code)); } else { stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, vgId, t.nodeId, (num + 1)); From 7d3a77bcaf7d7737d310b5a16c79b8cef970da60 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 14 Sep 2024 09:57:34 +0800 Subject: [PATCH 63/94] docs:[TD-31985]add schemaless docs that losted --- docs/zh/08-develop/04-schemaless.md | 36 +++++++++++++++++-- .../zh/14-reference/01-components/02-taosc.md | 2 +- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/zh/08-develop/04-schemaless.md b/docs/zh/08-develop/04-schemaless.md index 130d012e8b..17a377950c 100644 --- a/docs/zh/08-develop/04-schemaless.md +++ b/docs/zh/08-develop/04-schemaless.md @@ -13,7 +13,7 @@ import TabItem from "@theme/TabItem"; 值得注意的是，通过无模式写入方式创建的超级表及其对应的子表与通过 SQL 直接创建的超级表和子表在功能上没有区别，用户仍然可以使用 SQL 直接向其中写入数据。然而，由于无模式写入方式生成的表名是基于标签值按照固定的映射规则生成的，因此这些表名可能缺乏可读性，不易于理解。 -**采用无模式写入方式时会自动创建表，无须手动创建表。** +**采用无模式写入方式时会自动创建表，无须手动创建表。手动建表的话可能会出现未知的错误。** ## 无模式写入行协议 @@ -30,6 +30,7 @@ measurement,tag_set field_set timestamp - tag_set 格式形如 `=, =`，表示标签列数据，使用英文逗号分隔，与 field_set 之间使用一个半角空格分隔。 - field_set 格式形如 `=, =`，表示普通列，同样使用英文逗号来分隔，与 timestamp 之间使用一个半角空格分隔。 - timestamp 为本行数据对应的主键时间戳。 +- 无模式写入不支持含第二主键列的表的数据写入。 tag_set 中的所有的数据自动转化为 nchar 数据类型，并不需要使用双引号。 在无模式写入数据行协议中，field_set 中的每个数据项都需要对自身的数据类型进行描述，具体要求如下。 @@ -81,7 +82,38 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 需要注意的是，如果描述数据类型后缀时出现大小写错误，或者为数据指定的数据类型有误，均可能引发报错提示而导致数据写入失败。 -TDengine提供数据写入的幂等性保证，即用户可以反复调用API进行出错数据的写入操作。无模式写入TDengine的主要处理逻辑请参考TDengine的官方网站，此处不赘述。 +TDengine 提供数据写入的幂等性保证，即您可以反复调用 API 进行出错数据的写入操作。但是不提供多行数据写入的原子性保证。即在多行数据一批次写入过程中，会出现部分数据写入成功，部分数据写入失败的情况。 + +## 无模式写入处理规则 + +无模式写入按照如下原则来处理行数据： + +1. 将使用如下规则来生成子表名：首先将 measurement 的名称和标签的 key 和 value 组合成为如下的字符串 + + ```json + "measurement,tag_key1=tag_value1,tag_key2=tag_value2" + ``` + + - 需要注意的是，这里的 tag_key1, tag_key2 并不是用户输入的标签的原始顺序，而是使用了标签名称按照字符串升序排列后的结果。所以，tag_key1 并不是在行协议中输入的第一个标签。 + 排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名：“t_md5_val”。其中的 “t_” 是固定的前缀，每个通过该映射关系自动生成的表都具有该前缀。 + + - 如果不想用自动生成的表名，有两种指定子表名的方式(第一种优先级更高)。 + 1. 通过在taos.cfg里配置 smlAutoChildTableNameDelimiter 参数来指定（`@ # 空格 回车 换行 制表符`除外)。 + 1. 举例如下：配置 smlAutoChildTableNameDelimiter=- 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1-4。 + 2. 通过在taos.cfg里配置 smlChildTableName 参数来指定。 + 1. 举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略。 + +2. 如果解析行协议获得的超级表不存在，则会创建这个超级表（不建议手动创建超级表，不然插入数据可能异常）。 +3. 如果解析行协议获得子表不存在，则 Schemaless 会按照步骤 1 或 2 确定的子表名来创建子表。 +4. 如果数据行中指定的标签列或普通列不存在，则在超级表中增加对应的标签列或普通列（只增不减）。 +5. 如果超级表中存在一些标签列或普通列未在一个数据行中被指定取值，那么这些列的值在这一行中会被置为 NULL。 +6. 对 BINARY 或 NCHAR 列，如果数据行中所提供值的长度超出了列类型的限制，自动增加该列允许存储的字符长度上限（只增不减），以保证数据的完整保存。 +7. 整个处理过程中遇到的错误会中断写入过程，并返回错误代码。 +8. 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常，从3.0.3.0开始，自动检测顺序是否一致，该配置废弃。 +9. 由于sql建表表名不支持点号（.），所以schemaless也对点号（.）做了处理，如果schemaless自动建表的表名如果有点号（.），会自动替换为下划线（\_）。如果手动指定子表名的话，子表名里有点号（.），同样转化为下划线（\_）。 +10. taos.cfg 增加 smlTsDefaultName 配置（值为字符串），只在client端起作用，配置后，schemaless自动建表的时间列名字可以通过该配置设置。不配置的话，默认为 _ts。 +11. 无模式写入的数据超级表或子表名区分大小写。 +12. 无模式写入仍然遵循 TDengine 对数据结构的底层限制，例如每行数据的总长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB），标签值的总长度不超过16KB。 ## 时间分辨率识别 diff --git a/docs/zh/14-reference/01-components/02-taosc.md b/docs/zh/14-reference/01-components/02-taosc.md index dfcdf22aac..2563fa6811 100644 --- a/docs/zh/14-reference/01-components/02-taosc.md +++ b/docs/zh/14-reference/01-components/02-taosc.md @@ -33,7 +33,7 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在 |logKeepDays | 日志文件的最长保存时间; 缺省值: 0，表示无限保存; 大于 0 时，日志文件会被重命名为 taosdlog.xxx，其中 xxx 为日志文件最后修改的时间戳| |smlChildTableName | schemaless 自定义的子表名的 key, 无缺省值 | |smlAutoChildTableNameDelimiter | schemaless tag之间的连接符，连起来作为子表名，无缺省值 | -|smlTagName | schemaless tag 为空时默认的 tag 名字, 无缺省值 | +|smlTagName | schemaless tag 为空时默认的 tag 名字, 缺省值 "_tag_null" | |smlTsDefaultName | schemaless自动建表的时间列名字通过该配置设置, 缺省值 "_ts" | |enableCoreFile | crash 时是否生成 core 文件，0: 不生成， 1： 生成；缺省值：1 | |enableScience | 是否开启科学计数法显示浮点数; 0: 不开始, 1: 开启；缺省值：1 | From cd580c575842b5eee26cda2ecd3c830addd02685 Mon Sep 17 00:00:00 2001 From: dmchen Date: Sat, 14 Sep 2024 02:00:05 +0000 Subject: [PATCH 64/94] fix/check-state-when-restore-dnode --- include/util/taoserror.h | 1 + source/util/src/terror.c | 1 + 2 files changed, 2 insertions(+) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ba0930955c..2d9729771b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -366,6 +366,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_MND_ARBGROUP_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AA) #define TSDB_CODE_MND_ARBGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AB) #define TSDB_CODE_MND_ARB_TOKEN_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x03AC) +#define TSDB_CODE_MND_VNODE_NOT_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x03AD) // mnode-dnode-part2 #define TSDB_CODE_MND_TOO_MANY_DNODES TAOS_DEF_ERROR_CODE(0, 0x03B0) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ccf30438bd..b2e663979d 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -278,6 +278,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "The replica of mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_ARBGROUP_ALREADY_EXIST, "Arbitrator group already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ARBGROUP_NOT_EXIST, "Arbitrator group not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ARB_TOKEN_MISMATCH, "Arbitrator token mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VNODE_NOT_OFFLINE, "Vnode is not offline on this restoring dnode") // mnode-dnode-part2 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DNODES, "Too many dnodes") From d50c72d2f4236f2005b6234d6f7746c9dd5f6cf4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 14 Sep 2024 10:03:23 +0800 Subject: [PATCH 65/94] fix(stmt2/settbtags): fix memleak & can not set tags issue --- source/client/src/clientStmt2.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index ad8fc52b95..a1294b847b 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -194,6 +194,12 @@ static int32_t stmtUpdateBindInfo(TAOS_STMT2* stmt, STableMeta* pTableMeta, void pStmt->bInfo.tbSuid = pTableMeta->suid; pStmt->bInfo.tbVgId = pTableMeta->vgId; pStmt->bInfo.tbType = pTableMeta->tableType; + + if (!pStmt->bInfo.tagsCached) { + qDestroyBoundColInfo(pStmt->bInfo.boundTags); + taosMemoryFreeClear(pStmt->bInfo.boundTags); + } + pStmt->bInfo.boundTags = tags; pStmt->bInfo.tagsCached = false; tstrncpy(pStmt->bInfo.stbFName, sTableName, sizeof(pStmt->bInfo.stbFName)); @@ -984,10 +990,6 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) { return TSDB_CODE_SUCCESS; } - if (pStmt->bInfo.inExecCache) { - return TSDB_CODE_SUCCESS; - } - STableDataCxt** pDataBlock = (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { @@ -995,6 +997,10 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) { STMT_ERR_RET(TSDB_CODE_APP_ERROR); } + if (pStmt->bInfo.inExecCache && (!pStmt->sql.autoCreateTbl || (*pDataBlock)->pData->pCreateTbReq)) { + return TSDB_CODE_SUCCESS; + } + tscDebug("start to bind stmt tag values"); STMT_ERR_RET(qBindStmtTagsValue2(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, From 0ea38a15a8c9edd6a2b146698171f086d40d4f06 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Sat, 14 Sep 2024 10:26:14 +0800 Subject: [PATCH 66/94] fix: (last) nextRowIterGet compare pk --- source/dnode/vnode/src/tsdb/tsdbCache.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 6b61248d3a..9cdd4dcdb5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -2939,17 +2939,19 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI TSDBROW *max[4] = {0}; int iMax[4] = {-1, -1, -1, -1}; int nMax = 0; - TSKEY maxKey = TSKEY_MIN; + SRowKey maxKey = {.ts = TSKEY_MIN}; for (int i = 0; i < 3; ++i) { if (!pIter->input[i].stop && pIter->input[i].pRow != NULL) { - TSDBKEY key = TSDBROW_KEY(pIter->input[i].pRow); + STsdbRowKey tsdbRowKey = {0}; + tsdbRowGetKey(pIter->input[i].pRow, &tsdbRowKey); // merging & deduplicating on client side - if (maxKey <= key.ts) { - if (maxKey < key.ts) { + int c = tRowKeyCompare(&maxKey, &tsdbRowKey.key); + if (c <= 0) { + if (c < 0) { nMax = 0; - maxKey = key.ts; + maxKey = tsdbRowKey.key; } iMax[nMax] = i; From b649f1f1c41988563a35887144a063f34db80bb4 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sat, 14 Sep 2024 10:51:57 +0800 Subject: [PATCH 67/94] fix(stream):adj build file state res --- include/libs/executor/storageapi.h | 6 +-- include/libs/stream/tstreamFileState.h | 10 ++-- .../executor/src/streamtimewindowoperator.c | 41 ++++++++-------- source/libs/stream/src/streamUpdate.c | 4 +- source/libs/stream/src/tstreamFileState.c | 48 ++++++++++++------- 5 files changed, 62 insertions(+), 47 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 61ae034450..ff42643b75 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -408,9 +408,9 @@ typedef struct SStateStore { SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key); SStreamStateCur* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key); - struct SStreamFileState* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, - uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark, - const char* id, int64_t ckId, int8_t type); + int32_t (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, + GetTsFun fp, void* pFile, TSKEY delMark, const char* id, int64_t ckId, int8_t type, + struct SStreamFileState** ppFileState); void (*streamFileStateDestroy)(struct SStreamFileState* pFileState); void (*streamFileStateClear)(struct SStreamFileState* pFileState); diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index 8c005fa994..6f1a1b3b98 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -45,9 +45,9 @@ typedef int32_t (*_state_fun_get_fn)(SStreamFileState* pFileState, void* pKey, i typedef int32_t (*range_cmpr_fn)(const SSessionKey* pWin1, const SSessionKey* pWin2); -SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, - GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId, - int8_t type); +int32_t streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, + void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId, int8_t type, + struct SStreamFileState** ppFileState); void streamFileStateDestroy(SStreamFileState* pFileState); void streamFileStateClear(SStreamFileState* pFileState); bool needClearDiskBuff(SStreamFileState* pFileState); @@ -63,7 +63,7 @@ int32_t putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); SStreamSnapshot* getSnapshot(SStreamFileState* pFileState); void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState); -void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId); +int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId); int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list); int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark); @@ -89,7 +89,7 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen); SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen); -void recoverSesssion(SStreamFileState* pFileState, int64_t ckId); +int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId); void sessionWinStateClear(SStreamFileState* pFileState); void sessionWinStateCleanup(void* pBuff); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 6196647eff..b9484decdc 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -1991,10 +1991,12 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN pInfo->pUpdatedMap = NULL; pInfo->stateStore = pTaskInfo->storageAPI.stateStore; int32_t funResSize = getMaxFunResSize(&pOperator->exprSupp, numOfCols); - pInfo->pState->pFileState = pAPI->stateStore.streamFileStateInit( - tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, - pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH); - QUERY_CHECK_NULL(pInfo->pState->pFileState, code, lino, _error, terrno); + pInfo->pState->pFileState = NULL; + code = + pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, + compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), + pHandle->checkpointId, STREAM_STATE_BUFF_HASH, &pInfo->pState->pFileState); + QUERY_CHECK_CODE(code, lino, _error); pInfo->dataVersion = 0; pInfo->recvGetAll = false; @@ -2176,39 +2178,33 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in pSup->resultRowSize = keySize + getResultRowSize(pExpSup->pCtx, numOfOutput); int32_t lino = 0; int32_t code = createSpecialDataBlock(STREAM_CLEAR, &pSup->pScanBlock); - if (code) { - return code; - } + QUERY_CHECK_CODE(code, lino, _end); pSup->gap = gap; pSup->stateKeySize = keySize; pSup->stateKeyType = keyType; pSup->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); - if (pSup->pDummyCtx == NULL) { - return terrno; - } + QUERY_CHECK_NULL(pSup->pDummyCtx, code, lino, _end, terrno); pSup->stateStore = *pStore; pSup->pSessionAPI = pApi; initDummyFunction(pSup->pDummyCtx, pExpSup->pCtx, numOfOutput); pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState)); - if (!pSup->pState) { - return terrno; - } + QUERY_CHECK_NULL(pSup->pState, code, lino, _end, terrno); + *(pSup->pState) = *pState; pSup->stateStore.streamStateSetNumber(pSup->pState, -1, tsIndex); int32_t funResSize = getMaxFunResSize(pExpSup, numOfOutput); - pSup->pState->pFileState = pSup->stateStore.streamFileStateInit( + pSup->pState->pFileState = NULL; + code = pSup->stateStore.streamFileStateInit( tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize, funResSize, sesionTs, pSup->pState, - pTwAggSup->deleteMark, taskIdStr, pHandle->checkpointId, STREAM_STATE_BUFF_SORT); - QUERY_CHECK_NULL(pSup->pState->pFileState, code, lino, _end, terrno); + pTwAggSup->deleteMark, taskIdStr, pHandle->checkpointId, STREAM_STATE_BUFF_SORT, &pSup->pState->pFileState); + QUERY_CHECK_CODE(code, lino, _end); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pSup->pResultRows = tSimpleHashInit(32, hashFn); - if (!pSup->pResultRows) { - return terrno; - } + QUERY_CHECK_NULL(pSup->pResultRows, code, lino, _end, terrno); for (int32_t i = 0; i < numOfOutput; ++i) { pExpSup->pCtx[i].saveHandle.pState = pSup->pState; @@ -5348,10 +5344,11 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* int32_t funResSize = getMaxFunResSize(pSup, numOfCols); pInfo->stateStore = pTaskInfo->storageAPI.stateStore; - pInfo->pState->pFileState = pTaskInfo->storageAPI.stateStore.streamFileStateInit( + pInfo->pState->pFileState = NULL; + code = pTaskInfo->storageAPI.stateStore.streamFileStateInit( tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, - pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH); - QUERY_CHECK_NULL(pInfo->pState->pFileState, code, lino, _error, terrno); + pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH, &pInfo->pState->pFileState); + QUERY_CHECK_CODE(code, lino, _error); setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 9a1a30ac7b..cc80e27467 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -345,7 +345,7 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p void** pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; TSKEY maxTs = *(TSKEY*)taosArrayGet(pInfo->pTsBuckets, index); - if (ts < maxTs - pInfo->watermark && maxTs != INT64_MIN) { + if (maxTs != INT64_MIN && ts < maxTs - pInfo->watermark) { // this window has been closed. if (pInfo->pCloseWinSBF) { code = tScalableBfPut(pInfo->pCloseWinSBF, pInfo->pKeyBuff, buffLen, &res); @@ -585,6 +585,8 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { int32_t sBfSize = 0; if (tDecodeI32(&decoder, &sBfSize) < 0) return -1; pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void*)); + QUERY_CHECK_NULL(pInfo->pTsSBFs, code, lino, _error, terrno); + for (int32_t i = 0; i < sBfSize; i++) { SScalableBf* pSBf = NULL; code = tScalableBfDecode(&decoder, &pSBf); diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index aa237efd04..abb796b0b7 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -127,9 +127,9 @@ static void streamFileStateEncode(TSKEY* pKey, void** pVal, int32_t* pLen) { int32_t tmp = taosEncodeFixedI64(&buff, *pKey); } -SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, - GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId, - int8_t type) { +int32_t streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, + void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId, int8_t type, + SStreamFileState** ppFileState) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (memSize <= 0) { @@ -194,10 +194,11 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ // todo(liuyao) optimize if (type == STREAM_STATE_BUFF_HASH) { - recoverSnapshot(pFileState, checkpointId); + code = recoverSnapshot(pFileState, checkpointId); } else { - recoverSesssion(pFileState, checkpointId); + code = recoverSesssion(pFileState, checkpointId); } + QUERY_CHECK_CODE(code, lino, _error); void* valBuf = NULL; int32_t len = 0; @@ -208,14 +209,14 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ qDebug("===stream===flushMark read:%" PRId64, pFileState->flushMark); } taosMemoryFreeClear(valBuf); - return pFileState; + (*ppFileState) = pFileState; _error: if (code != TSDB_CODE_SUCCESS) { + streamFileStateDestroy(pFileState); qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - streamFileStateDestroy(pFileState); - return NULL; + return code; } void destroyRowBuffPos(SRowBuffPos* pPos) { @@ -806,8 +807,10 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { return code; } -void recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { +int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t winRes = TSDB_CODE_SUCCESS; if (pFileState->maxTs != INT64_MIN) { int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) ? INT64_MIN @@ -818,7 +821,7 @@ void recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileState->pFileStore, INT64_MAX); int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); - while (code == TSDB_CODE_SUCCESS) { + while (winRes == TSDB_CODE_SUCCESS) { if (pFileState->curRowCount >= recoverNum) { break; } @@ -826,22 +829,34 @@ void recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { void* pVal = NULL; int32_t vlen = 0; SSessionKey key = {0}; - code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &vlen); - if (code != TSDB_CODE_SUCCESS) { + winRes = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &vlen); + if (winRes != TSDB_CODE_SUCCESS) { break; } + + if (vlen != pFileState->rowSize) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); + } + SRowBuffPos* pPos = createSessionWinBuff(pFileState, &key, pVal, &vlen); - code = putSessionWinResultBuff(pFileState, pPos); - if (code != TSDB_CODE_SUCCESS) { + winRes = putSessionWinResultBuff(pFileState, pPos); + if (winRes != TSDB_CODE_SUCCESS) { break; } - code = streamStateSessionCurPrev_rocksdb(pCur); + winRes = streamStateSessionCurPrev_rocksdb(pCur); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } streamStateFreeCur(pCur); + return code; } -void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { +int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; int32_t winCode = TSDB_CODE_SUCCESS; @@ -896,6 +911,7 @@ _end: qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } streamStateFreeCur(pCur); + return code; } int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState) { return pFileState->selectivityRowSize; } From 12f9e268a676255c7efca006738858959c1d3c60 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Sat, 14 Sep 2024 11:10:57 +0800 Subject: [PATCH 68/94] fix dead lock --- source/libs/wal/src/walWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index b3c5f30ba3..6503ffdd02 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -508,7 +508,7 @@ int32_t walEndSnapshot(SWal *pWal) { taosArrayClear(pWal->toDeleteFiles); _exit: - taosThreadRwlockWrlock(&pWal->mutex); + taosThreadRwlockUnlock(&pWal->mutex); if (code) { wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code)); From 1b48080a94c470548645fad9e2a05f6fc5d83ccc Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Sat, 14 Sep 2024 11:47:20 +0800 Subject: [PATCH 69/94] fix: os return --- include/os/osSocket.h | 1 - source/client/src/clientImpl.c | 2 +- source/dnode/mnode/impl/src/mndUser.c | 3 +- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 2 +- source/libs/function/src/udfd.c | 5 ++- source/libs/transport/src/thttp.c | 2 +- source/libs/transport/src/transCli.c | 6 ++-- source/os/src/osSignal.c | 20 ++++++++--- source/os/src/osSocket.c | 38 ++++++++------------- source/os/src/osString.c | 12 +++---- 10 files changed, 48 insertions(+), 43 deletions(-) diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 48478c8f49..5ee3f30ddf 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -165,7 +165,6 @@ int32_t taosGetFqdn(char *); void tinet_ntoa(char *ipstr, uint32_t ip); uint32_t ip2uint(const char *const ip_addr); int32_t taosIgnSIGPIPE(); -uint32_t taosInetAddr(const char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len); uint64_t taosHton64(uint64_t val); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5fba447c8a..6c98714233 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2043,7 +2043,7 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) { int32_t idx = -1; iconv_t conv = taosAcquireConv(&idx, C2M); - if (!conv) return TSDB_CODE_TSC_INTERNAL_ERROR; + if (conv == (iconv_t)-1) return TSDB_CODE_TSC_INTERNAL_ERROR; for (int32_t i = 0; i < numOfCols; ++i) { int32_t type = pResultInfo->fields[i].type; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 7035a9c9c8..8ca3d59868 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -342,7 +342,8 @@ int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8 SIpV4Range range = {.ip = 0, .mask = 32}; int32_t code = taosGetIpv4FromFqdn(fqdn, &range.ip); if (code) { - //TODO + mError("failed to get ip from fqdn: %s at line %d since %s", fqdn, lino, tstrerror(code)); + TAOS_RETURN(TSDB_CODE_TSC_INVALID_FQDN); } mDebug("ip-white-list may update for user: %s, fqdn: %s", user, fqdn); SIpWhiteList **ppList = taosHashGet(pIpWhiteTab, user, strlen(user)); diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 72f2052867..51109a616b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -243,7 +243,7 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id code = metaGetTbTSchemaNotNull(p->pVnode->pMeta, suid, -1, 1, &p->pSchema); if (TSDB_CODE_SUCCESS != code) { tsdbWarn("stable:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", suid, idstr); - if(code != TSDB_CODE_OUT_OF_MEMORY) { + if(code == TSDB_CODE_NOT_FOUND) { return TSDB_CODE_PAR_TABLE_NOT_EXIST; } else { return code; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 6ae718c101..6554c5bdf5 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -85,7 +85,10 @@ int32_t udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfNa char mergeFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; char *mergeSuffix = "_merge"; snprintf(mergeFuncName, sizeof(mergeFuncName), "%s%s", processFuncName, mergeSuffix); - (void)(uv_dlsym(&udfCtx->lib, mergeFuncName, (void **)(&udfCtx->aggMergeFunc))); + int ret = uv_dlsym(&udfCtx->lib, mergeFuncName, (void **)(&udfCtx->aggMergeFunc)); + if (ret != 0) { + fnInfo("uv_dlsym function %s. error: %s", mergeFuncName, uv_strerror(ret)); + } return 0; } diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index e0e2a0c323..f62d728511 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -616,7 +616,7 @@ static void httpHandleReq(SHttpMsg* msg) { int32_t fd = taosCreateSocketWithTimeout(5000); if (fd < 0) { tError("http-report failed to open socket, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ", reason:%s", cli->addr, - cli->port, chanId, cli->seq, tstrerror(TAOS_SYSTEM_ERROR(errno))); + cli->port, chanId, cli->seq, tstrerror(terrno)); destroyHttpClient(cli); (void)taosReleaseRef(httpRefMgt, chanId); return; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a8f1d26f15..087e82d0ec 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1403,7 +1403,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT * 10); if (fd == -1) { tError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, - tstrerror(TAOS_SYSTEM_ERROR(errno))); + tstrerror(terrno)); cliHandleFastFail(conn, -1); return; } @@ -1883,9 +1883,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT * 10); if (fd == -1) { tGError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, - tstrerror(TAOS_SYSTEM_ERROR(errno))); + tstrerror(terrno)); cliHandleExcept(conn, -1); - errno = 0; + terrno = 0; return; } diff --git a/source/os/src/osSignal.c b/source/os/src/osSignal.c index 7060b408ca..f341f5422d 100644 --- a/source/os/src/osSignal.c +++ b/source/os/src/osSignal.c @@ -31,23 +31,35 @@ int32_t taosSetSignal(int32_t signum, FSignalHandler sigfp) { // SIGHUP doesn't exist in windows, we handle it in the way of ctrlhandler if (signum == SIGHUP) { - SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigfp, TRUE); + if(SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigfp, TRUE) == 0) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + return terrno; + } } else { - signal(signum, (FWinSignalHandler)sigfp); + if(signal(signum, (FWinSignalHandler)sigfp) == SIG_ERR) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } } return 0; } int32_t taosIgnSignal(int32_t signum) { if (signum == SIGUSR1 || signum == SIGHUP) return 0; - signal(signum, SIG_IGN); + if(signal(signum, SIG_IGN) == SIG_ERR) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } return 0; } int32_t taosDflSignal(int32_t signum) { if (signum == SIGUSR1 || signum == SIGHUP) return 0; - signal(signum, SIG_DFL); + if(signal(signum, SIG_DFL) == SIG_ERR) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } return 0; } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 851615fb7f..2065ba68fe 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -300,14 +300,18 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void } #endif - return setsockopt(pSocket->fd, level, optname, optval, optlen); +int ret = setsockopt(pSocket->fd, level, optname, optval, optlen); +if (ret == SOCKET_ERROR) { + int errorCode = WSAGetLastError(); + return terrno = TAOS_SYSTEM_WINSOCKET_ERROR(errorCode); +} #else int32_t code = setsockopt(pSocket->fd, level, optname, optval, (int)optlen); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - return code; + return 0; #endif } @@ -325,19 +329,6 @@ int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void #endif -uint32_t taosInetAddr(const char *ipAddr) { -#ifdef WINDOWS - uint32_t value; - int32_t ret = inet_pton(AF_INET, ipAddr, &value); - if (ret <= 0) { - return INADDR_NONE; - } else { - return value; - } -#else - return inet_addr(ipAddr); -#endif -} const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len) { const char* r = inet_ntop(AF_INET, &ipInt, dstStr, len); if (NULL == r) { @@ -943,8 +934,7 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { int iResult; iResult = WSAStartup(MAKEWORD(2, 2), &wsaData); if (iResult != 0) { - // printf("WSAStartup failed: %d\n", iResult); - return 0xFFFFFFFF; + return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); } #endif @@ -1008,7 +998,7 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { #endif *ip = 0xFFFFFFFF; - return 0xFFFFFFFF; + return TSDB_CODE_RPC_FQDN_ERROR; } #endif } @@ -1021,7 +1011,7 @@ int32_t taosGetFqdn(char *fqdn) { iResult = WSAStartup(MAKEWORD(2, 2), &wsaData); if (iResult != 0) { // printf("WSAStartup failed: %d\n", iResult); - return 1; + return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); } #endif char hostname[1024]; @@ -1077,8 +1067,8 @@ int32_t taosGetFqdn(char *fqdn) { int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - fprintf(stderr, "failed to get fqdn, code:%d, hostname:%s, reason:%s\n", ret, hostname, gai_strerror(ret)); - return -1; + //fprintf(stderr, "failed to get fqdn, code:%d, hostname:%s, reason:%s\n", ret, hostname, gai_strerror(ret)); + return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); } strcpy(fqdn, result->ai_canonname); freeaddrinfo(result); @@ -1140,13 +1130,13 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) { if ((fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) == INVALID_SOCKET) { terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno; } #if defined(WINDOWS) if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) { taosCloseSocketNoCheck1(fd); - return -1; + return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); } #elif defined(_TD_DARWIN_64) // invalid config @@ -1160,7 +1150,7 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) { if (-1 == setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { terrno = TAOS_SYSTEM_ERROR(errno); TAOS_SKIP_ERROR(taosCloseSocketNoCheck1(fd)); - return -1; + return terrno; } #endif diff --git a/source/os/src/osString.c b/source/os/src/osString.c index ffc64f3493..73ddeece3d 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -216,14 +216,14 @@ int32_t taosConvInit(void) { for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) { gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); - if ((iconv_t)-1 == gConv[M2C][i].conv || (iconv_t)0 == gConv[M2C][i].conv) { + if ((iconv_t)-1 == gConv[M2C][i].conv) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } } for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) { gConv[1 - M2C][i].conv = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); - if ((iconv_t)-1 == gConv[1 - M2C][i].conv || (iconv_t)0 == gConv[1 - M2C][i].conv) { + if ((iconv_t)-1 == gConv[1 - M2C][i].conv) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } @@ -251,13 +251,13 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) { *idx = -1; if (type == M2C) { iconv_t c = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); - if ((iconv_t)-1 == c || (iconv_t)0 == c) { + if ((iconv_t)-1 == c) { terrno = TAOS_SYSTEM_ERROR(errno); } return c; } else { iconv_t c = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); - if ((iconv_t)-1 == c || (iconv_t)0 == c) { + if ((iconv_t)-1 == c) { terrno = TAOS_SYSTEM_ERROR(errno); } return c; @@ -312,7 +312,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 int32_t idx = -1; iconv_t conv = taosAcquireConv(&idx, M2C); - if ((iconv_t)-1 == conv || (iconv_t)0 == conv) { + if ((iconv_t)-1 == conv) { return false; } @@ -350,7 +350,7 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { int32_t idx = -1; int32_t code = 0; iconv_t conv = taosAcquireConv(&idx, C2M); - if ((iconv_t)-1 == conv || (iconv_t)0 == conv) { + if ((iconv_t)-1 == conv) { return TSDB_CODE_APP_ERROR; } From 31354d47678ed2f7345598ab5854848056da600b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 14 Sep 2024 12:22:51 +0800 Subject: [PATCH 70/94] fix(stream): fix syntax error. --- source/dnode/vnode/src/tq/tqStreamTask.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 8f575b540f..8cd04b48f6 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -330,7 +330,7 @@ int32_t doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfIt } } else { if (code == TSDB_CODE_OUT_OF_MEMORY) { - tqError("s-task:%s failed to put data into inputQ, since out of memory"); + tqError("s-task:%s failed to put data into inputQ, since out of memory", id); } else { tqTrace("s-task:%s append input queue failed, code:inputQ is full, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer); From b494163f28682e054ac48ce39e43804d6bfcf361 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 14 Sep 2024 13:39:23 +0800 Subject: [PATCH 71/94] refactor: check return value. --- source/libs/executor/src/executor.c | 8 +++++-- source/libs/executor/src/operator.c | 1 - source/libs/executor/src/scanoperator.c | 23 ++++++++++++------- source/libs/stream/inc/streamInt.h | 2 +- source/libs/stream/src/streamBackendRocksdb.c | 16 +++++++++---- source/libs/stream/src/streamCheckStatus.c | 7 +++++- source/libs/stream/src/streamCheckpoint.c | 10 ++++++-- source/libs/stream/src/streamExec.c | 9 ++++++-- 8 files changed, 55 insertions(+), 21 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 0033e14a2d..cd43c5c99e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -362,7 +362,7 @@ int32_t qCreateStreamExecTaskInfo(qTaskInfo_t* pTaskInfo, void* msg, SReadHandle return code; } - code = qStreamInfoResetTimewindowFilter(pTaskInfo); + code = qStreamInfoResetTimewindowFilter(*pTaskInfo); if (code != TSDB_CODE_SUCCESS) { qDestroyTask(*pTaskInfo); } @@ -631,9 +631,13 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, // pSinkParam has been freed during create sinker. code = dsCreateDataSinker(pSinkManager, pSubplan->pDataSink, handle, pSinkParam, (*pTask)->id.str); + if (code) { + qError("s-task:%s failed to create data sinker, code:%s", (*pTask)->id.str, tstrerror(code)); + } } - qDebug("subplan task create completed, TID:0x%" PRIx64 "QID:0x%" PRIx64, taskId, pSubplan->id.queryId); + qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64 " code:%s", taskId, pSubplan->id.queryId, + tstrerror(code)); _error: // if failed to add ref for all tables in this query, abort current query diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 88b9f6bf55..fe2f3f8dfe 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -394,7 +394,6 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand } } - //pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan); code = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo, &pOperator); if (code) { pTaskInfo->code = code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4369b1df54..b6b5c5484e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3894,8 +3894,8 @@ static void destroyStreamScanOperatorInfo(void* param) { if (param == NULL) { return; } - SStreamScanInfo* pStreamScan = (SStreamScanInfo*)param; + SStreamScanInfo* pStreamScan = (SStreamScanInfo*)param; if (pStreamScan->pTableScanOp && pStreamScan->pTableScanOp->info) { destroyOperator(pStreamScan->pTableScanOp); } @@ -3914,7 +3914,10 @@ static void destroyStreamScanOperatorInfo(void* param) { cleanupExprSupp(&pStreamScan->tbnameCalSup); cleanupExprSupp(&pStreamScan->tagCalSup); - pStreamScan->stateStore.updateInfoDestroy(pStreamScan->pUpdateInfo); + if (pStreamScan->stateStore.updateInfoDestroy) { + pStreamScan->stateStore.updateInfoDestroy(pStreamScan->pUpdateInfo); + } + blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pUpdateRes); blockDataDestroy(pStreamScan->pDeleteDataRes); @@ -4127,16 +4130,13 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* } pInfo->pBlockLists = taosArrayInit(4, sizeof(SPackedData)); - if (pInfo->pBlockLists == NULL) { - code = terrno; - goto _error; - } + TSDB_CHECK_NULL(pInfo->pBlockLists, code, lino, _error, terrno); if (pHandle->vnode) { SOperatorInfo* pTableScanOp = NULL; code = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo, &pTableScanOp); if (pTableScanOp == NULL || code != 0) { - qError("createTableScanOperatorInfo error, errorcode: %d", pTaskInfo->code); + qError("createTableScanOperatorInfo error, code:%d", pTaskInfo->code); goto _error; } @@ -4180,6 +4180,7 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* // set the extract column id to streamHandle pAPI->tqReaderFn.tqReaderSetColIdList(pInfo->tqReader, pColIds); + SArray* tableIdList = NULL; code = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo, &tableIdList); QUERY_CHECK_CODE(code, lino, _error); @@ -4189,9 +4190,11 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* } else { taosArrayDestroy(pColIds); tableListDestroy(pTableListInfo); - pColIds = NULL; } + // clear the local variable to avoid repeatly free + pColIds = NULL; + // create the pseduo columns info if (pTableScanNode->scan.pScanPseudoCols != NULL) { code = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pInfo->pPseudoExpr, &pInfo->numOfPseudoExpr); @@ -4268,6 +4271,10 @@ _error: } if (pInfo != NULL) { + STableScanInfo* p = (STableScanInfo*) pInfo->pTableScanOp->info; + if (p != NULL) { + p->base.pTableListInfo = NULL; + } destroyStreamScanOperatorInfo(pInfo); } diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 350bd35490..a5c5c1b775 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -238,7 +238,7 @@ void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t up int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId, int64_t checkpointId, SRpcMsg* pMsg); -void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock); +int32_t flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock); #ifdef __cplusplus diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index d79e5eb143..d9b6671d9b 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -2573,11 +2573,15 @@ void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { } STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { - char* err = NULL; - char** cfNames = NULL; - size_t nCf = 0; + char* err = NULL; + char** cfNames = NULL; + size_t nCf = 0; + int32_t code = 0; + int32_t lino = 0; STaskDbWrapper* pTaskDb = taosMemoryCalloc(1, sizeof(STaskDbWrapper)); + TSDB_CHECK_NULL(pTaskDb, code, lino, _EXIT, terrno); + pTaskDb->idstr = key ? taosStrdup(key) : NULL; pTaskDb->path = statePath ? taosStrdup(statePath) : NULL; @@ -2592,6 +2596,7 @@ STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { pTaskDb->db = rocksdb_open(pTaskDb->pCfOpts[0], dbPath, &err); if (pTaskDb->db == NULL) { stError("%s open state-backend failed, reason:%s", key, err); + code = TSDB_CODE_STREAM_INTERNAL_ERROR; goto _EXIT; } @@ -2608,11 +2613,12 @@ STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err); if (err != NULL) { stError("%s failed to create column-family, %s, %" PRIzu ", reason:%s", key, dbPath, nCf, err); + code = TSDB_CODE_STREAM_INTERNAL_ERROR; goto _EXIT; } } - if (taskDbOpenCfs(pTaskDb, dbPath, cfNames, nCf) != 0) { + if ((code = taskDbOpenCfs(pTaskDb, dbPath, cfNames, nCf)) != 0) { goto _EXIT; } @@ -2625,6 +2631,8 @@ STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { return pTaskDb; _EXIT: + stError("%s taskDb open failed, %s at line:%d code:%s", key, __func__, lino, tstrerror(code)); + taskDbDestroy(pTaskDb, false); if (err) taosMemoryFree(err); if (cfNames) rocksdb_list_column_families_destroy(cfNames, nCf); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 6353904b07..76e74db33f 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -258,6 +258,11 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa } void* buf = rpcMallocCont(sizeof(SMsgHead) + len); + if (buf == NULL) { + stError("s-task:0x%x vgId:%d failed prepare msg, %s at line:%d code:%s", taskId, pMeta->vgId, __func__, __LINE__, tstrerror(code)); + return terrno; + } + ((SMsgHead*)buf)->vgId = htonl(vgId); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); @@ -268,7 +273,7 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo}; tmsgSendRsp(&rspMsg); - code = (code >= 0)? 0:code; + code = TMIN(code, 0); return code; } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index b0f6f45110..35d5ba4e08 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -365,7 +365,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock // The transfer of state may generate new data that need to dispatch to downstream tasks, // Otherwise, those new generated data by executors that is kept in outputQ, may be lost if this program crashed // before the next checkpoint. - flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock); + code = flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock); + if (code) { + return code; + } if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) { stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); @@ -398,7 +401,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock code = streamTaskBuildCheckpoint(pTask); // todo: not handle error yet } else { // source & agg tasks need to forward the checkpoint msg downwards stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num); - flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock); + code = flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock); + if (code) { + return code; + } // Put the checkpoint-trigger block into outputQ, to make sure all blocks with less version have been handled by // this task already. And then, dispatch check point msg to all downstream tasks diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 4fc00a6f59..88e40b247b 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -676,7 +676,7 @@ static int32_t doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock return code; } -void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock) { +int32_t flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock) { const char* id = pTask->id.idStr; // 1. transfer the ownership of executor state @@ -717,7 +717,12 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB } // 2. flush data in executor to K/V store, which should be completed before do checkpoint in the K/V. - doStreamTaskExecImpl(pTask, pCheckpointBlock, 1); + int32_t code = doStreamTaskExecImpl(pTask, pCheckpointBlock, 1); + if(code) { + stError("s-task:%s failed to exec stream task before checkpoint, code:%s", id, tstrerror(code)); + } + + return code; } /** From 6e167a347bc186b1f8cab988a73dba194c0eb400 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Sat, 14 Sep 2024 13:53:26 +0800 Subject: [PATCH 72/94] fix: check conv --- source/os/src/osString.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/os/src/osString.c b/source/os/src/osString.c index d3393f930f..68392b5050 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -289,7 +289,11 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) { } *idx = startId; - return gConv[type][startId].conv; + if ((iconv_t)0 == gConv[type][startId].conv) { + return (iconv_t)-1; + } else { + return gConv[type][startId].conv; + } } void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { From c9f0df925bdc0690bd5f652e2469eb2f0d350e4a Mon Sep 17 00:00:00 2001 From: guxiang02 Date: Sat, 14 Sep 2024 14:03:43 +0800 Subject: [PATCH 73/94] docs: update kafka&opcua TD-32095 TD-32090 --- docs/zh/06-advanced/05-data-in/05-opcua.md | 4 ++-- docs/zh/06-advanced/05-data-in/08-kafka.md | 6 +++++- docs/zh/06-advanced/05-data-in/kafka-03.png | Bin 6686 -> 46089 bytes 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/zh/06-advanced/05-data-in/05-opcua.md b/docs/zh/06-advanced/05-data-in/05-opcua.md index 36dfd08664..7ae7f153e9 100644 --- a/docs/zh/06-advanced/05-data-in/05-opcua.md +++ b/docs/zh/06-advanced/05-data-in/05-opcua.md @@ -169,8 +169,8 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下 - **采集模式**：可使用 `subscribe` 或 `observe` 模式。 - `subscribe`：订阅模式，变更时上报数据并写入 TDengine。 - `observe`：按照`采集间隔`时长，轮询读取点位最新值并写入 TDengine。 -- 采集间隔：默认为 10 秒，数据点位采集间隔，从上次采集数据结束后开始计时，轮询读取点位最新值并写入 TDengine。 -- **采集超时**：向 OPC 服务器读取点位数据时如果超过设定时间未返回数据，则读取失败，默认为 10 秒。 +- **采集间隔**：默认为 10 秒，数据点位采集间隔，从上次采集数据结束后开始计时，轮询读取点位最新值并写入 TDengine。仅在 **采集模式** 为 `observe` 时可配置。 +- **采集超时**：向 OPC 服务器读取点位数据时如果超过设定时间未返回数据，则读取失败，默认为 10 秒。仅在 **采集模式** 为 `observe` 时可配置。 当 **点位集** 中使用 **选择数据点位** 方式时，采集配置中可以配置 **点位更新模式** 和 **点位更新间隔** 来启用动态点位更新。**动态点位更新** 是指，在任务运行期间，OPC Server增加或删除了点位后，符合条件的点位会自动添加到当前任务中，不需要重启 OPC 任务。 diff --git a/docs/zh/06-advanced/05-data-in/08-kafka.md b/docs/zh/06-advanced/05-data-in/08-kafka.md index 5c6d9190a8..dd5512d615 100644 --- a/docs/zh/06-advanced/05-data-in/08-kafka.md +++ b/docs/zh/06-advanced/05-data-in/08-kafka.md @@ -32,7 +32,11 @@ TDengine 可以高效地从 Kafka 读取数据并将其写入 TDengine，以实 ### 3. 配置连接信息 -在 **连接配置** 区域填写 **bootstrap-servers**，例如：`192.168.1.92:9092`。 +**bootstrap-server**，例如：`192.168.1.92`。 + +**服务端口**，例如：`9092`。 + +多个 broker 地址时，在连接配置右下增加 **新增 Broker** 按钮，成对增加 bootstrap-server 和服务端口。 ![kafka-03.png](./kafka-03.png) diff --git a/docs/zh/06-advanced/05-data-in/kafka-03.png b/docs/zh/06-advanced/05-data-in/kafka-03.png index 31a3ead3edbd7c068dc71d1ff0e0e2a56643d458..a3257cc588b6c88b0571f022adcfca77b867c4a7 100644 GIT binary patch literal 46089 zcmdSARZyHu)HXV}y9N&sJh*!jEC~*SySuv+f;$8VFnF-R-Q7ZPcXxM(;jr^<_V-tv zn^SeJ{=T7#>GifPd3vp9(U6Id0RRBnXDKm7007<$0DuA^Lcjb+v8^8cjq$_Ej>>WGR4wFg!{s;{X1q-DB+YOhX zAer`t)Re@zS5?>4XE=i6glT{GfV*siotZa`F0AqdmJ&9=h>hAvw|{zR@7IK<>Ws^< z<1>U8xHPqt5I8DI+t5lbymncrv;*R=p?1D{T8u7W2zhO0bQc;K0IYx8m|FL0#OqH5 zu<)j0uk9TxFBRp#jlsX;&8vh066)})K*$$8JBe2TW#|BoSAlG7LeQ%KRzmy#O+$RU zNgUO~E887LOj_JJn8 z-^ii$B;&E=b$fQLXcS5G`9>}bD%6iy% zLg-%iIqle?L5n|a$fRm5dTT4xO%U>hWH$kA_#4IU!2t$>N$Oa#NdC^%YDN) zJ9hKr#Glr01lVxbad$VEbj(*}ilW4H1UA-GdFkly8@DLuLGF@%^}Ze+CAUQ(l)iRzwjS zl8{6TNRqxbVNX_=+o3}+Z9TcUx`Lu(Wb_QolM)gNEp&BtdDXWDIFUQl z&-6yhrK${U7J1;|tTqwS^u+-|+Sl>3<2frG7$frI`hPhF zFv{s0Z4Mw0!^?gZQ!K>wd(no_*`zAD|LtsF%*6DvBL-}MHNjM{hM?K`X5{cD0GuM5 z4;3+Ol<_(Ru|8K26BDS0J-+{Y>|Eg$4sqHllk`ybSiz2aPUWZn{UB%M<={oL87DGD zd_U;iMox$qMUL)~BlQXxVD_W%KWlE8|APM`Fo#Pwb(Kl*EtWlc>j|-8XVu%+4|xoU z=cI?+ww{FTAiB{@=ojRjJb;Np#r%{6kzV^`D-eW=0pbVX@?CBe5QiNaKx`(NO>^|p zN1tD(&AH#%{^#aNsp)|Z*7O5r0eHuKZCTbj4*Gw;@e7=8dO8npI?M;;GI65WebqTi zk2|q$!W?kEps<6}x2jV?9Z2b`)`UoWB~-(sL2&C|x|4i+g#1Y$YV=huo6R_L_i*xf zn4Rr)5Pn0vToV>P1jGNIPD{Z3n3|h+jr(n*3?K=&ovX5 zTQ6%E+@1cL#!?zmM{)?LM)Fl#vQ``+-5HI^Z5pVH`!N zS4reFJoOQJ+(0y$ZVAVc|AyD#t@_51AJEb-3oe_>EC;G*wI%z{CV8ADxrx4t5={&j zXQfbL(q$rFV1MnHb-1lF z82U}~huq!itW=A&f;DtF7QXz~*pSV3e2F;Rsqn*p6NU)|cP6;d0wM{-n`@a_X*67E zZs0JHy~#HDt*0Ejfu4_RAC}m~oXQ2T0Lv@ED1kEvp`Of11UGrOoYjp=TMho$UlZk7 zf$jV{kRY(geFDeja!`Y#;V~le`o;VGBmXEE6LQZ&2 zmVKqr9$j0VI}nzecy+&UGi572UPg%s-d%F*7BOJY4&RMA9Hi$y8if>#1|Pc5X6PH?lgR~< zQ7+P8O$0i=W0%))b-19${MY@Y>B~m1qP8zTbtJXAcuLA$J>QMPWZZ3PvgjDJe1zwm zog$aIQ>!VZFTz1}*m>VP@9Ot6j)UU+24gVn$1uQY4MV#OIm_#{HkH@e(x!GXT8T^#YYDF&J_%GE_T9j%StEn)_EJs% z6d8yHFe5DILseHnfuiK22tF631c#oguz}_2Ktw>dxYKS|{%Q|dF+9GKb0F-nltv6^ z7=T;!BNx;ymO0PwHx;rQttuaKR1{;VW8fmbbj}ng5%ZF&A8=Rq6aRaJW639+NW-wU zJ{u8YTfxr<57ECsNzPMskjt}zhV^;N{TuEL&6T^{XnS;|0yIpNYLarlyREWB;(kPj zfF>*jUy;*iN14{*mAIxS_w)(7W6 zJXiyRErp1(P_YsuJ!%n|54{kOF(UvcFl(FSUj@fgC?0G3=g711s3kj2M<`%BXd zR8D>CV`5!jT|!*=FuA9rKNP1Co2_?|uEDeo&Iqj+0r|LCv`|0Zz6O)Dqh24oZGa(Q zWHs!omCV#a>$UEU>y-+u;#1B-qJaC&_t5ls_NnuBAg@|Jz5$$_UjgvPCldI15IrCS zKASHeNmXsfPjn3zYL#}a-#^|N^-i84ixflS^#5MPg+k_K{IIe4lfHPis?2rh0>QhaKMP#NgV{}L zbTHi2)}U!w6gQ*oI?Pu^`F!f?yT|A;;dU5GGg4*SrXPo<`HLQudB^H=8K2M_u2pU~ zT%#&i7GZb}iwNlFQBn#7!s63d330m#qqByA5XJPMG#A}v{K`|LVUgwZs0qyM?YKAD zk=XGme|uk~-gxZ7X!)^^IWW@m-i$>S`)cKB8AIf*;)bP#5dXGwrwqlfOY)58c?G$EfKA)iYao)R zxnDFB847i^e>6%DnW{XFg=e9q?qK+Vd#8H1L#aEHY}I;csqUaBfP==kwB;BC{=%f= z7a)16`|O~rs%uPXW8&?lX#Gww5XK>l%h#-O{ToG#?M)+!Fk1^dlqI~Jko=RV`Bl;nPrJKN_*Fm%A#}iJ|}Ej$vg(5nf_k zd^>U6BppJ&8_SI1vOqXkOem7$7F)+v(-Ql5;$|LQ!`0K46QT7SjhLtS8*M)0ZwRPm zo1LpxAR3T~Qr9-bf`U0BUK#Z~;4NDrRAl6p_5w$tw9Q$$uKH{GfPXlc|=H2-qxO;56TqChE`u4Ls@0tWyl_{Bo)zp z)$w+Zq_r}Cy<3?UvJrq1ej)2-LFd?s&Ym+C$!O7r+Dfm z0}b{n2I{k*kLTjXqJ(64iX2pGl?6Uy*_PhotEY2qC$qkEtcc7_Zfgk3sswnq!eky! zs(!F~&=E!8(t};u8|safHs_PWJs|pZEG_q40U+D?%DK%OJd(n!ptk4zaJAEuvj*cw z2_~wEYdk#vI{m0`$(^Lb>8rw@wp^26lMVJ*bHA7=R?i73;-^zSFPR8qhQgH(WW>v* zx4UT>NYQjeCj;OCF!8Vy&{7LSC&XkeDHQw{Zq`dPTIz=>amYI9{N4t0oaReMXnEhg zzsvaVA~iN=ItXVxZkRpos{`S z?vl<(7pO?n(oTFN4J{dUyv}<1aq@f)bGAmxj!v~(LV)e`(1GZ~aY%5Pr+E7MDd?7b zd6@knMog%a1xl!Crb<0@f^=f~F%?qkpWpO2yar=O7V%llhn-VBKR<%W5v-R#h2M9PH{XZu@X#f?WV-95_|*b;CO_(7 zs{NuU^shpzb24D%w5rUOV28f_FdY)^S-#61hqcM17kk#@N`&UOBHUoOvrIe^9hRkyn z-5py21aYpTiRTES{&iHl#daRBrT_qyK^A=Z3;HmP@$0199!WFf+Kchy6(vRO7H#<$ zzzXBW3=&a-iWL zVk72ow-d$|yvO3^3tmpA&UCIZvq45)Ag~D}01(%b+&wi?-60FuLz@ZrgaeF46@zLo zdA|K&r=ti1t$T{hB@F{n_oDNzM`LXQ^O4W8U00Iu`w3A`f)h_YgdwqSba=;cSZSmF zELJp@|9fIsX~!>ExfuM#rg3eYRSLQKu*3KuEgYZ%iz%HPk!qtBHw-M;{KS~gCKAHp z>tUUlPT{%t&atQ{LV=RFacMU$ROzJ4>($OQWP*YhC$klv+-HSP zUpC&+%|VHkHj13JZTEePx&--5H{~<$oTz z@@0i6JD!)+BoVruBVtU zbO74y>G?<&y`gzD1_@j!1EJ3+qifQcz;D0PTO@tmESp}gC*dFAV0?+<(?#-+BUOjc zM>K(W=@Z@iTyFxbK6$qQB9-?2A=gpUzE5@d6!1U2IDD`G@Ewm0Xuj7S^s(8&G|0d8 z_6ZUP(A=$%1M`W1yIW-$o_Fs8g>9hGOmUs>26*jCrc3+g(Gc*`)5*#ZUW2D?KBTo{8o~-DJ5b__!HPhy>?N8CQDykjmGAV+4jax?-;G!7E+euSf zTOp#G8k*7X*n}69C8~9bgrDN(EAc74*!C(Pc ziMN71HG0YN>4twKuJk7k5)VWsrZ_t~5qfAj$u}-fN@lbdnt~6Z$G)4xculn+wwq&v z5?DunS`E~8MGK!Hm^CnZFfsI+OJfBPiaTwAttA|?KY2Ka%PeTF2$nTsGZ3z@=H`}& zjKTuip{6P7xA)(j_$=@;vc|Dz%d?DzP=d>2+A~MmlOBsxd*5bQ10#U7`3`@;ahOL~=v z?Hd4bp#%-ybnPyLFhB01#cBJeA?JUPP1cZ=ml22u^g3AmDT5#?hANf?&rFhK{?@;f z?%T&7M&ig^ns?US=CgRx01P!=+x0~2q74NtyESoQo};}(sBWHl)7)Xi;bfrqMC$e} zKQrkhvu=g(jwG7m4og8YTP}M>ROpb!*175RuW^gm=G)G>`2Awqjo~yUC6Ie464O0o zE2C_o4k0~Q6v`iNI5pgS!v0}|9UTib(YxU>^G{aL<*q9-7NSI+$v08zi;8F+=bl_z zI%*WZnuF6J+Y-bW-aS+QSI0ic10PU@Gn!q=1U0NXiW#kyP2_lbu8=|`PJQr)+~+bG zkWTRsG(a#t{j;3jXLjK$Zu;eqqz||%c6Z~Bw9-GTcfZ-k(m{?-T%E7r{c!rBSifBm za5M3iz0Fs^KWxGG&z+&`G&45uD!*Ous7J@INOe;FAy^`|-pJipj1YTg#;* zVtt$`=hWW_Jz8jf53?$+@vQM)zsdrdmC&{eUq5UwzPT{I_s;J`FP4o635S}#TX8zG ziNF=IT?k+U%(UFaiJ?m9K_P7Vc^4={Y2%b-SMElGYJfA% z`W*>LQ$5&jPU;pk@|YbmgV`;v@DGc4;~Y#sOvmA zo|n+5n7S+E3)JptqCO?enTna6&S$Q!s<$1Jy}Q98)u&5!Oj>75>9E~KE?5|Cz<lYO^hx8ryP5YW%kBtc0&K20})r^yif;V_ka$QQ-RPurU2h!z{%G~jL;8H~97Pla*%lxsWV6`{su9;U-^KWgk z?4USTR0feM`3{$Y-wSA_Yqofgv*h2*nEyQ8vxJvDdTyCM>-`0WhdjN!$Cr2XPm4@r znR~|O4#vw7zFm-`2}mL5^r8x?0H(R!9TS5Uo(~FKg#FD%ym(7$J6u|;-lEUnO~KZto{--oVbb{{vZAX&|(j!3TVe9u18sv4~x4V&X<)_ zTEK-947wfmTO-2|w$q__kHC8Tr8B;K_e>LSLe81)Oxp07X(p# z$w~huHa5RFHNQ^mEbjj{=zS6ROE-=JVZlPBwP}3|EoqX@B4P;(R?J=q*MANVR6zg= zjGY@@^T$K-TDzF>JtDs053bX~sawPRm^s)*-jV~cwr#LqxE}Q1j_Q(}kAIqj7}2B- z@w5F?i*p?2%LfOho7}~5!f^Q7L?WWz40QSITs%*5b(@!~|31 z0B|8vNwh5rpS{n>%c?$+XE4O6KlbRB{?4Lgm_}x9EkBoonC&@Qx?bi_$g@b-p%zny zxU)NJJc)lz)Tyl&BnhAwbYrUNHO~HYn@k*`WZXudw&Xam|Ykg zA)_%DklpI%c1#9u7tOz6ENx8eYAX?=rp^_*poo}}m;eK-}p zui_GG#YwVbW1;i*A-EdX2w_VIn7nGchkNYoyaNo^$BwN(3wjOkx3eM0}=0;1vUz~@rg;8muz+JIlb z8~0pDK@&ZEOZ-YEY#@Tl+V6o@Cjqx2KjaaB#fnuj%EXZ5}# zl<|TVQ)N65hBb|Z>XuO$3K&xF-xcBYvnR=7cZwJ4jRO`kyvFbUZrAWSvJPC=K^Ew_ z;7@<5zHEI^yv-nOj06XDVMR*$sBRNpDS+=9sG=);PY8JusHuo|7_It+23EhlpC?Sw z%^6+Gc^7>0Rxl;SD{~hh`UOFg|1$GcMV&|tgg+|-(OMT|AiElvCj&kx(AN`!%Xnt zV*5X6H65NMa(*EpvKZ&Dl$^1m2)zHV92~PtyZ!%(BK!gWFS1S=(-clZJQAq!t+T1t zHhWUTpfR&!@_?3bOJZojKh@?J3Q!ui*sohjAgP+;e^KMX<&1~-(vKR66MdiI~Q+MOckHR@Xf z9qj+p_$L?H-Oo4am52xa{DHT54|z{NrXKzerzowlwY9ay^QN|@28q>rz8Wc>CNUx5 z4I2Ox!pQizL-gOgUzVWzv}KT6tFUu_A0LcCE?oaCs-&beJu(F1bEFV3w2*)mm0c{~ z7MouXfSR0&B#yjzOPO6E3w-xDlc={doHm4z`oggICXT|$$Qb;n^q2J{i__B5lFDN% zo2wWZ8Tk?)jxD|U#c~d}s8vBEFDIjuvfgS^Qc?oIXVy|vSHDY1$Nk4%(i&d~EP*Hl z&}Qc5=9ZQys;!Uw#MvBxe+W<>d$5>JB-R)XkjcFX1~5aG-}Vg7i_DmLer2V@e6{&- z!^=+Fp#@uD|7BIB5$$Nn$zk{*;m|Zs?|l5nV)AV=E14c$wQp?&DUl<`R;AZ)kCjg} zv(YHA(E93-TLdY`7Zy+`M5N+tO#b%I_y1FAr{QO0OWsD0iAGpeWf3(d7QH*z5fSj$ zaKh9qp?F=xfJ|bhzu1UCEg;)VX_yd+b>(2r{n?9u_&U20-F#3XVHM@!yG%W?DSz#j z6Wdqhi?UUO3Mfsq)s2RU-1u73$$;v4pHZjFgY$pjOLna6!eprK*=vm6P2iMZ91fF= z`FzGeuLEc7C&9TG@?|idj&|eW?^HrU%zjPOuA`1l+-<%JnMDxi;F-SV+s1%l{F#P@ z#<4CF>w5yKr=<``8zamImq`_Av|j%0y-P?a2nDiE8b*d^)%YpxxQaJIAd)7b(}9$J zEAiLFvK4#H>}Y83m!6T@j-nS=d0p46nw(O`-=aZjXB8j*)C|&u^Gv?rwTf?&95^P9M88-xT!&BTJ_0QDWi1K{p+#CY{Eu5nR+sI& zra1DWc*LEN#Z1Wpk^G#AZ$!=(OnIzVj#di^Us7K*FIKZe=W?<%v*;k!W3|}qc(Gkl zG(FsWJkQ9i!>J8;(8v%RtB6<+zK#KJ&K#EoD2Q-A>-^(Ftxlg8nwVI4X3QC(e9wcPT=|BARy=wfkRMgZMr84@n{G!@LO&Yob%y(bM6 z)bRcN+Z*eB103mGnZo)dS_h|{rz*-+*<|_4fSmGaEhvekfB$XB~?}~+BXd zdjor@*t(LszY}Ttb_|BI{N%IN*hsq(jJp_e^*x!HE}{gw=$!1 zQxg+QX~Ra?XKp~}m-bHQPoLH5=<@RN%*+eH%-!7`2m~67&wXC;MBFL2oGCp&Kkrpk z4f<@>v9PYLI7 z66?Hr1Eo~1E6?Qxo+C{wF3ZJ1Yr0Eef8IKs%F^Ajl9U^}9xUAmF;Sj0+~r|=GR}9= zsVt*e&Ug)@A}PYL`zlg<%SG=1Vb!>$$?d~C zUHQWHWp9KilA~fEXOx>a!{|inVvDCOdjgvsI;#b!{ZN26*Z%ynw_k$74;lf3+xg}i z|JW1|QUJ#H_NWiM7%3uEbq8@cg&hgF)Ya5!V$oVCFLorJhgHI%;CDA0dgY-$<{?P< zJMlNUFI+4860MnT#<~D-h=nUX zI#Oj$1HTN9L;%y)R*gFyeNZRkxde|_!M3Lo%0&E`cTx@;y)U*KA>unm0`px7{;|tc z_;%^p{j!1U;Kbc-Zn98`MSYTYTZD8M^~bXjS?Bi*?S~tlx@s~Nws#_@?8$lq*Q7qO zh843v=;L2P*f%S?Ur%N)mD3Iw8GLrm|}7?P-ZT*)&B3_4IwYy6S4`hkG9D#b&p$$K49>lyVos&V$XwwDq1c4Oz6D zIfx{>&(rfhB_$<_#QehB7vjD{^2~ttph?xt?q?s-^w*_JEc|@7I3!`8gTWA-g*2HC zTbt3f-R~n|vBVE_@(Et94o?;Fj~9M@ex#~5qdy$&i+f&z=-sj(7EOmNt{D}T$|?)~ zSQ+^&LCbEC@-6C^EI2zhzYBTG=#=hT`S*3V;Jw{LwC<|?&NIor-949PNuf=J7Z#k7 z@~Mi~A(o}AT}hOxr3*mr@o|dUPAd&Yc$dBgJaX_oe5If-qD&55$*>t%K=< zjHaieJ+83Ot7rM?18)`IOvojE8Bzv{AJ?1C-4k%jf*GIpv)i}=JvdRk)~R@!%h&E8 zRz`ES8J-=Nmpcku+Cr9$t_tWND&K=>Up9Jlq$UAA{=1 zhC=~^u>tbhY^|l&fikH|h02x(vAw;YO*T_$pVi+2G&D3O3ZhPd?4-@CoyDzX zuC|9XqgXB|HJ8AKE z{6 zkWIHjh>kJX%tp8wa)+iZv}7Qou0F>`6ra`9MBF~OHxV&)s)~$RU~oVAd0GSS^70b3 znb@e?w*hoKThYbmJN`ZBb9^uP&JR#_;Z_U`7z{oyd_y71X+cCQ2R}ORaeIsLhdu?P z2Kpy&>s#!KxDy7yKYzOG?Kc3`wYB&{)#E20fz}qn)jm%_$HzLB*6QklZIIhOn!nH> z$&5(_gjQHz0bem2*LG*g$)*3`QmpIi)OBBf!d@{OQB>CJ?#O1!ZE;|atqnpENCNC0 z_g#}BT=UetAaS{rSyspUUl-fb%+Xo(-6pHJol@B%;Y*$i=+z`P7Q&mL)Op?p`}12z zQ&|$1vJMzLnoBF2fg>TiT|K3*nx2y0NA>1PG}6`}PHRO2wGnzauS|i+=~`yCvGzNe zV1c^h_dKUird&TcN|?8y0+)`hWo;HKtB%Z6b$u%v8yjo$JjNdRs`7rIU7jJ`xZIl6 zbvoL;$T8j|XkI}Xa`BjIXio;fpx3+;co>(kKQ89S5gtcH=XQ0zagrg^74rU5Z%2}W zQSGdMP@6`o?VYAA^aAq}5)x`Q7)H`K3?U$Xmcw_t^~mUE<_9NxU&9*Jt7r%*;=5{> zN*>UwIqRxbR8>dP$9P?@FYhg8#{=Gd-DwCYneZbSoeH1wopq=$smdv;%FZ^>l9}S; zASM`0#f;S-2|=n8nT?yDnRF<>v-6Zx)n2Nn$2mx15;WBkAL~WN<0RiZIsS^Yc&NT1 zBHs>+TTyMMmB8FU^y(hA+rd0poO&H5X};h6nikRJUv zgUj)e;H+1D;hj7~LNaHC{(1NP%=={gq_Nb~~Z>$G8T0?Tlf>6ub5AbEY zyAr-;sNWr;Xx6r@U7^SY8@)i3r^3rt?+21!+09U?yaMwo*8M3B1W4nXN6VbZ+@4mg1N!vL&Bgkuwy}g&x zTRbP`tW;G`Dl~r4$4KXSuR!HxWwn+EKWbV(-rm<2oTWdIea@@2C{}`s0Jdedo-6rY#U@84bUDPZ zaxJ_lF<`Pbv73%J;sHU^s*6lgV37eX;S3E|wFI!#YM41cNRI zZhww_3Xh&~6p_o;I$$rt$iqV&#GxxusD(c1Qv(P5>2c$Z&d8kvG~N~G!*lE7$koO= z`tH{aUmu^7edkmd#L4fUD|o;H<6Oe*~;+`4kZh3`#I6ncV& zJdb_d?v9_{gC+Dy2HWdZi)h(cL&rXPTZRLwc^u3uD4HMzLNqYrk*Ua)s$Mi^z5>$l;PVUcy+k1`?$>pxG<5u%^!K1 zYQNvEoxJh&>Pv5tp`1p_gkO(n?Cr;aJi?stxw>fIo}(ZrFaNVD)e^iORmjGQC3wpp zAzcG<>9{A2-j4S*$?LJ}gwVxXQFvOoZLEDbgQv;K($`<9AhT5%vTey6n+3zyr8N$? z*cn!DaM&VGU-@}vTS1mT^Gg+YHc7db;wC8UygzxiVJWVcnf;Dvwa*~-Vk9G4k;PLt z_De3{2Q7A&&JoPQ<#nEBGznY#VKRrH=QoUBFZO{+=l7CI(dQAT#15By3CgyibSLpKTz1N%*;~!ubZ!$Fjtgc zhyJD5;tKJ2K3+oVi zX_Meaa0_FZF(aj>I#=X5_k_jM4bWSo@i&VDl+^9U!#b_PW{%jK|`NI~xm<`TLiZ zx2QiHkwf;E`=%pzRh%z@ z;|`w2uM7^ylmaEU1=kF{?l)rv#M5-F3iwT@ojqu1H74cTe*iSnQ}CTN=5L)qajVXp6xTvUiW(HTK$zQHYe|*o%ZBa@g_o97AbW%dhia&vGIOpXx z89XySJ=U1Jj!#Oe6F=G}d0Yr;1U)IQyu12(C(3ZfsC?=PpSoY?s-=Y4U)Y&r@DG3? zWJ0#{xU7(LZ7qL`AK7E;LtrwJ?mh*SkrTdq)Na!eY#3wy$y4e5P;q&E&D?VOSl4`q z#*wEYE@b3oHMe}dkpZb>d_(OB8S#B;|K|I2D~hH7IOH33y$O)J4e46VMj39}+9sxr z-~r6fpS5?%2|g{ti*Lpnk1%Ug?>;*2Td%!9eN|VXMW=dAtims|B2TLbiA*Wj(G>AL ziKLP~UJq$YEl(wBPj%1dts}nM-fN-Wr!Xf+G6K5TBF#s`QOCkm9Rju<2MtwuYF9vt z-$0cfyxHcBQxr{C1gRhI41AyWSiHSy&e2#Hc$N{Q(IsLMpJ1^5U;-1;-`&ni*5*(&JSG+B=}8;g;zIIDqTc};2MIDu5M$)9j0Gz zf4o7(+o^b%coP>_+Lt$re*N*Z8H4J4JDnR6lqq)R7HL`^;+`fmnhV<_Ei8+j{Lq zLmujawVJXT9!j)#PSm)TG!U9QsJ?RopvND;kMl8&FEHTZEOIAXnQmeCA{eQcHns{)pwbLI#>?0 z;8^Z6m2D0^SV})G3M+LnXFy>GW(-dp9115-gR5`ftGa@(6}Y zm1fc1GiDr2(SfCahJC_jpJyMw49m`~DEZ8U@!rRlRssR$d^OMWxXm+nB}$ms)@39p z!7#Y>9+#~CATdBvmWu*!Q3Tkj7d>2x9zOg2vG*hUG8T=%Qd?) ztigi9IJUxH@VmQYw^bs~2V8a5%}>{8ZC=}Rc#MfO+#zW?wxVn%L&XiVi(`1lP-D&s z`zgXU%xZD|A9ha145M(6-~%S|Ro_|v@f;vtHLwiFf`x8vZa)0J(~=z9LP``uC>Agf zwE>u~ln7jF4uUVYMj7q*uNqV3m~p<^ZC78f6Rx#x!0NLHef{3nEfDm1=!5j%0HJCX z_SjLP+@B|tDtyy0AG{o`hv^S+a8wn#-#yZOZY(qD-69EO9+bUcnA5BR)cGhe)N0YsrDioPU_ zNhoVdx_T+WpMLy|FKr`hvhH4mY88+fDdV~XKL)=aY2CCm>s|i&k(QNxKX$j*FrdoA z)3fDWnMf#Y#|Ctp`yCe=O8W!(ycI3UH&?B%1S1GPu$l^MoTV)imh0E9U&bGna{h=v z#K_G2n^V-a{`t0W`}Qzir`l#F9fru@ZKeLxmlHurR2`2knfDBSvwxQxE zJ6&`LX68qeInkwQ&)Zyk!ezAcLA9BZa5LE3OG;mIL4hnhw6_UmUR(Y{mMilW zmc5tzcU9rE^^7;%zdb&4zKck69S5{{e?mYfjS{1T_}N05=PW=sl^dBkEjXjb?{Yt% z+{E7aTCFzU#1pVjXyiY7xuq#E^w{Qh`{x1{l`o8mIL7e*CTDk-c*G%`bxgnV4I$9D z1YOr3=x2oobwkW2RxduRBZcoONtVq=S8`+tvGT7tRql%-L9V0j8k{d zi&M-K$0(<-wGnh?9ja#niBo@Yv?j+=e(+;Qy7*30?XNmzxk+EL8Zo5&#SAld!&DtW zL5S+M=T#~@#;ec_iQ=i)_+xzWW32aQh~YfHB!iuJ?;SyzkgB%A&+RI`$)4#_@<+F? zni!N+rWErXw_>?6?@8e?zYZ1C!>;zatXY?2X8zX9ugP_se2}OX)$El~p#3e@mFHTl z`F50O`g-YMahGFM^pm5lwjoC*m?6wE-`IVps%C=JfdqhuhX*B6{bny{CjRjoG!}q)^i9FTBQK9eW(kI`y1;TUGw&q54nHV1j;DtsQ<@PY5hPB&kb$2?( zvx@tK{!=)`l2PZmlpn&%UM2SS%mh5+EwV;54typdRfW62Nq zzmTh;)CAkCIzTBfv@Bwj_kALs_j5bCYHcQWN0zF<#9YRLTdBc*%+G?P8kv0pSvpb! zZ1tm<7S;!dfgc+h8pH$d;CkFO?zqHc*11*Jsd7-oVz9(0hECAUt(7%BRo#93;eD<; zQwcPi^sL^?k4g~+DeHiqArCVQ6ap|9%WY?-Z9Y?M*NG(pEAy@1Q{#ce(`6|FMcvql z%7SE^zsBb9xZ{qiyxV+^-xGiO6JccWMyj3#Tgb&YCguv>?8JW)D!NySGamiGR{Ls_ zqf9`)>4pq4Np+*iz%T4Ha|}ZSy(=(XQfcr>Ky!hg%L)b+4gL9$z?PoF><2&|p2$Ef z&DJcu_dX(gRvM1X*zeW~9p}%`Fzo2$;_|pYL)-vJ+X+;0vpqAYLNk$0`;mGXxICMewH)z|%&29qy6NaN3$^1?ylD_kY zxOv_vnD*Kuw~mU6Lq8E9DJNXJy@9nNQ1owM@#PhW%ys_VR^T}HNvwk-2L;h+>6ms_ zR@|c>yvBk+v-S42asFa8^h+JAdF%fn?=8FH3f8UB#uGHSOM<(5kOU1L2(H21-6as5 z;KAM9p=m6*ySux)zbo1MoHOokxNG#_Lw7a3s#eugkIXr%eg+lg&DhNNAqx99jos~s z%{rT3zjzYT4Ctk`P#8@uP;g~NaYEpwAx^ziY zz{+4g{jI6_{DN?z^|7HVRO|}kA(r-znB72be6E?#fGpeK+6%U-NUY(s%;R7mm$}{v z`dy9zVXYnlnP8gx)K|K<%|pW@2R4WBW7E@cSa^D;SvAxn+EbO;lOa8$HRBCaDgutL z8a7LNkK7gL(FGYkPGiwfv$qfFIW{M}iWEKD9L2|CSfNS(ifkWrgQTdtDZ zLlEw2h4J&U$P$c}QjfyFTnoI9ub>&#f#{vIc2Yk)q>1zC8-~H0 z(%(w!U8FUCqQeyi?bPpt$s(d=YU3e|yGf(OwO`;RoXom zL@LH46s2z^F6Z~Y=t@?mACzKK{?lytjlZM6#_63=IJ<`mdawCJvkluRc(0B&tH`Cl zxWuNoc`Vb{{|Pc*V_`Me97Cbo*3z?{?$_$~9%N*cw|Pu+PLE>KaeC_^#{QGsn(P-5 z525{sp5IsDYlRWvf=q3z%pY{DHDV+wvY~>%$jg(qQLkYCZu*C25Iqe_VX%eQGPfQ+ zPbVNDxxWx%XXh)Yeo1S?l!w71_r5f^XN;{(VC|u?fD}!KA1c#Nf-F}ler|z$;c_3( z1AB#RU%hTyYq!g^J#KGNzy$VJj?W>dPn*k!i9QF>kCRL-NmvwI{F)n4I_2|Wnm5mf z^18O&2$FIt#$W7OSF0EKz&Y8nIpXO4Va~=zZS1)Mgf70cOS7 zbWeyS?XRG6WNz5xPHeD3#V#?$OA zcD+{!t)+K3((v8z`e0gzh;e&y1THe|V~*G|*z?PR5rj3QoE@E4cUEn6%ru;qo;iip zAi0Y~-g3t)wFRNMxfx0gfZ<@vRCRwVfvIHpB7KR5it6IzWMq`k-C*WxyQ*h6haR0e zRQeX2j4y+*2DWx_(X}3toLT`Lr)h&M7v<0>UY+IkLcrD8u>Tx3N4R{2HA~3poP>aP zKU9>C-E-CFES7M8GJbaQVNzLjIz}e8oyuVB1H=Mk4pXuz>>C{EP(+BU#HUZmR`k8t zIe9fDLP7$>IU;!Qdx$7uZ;<20DJk(mkQSJCm}xK^Up53{q0Q+hO}sBoBz$&H(ridy zj}sb3rns9ceI9ofv6ZJ&W>)nJ1XYXN+|+b}5?SBjtAB-YViW<=j}R3>qa}HtBR>x= zhHxQbsPu>H72TGz#VQfb&3k=v>W}aD{~#^>__ORQ{|Oft#}(l^qT;z6$@O|!RAMx= z!IFLszDgZI4@ykkbt`L{)XVfbIwzt6(&RAuf>~ZIpK9^AOP&6Fu!$B*CZu~Gw{1Ee z-SK+1UI&#_=$}FmyBqVZpWh8mZ~xE0QtDN`t>LL(j!5*Qr{Iz6VvUQ4JCxUoLSnMHLn2o2RCP`kUfn zwl8zWnil+-{vE>ZU?Dq;96k^%w2HU4wTVe#Ss5cWHMyNG0EPgCUi9Ky2>6r!!>whz z^kY44KE`Z9F)LHagiGPaf8OloBA3bxLvqNG^{bBgEUo98KNDGXiRv1~QRnQ+&&Wu7 zGm;M-8iv?qiqbSU5$*W~*cvqWBj7VH?Q15E;-ago#HRIo-=*wNz_WSHRT`hX-i=0C zfj!8{n4mRR-T5_2EVi_0$#YD;6RrLpq9YP?c9+y#P(#{XwLc9CglZ>!#05cnOUtVD zc^uJir1|a4cr)kbi{j!wv}_K%A>;S86smsilM_+-V}B&b;ST@p2pQIwvf-QObq$)& zJyHg`mDZi}2`>j=j8L#}wxAbytR)Wh{x}u_jE(V<%u#KFxyD9Nq(MA%dye{G%7sESkRz3dOy zFa&>ww)z^4nex74QBFhzCQ~!zAmgQ5e)VfQQ_$|KrBTeKg1zxI@@TWPlOI&aX_`ntRup6Tr1ke`FWFI`<5zgD7!j$9(Kc7D&z zR*WKD00%9bQCCe(C3SXg!GLTztK;eq2i&)KT<+Sm+3b4D#pk}xLn~qY*x!nYF}TNI z3Z%J8@8JsFg{w5k+(f~7h z?1lY_D9g@th)~J}6jGy0XL622Jf_pb4IXFBVk-)3;o3-U(D(P~# z%qq*)#o75rSl7qKhS~Q|wtq~_*L(eSh0*SM)WK@Vc+LtO7e9YXV{Tzx-f>saWCSkz z00Y3|^7Pra3{v_}V%u*@{-j?c=d>pSgRG)ri$; z_fJ@<`4VDbQd4EK!c(r z-mg0$2-J}e=wtd35X85yLE1oy`?Dj9-ZpM7RHZKh_p((d)Y+nHJ&DlbF(^{fbJ;af z#y+_xPY741uez4yIANX>bVI^-zEHmmKcKkdY$G^4_ti~_4`Oq<3_UCSd}q>Tei+gk zd_EEdVLIPO(ic&o9>tr5PGBzfyn9Xo9WB+pYWiH~-07I-V|%aPdYfEV^R}wRq`3%2 zh^{8QjyAknJ(hbRKCIs^H+z1M$3BJ|M-dAfH>lC_@bKuy6Ygxp@K}}mmBrw{{(Y7Z zFC#ihE668zzA!^S5e^1KNeOqkJ1Two2#kSrS%tU2IUhuxK^*&}Eb8ed zXC*5u8)3C5ARqu>a03DY{Dk#Rj?|5e#1cOuZ91yTn+F;EErfjl6DiSbS`yLvBhj7D z-<7omYmruF+FZb?YkA-_g464cMHf#UaNhhyw+A4 zR=u0K^+^RL@2FQFUIJ21(sh*c~`1yQ!aN8JG|o z*(^GYIW6QqV)@>*G%pH?>FoyS*KH#KvLBocZ|No)Y? ztM_QL*uX1VSv{Tv^w^nFm!v=#w++>0^xwBYEGaG{~i!{bn~ADNW*RnpfxLomh;@p*~ohYpw8T zR~8>l6PQderCJ!;cWWrusK3vajP!3IBX zR=plWuPt!hDjp3B5F*mn7i~1%uC|i=zF+s$3qZG+b}TI&ea(_Mjc`2e*=BC7&&kVa z%PPA?Yr9WFI9X=#Fjltv-fEQ9?2(zn?zq*Jlp}HTI-W?v$T&cx>~1;nj9vx0^F_1P zL0Q{vbKhSp3X*7hDvJ_=+^q)QOK8)I)Y zxS9Q)Ur!0rV74iszq~b6>*srO{qr^E#-qjOf^|{h`z$@8g0(kbHtQo1TvSa z_14pKC~toy#A0A%bdv@q4P>`^yp7%KHjHC(}53bk)`;L*@_}W?}E~m6K zQh1t~n6EN@BGt!CrS!aor0F{L;-vrRlsFE{KDWcujiUx=&b#CW{*YXJF4Jol0@q(bKB7gF?eq2<(c%Q+20_1d;o#Q&oF;J z%4Z|`;&a>YXhJ%qyD<@$2AA8@1sBogi$z9FU9W4-(;$gX=Wa#YM5a=&Eg)xLaFS2b zxEoovSpV~9(9fUO^=Td}3r(GPo1JH6gSc4l`KPc2-A?)<(>%z;!{PPx%;w5z;9B+) zl4V^O<5@Ki@tw0Z7psu68IjA>x!RVHQq5-B&_Rf_#vtr~Jfwldp^UoQK3;@_yu?-4 z?RdtOtyq9{_`Aw|J7&4rrNnea5fK3D5L@Bio(MsFUOVRS70T6olelv>!ZMz7u~BT~ zxWw6c-toHb&H0J+qLVh$j#Y-AtFyp0b@U3|e-9yUh5LB1j_^5aELsUyi(GTX_3)az z9(sYk(uRznKCmyXp(4C(L#o#3!#hUI9ZmcS$ws~F34Eb14b2b4Iog^*+~vBQH*LY=IW0{BNA@eo2>bE9{XCya|3_d56tv~Q?!=OOIEW@NS zm(&7I{UO#&2HgLNt+1_M+LLg9o_+5#d5{hZC4HPLBb*(2Imc}K=+OE}gw_-(pG9al zOV-6kEkg@sISf*&rKbHjT=9Gc0Ip8Y-p;=1Vnc-P4bovT=`qOg&ZQZx7hf%V`zt!A z!h0u++hxRkqhHtM7TFzcH^PriM7^V3Lhx=)yMb9Uy>6@>BW`P*Q1`LL$87_5_mBx1 z(c1LfSEUsB6+VF#K7j_aUj}K(WA+$r$stAP`-vnRmPEr1 zV1vXf3;_G0ysgZu{;9%`v6Jfo^d>0QrWqFT3@XFt$w2<;g1f?M(Wv$!hzyRQRHN>U z9IdUb+2m}kJP~S}UBuvIp|)L}qs@6Tjr4IIQ~pH2waX4}3}-3T(c^l6GUM*d%g@xQ z<_i`v=J`X7aku}8t?6iM=QBXQA~ev;%w*(*IA2eo3A);Rx+!E=_#Gki5|!b)J=`h( zvN7-H7wji0lJAR=mUyh&V1Mo3RCivjixb2!iI$^8{Pj;0`guKRZ52?NT7}f3DBRC? z7$GUi@0vh0Ol`f~`#f$awYhJ{&(?#*L5=!QmErYWMGW+i;~673-hJ|~LIQj2FatD5 zIUZ~;!=4aTu1e?q+BICk^F`GLVm|MO;p4~EIlFhlVs7W9_?$g``wk|ADH>2T!NQEm z{Kx71vy7TN!#DcQ;mP9>^co1VqS^7J6=XfIWG}M4q8XR{)^yw94R47jLE%kTrR3xu zBn_08!vD&RE6VojFq4W0dwb9d3cp+!9~_SH+c~SSa@2Cipx}u(ID&w!xj;2>*XR#{ zK%IVQ1n3!kiaU@nS)*}6_cetK|%^s{q+-;7cWZaV%RAL_2ha59|2 z0%Vd#n@)04Eh{Jy0@>jJVq)SvBTA#dn=We4k1Okq?Rv1ggrC#0R!&YxGGekf{0Aof z_-j>x@Y%_KaM{8#jm;FLXf80<*WP=!LQ+d1OQi>)G8y4lU7+Y6?QsIeO0>;eEjsOe zA!%!j=ujUBhRZr-hZ&F4vH)P`)6&r&|D}r$x*NP-j=o$pC9@z=?Q8M2TiB${)t#Po zh){>qFiW#82*=_5-QAR>ajn(32Sc$14Ti$0YzFBopyym?(@s~IFjrnxiP#9|QN`<> zQNr&nd-4^}(_IS;bQ04w{whE0#prllE3?kI?xsOlGgJDfMa!*U!r7r1{Nm4Ype)G- zdzKC?$@q1CGgRFuZ5#x5$Fq6<)=U#<;NyRpx_ggtto-wKgLvCbF+mkRuABRJt^Sp@ zbap?QPLF~LNPT;}c>4Ti1s-YQD@(~c8kwU4so*Axo29TF3MSmE~ zTUapNF_q{Q((c(x-yxU3@>y~|T8X=fZJ<*w-{yzDi2!3?J(Pk5e`=cB`)LW>*g|IK z!eLwD+7Z-*#aih`nR_7m6i1R6;*{>|ZFfv9y}dVd_sE2E!k+n~RlDz3Ru)_HSX!op zT51O}>8hwhc3GOJLsC+%Bsl=5V0M9*aCQWrs3(aXWqEx?K^{-oTE!W#I73PCX5NLl zx1-ar5wEkuI92&vWVy!DIac?`MgW*QedPn5}?>JL;wnRrK~Eil&+%FoB8cg z*Cit!U+J!R%n-7otRtRFMIsEVMEwufU*im^LYR!aC@&C*oLYEW8& zq}LrO1~ahR8{-zw_kDCT@E(sdhj1y5ro%*UBS$zVqiFEx7CWA|d3_Y=Ux@SVGW|`e znCwH)hQ^Yc%#QSH^i$xMV^$q*>LRbB-MdY4b!p6HQCb<>Sk9&nU-F;4`7ObrDDcFT z*Oo3-5=sKR<4Hsf0f`yX7)Z(86hz~$O~$eB`xjfnNpwyGF7Lbu-J&SdhtH$JU|RQr zoYS&J5^qBk-cfi}_aHx|-|>j+LNPI~M*NG_%j0hq*N*aS>#+A%un|GwY#Mr0=_waM z32Z&wVmkikZvyoH4X3DRtTQI;NDK^AVhU!QejWW%S8fAoJRamXPbxa|7U$QKi674` zL$>|fBp#oq)789VVUX#A*%GmNq?bpdDVnz&n=Oon9!)I0bsfJ*7nA6;O1FK8^&!+5 zA>_IyF~YD4-m{N*ErKk`;|s^l1QDdC{jFIl!_MTy~+kOw;h7lDxFNHg|>cj9%V4WpQCKIL1>*CT1a&!bFbgjVBc zgM-UCjidhkthUD!(xWz8UV%Quy$8}O|34b;6G-cD=fDOCya>3W$hqW<@yO5D1NIy{>IpK-+%Zlphv8&&WnmV zQr1SQz)zKXczOI|D$f46MMzgf#(EdAwhLIOas?F?9N$(yZ*S_q($9{VfbuxZucRcm z7XHtUFyH^mz?c6}96(B~tF6^KZT%~GG0wU6teu#acKqC@F)gb1uZh~HuUXY)NKV%+ z>*vhDBaqT>)?52?Ie(mKR=L#M?gSGTkivI{O`2VPFAENr)LK~VWw^c;<4egAggV_rgtDKb2u*uR%X@xNUHdY7&^C>rNmWB%V6 zi;kYx-VX3_wad2LjA4WJU;ZsCj%(JxQrW75^bTC-+LLZk6 z@&kq(4F3b$=$}uqm*1(*84|olpf~rAWE*TPIP`y{XDPqGt^adnWBtAV|6c!p{{Wm2 z&N|zS^q*0lh5t9oj3L0&hyKqn*LD1RKpsQ<|0$NHfc`acqr${8Soy2LJ020G>x`j| z@!z)*L-FxcJ3+=<(bCLOE6|HI0{{1F1GJ5vs$=&sjt-ZhK72^N2xkR6rnI!Qb%oot zH9ZOpC_PwE*ILYA$XxpI{QtgQv@I&)28Koe>KLHQPftg8u|R0RkvQmgvQOOqpOqBRWLbN1d}?8BZC%(lHa5nBg`du9AnxwY7fd5w-io6q zA%W}b?+>vUCj|#__@>l{Yj13)2d{WljjjR6JVIuU+&lza1~tGg;WNf|jQ-V$7>}En zoOHN5Tle$x1Nt^!V+`{Y7HLhVirEDIl0zpddNpEUV`F1&4PU9(;a^(HfS!;@3C!E% znCj|k90oN|W>r;HOWCoz%g;UdKkf7UBVBClD^BJU6%~7iiLj)|?i zii*>L;u_}x?ca%?Y!3Vf0L5e+zJG84Uh#Sl_G9_C<~~~xqyB`7i;MeTL#d#ss2{$+ zzyHPaZ^jv2)(kmS+rZ#*utzom>|JDPAS1{f5&S0j`FZR4YkYLHT}!jb-^ny9EG`}t z?CP`Gy=jzc7`w8G#{>}L2dO?tiTMO{;r`v(O%d@^q;{UQ3+muEjYL^*u20&Bci5O1 zXZPqhR<2^GUi6;yX+i>r%4Z`^I1gYpkX~L= z`)&L|)mh%PKXZ;hIcMD}Rh{fVs{t@UA^0YkP1$c$^{Hk#Bcyejm*@?K;zCOMd6Mju z@?QVjjKhd^^Jtl!%f~7P-)h{L4TDjtX`>5`O3))9ky7fYnDvhf?Z3qY7&_b`p5~n% z6%Ge6SOtSlWmUX(wj$NkyN)fB2|VU70{{p--b>;n-Pmx zgP<8@*dkGibblk{DGDw_eGze}LRz+RR&Ny)7IXW6xk=>g|Nhc=bBJF}KWmP^d~A4f zQ+t)tu(L*GDFyYUk62aBk7w|>S0`xZhnuF(RB4UGPY-y|)~@kZ{whmVjoo9G7?yq- zH=8+1;GHKzGCB7@`wHC4vRy2~2uAF0@Z*Q_D)B%=eJmNde2h=0pdcdEY(E0l9uP(u z3l)nvj6%Ykf9aJp?5I{|(eV9D!|MvG)SBmIR4a#vd%tT0rpTupF8XnvXn3ajpoY_h z#j&}BjFjvCD647=G)!BA_ms>-hQ7Eu-kbNh4R5Yay*CV8%5*dJv$&a)1{3BA`w4xa zIO5=uS$M)HO_DzgtlRP+7cP|^m6nFj*U&)=p6*S3`;z%z6Q5mVJpen>Lrl#3{Y2Rl z8~3iJ?WNa2HjQNov zmRFMS^S=dMqf4p1@BjFIL13|Q9g-3S;GJIs~)Au)43KJ_jGlN_lQmeC`_4KH}I< zzf%B4S5qD1n=lL^nn4%ygKKGV~uQ&?9Er- z$d%vtMxth}HAR#W%?%Ac*tJLl+PRm6J{Fnl>iEkYkUDgBiy@P$e`aT=`_bO^@h( zp%M{+we@R2n`vn@TFlxkcrR!miaanVZDAq`4f$JzOLY=HR2Gc zAG9VX+lg{u%T_mT@gIxt!TehXEf)DQYsjG5p31P3cYKRq6+@3KX`G7A`U{8PSS68u zQ-w%ZMhVWK?n5+JN#3Z(8#Jtv-ak!Q2<>xyCM6njm z8=j+RFUP7(p5^Gl91TGl8jF--wdNGE9c<+he(w?R{i1(qZRklssQ@*&+a(ngUe71T z;o5Dyc#k{^6|rGFVX_!L?`_KYh+}bw-mzGDq0>;^P7Gnf98t)R#$0TQ1RC5|A#ahA zcS&Erp0#^fP@`IQ&%}S%+g%B2dt8aqs)BNQJm?<89ZBLfaQ8luRI6s+N*7RV*j+p4 zb~zGog}n4t)#yGwOUyEqTt%#Oqu49SimEa^(8~S3wY|A=9Mx_qz7X<6 zg!c^s?ZlvSfPD9kuC7LfoHMw8kHfXTRxmfzl1f@;BkKi`-p`3BiwXZ({)LoUOeko- z6%6kwNSf`2K<*KtQip(=Vf}jJ@@h3!hiqv?7W{@N9?Od@TMVBDW-+je>{o|xwWqZA#111uIA6ujt zAf+-ghao}X#J$jAwxOp}!bHCR0n&;is9YB^px zd5r)0s^F?+U%#4YqAJs=ArDbP9)fbF!8d%w$MGGGjRa$3d@2+SM*h=<2=U|4vr;^k z(ZPvtsEjFyhcFC2(HC~{vqQlnxC15)78(i^@&%a+=k3CS^Rv*)*;{6Fc)jCfBT_e0 zFX0Y@p`}Ny)rXS+ar6qKG=4E1Wka6=jF$Ams-!2;l}6#QpvpeD&R)NIoUvGICbjY{KryfmqA+<5KV*{0J;h~Nwg zw&v>roTPit=RnuuQuNd^o5G^v;!+0LVzzyAB7Qq?O$~c}J@l_=^whSs1YZF(ba(hE9ZJE*s+!AaIES9rZt8C`~UlPOP|Nq2@i<8}7n`EBR9 zt5XpyVdix6e3)5%I$lL0!ZXtQgypNMvwU%okO;z?6L;%i6Sy(NCuc#-W@GaQ4uUCQ zz&w|N;4{z5i2Iv{41U=7gICzuE8q6CAjh861SiT-d8)TOISMh%2 zKnc%mqJ*4|RmH?)VDbuW z0YC^K?%?qV;JH6ST-O<2eo)lq4#RsotGYA-9&vg3iWB){Ny!Ey!p{#9k;E=U#BX5f z=;$Wc{i7vwYszxUayI`2!lM@z(_tDz{)M8;5bMVFB+1WDT_6nu)54e8(b~52ht!yu za1_GO=NmmkJn_wW6Zw;+an>=P!_BDg+XQNxq|ywuU&8!paL*e*6{6I#em%HuS zCFQhfGvHJ+HXpS1&1+TYc@T3qWdX&~Zv?TA)bphZ&LNaB_syeP`-Wyd8ZSmhzTQ<$ z^wJOqoEl#BJHJo03`wS}mEkFKYT@;ucIDtM5^3a2F|?1%reOpu)1(y zL$bF=CE{Be8&gyJm6()tQg@1YUGL%y^;^it+uHdZ$eukE@dAyA?;{J#*wWI0SR=?a z1x`UjPy9>4XjlKgeBy=VP} zdbC0VbJo46h;jGTF{7Xp@MWZgVC_nMrDMN8>C>hBLT5l{`SXyC&SJys7&Ty3*F_{U ze2g@^)ZpvellXHkYNDRyjQlu1eXXqUT1O|A_Pj)8fNn(fbmsVF=qvv;(h1Y~yX!{+ zfv__;P|fU>hjrXGMKyU*yx3IziX-x)et4CrR=fY9GM}`D4Fe41W+sHZS5CA2^mQf2 z*>Sd?ITbpYFa4HA063W1#>S?)x~d|#kt2K%@>=&F`mS{`pg5 z*T4g<%?^Z2`y+-)j08`Q{Yg(Vri`nWKD&*+e}_~Ki3-7-&s`k-O^=hbDicE#x1^Rf z$mlHNW$pHs_+!1^`crOUHi#Z4!A7mYez4x_5iMh(uN{9}wina_^9;JUuxA)-<_Z6% zG1T7=pxTa)P`fLH)@|_=D8WsD#2@&W~db8A|;#XhGm z1M%|m4wn&UDa>iQ*xK@tlk;$3_4f}+KAZ@{`U$t=?~&r+dG5;nC%vQX!tK0<>`OC; z2%@9=mBb2hC_u`wcL4wViFDz>>hqp$avd#PhKeOgmH=hE2LL*`*2w3Z!-3Smj`xvX z)F9zn`K#K{jb5Sa^-{f1C(AYVw#V-_>g0S#g*XiA^;dIEK?*Jzer#0-deA`a)VMn| zPbypFQ%iX-cZwRv?oiWA+f#{+#a)4H3Rj7JB=l$PmW8?Ftg~M*y=w*XA1(AO)Ql~? zYJYGrp`U*UfL8r}*YS8tJMdQS1tL3?@`0n9XE^r-B!Yf4@9{1hIAX$#lIWuFLfjV} z3&(J|#!lID`L0Ii^U_ljdxIr^rO9bWR$qW1sRy_=4#z@5U2?XoB0TYGK;3+x*-e zdRNN$o#PX?n6!Cp&i$J%jjR+rEPf>?ZI=(Y5fH<8Lgtjk4(S?;jb7z=K>p z$RXgDtDTRjg&N4aN_AX+KfAeh!PyF@>la!9&xR&qx*W&^pu_E{b52sM2TB3S1z{7k z`qsMUhUVw!s1+e+B^(8Q12CSIXd2&SrK@$G)N?ZC__z3-OEeTzi4U~qY^W$G9$7b% z6wqy)99ZWUeY{H3Lol)_ly z$L8mg*sPtI{{?rlZrrA83`w7QZM+>e+Big+Gb;>IrxE*!>A3F#C#l6K!Ck$pqhx6pVk))&%C`54U3s&bz7( zWG^+H9pg4UAk-uzSUgBcJ|qO}E8ug$b6TpxDnMI(GRa3!WzYl=Bohe<30C~h>4HkT z5g1nH`F_fVvut*!QItKKS24Sq;@5ctmr zoVBej!){bLdUBTWW^P8Q&I-7MacG1wef^*4#Ngx);#4|tnS)y>C8-oJKmN4nD`==_ zef!0AV|2MQ>G?1}t2+U-ZAjsrd)dg^|OG^YNCoM;EnByqS0s<|~1wvSe zW=csM-)&52DDxQ|Ryo zs3=KQ4PO%PHgu_9GBLcvSNQPr;!iVtF2Zqbq)yUVaVYj2eC}SKt8Rmlp1NIL_V4DY zRG;|L+bE%oYK@kBkN-~B>*V-LT$KexL^U4%%av<(%|(QUV^@#En+8q(PWXBxU01d1 zsh@J%d$4**!4ZCd?>kw7GwaSdhlqqopwU@oMXW=jMMK2(4!wVL1d--xw!()*pE{gZ z`8o1o#-$Za4d46O$LksiCkTrO51U^^V=V#KM2>EWr)m(;@Nsd00oNPzDYmEpf?2PJ zK-S}6@dR>w6EXVqpjx8BkoavFB4KVK10-ire|oIW%kX}1SEbN-g`=UPGdH(BJR~fv zuqiDs4<6lJoSvSXnfVT@pIHaLhAaUxQdb#xA6st)wSZI}l37@IrY|k+v)X?Dke_^D zA9CDKQqoY@qD1&rA2OVOw7!<lCnW22fhH4;_U6Jpk?mLMGdUWJSspE}LX_ zLb1Fz@ZU8nkCdYXLrWRH$q}YlbFFkGYsC3&LnGQkp){@>MfZ>;*cro)>5FQ;r7?h& zt5r)GSX@j~l%&JyA%1;9$<8*B=|}C;WV9H$ zjQG$=>Unmx2Z<)7ZpY>DN@Zeg4^Og?MdILL-9pUCUZRQ;mt?T%RN#@#$5&I7&FT z{9YI3bhQI}+tqdjR-%#5rhKcak`lX*+&1iGgq(4JD2LegmSG$Yc0*kqCkMxm&h@7! z&%tVdE1;pdcyWOU28*y%S`k7IR)fL`kJ;dT73o^%H~#!dsILbgkSn$g4f7`}={E`W zOA!BZ1-zcRW2`o!7Ofic-bjfL=o!GFJSRjt9{*CX*6!R|YYf*`!A9=|h%a$J!AgCC zs{iv8=o_>hP?$U26OoI@xm7%Q9r;-lzfu3^!w2~K7*i@}$oFtIh~<2B!z~HU9>R%^ zruNsp=?VLHt_S{ zbbM`fJf($GYyYzV39Pe#GRSw|Zh3;CBszH;=m8Rz`TD72D2W|VE{AC+s6l6*Je~`gEI-VLgF)IEMuCTcT>LwmVEGD& z%+$@Z*OYQD`mom8Abvpv1Hp*bSFfW&kMw>PA$CA4j_xzzDiYYY^LWDNdRBSJBLdTM zIJ;yH!(`hEx2}*#d5|z)7xGx|%j|QqsUw!!Wf!Q!(+@%&LnpAM(Cd^NrMn37E&k!E zEKX;Zn$fxBRd+h2anbA@YdW!1s50B#l-~; z89Be8V0UN7&=A)BksL(UQ;!X5TRkn?2A5lj`BZdQm_Me{)M6x7`iul*z~{RlkJ@aUnTtH4W}Dl6X_vwu-=3xZ5^JHGE= z@R92|4KrtlCMF0-Y9Yr7oB6TX>gja_hpN6@(Bz0Ys=Nb)=6Ne^UdxwlUPmNO`rius zmMuEteMK~WNsay#he3FEKWNgx-EixQCT`%V4+{1}(lxN?Yj@5}_qZSCZg|AjV%4W0 zsJa}uA#BIk4M^dR!_A{c&BFp_vpha%Vt!g%0qMAN& zP?jDKe*vFLPL5;l&Xnt@eBDsO!{lTK5=*uRGHK|+<2A=x{FnC8KplINrkldqQHjm{dOB^ z^7+W#4#N9|MM!gY>D}@)vA&jcnFP&C z*sN3%*Sq`M*5eq@?A~;>QzJV(ohJLLh_(POOZW4-V-IXCmWa2%{V8zL%uT)Y^4@d$ywW37T*bWmq~&@LEeh$?35CTM>uNkjWxLG{$~lBDdz) zcA~ac+Ivoz{P=hNNkB#~k_11Py}YbUj6liLeXSj;pOXvcHif_7;6R2cQONVoR^<6; z+$}ffS>%4*KR3T;1HTLAxl_VD%6Ma=w!7wyObjZ?$35VKo^evz{Nu&2qWPow8fLYR zQ5jsoz_+_?a}brd%F(xU?~^J>LN2}gmy$>y+qWBh)`}N}Y^O)P^p@d}_!+`a$Bc}r zw4J|5h-fQk54iw>XEqHpHgnUH++bC2L37UTcp)RTYYa)J|Q4j>moKH2qErV(;CLKo<2u zHWz4JM!D~vaJusaoX>~0a^S1%ystD*cjMQG+$Z?BV5`;n%c;rH;iKOscF#U9n{5vR zt6rV*JYaT7pn=<7)wNupG_wZZa$B(9rT965|w3v|$pXT+5glju(@30eMLOl|qDa!wQ-BsXv!N8>Gu2vfzq6*ziWNA$-d#O#7 zsi?qiM0)!J&Nk9&);K7 zRgL3=Bk^q1)#7SmSuB)%VzD9}1;Q>3f&|&5+ib4IgI?|JDJcMP!1V!*X3yY+N?|;y_?ZhVpLWzVVo`VybCeWU8&w}l1&-@z;0nx+ICYM-(9bzvY?OW&u-Bd zsV+)zjx6nSEC0%+B}uW$T66wDkE>$wWVGMYNx`j$Loq!&6G!=}TXd-3sdk!}Z)tT& z4Ip)4e*UF3I5aO380{sFLq+dh$?dqe-Dg2Z+^sM}ZS6Qt9(AWze#8S?V%uzosit5F zzV5N$ZiN`>&3WzHoIXV8f*Hx1-Wk^GSwGfUB$l-th0-%?>UaAplDnKK>O4%Wu1sfK zjy#od>k#4!Nq9db%0o}4N8tBVX@U*U=~HwUo0=`;XZ?0|@SC*CD=L_OC@7_9x9XeS zD{V@K;WEuVJtUNuKLZ#P;2;1P7!fZo3Cdw}i!{!XZYiO#4K=`oJ{{1fxSXymPS>?H zHSs1AGRSEv)p*`dL@OUHy6Bm*P&+((1&j)|eZ}^kc$hz~Nzkc0lkrykJLP5a03rB@ zL`ej++guaqAxqkHQMV-)+K4eh@5fQE6Zw~dTe~-MS+;|W+(rY7iGg|vCr3n+h4c}f zkgEnCV5~FdX~H&tJ?=dUX>s1L#jo^HD%zZ)(_)H!m6e^B5Mikyquy6)d)8!R9LeSP zISXe%e7|)5?4h%87yEkU(Uzyc%SrEJIiY5?q$Jakn&|1qxy-X2#UnjSij}7KK7}35 zY`7aq@a#nM^zNlaUhw%Snp{Hg(L;zdC8^fdyVvGrKw{8hY=XqR)lqOi<9T~l*WDatG($`eg))FC!^P zM#P8otwxB7Itz8ouXR3mv#L6aNoqv2uaPOU+I7Y+ceieN@mFZ3u8m}2$pnNwvPmO5 zdGD4UOm16Ry?P%qtgqN;-5sBis^Zj2FQBUsk?h^cPfubpQ$r)hWHukelv@MhZqI*j z)V!^*TOyCX>TuCP5B$^6=JT4b>hiQv@Nz^>>N?7A&7g7_|4JNTIr_97kkabbi*^Ag zHt}=S>GE;KPHa9TH7*6p#vAiMQ#N^iW@=%9<6|l*(U4JgV}3nfDAlvs3`dUj@^<3Z zNLIN+$K@PhMC8leXg#X+9ZiOBgQ z!KFv~wwHpDi+$+OZX6tOW&&>1ong3c`)S0$Xk1QKN`}On5RX+WRhOak+gaV0)%(v+ z@~`&2k;IFD>sW7i{wE=S+{EQ}3)p=&lOmX(0n25SM1$r1IO}~rz?p_`KJ0rLz;0B+ zK3^NJhUWm;5v5W#$}Z0D`BFjC?JyEI+`~ElFpkhG{ffU0yHOdtfQ|+X*xC~Jb3hH5 z)r$L_Xq9!2fISyT6*+xe3LcxoNuwWb!Wo0GrYR^@CI(Y++i6Tkrh0@v#(4pY{zXye+0xGvQ5tI3`y2!UM5IU`vf>hLd4&jb@?(ok@xVgyELMLNb_hD7bzFn@fP`VjV zYvFuMIksV&;2bNvGaj2YK22xe4tXANhSOEo(>pZC!iAtdCuzQ5j7FTDxSPee|8Y+7LBVS+ulN1>sQT=-6k zRn;gs92&aU+YcYjpM2TGk91Awh9ceDV`Mc6nDc}7=2+Rxm^J5FT~d=Uz)%k44l&!M zGGQVvZc~jpS?5vbN5u_$-o{qgZm`8kP-d|G+}xVv#&}4tS&}4=bb&1p-@|_N#WhOC z8Dtl9E9`rYTBiSEiml_u>h$>>k@^GJ!rWvIxzoZfrR7v@4jLV> zuTm^ziFO$%!q@Vta7n#@Cosy|Sy;HUkm>i{Ii&2_ zdOj74)m5)Lzl{l$Lo7Er52NWeBhs?U4X5Icu{V{~?|MFNOAU|^ScG(1W796Vr=7W@ zX?&E;<#(KgO`Tv*<+hh~tNr%58OWs*wh;YB5b*<3dl;hWR2n3|zt-?g8Z`>uLfZ|5 zM~yY?CAkCM&)kn3vnDt@G%YR%V$U!^5r$h9gS;KD#|d}56;3C|jLfC_!E?`YMr9sY z0a^7G#qBnlyXQfiVQbpPr6%7He?Q{NyI$_o@9SJ<>en|u(n9>Fe}v$L>?;yQ(F#mF zblT0IHuzUygK<&&Z7!qbM8!_a5L7h(G+dK0Y5zao6crrAw`e6Z?}_u{1S^FdWH+q+Nc50E*TUXtvV~gA0h?T)fxeB6&D^_VN`@Ys z5&aTt=^6)cQ7Gt03y(I6WFw;UoI4K*S>v%$9mlGfO^Fv&n>5dN^)T9$b8jfh=OccD z=vG!gt`+oN=vqBgkHC&_YlB>SBy9x?Gz z=aQq$+-vNkDY2uxFt>WrYVZe2MqdfZ0ZfgjvTsT8pk;-lWq)cVe6^x4JWA$9iy&yV z=sBg1Ijh7ZzrY7^Rm#W!>LRF3aQ!shIze z)CGIEFP^09zo-91%D@oLKM;I}(1qMIt?7E zGp8-NQ%9Ab00Ecfhqag<<8Fa*(F0sznUC0UAKiHyI4XlZ+tjsXmU)~;fF!HoY5H7B!W;u=UcOoq*WPawdTSK zC$(qTyP9h;xEQ}Ti(oCO+pphCV!6rOy&leAkT-vFj|R_|Ch$gDwbJ0q+mY`+j#`<% zh@coixO2%{?u?)%$YiR+46)SwZfn;nW|FFeLP|o%XpWMuMXC3X}IxAq_cwsNh_JW1WL@CX-nptIEc{II|8KM^hxUXYq=RgswK zq=nAwexP24%O zcyAKUV^vybAxJxumQA5wR;SjMKB6sSk4oP1W%{SG4$H3lR$P5IJ=}_!;F}bAv>~Em)1pU*tK8U;uZK}i-;3uB9sn-ChBaN2? zZ9@k9bG7j%*)(^CwX*WXNcE7@jzCO{**_?6alyfdWhB^I<>$L z=VEz)X1}CAI;PC@)>#)M)A-SBVRp8nepuLfO`C4lqYGhBstZsU7#y?35TWSzr>LzcH5_?YNBI}=w-Xvn5n4M}E3=lBTs-RoC#-y z#8SMaf8&L9Y;$uHxcXkBqN0C8bnvyipdv3GecXB>kuToV207rE4=LG!LKILC*%v44 z!5?6BYNO6>3k>_`_)qhhqxeA9vKJ_CdCv!m5a7Acwu&8gyZ=TB-h3C;fHtgs^X(c_ z1VsD~vVzDkIKY$qZ_G$aPJjI|u=@$;p)FgMz!1pt@9%mUST ze?Y3nlOo0!#ANOLKVzbR4Ftdor*2KYDGvrE5&lJbMg6!G=)5+}JdOz@PqYCihgys} z1{;xD0%ACTT?oLmtbS(N(R(%-V57(k4`oqus#i=yk7kpARRu4f$CZ>}1D^+tod!^q zC1zwe7s7wNozy`f28%|2>V*Wh*Y~sEKZ|+lu=zU$)7<1X-QV9uFDUi)VdsIwgqV|K zW8~^bq?}r{>vhD15`xSl6zq(Xz|_n+x@XtsXQOiRvhRE8zy#FwMOxqWtV>;eFq-bO zu^~S?*09pTwl;|OdoE<}aLM7~zhqi!PCSK?j~@0OSfIHu&6VOKd5rjOx+ucw_qCfK zd5F!D77p~`=03=30wF_?`5a>G);GRd(bXA9WSiqZ>gM`2?bBK9pu>It7L-G^EkcVt zX7KAsl6s`AP^64~wd+Vl*cLXEoH&|OXgT=szk6=`*n4$#tBbeh&6~#%(z@VPCazS9 zXev|THQ>v{u)gH|#dHD{A}Q$7pBF=Y;UtiCsRNOq;lTDMgfkbnzuO`Gn5Yi$DFD6% za3;F8zguTrZ;m1A@>a7QL9Ztz>zx~Y<%?6DgcoPUKcY#DEMT20ue`$NMsW9!wXDz>^=T6LznDJEfSh^l~NQ=Qw z@{^I#)Bc~IUU(aPBCKTH*-SNUZCAtb`E>AQmHNL7fYZ!;Kbur7)}CAB;>>TWbYF5w6VJHOC^88g9)zK>Vby_3P* ze;tQs6_baj)xc~pX*-^iJ6RH@ghgBWA;?On{zJzEBx=2~?&@L2y6Fw1Y}S8%PVq&d zf#qo)?VUz)2sd~V+MUE#rfS1M=>=oC>UTQ()zz7t@8{;fx9bH%*z)B{ zQbnr4q#-jk@S)T^4g2STU5$j1)(LXRzokEEJH1Zfrp0N(%U!|2KHebrdB5T8(0yw} za(w~%VcIp9%jC+6p-?KnKRxvCH~Ytf+4aCgM83)|Po8831)e2y*XR}9-1S1bved3+ zG6u%Hc|R;b5lN}&ceag2h7lVZpM+Pe-lnY3%cdNzk#+vOW16Zbf2f+Y&&6`~TI*KU z5n+F>YdO;S)|3M-g)!c6IYc55Q!rj~jsE8*vDPp}T+%FVqEO7BWN@t`7M_HjuByrV z#CNE8@r6HdQcdhCtmAQ+DqlLHLSpthyTXAqHs_N_LAWual1M_z^%bBzxz*?Q#N95& z0!elLE^0O)(M?3G9cqMSs^Tl0s4vuarZCw%ZoVX{k*nUP9zH6?sh+%(t)v{nG9u5s zTY%K-hR)`iNb*1$U0hHoh|a3V?O&YZ%eVVkm&P!P0Gw|^|;nUE=@X`g@HQv|KVs#R}5QPvvuV^zfM^RpIqIs@lqZw-zAB=s1f4`QkYiW#P! zuT`|$A%gRkH>gBF0c^{IOXss)BSp)?q*F=lNZj?Ueq4G>3U5Zc;w35iUp&paGxau4 zSl@1P!}n7p^|iI$Cpafv-M{s2Z%iOvR7-J@Hf-z3ybXZ?V?ple@i$jS-Bxt)Q2M%6 z;~JGKf$J--bqB1aNKn6rjq*N7VZ!5kyN86C8H7HK1%zxM9WUar4Z@NR_*wD{aX7&= zGiRe23WDhQa*Ew8g3A@lKYt~3&wqh47QvRf8GKan4EOL~Ok&R!9#o}o= zg(wlG!V5^_GsDoE&5E2}o7)En^d=}fVw@VpDvQmJT>d$lYKXl{ZYPk!sbn;9-0>o&7Hz*;l zO1fCb0CL4^gBE{iJbOu)+t@CBms?H6eZERNVq+)ssZ3JFE-yTV zGN(-s-T;<(wXy4Tcp@cF-epnEo~e$> zK$S=O$I=raZ_O}Wc~w?>d<<0fa2=6PY?a2GjqWuKilDPUSh!v=PpQg(mqV5o@ar?y+7%Vng45;+gN z$O}1=p4RQi4M`cISbuXaK5`Z1W;YKLe&e3b@rmR1IppUmi`y%2d7dv(z`M#!2F8&0 zjN-K5Ux#Qk<=c7Z?nLLR>BnsbO&4Act9O#>zS6llIhUp}LU1!H z7B$7Y3X48?vHJ#hudd3MIAjHZjZDgY4(idaio0I&j!R`tT}YP^j9->_nYUY8OL6k& z%2u7oQVw$A!|~RT6Y2xY@&oARW_DX6|lK^lF9qJ_5C`azuRfQF~-fPMNw&WI-3#ChVkEzJgg7OpfNm@|Fn{M*;0#! zHqKB}l&WozIMxt2wra=*=%xsRqnXBI8B=xDdFYw^wX&Fr?3*B44XwA}H6g&B7Qr62 zCg}Vs$~N#|L>V>e(7jbMTGx{fa|}K*^o+{orlcQz38t{;Y}W*IhBnNW=wK{}_v_?J z(hzkBQKn~nuPVGpme`%G9HB+p0X%^F&ri1=KmL$$vUhm?eH%!IbO$ZM&3N>wav{R^ zd~?YS>lonc;x;>y4(`P^A2(T@P3s-L6i)$cX;j5`widj|Lk=Tk!hFsljWns`VA4vd zU-0=6O~`aI3)1bH*IjmK=;?gSlEmir4!PfWfH7x0kPp5Gvu#D%e!evRx|vqhO8oa( zLsQ#*A#uR0rXIxcX-IEC#GQDoF-%lkY z3h!>|axfkt0T;Q{&+E`C%H+=ruQqF)Ybpzm1>IeX_0v;q!>uU@qm_(;=;_e3d1DwQ<8c4g<@+a&HRUVtEW|xsrh;A%)dq`nC%yBxhG9`5OK-@FeXKe9sGjt~N z#ImH={doFhvf9g3*1We>`^+zNDI0lCl%uMYwDNRv=4Q-=u2N+0)G7FXs;aWXrO}_M z9a?_+)TsUMk_h#$BT;HjgU8Uwv1)VPSg6>-^GRv056v7FGc%2SRXurK(w0SZkQQ3+ z)DsH#au!P4&phmo&4ccpgw>!63l|fzmP;z|?FOs31&X`<9(22p^wOxL7p#I@+m=W* zT?TRHSKu>Bd#~zl)trYqXxKJ`^{+(c&U}t6wG~sNROFOv|1xxjceoXqm_nzd4&%Q1 zNH5>}plRwXn%`cc!?c_&9U9Q`>Mym>6&qs{l5OsPfB`7o4x~v zFk?+ja`4j*1*3hr-LF;1u(g#Q(uuFzo)p=V_6f}zl5`w#nT8d6+&RF0zPwtbt|K2m zhOs(S*b8P|gESn0q89H-fS)yFmthas&X?FpM>10Zj{BW7g9cz29DbetICRTzYHB3A z{ei$e(GCj5$yP)YxOC*+FPUGY#a!=c&RgvOT#}bRT88#gtpppXE1Abh$TpY~WZXkEC2~vxq{&ruHmm$Bcb6e` zR#Ua-|5V#o=JET2Kd#7sIA20$O@_VqI2$k~o%RS7dPOGEa9%17JYB({m2h^>$++~K z-6}^$tMw0B8!tDrjU^&9jZAkS&%zfN@hj_kt#rIuo9)*Smprt72_DnklsY*X_}1XB zU2Lm^DcC@@>&z5IRGqhEzokINvd$NDK|+ip|KyTr&ewyAD)iGV4JoR-U7k9h@4ZwH z9E;945+T*M)2cS-t4C^iwnWzK)?HE=JfAvWUL1X9swcAG^}xS;iP ziDvS|Aos!!I^=V{Zp6q_SYcc%^ay&uzUt|v$q;kP(P=mSn`m>TnB*rD)cZhgVZ zolx)diRJrwYCNzXRI{;~8-5IW_gg-}!#V>0l`e3pdM?%duR#v2O5cmzG#6!CQ;7Kj z6vVcftR#!7bmEre*X<;DUdf>cJWvYa21~$JC#y6EY)mjat>%ijR!asty70LBT@%60 zDnnM)N^QkoL>JPT4OmiyihV)p;}v&GyM#}tmq>e)8+C!~K~GP#B?W*|w~c~=(K`Bp z_dfb;#{ojsqJvmc53~8UP8k|~q<0vvd^rfpeCNm-%<1FnD=sO%$0m2W?@n6oZ#V&Y z<`OtRjE228^Ag5>pY0NsUlS0f=MM0ufOAOSnYirN#jKeZjIGRjS!M5j<+qS5%6Z8F zTiJ1TQ4cOyPlOzaBu(f_xD1~aT7{WO@COXEYCt!`Z13HB=IkT^-i5J@$VkI{RqU|_ zYKxddCwEcatD|8a+m2xs_PEvdzqOi$UQc(|UpG9YHaOpd7sMVM(Q>Ezcg_!!^IugI zMLwlo&XDFA>MLuZd{E^!hW1r6{y^^Z(6LbsAO6&lK}oMlwQ>H{n+U@jnv1U_K|}^R zF)=YZ=!K)!q1R#_wt8Z3(9G=_38gSrWp}bIN5dL;C|HU&9|Z{|ywy5laW}zunMO3l zp@N-~bYR`pRf7+m5XqrM6W@<3BE?@YDgF{d@i4I}UgU}>h&MifadvAG9qKnUV|6l6 z-6BC-XhpC(L$X(Xt^w>^IGqRnpUfg1@M$xVYE)X|Hjb7;>hd3HXTPp`HwVra3c}mV zY6IgFOSAg6Ni>2{&8g3=!M$ae$LHOuNG&OQX`{#ABw&N4y*2X3o~%I-)>uljruGel zlWI=IK;IX=;P+;rXWY^txHS8RhYx&e)OPo&DveHk=!=RbPA10?@u3H=BtsU$6t-X0 zv7>zkP8NGS;}N;0ATxk`5W~CyM#HwE=R&2~5OuQ@cxMkzEN66`%7%;ohaf7Y8N84Y z$1qP}8GCN4Stin~lBbpzKxr&@etN)#(a1f!99vUUln2gN3V6CuY9g`l>WT#+|~Yy4xXTe<+Xc8AHk8iN8V$~9|L9!L*CB6O6ZRGNV_-w zCi^AH{BBiV6idI6JtucORa-t^fp3bqMg6lUFz)lEtXoV%bcb0&>n?8v&QFO*y6e&q z?yYXk)s=qs(-ax`!zt|9DWzHrX8+rt3-)bFEx)dQ8j3>kK`R4XXJu)=cw;BSCkI+Mpg1ycCCbGS3*gf^zJtYF^8lIB4QOp+JXHh60D@kX67M>`qHf z*`ecMhF`Ek#UBvGyfT}&MG3izMtUnEFXu`m3}_bY@*cJXQBkiB48=}{^dy7_yQyZF zVTx=j#2@gAVu)-*Kmh0=dk7HBVq8!BKo3{ffYwqkfvO#-kqpE^?fjZGgHYz zJ9KCEru($}pErIfi-1Hy<8N@>HP9izQp>1y}X&|#g30pCAFm~Eq&0ozj80;BvlS@|Lg zhIuFvRyeKnc^0}JJ%se%j@J&`KcA?4+7cPyJx0SWU+>w^ulx!#RA#o@M#CSKe2SSN$Wh~U@TO4K5=u6cdeZrK@SyWKkHS|WDvy8 z1p!qiS{SVYJ(XPs`=pm7vadrt<9)fovxxZxD}QdODn_Po@1P)rjek9PL9msdX=53=%jzE)~XO~Y&4i~UO(cDn!wOYcb8t4KOt{^-p1S5!ZK)KwEFO1 zZ5oROB%#sT_?z_|GKHw-kQK^?jL=y`J)KtgyC?;s%+TK!Yg#&940JA*5%On+#k+57 zYu^ybrv}qpON2K!wN4sU@^#O4#R=kE#2fI83q!~Scra+CeM{Yoj~(BgrIZ;P+soLJ zeLnR{IH-jY`8cjrpF03zQTdp8zbbw3t3J0d?`v!Q60FHidt-b)x4!-nL%uqCFh4Eb zEugOdlfccZc<6`B$(fVQGfO(JHYs0%(!YtZI^WGZXP*+*aK_)`=n&$PAM?sAVJ_Wo zFDKL|TX(kEJll-?!<0M9rrOcXaAjrAo`ooYkoqk>`a->Ty+_9pEQ=jJ1N6nuDtv)_ zPQ^w(U-Lg76xk%7ywaVT95->@jaRzoDjqyk%8wdZFE5Y;G19>&eb?@I4Ww8yOAp+O z?v8AP2VQ>mz`n#xtO2L7|Fqyt+{n`G@j+6irnO?8@_z_9vQyJfQW{$krfK-~SWiF1 zmER;<)V?*F3X`IPN_XJ0#>A?L`e1xLZ#*HAZ;HSK4lp)SU6=>fGdgmNiH9Dl6|TOs z;4JLXGDe@PA1ewGNq@vR<=-rt{wZqDMcSh7E>Ab)*d_gD5J>mGZyCfG(?h!M-1na` zDbir0}vK>gBYdj4<|-Cei=i^mi`r+*Dw%(75TI*=QQ43aWwL@r=&T3m*e1z+tc4F(M%{{n4Cgt;`O5O819CEx?N#T94a>7KcZ%E#N5yN~ZWt)!mq20ZS z;=PhS8wx3b>DRycZlUCL>*%gmF>Ev$>Mesc>YGe{3SBxo`8TX&qPulus-FEYHi(3T zKXrPPTJp91)F7jFna63t_gNidTaS|>pQ@M5R%jC_voqk2woM<2oupe!$8#-D+s?m5 zlPqqOXVCr+(_1RmWju^0^@A5^nu*UZJ)I&r=*L9=m+3%6Fhj##$eJ%&;V<9S|6xuL z5!^{;mof}y{9*6;-~KhBEtM%R>%vNlD?;^!1< zG1TPj;64N4D@z6K5#N{NIScfxj6=#;FN-kzk}vxwhLtHS^ny;Vjd%=AD!-LvAD8xi z!-;2X?7p1hzp?Rvo5jsNYo^I|sFt@IIa^qJ|HmX!ne)ks6(@(rY;q5_R~IKxGLKo9 zQ85H&go=D$-(+?fCIZMv?;vI*n3|QbN^hw8+hD`}n#0ugF~*D3%IL?s!WJB|7OLBF z6aS!qgC(=iq%v^fR9##LJ zX8o;z*f)oAfnVDH3kt3puT7iRY#fWKqpDMg--?PC7< z*@OBw2PfnEz3~6vpQNZ7O|KuP2dE9^gC z{)@t-zm0e@j56L#3q;-q9|e=X75rGSWMn#h?QhzZvT2pnn~M`Xn4u`Gy9s5=>-|4n zSH2jq@ZQ(_tGIBy2tFxODrJ-P(_BP`|IbnI?ISJwRX#idLZ`y{8Zfs*vZ#IzM9TLl z_`1@+bHnHatuH#t9(Q@uOJU|hDKdhnUyHH8(a}*|-`95^+~cv^Xc{1c@*!_-CXe~K zmw~$R{B#s6i|vJ1@ttFExT!>XQexMymq`Xhxcckc9k=)P#?7NfiXbcPVS!c8G>S?~ zuX4_Np@Ze6ExdnCuFcwht^x$=S|U1WmIE0wAn?=Agtz=40Ad)4kO!^mnY|>32 z$*qve%E|(LxBPx41h}ZM5N|6Ah5=zhnXOL0&~AL2n%cs|&ei1SFS1hLxd9(<^-y3c z9&tA$exi67QBg1cec7Ayb!}}eBx^(LvV$k<(9n=3GZPaNGqXG<;iqkmEa4`Q(u0Xf z1O)^XNyX@fH1)FrotBm9tl^JOP6}myqJ1$HkxE_LVowaVu+Rp`ncQ5IhbC62f~^tNtQ31c3UHaN7FA$rGi< vCE}6Nli?tD25_f6N?crgbB{gLbnpDH$H9JdR#Uc=7uaAGorh&g*3ka~&klvp literal 6686 zcmdTpS2$cPlH|7ri zf4}?P?>^m!`*8QeIqQ7;ti9@9d#|;^RaN8(9#KC+K|vu{b?Uqz& zQKojszFJr|=`fv&_MG`kYU=Cdmwl-{BZb?X%CUeVGUrSygyQBIO<#f|Yf;#-e?#J6YW6r)6WB-5vNpEopb5M9xlxtMa*l8#Em}EiQa%}H< zx3w}0BcsnRk=KCxk&+CB`a+g`ZTVSucee^)A%Sm=j3mUDQPR@1#>b;x?w35QtFMcN zjsExiOO3XY0v;_Ej;5416!pM8L+W8ROzAC>2k>IZh`<}aHi=I((hS61>wn_YtR_W? zN$yA5W2~TnK_ynY*pxR6lzvbo`(=CvN&il2SXs%HPq|)SU#|~S-QJ-pD^c7d#l4J+ zi;G#7ApS_srOnoFQ5*R{O)0V4q~|C$i`9fPZM^7*$b%ftLdXwFO3El9^U;NWmsPZ~ z>bcC*(S7+&=6)2#cNjBJ(vrv&{owcqhw8&Wf5s*LuYe3sVk?_1sGIyO-^nF3U!$U| z*xCTSsxjI}!4A7^$4vc4BP*m`_?iC6L4{spbD%dcG` zc;QS>X`s^*w7o;Q+rFU=<8tK#g@>j^cjnTWlvwPMf-kUNSLC#z`A4)!LdUwppanU* zA>t6Xp-$hpEdmBPxhPE3Uom?iLdxC2px#$Re{p)%X|Dp?-GKhQyQDxo5xA%*qj$=R zJCmqfsgyxATUIO0-j5XVww}i=ZOEd$Zr;n2wVkUDn#E86zG7dASP1|R$4GdqirW~A5qrFyNxeVK_&^v}nKKLpvrx3AA>c+A1T1VJ zF8!ogpzh>juasn3@L`^&$w80d_5*!vRar5T)^8{mH*vdanBa(*?7dMo)&%}Ape1U2 zl+mQpoh7GZ*Q_sh6tsPgL(DWu{##4F^s!oAenlveF3R;x?P7Q79-E5Y5ur~{w!L9n zE2oAsm0F(QnZCsi$H?S|cTLu{iCj39-0q3Um_nGD!n$TW_-Pfn zwd)@{#;nsxyGXBSlu`H_dnrTN7@hdY%ehPO64vxup_yBX>%oL%TF zkYC`(t$rFKr@D7bk~B85M}QjXGARz2H+A;16C7Ww19Q0eEho_XJ&UH7+}#3II_KR- zvK>*zo1DI|UvCrR>p?)Xt_|P=I#mWPX537kCJMWXtdz`i&Tl*0rb!_?kGjLc*TRKI zr)aOMTo|lYsO-!tR1svEstq6&^-v9=9)kw zlRR8}9z7%dnLeF-b?vON^O?zrj=LR09r~9`?S8iyj8j=!Cvc^apyq?oWYo=cL(Q2$ zdoB14b$>}DR>M8&5{f0jtslUA9MmwMNAVTR!S2+MT7Mt$*8w0D2GB6bdBT23D4m}FjO z_VvL)rW$1Pqr@+o#XU->Ypjvt3q--McQiqYQ(HB!iuS%WNs-`+u+Zyv75(h1?`H*+ z**AWu?Nx@koT+s%2c1pImY1?eENdD(h+ecnjtul70*hG>0V?jDp2vk%Ey#iZZkyTz zePFrIOi}V*HL3hx0;Um|L6T?z?XDiuNOBv2m}sbh9V>|oB6f<0uK*~S?#-4qHu?Fj zJ}uyTOK`ITudW_He$2_rYNLKs#sJ`!Z4nj`np#j&bK;sF%!wVd%sgZdM~a~T?G3}5 zn-rsy*|Ge=)@y5LXJ=;E(jx%iLqSzy^J$FmM5C$(on%6(@+bt2ksr-yJJ6CA-gVodB1ZLYvlIWq(|E%|k1h!>(w~kHaV{u^&gfv$rPlE61mm5b z0KKsP1lhHhd56i95rw5BJgbblBgUr=1(mOBAB84;Pi;^kJUGl1SZ$uMP<9#o)Qu(| zxQ{ZS^u!x(?L0DJfZVSNS>c*8F|FurGXzF2LrAd?t|Y2kf$=T;P6S7jCkN*vJE?0n zrY3jhry}?4e!N(C>~cHwG{?E=N790=h{sZG)Jd(B@5umB(T|h~Q_~fc_Wpn(o&pNC zBIjC9`5?^hn|@I4pAj*)(m z>pFUrN_I!MwX%XAtu~FcMBm>GZDOvul^MO`38GKa=9k6hl${N@cZMtkUi(&pyWKe@ zlB1#9retpU_0Rl^;TSg;Q=qmV*nWYE18-1p&ew%tJ#KSi3+C+uHJxX_VRmb&^B#9L zSdbDPm$_(`6PoQ@hZ1_X&@4|AhtOphAw6Dj&X4fVtpTTg3S%Qee+`<}b0)S4 z#RfXaT%W@aR7FzeDSVSl?&L9i^0p(OygFmLX5Qb)|-wWZ1iyyHir3( z=&k*Pc5Fpu3?`_??WRa9BD5iZ7HYLFHX8hE(sI~STp~QOqr)Zd9=rysyjPu>qO{B$ z#jz}hBVX&3mo7+Pcc2u>38e_e5pEVD>{eG-R6!wFz+hS)H08`fu9FC&8C!8NOUVL=cox#J5WOj@KwNKLE?l*Jnct2SJ^qNVuTbzWY1R4r01e$sz;aUOUt9$ zU^_lXNI%FwVqMKM4aYdXK>IrpOX6kzSBQRzok7hFJLG3bh%qw6>fbU8$`@c}Ukfe# zAuV4ZSoS5=f?_Cu`WRx&8c$s{#m&*12;u4ShId4FTd-LO@cqk`je5TsdFkA+BScC1 zrhEHhJLvuIKerN7T!LtH*S<)kfiL0ic@^u^7-+&eQ~SD{y=?nRI>)KJD6DyP#Cm2r&|->jL0vq+i~j1H zJRyNP5aBU%$@jUbE&r(f3Ou6M>~3gV5@smGs{-$W*Wtj(=<5nSf9( z9K&ngABwd{{mjP9>_sq2YD5$G3x5NY?DtbtC9PWX**;#B5La8!f$31~XMSx?J(Qe` zn7*5+j=&-AIlj(TBVQXnz5$B_0oKrRB?b`-!C(|qEywlAH^q_3JG~7?3bJZVSslai zBCOg{{$dE~p{3EYe+~lS+hqA3D3&zAr-ZX#c*AW@NOGu zUfIIqVBglhAq#nOJE1Z;vkT1AdEB0mr*m*y4n$@M((xqjnebVl;?c@|nI}|jyg_lZ zP!d%7+&n6IHGSGgD=58wi2fTUj>$txfQ~mrvCB7Px&%bKKlcl!x;_qZ%eR`F>+cAE zcydx3HM&uVp1Oas_ESnNkEuvgD~8;0d>o6|T2I%}?4Y6+Cg}gU0#^Ne_8SXc*-?pc zFAsre%PxH%YqqlTkfPQI)lsCKe+-=d^stv@W`eNr<2GlJI`%{#ogXAy_5xnkt}w8(;cgGe z@Ngicwc6m~ZZ|5`P*;9h+amTIV;{|(#+h}f$L^~;=T!yYCiv?qcv)7r0Gk;+y((Vd zSESkZYE)HlBiAYj;G?HmNgg@q`_~Y4v#rdwQf^VmD_4T#ek&2JG4l9Po-Evt zhF`*{A|@x_OUPi(1D;e-`a*|$r2nRJr*XMHhm=0is27?`3HgaeCo@}Xt%GIItbW%@-+@bnrFV>Sc!85>Po){k& z{--yY)y7OVwfS>V`%7BMe*1b5nC)8Y^1CgGjy-k&GMrxz_!6sZ`(c-CWZS>EKde`8E@Q1#E}wJ zQPcVIEj5shlJjvqH-Die^INroWcV7*ScM2X#u0~$<&hO~lR9&yZZV8^)=Z&n2);n; ztBwB}o3AXUDtsF_F}{a&@R%ZaZUZ&iC^MMmSBS&plr)~Ij%POR3JNp!Y0AeF_+}EV z&$=1i)eh&4jX6t}YL+$yHet)N;=t&Oa$4#*b8(ILi&y-12Zsf-*xi`T{#i^2zAICE z&Unp+u=HBl-@L88tr;0JPCMqMAU4OY$N_tcJ3*|OAp;;^H@$C9Xz0EZUBE9ygXNCE z5ReSg*jy4dfB15}Vb4eOx{t+nd0uuihuPPQjf0g|H(5X(b@#yC1qS)qL`(t;j8Wl9&ml^-uD@gzCJXhqbML}q1uXN|2 z)H?zSu7Rll&?K&qEN^4Tuxv;=AHDD!FcZY1?~hz29b-)G_v2A62`>-6(BX)}-FYFwf~m_Ei=bwH(<2xrBrbaOV#zG;tIivop; zyj;h!%f*pHk1%u#a0%W6o-4DMCe~it#Etv{u;Mv-IJ4aXD-G#i_hvN#2DS)ClZn3$vY})SXMoyJC!! z*q*((&neaS9tw@`y0PnT_cZA)M-_*hC60V~AxH9mPBMVk+1$2qAx`g31FuS}PBzdr z2V?Pha2r7D&3;>jTdOIrL9Jm94&`))YZS-yz?@J+`|nJUl9c<(aFk+JG7>oO1D;^#dQ^*1C!Q1LsM6>uqMF6fE;ou!_-{|I`eZ_=QAQ4 z0DYP9;=WaPsC6EIdz`>d#xArSeJ-E))zsT~(vt_pT_HBzvk5tF_JuF$Vxe=Qzo$np zRC^L6TGFoKMwrJ_Jnt6go7T_VPvqD)MV7n-pd+mu{Hqb^^6?k-iu}t)T(?IVGNy&) zbMIr+4Ym5}PalOc*R(aSHzIFs$6gdtz8~}k=ruK!Jp}I6#M~{jY}W#QpO7VHv$LIu z8AmBePuECf>u#&C_2JTM(Gx3xmqsT|!Cm2Rv(p(W7M{-e2FM)V3eB4=GO1)Fm@n@t z6Ye%}fm-xehiWZHaIJk^-0qo7x#f`ux6yO5+(OYjRlKA=nbjRx>-V@Et=bIYwk3?Y zxr2pnf%~JRt<5o18U~Fk7tV!jnDk`73rA{avE+sKUjzm-$d z&;s6|luaUw)~Wv>Drx_lk1L_(kM-e$v(7o%GXHZ5SrFjv6mPz1L8j?SahVJHnXg~h z31^i?-LrI5h%6bDO@xCAzh-7;DzskwJt${)P7{PVk4;U<$$Qt5<_zl~)I!(|>yYy`0>a1)-oZ%XV|n z`24;st0gk)K-$R^2P^2lSuk7fG)ate#2BA#-($kH>h7iSPXRR!mx>mD#oen~TdANp zgRG%>ZK2s^8zAxb;^vPhM7v@acYx(VGC=oijGtOwr6Bu0C Date: Sat, 14 Sep 2024 15:52:30 +0800 Subject: [PATCH 74/94] fix(stmt2/gettags): return zero tags for normal tables --- source/libs/parser/src/parInsertStmt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index b67b7fc583..57f939d6d6 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -892,11 +892,11 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA if (NULL == tags) { return TSDB_CODE_APP_ERROR; } - + /* if (pDataBlock->pMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pMeta->tableType != TSDB_CHILD_TABLE) { return TSDB_CODE_TSC_STMT_API_ERROR; } - + */ SSchema* pSchema = getTableTagSchema(pDataBlock->pMeta); if (tags->numOfBound <= 0) { *fieldNum = 0; From 0a19bc974bd0ed13e8f49824ec11236d374c946f Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Sat, 14 Sep 2024 15:59:01 +0800 Subject: [PATCH 75/94] set wr lock to write priority --- source/libs/wal/src/walMgmt.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 31085e7919..3bb4b9d747 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -81,6 +81,15 @@ void walCleanUp() { } } +static int32_t walInitLock(SWal *pWal) { + TdThreadRwlockAttr attr; + (void)taosThreadRwlockAttrInit(&attr); + (void)taosThreadRwlockAttrSetKindNP(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + (void)taosThreadRwlockInit(&pWal->mutex, &attr); + (void)taosThreadRwlockAttrDestroy(&attr); + return 0; +} + SWal *walOpen(const char *path, SWalCfg *pCfg) { SWal *pWal = taosMemoryCalloc(1, sizeof(SWal)); if (pWal == NULL) { @@ -88,7 +97,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } - if (taosThreadRwlockInit(&pWal->mutex, NULL) < 0) { + if (walInitLock(pWal) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosMemoryFree(pWal); return NULL; From ae9eaf9d1a9b11f6dc16c3e75d78433d26ebd4fb Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Sat, 14 Sep 2024 16:14:56 +0800 Subject: [PATCH 76/94] osSys --- include/os/osSysinfo.h | 2 +- source/os/src/osSysinfo.c | 48 +++++++++++++++++++++++++++------------ 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index a6d8a6502f..fb6acf2a0e 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -80,7 +80,7 @@ typedef struct { SysNameInfo taosGetSysNameInfo(); bool taosCheckCurrentInDll(); -int taosGetlocalhostname(char *hostname, size_t maxLen); +int32_t taosGetlocalhostname(char *hostname, size_t maxLen); #ifdef __cplusplus } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index fc6296bc04..4d97232806 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -93,6 +93,10 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { path[len - 1] = 'p'; HANDLE file = CreateFile(path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file == INVALID_HANDLE_VALUE) { + FreeLibrary(dll); + return EXCEPTION_CONTINUE_SEARCH; + } MINIDUMP_EXCEPTION_INFORMATION mei; mei.ThreadId = GetCurrentThreadId(); @@ -333,7 +337,10 @@ bool getWinVersionReleaseName(char *releaseName, int32_t maxLen) { UINT uLen; VS_FIXEDFILEINFO *pFileInfo; - GetWindowsDirectory(szFileName, MAX_PATH); + int ret = GetWindowsDirectory(szFileName, MAX_PATH); + if (ret == 0) { + return false; + } wsprintf(szFileName, L"%s%s", szFileName, L"\\explorer.exe"); dwLen = GetFileVersionInfoSize(szFileName, &dwHandle); if (dwLen == 0) { @@ -373,12 +380,12 @@ int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t if(sName) snprintf(sName, maxLen, "macOS"); if (sysctl(osversion_name, 2, osversion, &osversion_len, NULL, 0) == -1) { - return -1; + return TAOS_SYSTEM_ERROR(errno); } uint32_t major, minor; - if (sscanf(osversion, "%u.%u", &major, &minor) != 2) { - return -1; + if (sscanf(osversion, "%u.%u", &major, &minor) == EOF) { + return TAOS_SYSTEM_ERROR(errno); } if (major >= 20) { major -= 9; // macOS 11 and newer @@ -433,8 +440,11 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { #ifdef WINDOWS char value[100]; DWORD bufferSize = sizeof(value); - RegGetValue(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "ProcessorNameString", + LSTATUS ret = RegGetValue(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "ProcessorNameString", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + if (ret != ERROR_SUCCESS) { + return TAOS_SYSTEM_ERROR(ret); + } tstrncpy(cpuModel, value, maxLen); SYSTEM_INFO si; memset(&si, 0, sizeof(SYSTEM_INFO)); @@ -696,7 +706,7 @@ int32_t taosGetTotalMemory(int64_t *totalKB) { MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { - return -1; + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); } *totalKB = memsStat.ullTotalPhys / 1024; @@ -705,6 +715,9 @@ int32_t taosGetTotalMemory(int64_t *totalKB) { return 0; #else *totalKB = (int64_t)(sysconf(_SC_PHYS_PAGES) * tsPageSizeKB); + if(*totalKB <= 0) { + return TAOS_SYSTEM_ERROR(errno); + } return 0; #endif } @@ -760,7 +773,7 @@ int32_t taosGetSysMemory(int64_t *usedKB) { MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { - return -1; + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); } int64_t nMemFree = memsStat.ullAvailPhys / 1024; @@ -773,6 +786,9 @@ int32_t taosGetSysMemory(int64_t *usedKB) { return 0; #else *usedKB = sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB; + if(*usedKB <= 0) { + return TAOS_SYSTEM_ERROR(errno); + } return 0; #endif } @@ -832,7 +848,7 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int if (write_bytes) *write_bytes = 0; return 0; } - return -1; + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); #elif defined(_TD_DARWIN_64) if (rchars) *rchars = 0; if (wchars) *wchars = 0; @@ -1022,7 +1038,10 @@ void taosKillSystem() { int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { #ifdef WINDOWS GUID guid; - CoCreateGuid(&guid); + HRESULT h = CoCreateGuid(&guid); + if (h != S_OK) { + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + } snprintf(uid, uidlen, "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", guid.Data1, guid.Data2, guid.Data3, guid.Data4[0], guid.Data4[1], guid.Data4[2], guid.Data4[3], guid.Data4[4], guid.Data4[5], guid.Data4[6], guid.Data4[7]); @@ -1226,14 +1245,13 @@ SysNameInfo taosGetSysNameInfo() { } char localHostName[512]; - taosGetlocalhostname(localHostName, 512); + TAOS_SKIP_ERROR(taosGetlocalhostname(localHostName, 512)); TdCmdPtr pCmd = taosOpenCmd("scutil --get LocalHostName"); tstrncpy(info.nodename, localHostName, sizeof(info.nodename)); return info; #else SysNameInfo info = {0}; - struct utsname uts; if (!uname(&uts)) { tstrncpy(info.sysname, uts.sysname, sizeof(info.sysname)); @@ -1269,7 +1287,7 @@ bool taosCheckCurrentInDll() { } #ifdef _TD_DARWIN_64 -int taosGetMaclocalhostnameByCommand(char *hostname, size_t maxLen) { +int32_t taosGetMaclocalhostnameByCommand(char *hostname, size_t maxLen) { TdCmdPtr pCmd = taosOpenCmd("scutil --get LocalHostName"); if (pCmd != NULL) { if (taosGetsCmd(pCmd, maxLen - 1, hostname) > 0) { @@ -1281,10 +1299,10 @@ int taosGetMaclocalhostnameByCommand(char *hostname, size_t maxLen) { } taosCloseCmd(&pCmd); } - return -1; + return TAOS_SYSTEM_ERROR(errno); } -int getMacLocalHostNameBySCD(char *hostname, size_t maxLen) { +int32_t getMacLocalHostNameBySCD(char *hostname, size_t maxLen) { SCDynamicStoreRef store = SCDynamicStoreCreate(NULL, CFSTR(""), NULL, NULL); CFStringRef hostname_cfstr = SCDynamicStoreCopyLocalHostName(store); if (hostname_cfstr != NULL) { @@ -1298,7 +1316,7 @@ int getMacLocalHostNameBySCD(char *hostname, size_t maxLen) { } #endif -int taosGetlocalhostname(char *hostname, size_t maxLen) { +int32_t taosGetlocalhostname(char *hostname, size_t maxLen) { #ifdef _TD_DARWIN_64 int res = getMacLocalHostNameBySCD(hostname, maxLen); if (res != 0) { From 045a04184406519537442bdc3f80c71b7c8cbf85 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Sat, 14 Sep 2024 16:25:07 +0800 Subject: [PATCH 77/94] fix: osSys --- source/os/src/osSystem.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index a8a9ff681b..fe52369a53 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -31,7 +31,7 @@ void WINAPI windowsServiceCtrlHandle(DWORD request) { ServiceStatus.dwCurrentState = SERVICE_STOP_PENDING; if (!SetServiceStatus(hServiceStatusHandle, &ServiceStatus)) { DWORD nError = GetLastError(); - printf("failed to send stopped status to windows service: %d", nError); + fprintf(stderr, "failed to send stopped status to windows service: %d", nError); } break; default: @@ -50,19 +50,19 @@ void WINAPI mainWindowsService(int argc, char** argv) { hServiceStatusHandle = RegisterServiceCtrlHandler("taosd", &windowsServiceCtrlHandle); if (hServiceStatusHandle == 0) { DWORD nError = GetLastError(); - printf("failed to register windows service ctrl handler: %d", nError); + fprintf(stderr, "failed to register windows service ctrl handler: %d", nError); } ServiceStatus.dwCurrentState = SERVICE_RUNNING; if (SetServiceStatus(hServiceStatusHandle, &ServiceStatus)) { DWORD nError = GetLastError(); - printf("failed to send running status to windows service: %d", nError); + fprintf(stderr, "failed to send running status to windows service: %d", nError); } if (mainWindowsFunc != NULL) mainWindowsFunc(argc, argv); ServiceStatus.dwCurrentState = SERVICE_STOPPED; if (!SetServiceStatus(hServiceStatusHandle, &ServiceStatus)) { DWORD nError = GetLastError(); - printf("failed to send stopped status to windows service: %d", nError); + fprintf(stderr, "failed to send stopped status to windows service: %d", nError); } } void stratWindowsService(MainWindows mainWindows) { @@ -140,17 +140,27 @@ void taosCloseDll(void* handle) { } #endif -int taosSetConsoleEcho(bool on) { +int32_t taosSetConsoleEcho(bool on) { #if defined(WINDOWS) HANDLE hStdin = GetStdHandle(STD_INPUT_HANDLE); + if (hStdin == INVALID_HANDLE_VALUE) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + return terrno; + } DWORD mode = 0; - GetConsoleMode(hStdin, &mode); + if(!GetConsoleMode(hStdin, &mode)){ + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + return terrno; + } if (on) { mode |= ENABLE_ECHO_INPUT; } else { mode &= ~ENABLE_ECHO_INPUT; } - SetConsoleMode(hStdin, mode); + if(!SetConsoleMode(hStdin, mode)) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + return terrno; + } return 0; #else From 1ccc74d768ceb74ffd1b0299fc1df4cfa307d07e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 14 Sep 2024 17:28:24 +0800 Subject: [PATCH 78/94] fix: add memory alloc validation --- source/libs/catalog/src/ctgCache.c | 3 +++ source/libs/catalog/src/ctgDbg.c | 4 ++++ source/libs/executor/src/mergejoinoperator.c | 3 +++ source/libs/scheduler/src/schRemote.c | 6 ++++++ 4 files changed, 16 insertions(+) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 0e54ac77a2..9a72a6d89f 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -3710,6 +3710,9 @@ int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVg } *pVgroup = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + if (NULL == *pVgroup) { + CTG_ERR_JRET(terrno); + } CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, NULL, dbCache->vgCache.vgInfo, pTableName, *pVgroup)); _return: diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index a95df06307..0ac606d4d6 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -291,6 +291,10 @@ int32_t ctgdHandleDbgCommand(char *command) { } char *dup = taosStrdup(command); + if (NULL == dup) { + CTG_RET(terrno); + } + char *option = NULL; char *param = NULL; diff --git a/source/libs/executor/src/mergejoinoperator.c b/source/libs/executor/src/mergejoinoperator.c index 4715aa8b96..af5e4ed235 100644 --- a/source/libs/executor/src/mergejoinoperator.c +++ b/source/libs/executor/src/mergejoinoperator.c @@ -149,6 +149,9 @@ int32_t mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* len = varDataTLen(p1); } char* p2 = taosMemoryMalloc(len); + if (NULL == p2) { + MJ_ERR_RET(terrno); + } TAOS_MEMCPY(p2, p1, len); code = colDataSetVal(pDst, numOfRows, p2, false); if (code) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 40391cea7e..7ff3d1386c 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -267,6 +267,9 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD if (pMsg->pData) { SDecoder coder = {0}; SSubmitRsp2 *rsp = taosMemoryMalloc(sizeof(*rsp)); + if (NULL == rsp) { + SCH_ERR_JRET(terrno); + } tDecoderInit(&coder, pMsg->pData, msgSize); code = tDecodeSSubmitRsp2(&coder, rsp); tDecoderClear(&coder); @@ -957,6 +960,9 @@ int32_t schUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, SQueryNodeAddr *addr pMsgSendInfo->target.type = TARGET_TYPE_VNODE; pMsgSendInfo->target.vgId = addr->nodeId; pMsgSendInfo->target.dbFName = taosStrdup(pTask->plan->dbFName); + if (NULL == pMsgSendInfo->target.dbFName) { + return terrno; + } } return TSDB_CODE_SUCCESS; From f684e12c27b46d750884da2c411b9b1e3638a92b Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Sat, 14 Sep 2024 18:11:36 +0800 Subject: [PATCH 79/94] pointer check --- source/client/src/clientImpl.c | 4 ++ source/client/src/clientRawBlockWrite.c | 8 ++++ source/libs/catalog/src/catalog.c | 4 ++ source/libs/catalog/src/ctgAsync.c | 4 ++ source/libs/command/src/command.c | 4 ++ source/libs/executor/src/sysscanoperator.c | 4 ++ source/libs/function/src/tudf.c | 3 ++ source/libs/function/src/udfd.c | 9 ++++ source/libs/qcom/src/queryUtil.c | 14 ++++++ source/libs/qcom/src/querymsg.c | 51 ++++++++++++++++++++++ 10 files changed, 105 insertions(+) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5fba447c8a..5ba3a9f786 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2240,6 +2240,10 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int } else if (tTagIsJson(data)) { char* jsonString = NULL; parseTagDatatoJson(data, &jsonString); + if(jsonString == NULL) { + tscError("doConvertJson error: parseTagDatatoJson failed"); + return terrno; + } STR_TO_VARSTR(dst, jsonString); taosMemoryFree(jsonString); } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 43e17b381b..8a888a2a47 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -417,6 +417,10 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { } char* pJson = NULL; parseTagDatatoJson(pTag, &pJson); + if(pJson == NULL) { + uError("parseTagDatatoJson failed, pJson == NULL"); + goto end; + } cJSON* tag = cJSON_CreateObject(); RAW_NULL_CHECK(tag); STagVal* pTagVal = taosArrayGet(pTagVals, 0); @@ -727,6 +731,10 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { goto end; } parseTagDatatoJson(vAlterTbReq.pTagVal, &buf); + if(buf == NULL) { + uError("parseTagDatatoJson failed, buf == NULL"); + goto end; + } } else { if (vAlterTbReq.tagType == TSDB_DATA_TYPE_VARBINARY) { buf = taosMemoryCalloc(vAlterTbReq.nTagVal * 2 + 2 + 3, 1); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index dff607f464..46094fd713 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -448,6 +448,10 @@ int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, char* pJson = NULL; parseTagDatatoJson(pTag, &pJson); + if(NULL == pJson) { + taosArrayDestroy(pTagVals); + CTG_ERR_JRET(terrno); + } STagVal tagVal; tagVal.cid = 0; tagVal.type = TSDB_DATA_TYPE_JSON; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 9940474891..520c715b01 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -2093,6 +2093,10 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* char* pJson = NULL; parseTagDatatoJson(pTag, &pJson); + if (NULL == pJson) { + taosArrayDestroy(pTagVals); + CTG_ERR_JRET(terrno); + } STagVal tagVal; tagVal.cid = 0; tagVal.type = TSDB_DATA_TYPE_JSON; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 2f756e8121..eb74b81141 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -565,6 +565,10 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { if (tTagIsJson(pTag)) { char* pJson = NULL; parseTagDatatoJson(pTag, &pJson); + if(NULL == pJson) { + qError("failed to parse tag to json, pJson is NULL"); + return terrno; + } *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson); taosMemoryFree(pJson); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index bf1153f412..18344d9e79 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1114,6 +1114,10 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, if (tagType == TSDB_DATA_TYPE_JSON) { char* tagJson = NULL; parseTagDatatoJson(tagData, &tagJson); + if (tagJson == NULL) { + code = terrno; + goto _end; + } tagVarChar = taosMemoryMalloc(strlen(tagJson) + VARSTR_HEADER_SIZE); QUERY_CHECK_NULL(tagVarChar, code, lino, _end, terrno); memcpy(varDataVal(tagVarChar), tagJson, strlen(tagJson)); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 0510b6cff5..c2cd8b8c3d 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -900,6 +900,9 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo udfCol->colData.fixLenCol.dataLen = colDataGetLength(col, udfBlock->numOfRows); int32_t dataLen = udfCol->colData.fixLenCol.dataLen; udfCol->colData.fixLenCol.data = taosMemoryMalloc(udfCol->colData.fixLenCol.dataLen); + if (NULL == udfCol->colData.fixLenCol.data) { + return terrno; + } char *data = udfCol->colData.fixLenCol.data; memcpy(data, col->pData, dataLen); } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 6ae718c101..bef0d06501 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -92,6 +92,9 @@ int32_t udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfNa int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) { int32_t err = 0; SUdfCPluginCtx *udfCtx = taosMemoryCalloc(1, sizeof(SUdfCPluginCtx)); + if (NULL == udfCtx) { + return terrno; + } err = uv_dlopen(udf->path, &udfCtx->lib); if (err != 0) { fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); @@ -606,6 +609,9 @@ int32_t udfdInitUdf(char *udfName, SUdf *udf) { int32_t udfdNewUdf(SUdf **pUdf, const char *udfName) { SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); + if (NULL == udfNew) { + return terrno; + } udfNew->refCount = 1; udfNew->lastFetchTime = taosGetTimestampMs(); strncpy(udfNew->name, udfName, TSDB_FUNC_NAME_LEN); @@ -1105,6 +1111,9 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) { taosArrayDestroy(retrieveReq.pFuncNames); SUdfdRpcSendRecvInfo *msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo)); + if(NULL == msgInfo) { + return terrno; + } msgInfo->rpcType = UDFD_RPC_RETRIVE_FUNC; msgInfo->param = udf; if(uv_sem_init(&msgInfo->resultSem, 0) != 0) { diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 30902269c3..e08d4cef87 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -514,6 +514,9 @@ end: taosArrayDestroy(pTagVals); if (string == NULL) { string = taosStrdup(TSDB_DATA_NULL_STR_L); + if(string == NULL) { + qError("failed to strdup null string"); + } } *jsonStr = string; } @@ -629,6 +632,7 @@ int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { (*pDst)->flags = pSrc->flags; if (pSrc->name) { (*pDst)->name = taosStrdup(pSrc->name); + if (NULL == (*pDst)->name) goto _exit; } (*pDst)->uid = pSrc->uid; (*pDst)->btime = pSrc->btime; @@ -636,21 +640,25 @@ int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { (*pDst)->commentLen = pSrc->commentLen; if (pSrc->comment) { (*pDst)->comment = taosStrdup(pSrc->comment); + if (NULL == (*pDst)->comment) goto _exit; } (*pDst)->type = pSrc->type; if (pSrc->type == TSDB_CHILD_TABLE) { if (pSrc->ctb.stbName) { (*pDst)->ctb.stbName = taosStrdup(pSrc->ctb.stbName); + if (NULL == (*pDst)->ctb.stbName) goto _exit; } (*pDst)->ctb.tagNum = pSrc->ctb.tagNum; (*pDst)->ctb.suid = pSrc->ctb.suid; if (pSrc->ctb.tagName) { (*pDst)->ctb.tagName = taosArrayDup(pSrc->ctb.tagName, NULL); + if (NULL == (*pDst)->ctb.tagName) goto _exit; } STag* pTag = (STag*)pSrc->ctb.pTag; if (pTag) { (*pDst)->ctb.pTag = taosMemoryMalloc(pTag->len); + if(NULL == (*pDst)->ctb.pTag) goto _exit; memcpy((*pDst)->ctb.pTag, pTag, pTag->len); } } else { @@ -658,11 +666,17 @@ int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { (*pDst)->ntb.schemaRow.version = pSrc->ntb.schemaRow.nCols; if (pSrc->ntb.schemaRow.nCols > 0 && pSrc->ntb.schemaRow.pSchema) { (*pDst)->ntb.schemaRow.pSchema = taosMemoryMalloc(pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); + if (NULL == (*pDst)->ntb.schemaRow.pSchema) goto _exit; memcpy((*pDst)->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); } } return TSDB_CODE_SUCCESS; + +_exit: + tdDestroySVCreateTbReq(*pDst); + taosMemoryFree(*pDst); + return terrno; } void freeDbCfgInfo(SDbCfgInfo* pInfo) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index b5b660a51b..8725496ed2 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -85,6 +85,9 @@ int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int3 int32_t bufLen = tSerializeSTableInfoReq(NULL, 0, &infoReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSTableInfoReq(pBuf, bufLen, &infoReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -112,6 +115,9 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSUseDbReq(pBuf, bufLen, &usedbReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -133,6 +139,9 @@ int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t int32_t bufLen = tSerializeSQnodeListReq(NULL, 0, &qnodeListReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSQnodeListReq(pBuf, bufLen, &qnodeListReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -154,6 +163,9 @@ int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t int32_t bufLen = tSerializeSDnodeListReq(NULL, 0, &dnodeListReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSDnodeListReq(pBuf, bufLen, &dnodeListReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -174,6 +186,9 @@ int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t int32_t bufLen = tSerializeSServerVerReq(NULL, 0, &req); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSServerVerReq(pBuf, bufLen, &req) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -195,6 +210,9 @@ int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t int32_t bufLen = tSerializeSDbCfgReq(NULL, 0, &dbCfgReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSDbCfgReq(pBuf, bufLen, &dbCfgReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -216,6 +234,9 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t int32_t bufLen = tSerializeSUserIndexReq(NULL, 0, &indexReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSUserIndexReq(pBuf, bufLen, &indexReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -247,8 +268,13 @@ int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int3 int32_t bufLen = tSerializeSRetrieveFuncReq(NULL, 0, &funcReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + taosArrayDestroy(funcReq.pFuncNames); + return terrno; + } if(tSerializeSRetrieveFuncReq(pBuf, bufLen, &funcReq) < 0) { + taosArrayDestroy(funcReq.pFuncNames); return TSDB_CODE_TSC_INVALID_INPUT; } @@ -270,6 +296,9 @@ int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32 int32_t bufLen = tSerializeSGetUserAuthReq(NULL, 0, &req); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if (tSerializeSGetUserAuthReq(pBuf, bufLen, &req) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -290,6 +319,9 @@ int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_ int32_t bufLen = tSerializeSTableIndexReq(NULL, 0, &indexReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSTableIndexReq(pBuf, bufLen, &indexReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -314,6 +346,9 @@ int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t int32_t bufLen = tSerializeSTableCfgReq(NULL, 0, &cfgReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSTableCfgReq(pBuf, bufLen, &cfgReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -335,6 +370,9 @@ int32_t queryBuildGetViewMetaMsg(void *input, char **msg, int32_t msgSize, int32 int32_t bufLen = tSerializeSViewMetaReq(NULL, 0, &req); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSViewMetaReq(pBuf, bufLen, &req) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -357,6 +395,9 @@ int32_t queryBuildGetTableTSMAMsg(void *input, char **msg, int32_t msgSize, int3 int32_t bufLen = tSerializeTableTSMAInfoReq(NULL, 0, &req); void * pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeTableTSMAInfoReq(pBuf, bufLen, &req) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -379,6 +420,10 @@ int32_t queryBuildGetTSMAMsg(void *input, char **msg, int32_t msgSize, int32_t * int32_t bufLen = tSerializeTableTSMAInfoReq(NULL, 0, &req); void * pBuf = (*mallcFp)(bufLen); + if(pBuf == NULL) + { + return terrno; + } if(tSerializeTableTSMAInfoReq(pBuf, bufLen, &req) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -396,6 +441,9 @@ int32_t queryBuildGetStreamProgressMsg(void* input, char** msg, int32_t msgSize, int32_t len = tSerializeStreamProgressReq(NULL, 0, input); void* pBuf = (*mallcFp)(len); + if (NULL == pBuf) { + return terrno; + } if(tSerializeStreamProgressReq(pBuf, len, input) < 0) { @@ -666,6 +714,9 @@ int32_t queryProcessGetSerVerRsp(void *output, char *msg, int32_t msgSize) { } *(char **)output = taosStrdup(out.ver); + if (NULL == *(char **)output) { + return terrno; + } return code; } From c4a49a7bd9098dc0a0f1127830c3f807dbacc58a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 15 Sep 2024 00:45:45 +0800 Subject: [PATCH 80/94] refactor: do some internal refactor. --- source/libs/stream/src/streamCheckpoint.c | 80 +++++++++---------- source/libs/stream/src/streamMeta.c | 94 ----------------------- source/libs/stream/src/streamStartTask.c | 58 +++++++++++++- source/libs/stream/src/streamTask.c | 2 +- source/libs/stream/src/streamUtil.c | 51 ++++++++++++ 5 files changed, 148 insertions(+), 137 deletions(-) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 35d5ba4e08..1a8b5c1028 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -209,29 +209,18 @@ int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStream return code; } -int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { - SSDataBlock* pDataBlock = taosArrayGet(pBlock->blocks, 0); - if (pDataBlock == NULL) { - return TSDB_CODE_INVALID_PARA; - } - - int64_t checkpointId = pDataBlock->info.version; - int32_t transId = pDataBlock->info.window.skey; - const char* id = pTask->id.idStr; - int32_t code = TSDB_CODE_SUCCESS; - int32_t vgId = pTask->pMeta->vgId; - int32_t taskLevel = pTask->info.taskLevel; +static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t checkpointId, SStreamDataBlock* pBlock, + int32_t transId) { + int32_t code = 0; + int32_t vgId = pTask->pMeta->vgId; + int32_t taskLevel = pTask->info.taskLevel; + const char* id = pTask->id.idStr; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; - - streamMutexLock(&pTask->lock); if (pTask->chkInfo.checkpointId > checkpointId) { stError("s-task:%s vgId:%d current checkpointId:%" PRId64 " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); - streamMutexUnlock(&pTask->lock); - - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } @@ -239,37 +228,33 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock stError("s-task:%s vgId:%d checkpointId:%" PRId64 " transId:%d, has been marked failed, failedId:%" PRId64 " discard the checkpoint-trigger block", id, vgId, checkpointId, transId, pActiveInfo->failedId); - streamMutexUnlock(&pTask->lock); - - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } if (pTask->chkInfo.checkpointId == checkpointId) { { // send checkpoint-ready msg to upstream - SRpcMsg msg = {0}; + SRpcMsg msg = {0}; SStreamUpstreamEpInfo* pInfo = NULL; streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId, &pInfo); if (pInfo == NULL) { - streamMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_TASK_NOT_EXIST; } code = initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg); if (code == TSDB_CODE_SUCCESS) { code = tmsgSendReq(&pInfo->epSet, &msg); + if (code) { + stError("s-task:%s vgId:%d failed send chkpt-ready msg to upstream, code:%s", id, vgId, tstrerror(code)); + } } } stWarn( - "s-task:%s vgId:%d recv already finished checkpoint msg, send checkpoint-ready to upstream:0x%x to resume the " - "interrupted checkpoint", + "s-task:%s vgId:%d recv already finished checkpoint-trigger, send checkpoint-ready to upstream:0x%x to resume " + "the interrupted checkpoint", id, vgId, pBlock->srcTaskId); streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); - streamMutexUnlock(&pTask->lock); - - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } @@ -278,9 +263,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64 " discard", id, vgId, pActiveInfo->activeId, checkpointId); - streamMutexUnlock(&pTask->lock); - - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } else { // checkpointId == pActiveInfo->activeId if (pActiveInfo->allUpstreamTriggerRecv == 1) { @@ -288,8 +270,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock "s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, " "checkpointId:%" PRId64 " transId:%d", id, vgId, checkpointId, transId); - streamMutexUnlock(&pTask->lock); - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } @@ -298,7 +278,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock for (int32_t i = 0; i < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++i) { STaskCheckpointReadyInfo* p = taosArrayGet(pActiveInfo->pReadyMsgList, i); if (p == NULL) { - streamMutexUnlock(&pTask->lock); return TSDB_CODE_INVALID_PARA; } @@ -306,9 +285,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64 ", prev recvTs:%" PRId64 " discard", pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs); - - streamMutexUnlock(&pTask->lock); - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } } @@ -316,7 +292,33 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock } } + return 0; +} + +int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { + int64_t checkpointId = 0; + int32_t transId = 0; + const char* id = pTask->id.idStr; + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = pTask->pMeta->vgId; + int32_t taskLevel = pTask->info.taskLevel; + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + + SSDataBlock* pDataBlock = taosArrayGet(pBlock->blocks, 0); + if (pDataBlock == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + checkpointId = pDataBlock->info.version; + transId = pDataBlock->info.window.skey; + + streamMutexLock(&pTask->lock); + code = doCheckBeforeHandleChkptTrigger(pTask, checkpointId, pBlock, transId); streamMutexUnlock(&pTask->lock); + if (code) { + streamFreeQitem((SStreamQueueItem*)pBlock); + return code; + } stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64 ", transId:%d current active checkpointId:%" PRId64, @@ -367,6 +369,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock // before the next checkpoint. code = flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock); if (code) { + streamFreeQitem((SStreamQueueItem*)pBlock); return code; } @@ -675,10 +678,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV } streamMetaWLock(pMeta); - - if (streamMetaCommit(pMeta) < 0) { - // persist to disk - } + code = streamMetaCommit(pMeta); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 514e25c689..d8249666c3 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -458,9 +458,6 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, code = createMetaHbInfo(pRid, &pMeta->pHbInfo); TSDB_CHECK_CODE(code, lino, _err); - pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL); - TSDB_CHECK_NULL(pMeta->qHandle, code, lino, _err, terrno); - code = bkdMgtCreate(tpath, (SBkdMgt**)&pMeta->bkdChkptMgt); TSDB_CHECK_CODE(code, lino, _err); @@ -629,9 +626,6 @@ void streamMetaCloseImpl(void* arg) { taosMemoryFree(pMeta->path); streamMutexDestroy(&pMeta->backendMutex); - taosCleanUpScheduler(pMeta->qHandle); - taosMemoryFree(pMeta->qHandle); - bkdMgtDestroy(pMeta->bkdChkptMgt); pMeta->role = NODE_ROLE_UNINIT; @@ -1261,40 +1255,6 @@ void streamMetaStartHb(SStreamMeta* pMeta) { streamMetaHbToMnode(pRid, NULL); } -void streamMetaRLock(SStreamMeta* pMeta) { - // stTrace("vgId:%d meta-rlock", pMeta->vgId); - int32_t code = taosThreadRwlockRdlock(&pMeta->lock); - if (code) { - stError("vgId:%d meta-rlock failed, code:%s", pMeta->vgId, tstrerror(code)); - } -} - -void streamMetaRUnLock(SStreamMeta* pMeta) { - // stTrace("vgId:%d meta-runlock", pMeta->vgId); - int32_t code = taosThreadRwlockUnlock(&pMeta->lock); - if (code != TSDB_CODE_SUCCESS) { - stError("vgId:%d meta-runlock failed, code:%s", pMeta->vgId, tstrerror(code)); - } else { - // stTrace("vgId:%d meta-runlock completed", pMeta->vgId); - } -} - -void streamMetaWLock(SStreamMeta* pMeta) { - // stTrace("vgId:%d meta-wlock", pMeta->vgId); - int32_t code = taosThreadRwlockWrlock(&pMeta->lock); - if (code) { - stError("vgId:%d failed to apply wlock, code:%s", pMeta->vgId, tstrerror(code)); - } -} - -void streamMetaWUnLock(SStreamMeta* pMeta) { - // stTrace("vgId:%d meta-wunlock", pMeta->vgId); - int32_t code = taosThreadRwlockUnlock(&pMeta->lock); - if (code) { - stError("vgId:%d failed to apply wunlock, code:%s", pMeta->vgId, tstrerror(code)); - } -} - int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) { QRY_PARAM_CHECK(pList); @@ -1398,60 +1358,6 @@ int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta) { return 0; } -int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { - int32_t code = TSDB_CODE_SUCCESS; - int64_t now = taosGetTimestampMs(); - int64_t startTs = 0; - bool hasFillhistoryTask = false; - STaskId hId = {0}; - - stDebug("vgId:%d add start failed task:0x%x", pMeta->vgId, taskId); - - streamMetaRLock(pMeta); - - STaskId id = {.streamId = streamId, .taskId = taskId}; - SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); - - if (ppTask != NULL) { - startTs = (*ppTask)->taskCheckInfo.startTs; - hasFillhistoryTask = HAS_RELATED_FILLHISTORY_TASK(*ppTask); - hId = (*ppTask)->hTaskInfo.id; - - streamMetaRUnLock(pMeta); - - // add the failed task info, along with the related fill-history task info into tasks list. - code = streamMetaAddTaskLaunchResult(pMeta, streamId, taskId, startTs, now, false); - if (hasFillhistoryTask) { - code = streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false); - } - } else { - streamMetaRUnLock(pMeta); - - stError("failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", - streamId, taskId, pMeta->vgId); - code = TSDB_CODE_STREAM_TASK_NOT_EXIST; - } - - return code; -} - -void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs) { - int32_t startTs = pTask->execInfo.checkTs; - int32_t code = streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, failedTs, false); - if (code) { - stError("s-task:%s failed to add self task failed to start, code:%s", pTask->id.idStr, tstrerror(code)); - } - - // automatically set the related fill-history task to be failed. - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - STaskId* pId = &pTask->hTaskInfo.id; - code = streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, failedTs, false); - if (code) { - stError("s-task:0x%" PRIx64 " failed to add self task failed to start, code:%s", pId->taskId, tstrerror(code)); - } - } -} - void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId, int64_t startTs) { const char* id = pTask->id.idStr; diff --git a/source/libs/stream/src/streamStartTask.c b/source/libs/stream/src/streamStartTask.c index 98c6534b46..0858f57414 100644 --- a/source/libs/stream/src/streamStartTask.c +++ b/source/libs/stream/src/streamStartTask.c @@ -30,8 +30,8 @@ typedef struct STaskInitTs { } STaskInitTs; static int32_t prepareBeforeStartTasks(SStreamMeta* pMeta, SArray** pList, int64_t now); -static bool allCheckDownstreamRsp(SStreamMeta* pMeta, STaskStartInfo* pStartInfo, int32_t numOfTotal); -static void displayStatusInfo(SStreamMeta* pMeta, SHashObj* pTaskSet, bool succ); +static bool allCheckDownstreamRsp(SStreamMeta* pMeta, STaskStartInfo* pStartInfo, int32_t numOfTotal); +static void displayStatusInfo(SStreamMeta* pMeta, SHashObj* pTaskSet, bool succ); // restore the checkpoint id by negotiating the latest consensus checkpoint id int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { @@ -505,3 +505,57 @@ void streamTaskSetReqConsenChkptId(SStreamTask* pTask, int64_t ts) { stDebug("s-task:%s set req consen-checkpointId flag, prev transId:%d, ts:%" PRId64, pTask->id.idStr, prevTrans, ts); } + +int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t now = taosGetTimestampMs(); + int64_t startTs = 0; + bool hasFillhistoryTask = false; + STaskId hId = {0}; + + stDebug("vgId:%d add start failed task:0x%x", pMeta->vgId, taskId); + + streamMetaRLock(pMeta); + + STaskId id = {.streamId = streamId, .taskId = taskId}; + SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + + if (ppTask != NULL) { + startTs = (*ppTask)->taskCheckInfo.startTs; + hasFillhistoryTask = HAS_RELATED_FILLHISTORY_TASK(*ppTask); + hId = (*ppTask)->hTaskInfo.id; + + streamMetaRUnLock(pMeta); + + // add the failed task info, along with the related fill-history task info into tasks list. + code = streamMetaAddTaskLaunchResult(pMeta, streamId, taskId, startTs, now, false); + if (hasFillhistoryTask) { + code = streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false); + } + } else { + streamMetaRUnLock(pMeta); + + stError("failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", + streamId, taskId, pMeta->vgId); + code = TSDB_CODE_STREAM_TASK_NOT_EXIST; + } + + return code; +} + +void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs) { + int32_t startTs = pTask->execInfo.checkTs; + int32_t code = streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, failedTs, false); + if (code) { + stError("s-task:%s failed to add self task failed to start, code:%s", pTask->id.idStr, tstrerror(code)); + } + + // automatically set the related fill-history task to be failed. + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + STaskId* pId = &pTask->hTaskInfo.id; + code = streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, failedTs, false); + if (code) { + stError("s-task:0x%" PRIx64 " failed to add self task failed to start, code:%s", pId->taskId, tstrerror(code)); + } + } +} diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 9a324084ff..365710f8a7 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -801,8 +801,8 @@ bool streamTaskSetSchedStatusWait(SStreamTask* pTask) { pTask->status.schedStatus = TASK_SCHED_STATUS__WAITING; ret = true; } - streamMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); return ret; } diff --git a/source/libs/stream/src/streamUtil.c b/source/libs/stream/src/streamUtil.c index b79ca32ff3..cef2ba35e7 100644 --- a/source/libs/stream/src/streamUtil.c +++ b/source/libs/stream/src/streamUtil.c @@ -35,3 +35,54 @@ void streamMutexDestroy(TdThreadMutex *pMutex) { stError("%p mutex destroy, code:%s", pMutex, tstrerror(code)); } } + +void streamMetaRLock(SStreamMeta* pMeta) { + // stTrace("vgId:%d meta-rlock", pMeta->vgId); + int32_t code = taosThreadRwlockRdlock(&pMeta->lock); + if (code) { + stError("vgId:%d meta-rlock failed, code:%s", pMeta->vgId, tstrerror(code)); + } +} + +void streamMetaRUnLock(SStreamMeta* pMeta) { + // stTrace("vgId:%d meta-runlock", pMeta->vgId); + int32_t code = taosThreadRwlockUnlock(&pMeta->lock); + if (code != TSDB_CODE_SUCCESS) { + stError("vgId:%d meta-runlock failed, code:%s", pMeta->vgId, tstrerror(code)); + } else { + // stTrace("vgId:%d meta-runlock completed", pMeta->vgId); + } +} + +void streamMetaWLock(SStreamMeta* pMeta) { + // stTrace("vgId:%d meta-wlock", pMeta->vgId); + int32_t code = taosThreadRwlockWrlock(&pMeta->lock); + if (code) { + stError("vgId:%d failed to apply wlock, code:%s", pMeta->vgId, tstrerror(code)); + } +} + +void streamMetaWUnLock(SStreamMeta* pMeta) { + // stTrace("vgId:%d meta-wunlock", pMeta->vgId); + int32_t code = taosThreadRwlockUnlock(&pMeta->lock); + if (code) { + stError("vgId:%d failed to apply wunlock, code:%s", pMeta->vgId, tstrerror(code)); + } +} + +void streamSetFatalError(SStreamMeta* pMeta, int32_t code, const char* funcName, int32_t lino) { + int32_t oldCode = atomic_val_compare_exchange_32(&pMeta->fatalInfo.code, 0, code); + if (oldCode == 0) { + pMeta->fatalInfo.ts = taosGetTimestampMs(); + pMeta->fatalInfo.threadId = taosGetSelfPthreadId(); + tstrncpy(pMeta->fatalInfo.func, funcName, tListLen(pMeta->fatalInfo.func)); + pMeta->fatalInfo.line = lino; + stInfo("vgId:%d set global fatal error, code:%s %s line:%d", pMeta->vgId, tstrerror(code), funcName, lino); + } else { + stFatal("vgId:%d existed global fatal eror:%s, failed to set new fatal error code:%s", pMeta->vgId, code); + } +} + +int32_t streamGetFatalError(const SStreamMeta* pMeta) { + return atomic_load_32((volatile int32_t*) &pMeta->fatalInfo.code); +} From 3332a0b822c41fd593e16444fa95e93d4da3fde5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 15 Sep 2024 01:19:30 +0800 Subject: [PATCH 81/94] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 26 +++++++---- source/dnode/mnode/impl/src/mndStream.c | 52 +++++++++++----------- source/dnode/vnode/src/tqCommon/tqCommon.c | 40 ++++++++++------- source/libs/stream/src/streamMeta.c | 22 +++++---- source/libs/stream/src/streamUtil.c | 5 ++- 5 files changed, 86 insertions(+), 59 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index b77c8535f1..29759bc561 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -494,6 +494,14 @@ typedef struct SScanWalInfo { tmr_h scanTimer; } SScanWalInfo; +typedef struct SFatalErrInfo { + int32_t code; + int64_t ts; + int32_t threadId; + int32_t line; + char func[128]; +} SFatalErrInfo; + // meta typedef struct SStreamMeta { char* path; @@ -523,14 +531,13 @@ typedef struct SStreamMeta { int32_t numOfStreamTasks; // this value should be increased when a new task is added into the meta int32_t numOfPausedTasks; int64_t rid; - - int64_t chkpId; - int32_t chkpCap; - SArray* chkpSaved; - SArray* chkpInUse; - SRWLatch chkpDirLock; - void* qHandle; // todo remove it - void* bkdChkptMgt; + SFatalErrInfo fatalInfo; // fatal error occurs, stream stop to execute + int64_t chkpId; + int32_t chkpCap; + SArray* chkpSaved; + SArray* chkpInUse; + SRWLatch chkpDirLock; + void* bkdChkptMgt; } SStreamMeta; typedef struct STaskUpdateEntry { @@ -776,6 +783,9 @@ void streamMetaRLock(SStreamMeta* pMeta); void streamMetaRUnLock(SStreamMeta* pMeta); void streamMetaWLock(SStreamMeta* pMeta); void streamMetaWUnLock(SStreamMeta* pMeta); +void streamSetFatalError(SStreamMeta* pMeta, int32_t code, const char* funcName, int32_t lino); +int32_t streamGetFatalError(const SStreamMeta* pMeta); + void streamMetaResetStartInfo(STaskStartInfo* pMeta, int32_t vgId); int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pTaskList); void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index e0b8caa938..1fb398d070 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1143,18 +1143,16 @@ int32_t extractStreamNodeList(SMnode *pMnode) { return taosArrayGetSize(execInfo.pNodeList); } -static bool taskNodeIsUpdated(SMnode *pMnode) { - bool allReady = true; - SArray *pNodeSnapshot = NULL; - - // check if the node update happens or not - streamMutexLock(&execInfo.lock); +static int32_t doCheckForUpdated(SMnode *pMnode, SArray **ppNodeSnapshot) { + bool allReady = false; + bool nodeUpdated = false; + SVgroupChangeInfo changeInfo = {0}; int32_t numOfNodes = extractStreamNodeList(pMnode); + if (numOfNodes == 0) { mDebug("stream task node change checking done, no vgroups exist, do nothing"); execInfo.ts = taosGetTimestampSec(); - streamMutexUnlock(&execInfo.lock); return false; } @@ -1166,43 +1164,46 @@ static bool taskNodeIsUpdated(SMnode *pMnode) { if (pNodeEntry->stageUpdated) { mDebug("stream task not ready due to node update detected, checkpoint not issued"); - streamMutexUnlock(&execInfo.lock); return true; } } - int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot); + int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, ppNodeSnapshot); if (code) { mError("failed to get the vgroup snapshot, ignore it and continue"); } if (!allReady) { mWarn("not all vnodes ready, quit from vnodes status check"); - taosArrayDestroy(pNodeSnapshot); - streamMutexUnlock(&execInfo.lock); return true; } - SVgroupChangeInfo changeInfo = {0}; - code = mndFindChangedNodeInfo(pMnode, execInfo.pNodeList, pNodeSnapshot, &changeInfo); + code = mndFindChangedNodeInfo(pMnode, execInfo.pNodeList, *ppNodeSnapshot, &changeInfo); if (code) { - streamMutexUnlock(&execInfo.lock); - return false; + nodeUpdated = false; + } else { + nodeUpdated = (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0); + if (nodeUpdated) { + mDebug("stream tasks not ready due to node update"); + } } - bool nodeUpdated = (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0); - mndDestroyVgroupChangeInfo(&changeInfo); - taosArrayDestroy(pNodeSnapshot); - - if (nodeUpdated) { - mDebug("stream tasks not ready due to node update"); - } - - streamMutexUnlock(&execInfo.lock); return nodeUpdated; } +// check if the node update happens or not +static bool taskNodeIsUpdated(SMnode *pMnode) { + SArray *pNodeSnapshot = NULL; + + streamMutexLock(&execInfo.lock); + bool updated = doCheckForUpdated(pMnode, &pNodeSnapshot); + streamMutexUnlock(&execInfo.lock); + + taosArrayDestroy(pNodeSnapshot); + return updated; +} + static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { bool ready = true; if (taskNodeIsUpdated(pMnode)) { @@ -1993,7 +1994,7 @@ static int32_t mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeLis if (pInfo->pUpdateNodeList == NULL || pInfo->pDBMap == NULL) { mndDestroyVgroupChangeInfo(pInfo); - return terrno; + TSDB_CHECK_NULL(NULL, code, lino, _err, terrno); } int32_t numOfNodes = taosArrayGetSize(pPrevNodeList); @@ -2048,6 +2049,7 @@ static int32_t mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeLis return code; _err: + mError("failed to find node change info, code:%s at %s line:%d", tstrerror(code), __func__, lino); mndDestroyVgroupChangeInfo(pInfo); return code; } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 3f4329f22b..116acd4636 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -160,6 +160,13 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM tDecoderClear(&decoder); + int32_t gError = streamGetFatalError(pMeta); + if (gError != 0) { + tqError("vgId:%d global fatal occurs, code:%s, ts:%" PRId64 " func:%s", pMeta->vgId, tstrerror(gError), + pMeta->fatalInfo.ts, pMeta->fatalInfo.func); + return 0; + } + // update the nodeEpset when it exists streamMetaWLock(pMeta); @@ -290,8 +297,11 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM tqDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId, updateTasks, (numOfTasks - updateTasks)); } else { - if (streamMetaCommit(pMeta) < 0) { - // persist to disk + if ((code = streamMetaCommit(pMeta)) < 0) { + // always return true + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return TSDB_CODE_SUCCESS; } streamMetaClearSetUpdateTaskListComplete(pMeta); @@ -754,8 +764,9 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored } streamMetaWUnLock(pMeta); + // always return success when handling the requirement issued by mnode during transaction. - return code; + return TSDB_CODE_SUCCESS; } static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { @@ -1197,10 +1208,6 @@ int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { ret int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } -int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) { - return doProcessDummyRspMsg(pMeta, pMsg); -} - int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont; @@ -1221,14 +1228,13 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t vgId = pMeta->vgId; int32_t code = 0; SStreamTask* pTask = NULL; - SRestoreCheckpointInfo req = {0}; char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); int64_t now = taosGetTimestampMs(); + SDecoder decoder; + SRestoreCheckpointInfo req = {0}; - SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, len); - if (tDecodeRestoreCheckpointInfo(&decoder, &req) < 0) { tqError("vgId:%d failed to decode set consensus checkpointId req, code:%s", vgId, tstrerror(code)); tDecoderClear(&decoder); @@ -1239,16 +1245,15 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); if (pTask == NULL || (code != 0)) { - tqError( - "vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", - pMeta->vgId, req.taskId); + tqError("vgId:%d process consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", + pMeta->vgId, req.taskId); // ignore this code to avoid error code over write int32_t ret = streamMetaAddFailedTask(pMeta, req.streamId, req.taskId); if (ret) { tqError("s-task:0x%x failed add check downstream failed, core:%s", req.taskId, tstrerror(ret)); } - return code; + return 0; } // discard the rsp, since it is expired. @@ -1272,7 +1277,7 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); - return TSDB_CODE_STREAM_INTERNAL_ERROR; + return 0; } SConsenChkptInfo* pConsenInfo = &pTask->status.consenChkptInfo; @@ -1299,10 +1304,13 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { if (pMeta->role == NODE_ROLE_LEADER) { code = tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, req.streamId, req.taskId); + if (code) { + tqError("s-task:0x%x vgId:%d failed start task async, code:%s", req.taskId, vgId, tstrerror(code)); + } } else { tqDebug("vgId:%d follower not start task:%s", vgId, pTask->id.idStr); } streamMetaReleaseTask(pMeta, pTask); - return code; + return 0; } \ No newline at end of file diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d8249666c3..f4202667ff 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -923,32 +923,38 @@ int32_t streamMetaBegin(SStreamMeta* pMeta) { streamMetaWLock(pMeta); int32_t code = tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + if (code) { + streamSetFatalError(pMeta, code, __func__, __LINE__); + } streamMetaWUnLock(pMeta); return code; } int32_t streamMetaCommit(SStreamMeta* pMeta) { - int32_t code = 0; - code = tdbCommit(pMeta->db, pMeta->txn); + int32_t code = tdbCommit(pMeta->db, pMeta->txn); if (code != 0) { - stError("vgId:%d failed to commit stream meta", pMeta->vgId); - return code; + streamSetFatalError(pMeta, code, __func__, __LINE__); + stFatal("vgId:%d failed to commit stream meta, code:%s, line:%d", pMeta->vgId, tstrerror(code), + pMeta->fatalInfo.line); } code = tdbPostCommit(pMeta->db, pMeta->txn); if (code != 0) { - stError("vgId:%d failed to do post-commit stream meta", pMeta->vgId); + streamSetFatalError(pMeta, code, __func__, __LINE__); + stFatal("vgId:%d failed to do post-commit stream meta, code:%s, line:%d", pMeta->vgId, tstrerror(code), + pMeta->fatalInfo.line); return code; } code = tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); if (code != 0) { - stError("vgId:%d failed to begin trans", pMeta->vgId); - return code; + streamSetFatalError(pMeta, code, __func__, __LINE__); + stFatal("vgId:%d failed to begin trans, code:%s, line:%d", pMeta->vgId, tstrerror(code), pMeta->fatalInfo.line); + } else { + stDebug("vgId:%d stream meta file commit completed", pMeta->vgId); } - stDebug("vgId:%d stream meta file commit completed", pMeta->vgId); return code; } diff --git a/source/libs/stream/src/streamUtil.c b/source/libs/stream/src/streamUtil.c index cef2ba35e7..5bf9370cb7 100644 --- a/source/libs/stream/src/streamUtil.c +++ b/source/libs/stream/src/streamUtil.c @@ -77,9 +77,10 @@ void streamSetFatalError(SStreamMeta* pMeta, int32_t code, const char* funcName, pMeta->fatalInfo.threadId = taosGetSelfPthreadId(); tstrncpy(pMeta->fatalInfo.func, funcName, tListLen(pMeta->fatalInfo.func)); pMeta->fatalInfo.line = lino; - stInfo("vgId:%d set global fatal error, code:%s %s line:%d", pMeta->vgId, tstrerror(code), funcName, lino); + stInfo("vgId:%d set fatal error, code:%s %s line:%d", pMeta->vgId, tstrerror(code), funcName, lino); } else { - stFatal("vgId:%d existed global fatal eror:%s, failed to set new fatal error code:%s", pMeta->vgId, code); + stFatal("vgId:%d existed fatal error:%s, ts:%" PRId64 " failed to set new fatal error code:%s", pMeta->vgId, + pMeta->fatalInfo.ts, tstrerror(code)); } } From 1d2c00a4fff9fe1d788a09f14bc73bba0f5389f2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 15 Sep 2024 01:25:16 +0800 Subject: [PATCH 82/94] refactor: do some internal refactor. --- source/libs/stream/src/streamUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamUtil.c b/source/libs/stream/src/streamUtil.c index 5bf9370cb7..4c481e6041 100644 --- a/source/libs/stream/src/streamUtil.c +++ b/source/libs/stream/src/streamUtil.c @@ -80,7 +80,7 @@ void streamSetFatalError(SStreamMeta* pMeta, int32_t code, const char* funcName, stInfo("vgId:%d set fatal error, code:%s %s line:%d", pMeta->vgId, tstrerror(code), funcName, lino); } else { stFatal("vgId:%d existed fatal error:%s, ts:%" PRId64 " failed to set new fatal error code:%s", pMeta->vgId, - pMeta->fatalInfo.ts, tstrerror(code)); + tstrerror(pMeta->fatalInfo.code), pMeta->fatalInfo.ts, tstrerror(code)); } } From b4277e0e6532023e9b9eb2bc5b1d07f01cbc67f2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 15 Sep 2024 16:15:11 +0800 Subject: [PATCH 83/94] refactor:do some internal refactor. --- include/libs/stream/tstream.h | 2 +- source/libs/stream/src/streamCheckStatus.c | 11 +- source/libs/stream/src/streamCheckpoint.c | 179 +++++++++------- source/libs/stream/src/streamDispatch.c | 214 +++++++++++--------- source/libs/stream/src/streamHb.c | 6 +- source/libs/stream/src/streamSched.c | 34 ++-- source/libs/stream/src/streamStartHistory.c | 21 +- source/libs/stream/src/streamTaskSm.c | 4 +- source/libs/stream/src/streamTimer.c | 18 +- 9 files changed, 274 insertions(+), 215 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 29759bc561..cb10aeb6a0 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -801,7 +801,7 @@ void streamTaskSetReqConsenChkptId(SStreamTask* pTask, int64_t ts); // timer int32_t streamTimerGetInstance(tmr_h* pTmr); -void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, +void streamTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* pParam, void* pHandle, tmr_h* pTmrId, int32_t vgId, const char* pMsg); void streamTmrStop(tmr_h tmrId); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 76e74db33f..2688617823 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -303,13 +303,8 @@ void streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s start check-rsp monitor, ref:%d ", pTask->id.idStr, ref); - - if (pInfo->checkRspTmr == NULL) { - pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer); - } else { - streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, - "check-status-monitor"); - } + streamTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, + "check-status-monitor"); streamMutexUnlock(&pInfo->checkInfoLock); } @@ -860,7 +855,7 @@ void rspMonitorFn(void* param, void* tmrId) { handleTimeoutDownstreamTasks(pTask, pTimeoutList); } - streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, + streamTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor"); streamMutexUnlock(&pInfo->checkInfoLock); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 1a8b5c1028..769116264d 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -347,12 +347,8 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref); streamMetaAcquireOneTask(pTask); - - if (pTmrInfo->tmrHandle == NULL) { - pTmrInfo->tmrHandle = taosTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer); - } else { - streamTmrReset(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); - } + streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + "trigger-recv-monitor"); pTmrInfo->launchChkptId = pActiveInfo->activeId; } else { // already launched, do nothing stError("s-task:%s previous checkpoint-trigger monitor tmr is set, not start new one", pTask->id.idStr); @@ -893,48 +889,11 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { return code; } -void checkpointTriggerMonitorFn(void* param, void* tmrId) { - SStreamTask* pTask = param; - int32_t vgId = pTask->pMeta->vgId; - int64_t now = taosGetTimestampMs(); - const char* id = pTask->id.idStr; - +static int32_t doChkptStatusCheck(SStreamTask* pTask) { + const char* id = pTask->id.idStr; + int32_t vgId = pTask->pMeta->vgId; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; - SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; - - if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stError("s-task:%s source task should not start the checkpoint-trigger monitor fn, ref:%d quit", id, ref); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - - // check the status every 100ms - if (streamTaskShouldStop(pTask)) { - int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); - stDebug("s-task:%s vgId:%d quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - - if (++pTmrInfo->activeCounter < 50) { - streamTmrReset(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); - return; - } - - pTmrInfo->activeCounter = 0; - stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now); - - streamMutexLock(&pTask->lock); - SStreamTaskState pState = streamTaskGetStatus(pTask); - if (pState.state != TASK_STATUS__CK) { - int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); - stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); - - streamMutexUnlock(&pTask->lock); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } + SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; // checkpoint-trigger recv flag is set, quit if (pActiveInfo->allUpstreamTriggerRecv) { @@ -942,48 +901,44 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { stDebug("s-task:%s vgId:%d all checkpoint-trigger recv, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); - streamMutexUnlock(&pTask->lock); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - - streamMutexUnlock(&pTask->lock); - - streamMutexLock(&pActiveInfo->lock); - - // send msg to retrieve checkpoint trigger msg - SArray* pList = pTask->upstreamInfo.pList; - SArray* pNotSendList = taosArrayInit(4, sizeof(SStreamUpstreamEpInfo)); - if (pNotSendList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - stDebug("s-task:%s start to triggerMonitor, reason:%s", id, tstrerror(terrno)); - streamMutexUnlock(&pActiveInfo->lock); - - stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id); - streamTmrReset(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); - return; +// streamMutexUnlock(&pTask->lock); +// streamMetaReleaseTask(pTask->pMeta, pTask); + return -1; } if ((pTmrInfo->launchChkptId != pActiveInfo->activeId) || (pActiveInfo->activeId == 0)) { - streamMutexUnlock(&pActiveInfo->lock); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d checkpoint-trigger retrieve by previous checkpoint procedure, checkpointId:%" PRId64 ", quit, ref:%d", id, vgId, pTmrInfo->launchChkptId, ref); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; +// streamMutexUnlock(&pActiveInfo->lock); +// streamMetaReleaseTask(pTask->pMeta, pTask); + return -1; } // active checkpoint info is cleared for now if ((pActiveInfo->activeId == 0) || (pActiveInfo->transId == 0) || (pTask->chkInfo.startTs == 0)) { - streamMutexUnlock(&pActiveInfo->lock); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d active checkpoint may be cleared, quit from retrieve checkpoint-trigger send tmr, ref:%d", id, vgId, ref); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; +// streamMutexUnlock(&pActiveInfo->lock); +// streamMetaReleaseTask(pTask->pMeta, pTask); + return -1; + } + + return 0; +} + +static int32_t doFindNotSendUpstream(SStreamTask* pTask, SArray* pList, SArray** ppNotSendList) { + const char* id = pTask->id.idStr; + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + + SArray* pNotSendList = taosArrayInit(4, sizeof(SStreamUpstreamEpInfo)); + if (pNotSendList == NULL) { + stDebug("s-task:%s start to triggerMonitor, reason:%s", id, tstrerror(terrno)); + return terrno; } for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { @@ -1007,13 +962,87 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { void* px = taosArrayPush(pNotSendList, pInfo); if (px == NULL) { stError("s-task:%s failed to record not send info, code: out of memory", id); + taosArrayDestroy(pNotSendList); + return terrno; } } } + *ppNotSendList = pNotSendList; + return 0; +} + +void checkpointTriggerMonitorFn(void* param, void* tmrId) { + SStreamTask* pTask = param; + int32_t vgId = pTask->pMeta->vgId; + int64_t now = taosGetTimestampMs(); + const char* id = pTask->id.idStr; + + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; + + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stError("s-task:%s source task should not start the checkpoint-trigger monitor fn, ref:%d quit", id, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + // check the status every 100ms + if (streamTaskShouldStop(pTask)) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stDebug("s-task:%s vgId:%d quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + if (++pTmrInfo->activeCounter < 50) { + streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); + return; + } + + pTmrInfo->activeCounter = 0; + stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now); + + streamMutexLock(&pTask->lock); + SStreamTaskState state = streamTaskGetStatus(pTask); + streamMutexUnlock(&pTask->lock); + + if (state.state != TASK_STATUS__CK) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stDebug("s-task:%s vgId:%d status:%s not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, + vgId, state.name, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + streamMutexLock(&pActiveInfo->lock); + + int32_t code = doChkptStatusCheck(pTask); + if (code) { + streamMutexUnlock(&pTask->lock); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + // send msg to retrieve checkpoint trigger msg + SArray* pList = pTask->upstreamInfo.pList; + SArray* pNotSendList = NULL; + + code = doFindNotSendUpstream(pTask, pList, &pNotSendList); + if (code) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stDebug("s-task:%s failed to find not send upstream, code:%s, out of tmr, ref:%d", id, tstrerror(code), ref); + streamMutexUnlock(&pActiveInfo->lock); + streamMetaReleaseTask(pTask->pMeta, pTask); + + taosArrayDestroy(pNotSendList); + return; + } + // do send retrieve checkpoint trigger msg to upstream int32_t size = taosArrayGetSize(pNotSendList); - int32_t code = doSendRetrieveTriggerMsg(pTask, pNotSendList); + code = doSendRetrieveTriggerMsg(pTask, pNotSendList); if (code) { stError("s-task:%s vgId:%d failed to retrieve trigger msg, code:%s", pTask->id.idStr, vgId, tstrerror(code)); } @@ -1023,7 +1052,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { // check every 100ms if (size > 0) { stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id); - streamTmrReset(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); + streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); } else { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stDebug("s-task:%s all checkpoint-trigger recved, quit from monitor checkpoint-trigger tmr, ref:%d", id, ref); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 5f7b8dd39e..0f0015c7d9 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -634,12 +634,8 @@ static void doMonitorDispatchData(void* param, void* tmrId) { void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) { int32_t vgId = pTask->pMeta->vgId; - if (pTask->msgInfo.pRetryTmr != NULL) { - streamTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId, - "dispatch-monitor-tmr"); - } else { - pTask->msgInfo.pRetryTmr = taosTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer); - } + streamTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId, + "dispatch-monitor"); } int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int64_t groupId, @@ -888,77 +884,42 @@ int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32 return TSDB_CODE_SUCCESS; } -static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { - SStreamTask* pTask = param; - int32_t vgId = pTask->pMeta->vgId; - const char* id = pTask->id.idStr; +static int32_t doTaskChkptStatusCheck(SStreamTask* pTask, int32_t num) { SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptReadyMsgTmr; + const char* id = pTask->id.idStr; + int32_t vgId = pTask->pMeta->vgId; - // check the status every 100ms - if (streamTaskShouldStop(pTask)) { - int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); - stDebug("s-task:%s vgId:%d status:stop, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - - if (++pTmrInfo->activeCounter < 50) { - streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, - "chkpt-ready-monitor"); - return; - } - - pTmrInfo->activeCounter = 0; - stDebug("s-task:%s in sending checkpoint-ready msg monitor tmr", id); - - streamMutexLock(&pTask->lock); - SStreamTaskState pState = streamTaskGetStatus(pTask); - if (pState.state != TASK_STATUS__CK) { - int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); - stDebug("s-task:%s vgId:%d status:%s not in checkpoint, quit from monitor checkpoint-ready send, ref:%d", id, vgId, - pState.name, ref); - streamMutexUnlock(&pTask->lock); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - streamMutexUnlock(&pTask->lock); - - streamMutexLock(&pActiveInfo->lock); - - SArray* pList = pActiveInfo->pReadyMsgList; - int32_t num = taosArrayGetSize(pList); if (pTmrInfo->launchChkptId != pActiveInfo->activeId) { - streamMutexUnlock(&pActiveInfo->lock); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d ready-msg send tmr launched by previous checkpoint procedure, checkpointId:%" PRId64 ", quit, ref:%d", id, vgId, pTmrInfo->launchChkptId, ref); - - streamMetaReleaseTask(pTask->pMeta, pTask); - return; + return -1; } // active checkpoint info is cleared for now if ((pActiveInfo->activeId == 0) || (pActiveInfo->transId == 0) || (num == 0) || (pTask->chkInfo.startTs == 0)) { - streamMutexUnlock(&pActiveInfo->lock); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d active checkpoint may be cleared, quit from readyMsg send tmr, ref:%d", id, vgId, ref); - - streamMetaReleaseTask(pTask->pMeta, pTask); - return; + return -1; } - SArray* pNotRspList = taosArrayInit(4, sizeof(int32_t)); - if (taosArrayGetSize(pTask->upstreamInfo.pList) != num) { - streamMutexUnlock(&pActiveInfo->lock); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d upstream number:%d not equals sent readyMsg:%d, quit from readyMsg send tmr, ref:%d", id, vgId, (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList), num, ref); + return -1; + } - streamMetaReleaseTask(pTask->pMeta, pTask); - return; + return 0; +} + +static int32_t doFindNotConfirmUpstream(SArray** ppNotRspList, SArray* pList, int32_t num, int32_t vgId, int32_t level, + const char* id) { + SArray* pTmp = taosArrayInit(4, sizeof(int32_t)); + if (pTmp == NULL) { + return terrno; } for (int32_t i = 0; i < num; ++i) { @@ -971,63 +932,138 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { continue; } - void* p = taosArrayPush(pNotRspList, &pInfo->upstreamTaskId); + void* p = taosArrayPush(pTmp, &pInfo->upstreamTaskId); if (p == NULL) { stError("s-task:%s vgId:%d failed to record not rsp task, code: out of memory", id, vgId); + return terrno; } else { stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, - pTask->info.taskLevel, pInfo->upstreamTaskId); + level, pInfo->upstreamTaskId); } } - int32_t checkpointId = pActiveInfo->activeId; + *ppNotRspList = pTmp; + return 0; +} - int32_t notRsp = taosArrayGetSize(pNotRspList); - if (notRsp > 0) { // send checkpoint-ready msg again - for (int32_t i = 0; i < taosArrayGetSize(pNotRspList); ++i) { - int32_t* pTaskId = taosArrayGet(pNotRspList, i); - if (pTaskId == NULL) { +static void doSendChkptReadyMsg(SStreamTask* pTask, SArray* pNotRspList, int64_t checkpointId, SArray* pReadyList) { + int32_t code = 0; + int32_t num = taosArrayGetSize(pReadyList); + const char* id = pTask->id.idStr; + + for (int32_t i = 0; i < taosArrayGetSize(pNotRspList); ++i) { + int32_t* pTaskId = taosArrayGet(pNotRspList, i); + if (pTaskId == NULL) { + continue; + } + + for (int32_t j = 0; j < num; ++j) { + STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pReadyList, j); + if (pReadyInfo == NULL) { continue; } - for (int32_t j = 0; j < num; ++j) { - STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pList, j); - if (pReadyInfo == NULL) { - continue; - } + if (*pTaskId == pReadyInfo->upstreamTaskId) { // send msg again - if (*pTaskId == pReadyInfo->upstreamTaskId) { // send msg again - - SRpcMsg msg = {0}; - int32_t code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, - pReadyInfo->childId, checkpointId, &msg); + SRpcMsg msg = {0}; + code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, + pReadyInfo->childId, checkpointId, &msg); + if (code == TSDB_CODE_SUCCESS) { + code = tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg); if (code == TSDB_CODE_SUCCESS) { - code = tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg); - if (code == TSDB_CODE_SUCCESS) { - stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x again", id, pTask->info.taskLevel, - pReadyInfo->upstreamTaskId); - } else { - stError("s-task:%s failed to send checkpoint-ready msg, try nex time in 10s", id); - } + stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x again", id, pTask->info.taskLevel, + pReadyInfo->upstreamTaskId); } else { - stError("s-task:%s failed to prepare the checkpoint-ready msg, try nex time in 10s", id); + stError("s-task:%s failed to send checkpoint-ready msg, try nex time in 10s", id); } + } else { + stError("s-task:%s failed to prepare the checkpoint-ready msg, try nex time in 10s", id); } } } + } +} - streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, +static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { + SStreamTask* pTask = param; + int32_t vgId = pTask->pMeta->vgId; + const char* id = pTask->id.idStr; + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptReadyMsgTmr; + SArray* pNotRspList = NULL; + + // check the status every 100ms + if (streamTaskShouldStop(pTask)) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stDebug("s-task:%s vgId:%d status:stop, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + if (++pTmrInfo->activeCounter < 50) { + streamTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + "chkpt-ready-monitor"); + return; + } + + // reset tmr + pTmrInfo->activeCounter = 0; + stDebug("s-task:%s in sending checkpoint-ready msg monitor tmr", id); + + streamMutexLock(&pTask->lock); + SStreamTaskState state = streamTaskGetStatus(pTask); + streamMutexUnlock(&pTask->lock); + + // 1. check status in the first place + if (state.state != TASK_STATUS__CK) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stDebug("s-task:%s vgId:%d status:%s not in checkpoint, quit from monitor checkpoint-ready, ref:%d", id, vgId, + state.name, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + streamMutexLock(&pActiveInfo->lock); + + SArray* pList = pActiveInfo->pReadyMsgList; + int32_t num = taosArrayGetSize(pList); + int32_t code = doTaskChkptStatusCheck(pTask, num); + if (code) { + streamMutexUnlock(&pActiveInfo->lock); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + code = doFindNotConfirmUpstream(&pNotRspList, pList, num, vgId, pTask->info.taskLevel, id); + if (code) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stError("s-task:%s failed to find not rsp checkpoint-ready downstream, code:%s, out of tmr, ref:%d", id, + tstrerror(code), ref); + streamMutexUnlock(&pActiveInfo->lock); + streamMetaReleaseTask(pTask->pMeta, pTask); + + taosArrayDestroy(pNotRspList); + return; + } + + int32_t checkpointId = pActiveInfo->activeId; + int32_t notRsp = taosArrayGetSize(pNotRspList); + doSendChkptReadyMsg(pTask, pNotRspList, checkpointId, pList); + + if (notRsp > 0) { // send checkpoint-ready msg again + streamTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); streamMutexUnlock(&pActiveInfo->lock); } else { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stDebug( - "s-task:%s vgId:%d recv of checkpoint-ready msg confirmed by all upstream task(s), clear checkpoint-ready msg " - "and quit from timer, ref:%d", + "s-task:%s vgId:%d checkpoint-ready msg confirmed by all upstream task(s), clear checkpoint-ready msg and quit " + "from timer, ref:%d", id, vgId, ref); streamClearChkptReadyMsg(pActiveInfo); streamMutexUnlock(&pActiveInfo->lock); + // release should be the last execution, since pTask may be destroy after it immidiately. streamMetaReleaseTask(pTask->pMeta, pTask); } @@ -1085,12 +1121,8 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { stDebug("s-task:%s start checkpoint-ready monitor in 10s, ref:%d ", pTask->id.idStr, ref); streamMetaAcquireOneTask(pTask); - if (pTmrInfo->tmrHandle == NULL) { - pTmrInfo->tmrHandle = taosTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer); - } else { - streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, - "chkpt-ready-monitor"); - } + streamTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + "chkpt-ready-monitor"); // mark the timer monitor checkpointId pTmrInfo->launchChkptId = pActiveInfo->activeId; diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 703e6a3256..72d2e89936 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -279,7 +279,7 @@ void streamMetaHbToMnode(void* param, void* tmrId) { } if (!waitForEnoughDuration(pMeta->pHbInfo)) { - streamTmrReset(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr, vgId, + streamTmrStart(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr, vgId, "meta-hb-tmr"); code = taosReleaseRef(streamMetaId, rid); @@ -301,7 +301,7 @@ void streamMetaHbToMnode(void* param, void* tmrId) { } streamMetaRUnLock(pMeta); - streamTmrReset(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr, pMeta->vgId, + streamTmrStart(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr, pMeta->vgId, "meta-hb-tmr"); code = taosReleaseRef(streamMetaId, rid); @@ -317,7 +317,7 @@ int32_t createMetaHbInfo(int64_t* pRid, SMetaHbInfo** pRes) { return terrno; } - pInfo->hbTmr = taosTmrStart(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer); + streamTmrStart(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer, &pInfo->hbTmr, 0, "stream-hb"); pInfo->tickCounter = 0; pInfo->msgSendTs = -1; pInfo->hbCount = 0; diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 2d54547aa2..63e24b0975 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -20,13 +20,14 @@ static void streamTaskResumeHelper(void* param, void* tmrId); static void streamTaskSchedHelper(void* param, void* tmrId); void streamSetupScheduleTrigger(SStreamTask* pTask) { - if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) { + int64_t delaySchema = pTask->info.delaySchedParam; + if (delaySchema != 0 && pTask->info.fillHistory == 0) { int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); stDebug("s-task:%s setup scheduler trigger, ref:%d delay:%" PRId64 " ms", pTask->id.idStr, ref, pTask->info.delaySchedParam); - pTask->schedInfo.pDelayTimer = - taosTmrStart(streamTaskSchedHelper, (int32_t)pTask->info.delaySchedParam, pTask, streamTimer); + streamTmrStart(streamTaskSchedHelper, (int32_t)delaySchema, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, + pTask->pMeta->vgId, "sched-tmr"); pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE; } } @@ -76,13 +77,8 @@ void streamTaskResumeInFuture(SStreamTask* pTask) { // add one ref count for task streamMetaAcquireOneTask(pTask); - - if (pTask->schedInfo.pIdleTimer == NULL) { - pTask->schedInfo.pIdleTimer = taosTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer); - } else { - streamTmrReset(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer, - &pTask->schedInfo.pIdleTimer, pTask->pMeta->vgId, "resume-task-tmr"); - } + streamTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer, &pTask->schedInfo.pIdleTimer, + pTask->pMeta->vgId, "resume-task-tmr"); } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -125,7 +121,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) { stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger); if (streamTaskShouldStop(pTask) || streamTaskShouldPause(pTask)) { - stDebug("s-task:%s jump out of schedTimer", id); + stDebug("s-task:%s should stop, jump out of schedTimer", id); return; } @@ -139,9 +135,8 @@ void streamTaskSchedHelper(void* param, void* tmrId) { if (code) { stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", nextTrigger); - streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr"); terrno = code; - return; + goto _end; } pTrigger->type = STREAM_INPUT__GET_RES; @@ -149,10 +144,9 @@ void streamTaskSchedHelper(void* param, void* tmrId) { if (pTrigger->pBlock == NULL) { taosFreeQitem(pTrigger); - stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", + stError("s-task:%s failed to build retrieve data trigger, code:out of memory, try again in %dms", id, nextTrigger); - streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr"); - return; + goto _end; } atomic_store_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE); @@ -160,8 +154,8 @@ void streamTaskSchedHelper(void* param, void* tmrId) { code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger); if (code != TSDB_CODE_SUCCESS) { - streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr"); - return; + stError("s-task:%s failed to put retrieve block into trigger, code:%s", pTask->id.idStr, tstrerror(code)); + goto _end; } code = streamTrySchedExec(pTask); @@ -171,5 +165,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) { } } - streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr"); +_end: + streamTmrStart(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, + "sched-run-tmr"); } diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index 21f0168434..1290aef6a3 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -80,6 +80,7 @@ int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) { } void streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) { + int32_t vgId = pTask->pMeta->vgId; int32_t numOfTicks = idleDuration / SCANHISTORY_IDLE_TIME_SLICE; if (numOfTicks <= 0) { numOfTicks = 1; @@ -100,14 +101,8 @@ void streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s scan-history resumed in %.2fs, ref:%d", pTask->id.idStr, numOfTicks * 0.1, ref); - - if (pTask->schedHistoryInfo.pTimer == NULL) { - pTask->schedHistoryInfo.pTimer = - taosTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer); - } else { - streamTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, - &pTask->schedHistoryInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); - } + streamTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, + &pTask->schedHistoryInfo.pTimer, vgId, "history-task"); } int32_t streamTaskStartScanHistory(SStreamTask* pTask) { @@ -337,7 +332,7 @@ void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, i stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d", pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes); - streamTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer, + streamTmrStart(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); } } @@ -391,7 +386,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { pHTaskInfo->tickCount -= 1; if (pHTaskInfo->tickCount > 0) { - streamTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer, + streamTmrStart(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); streamMetaReleaseTask(pMeta, pTask); return; @@ -519,7 +514,7 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) { } stDebug("s-task:%s set timer active flag, task timer not null", idStr); - streamTmrReset(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer, &pTask->hTaskInfo.pTimer, + streamTmrStart(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer, &pTask->hTaskInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); } @@ -621,8 +616,8 @@ void doExecScanhistoryInFuture(void* param, void* tmrId) { // release the task. streamMetaReleaseTask(pTask->pMeta, pTask); } else { - streamTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, - &pTask->schedHistoryInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); + streamTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, + &pTask->schedHistoryInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); } } diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 06286479a3..3709c4dfbd 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -491,8 +491,8 @@ static void keepPrevInfo(SStreamTaskSM* pSM) { int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) { SStreamTask* pTask = pSM->pTask; - const char* id = pTask->id.idStr; - int32_t code = 0; + const char* id = pTask->id.idStr; + int32_t code = 0; // do update the task status streamMutexLock(&pTask->lock); diff --git a/source/libs/stream/src/streamTimer.c b/source/libs/stream/src/streamTimer.c index 5e12f51c9d..8b77fe7cb1 100644 --- a/source/libs/stream/src/streamTimer.c +++ b/source/libs/stream/src/streamTimer.c @@ -40,11 +40,23 @@ int32_t streamTimerGetInstance(tmr_h* pTmr) { return TSDB_CODE_SUCCESS; } -void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, +void streamTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* pParam, void* pHandle, tmr_h* pTmrId, int32_t vgId, const char* pMsg) { - bool ret = taosTmrReset(fp, mseconds, param, handle, pTmrId); - if (ret) { + if (*pTmrId == NULL) { + *pTmrId = taosTmrStart(fp, mseconds, pParam, pHandle); + if (*pTmrId == NULL) { + stError("vgId:%d start %s tmr failed, code:%s", vgId, pMsg, tstrerror(terrno)); + return; + } + } else { + bool ret = taosTmrReset(fp, mseconds, pParam, pHandle, pTmrId); + if (ret) { + stError("vgId:%d start %s tmr failed, code:%s", vgId, pMsg, tstrerror(terrno)); + return; + } } + + stDebug("vgId:%d start %s tmr succ", vgId, pMsg); } void streamTmrStop(tmr_h tmrId) { From f0289293f327c97896c2ccf341081d2e6138781c Mon Sep 17 00:00:00 2001 From: sheyanjie-qq <249478495@qq.com> Date: Mon, 16 Sep 2024 10:16:42 +0800 Subject: [PATCH 84/94] mod python install doc --- docs/zh/08-develop/01-connect/index.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/zh/08-develop/01-connect/index.md b/docs/zh/08-develop/01-connect/index.md index 5cecd245e5..b16e96922f 100644 --- a/docs/zh/08-develop/01-connect/index.md +++ b/docs/zh/08-develop/01-connect/index.md @@ -132,7 +132,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 pip3 install taospy[ws] ``` - - **安装验证** +- **安装验证** 对于原生连接，需要验证客户端驱动和 Python 连接器本身是否都正确安装。如果能成功导入 `taos` 模块，则说明已经正确安装了客户端驱动和 Python 连接器。可在 Python 交互式 Shell 中输入： @@ -198,18 +198,18 @@ taos = { version = "*", default-features = false, features = ["ws"] } - **安装** - 使用 npm 安装 Node.js 连接器 - ``` - npm install @tdengine/websocket - ``` + ``` + npm install @tdengine/websocket + ``` :::note Node.js 目前只支持 Websocket 连接 - **安装验证** - 新建安装验证目录，例如：`~/tdengine-test`，下载 GitHub 上 [nodejsChecker.js 源代码](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/nodejsChecker.js)到本地。 - 在命令行中执行以下命令。 - ```bash - npm init -y - npm install @tdengine/websocket - node nodejsChecker.js - ``` + ```bash + npm init -y + npm install @tdengine/websocket + node nodejsChecker.js + ``` - 执行以上步骤后，在命令行会输出 nodeChecker.js 连接 TDengine 实例，并执行简单插入和查询的结果。 From 98442b4383b791a2347b5009bb82e7d733425596 Mon Sep 17 00:00:00 2001 From: huskar-t <1172915550@qq.com> Date: Mon, 16 Sep 2024 11:42:52 +0800 Subject: [PATCH 85/94] docs: update http status code --- .../14-reference/05-connector/60-rest-api.mdx | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/zh/14-reference/05-connector/60-rest-api.mdx b/docs/zh/14-reference/05-connector/60-rest-api.mdx index a804229d27..eae9fba1c4 100644 --- a/docs/zh/14-reference/05-connector/60-rest-api.mdx +++ b/docs/zh/14-reference/05-connector/60-rest-api.mdx @@ -123,8 +123,8 @@ curl -L -u username:password -d "" :/rest/sql/[db_name][?tz=timez | **说明** | **httpCodeServerError false** | **httpCodeServerError true** | |--------------------|-------------------------------|---------------------------------------| | taos_errno() 返回 0 | 200 | 200 | -| taos_errno() 返回 非0 | 200（除鉴权错误） | 500 （除鉴权错误和 400/502 错误） | -| 参数错误 | 400 (仅处理 HTTP 请求 URL 参数错误) | 400 （处理 HTTP 请求 URL 参数错误和 taosd 返回错误） | +| taos_errno() 返回 非0 | 200（除鉴权错误） | 500 （除鉴权错误和 400/502/503 错误） | +| 参数错误 | 400（仅处理 HTTP 请求 URL 参数错误） | 400 （处理 HTTP 请求 URL 参数错误和 taosd 返回错误） | | 鉴权错误 | 401 | 401 | | 接口不存在 | 404 | 404 | | 集群不可用错误 | 502 | 502 | @@ -132,27 +132,29 @@ curl -L -u username:password -d "" :/rest/sql/[db_name][?tz=timez 返回 400 的 C 错误码为： -- TSDB_CODE_TSC_SQL_SYNTAX_ERROR ( 0x0216) -- TSDB_CODE_TSC_LINE_SYNTAX_ERROR (0x021B) +- TSDB_CODE_TSC_SQL_SYNTAX_ERROR (0x0216) +- TSDB_CODE_TSC_LINE_SYNTAX_ERROR (0x021B) - TSDB_CODE_PAR_SYNTAX_ERROR (0x2600) - TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE (0x060B) -- TSDB_CODE_TSC_VALUE_OUT_OF_RANGE (0x0224) +- TSDB_CODE_TSC_VALUE_OUT_OF_RANGE (0x0224) - TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE (0x263B) 返回 401 的错误码为： -- TSDB_CODE_MND_USER_ALREADY_EXIST (0x0350) -- TSDB_CODE_MND_USER_NOT_EXIST ( 0x0351) -- TSDB_CODE_MND_INVALID_USER_FORMAT (0x0352) -- TSDB_CODE_MND_INVALID_PASS_FORMAT (0x0353) +- TSDB_CODE_MND_USER_ALREADY_EXIST (0x0350) +- TSDB_CODE_MND_USER_NOT_EXIST (0x0351) +- TSDB_CODE_MND_INVALID_USER_FORMAT (0x0352) +- TSDB_CODE_MND_INVALID_PASS_FORMAT (0x0353) - TSDB_CODE_MND_NO_USER_FROM_CONN (0x0354) - TSDB_CODE_MND_TOO_MANY_USERS (0x0355) - TSDB_CODE_MND_INVALID_ALTER_OPER (0x0356) - TSDB_CODE_MND_AUTH_FAILURE (0x0357) -返回 403 的错误码为： +返回 502 的错误码为： -- TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED (0x0020) +- TSDB_CODE_RPC_NETWORK_UNAVAIL (0x000B) + +错误码和错误描述请参考[错误码](../../../reference/error-code) ### HTTP body 结构 @@ -333,6 +335,8 @@ curl --location 'http://:/rest/sql' \ - code：（`int`）错误码。 - desc：（`string`）错误描述。 +错误码和错误描述请参考[错误码](../../../reference/error-code) + #### 返回key-value形式数据 当指定 url 参数 `row_with_meta=true` 时，返回的 data 中的数据会从数组的形式变成对象的形式，对象的 key 为列名，value 为数据，如下所示： From 8ca87f18e3a148d34ffaf093905913bbb89b35ff Mon Sep 17 00:00:00 2001 From: sheyanjie-qq <249478495@qq.com> Date: Mon, 16 Sep 2024 12:31:39 +0800 Subject: [PATCH 86/94] add c error code ref --- docs/zh/14-reference/05-connector/60-rest-api.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/zh/14-reference/05-connector/60-rest-api.mdx b/docs/zh/14-reference/05-connector/60-rest-api.mdx index a804229d27..e1211bc252 100644 --- a/docs/zh/14-reference/05-connector/60-rest-api.mdx +++ b/docs/zh/14-reference/05-connector/60-rest-api.mdx @@ -118,7 +118,8 @@ curl -L -u username:password -d "" :/rest/sql/[db_name][?tz=timez ### HTTP 响应码 -从 `TDengine 3.0.3.0` 开始 `taosAdapter` 提供配置参数 `httpCodeServerError` 用来设置当 C 接口返回错误时是否返回非 200 的http状态码 +从 `TDengine 3.0.3.0` 开始 `taosAdapter` 提供配置参数 `httpCodeServerError` 用来设置当 C 接口返回错误时是否返回非 200 的http状态码。 +无论是否设置此参数，响应 body 里都有详细的错误码和错误信息，具体请参考 [错误](../rest-api/#错误) 。 | **说明** | **httpCodeServerError false** | **httpCodeServerError true** | |--------------------|-------------------------------|---------------------------------------| From e87bd926dab8e04c0381c9cc400b7f9c95a55e6d Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 18 Sep 2024 09:18:30 +0800 Subject: [PATCH 87/94] fix: Memory detection in macro definition --- include/os/osSystem.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 01fad7ad97..44910ba94d 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -81,8 +81,11 @@ int32_t taosResetTerminalMode(); unw_get_reg(&cursor, UNW_REG_IP, &pc); \ fname[0] = '\0'; \ (void)unw_get_proc_name(&cursor, fname, sizeof(fname), &offset); \ - size += 1; \ array[size] = (char *)taosMemoryMalloc(sizeof(char) * STACKSIZE + 1); \ + if(NULL == array[size]) { \ + break; \ + } \ + size += 1; \ snprintf(array[size], STACKSIZE, "0x%lx : (%s+0x%lx) [0x%lx]\n", (long)pc, fname, (long)offset, (long)pc); \ } \ if (ignoreNum < size && size > 0) { \ From 5096f8008a5a165ea83a3b11b4342b8c2035a8a9 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Wed, 18 Sep 2024 11:15:12 +0800 Subject: [PATCH 88/94] fix:[TD-32137] Avoid double free when error occurs. --- source/libs/scalar/src/filter.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 588c252669..752bb2c0f5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -2866,32 +2866,35 @@ _return: } int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray *group) { - size_t groupSize = taosArrayGetSize(group); + size_t groupSize = taosArrayGetSize(group); + int32_t code = TSDB_CODE_SUCCESS; info->groupNum = (uint32_t)groupSize; if (info->groupNum > 0) { info->groups = taosMemoryCalloc(info->groupNum, sizeof(*info->groups)); if (info->groups == NULL) { - info->groupNum = 0; - FLT_ERR_RET(terrno); + FLT_ERR_JRET(terrno); } } for (size_t i = 0; i < groupSize; ++i) { SFilterGroup *pg = taosArrayGet(group, i); if (NULL == pg) { - FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); } pg->unitFlags = taosMemoryCalloc(pg->unitNum, sizeof(*pg->unitFlags)); if (pg->unitFlags == NULL) { pg->unitNum = 0; - FLT_ERR_RET(terrno); + FLT_ERR_JRET(terrno); } info->groups[i] = *pg; } - return TSDB_CODE_SUCCESS; + +_return: + info->groupNum = 0; + return code; } int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum) { From 1b08708fce3b98edcc8d3b33c0ba4c60a9c343b8 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 18 Sep 2024 11:13:16 +0800 Subject: [PATCH 89/94] enh: add GIT_SHALLOW for cmakelists of third party libs --- cmake/addr2line_CMakeLists.txt.in | 5 +++-- cmake/apr-util_CMakeLists.txt.in | 3 ++- cmake/apr_CMakeLists.txt.in | 1 + cmake/cjson_CMakeLists.txt.in | 3 ++- cmake/cos_CMakeLists.txt.in | 1 + cmake/crashdump_CMakeLists.txt.in | 5 +++-- cmake/curl_CMakeLists.txt.in | 1 + cmake/geos_CMakeLists.txt.in | 1 + cmake/gnuregex_CMakeLists.txt.in | 1 + cmake/gtest_CMakeLists.txt.in | 3 ++- cmake/iconv_CMakeLists.txt.in | 3 ++- cmake/libdwarf_CMakeLists.txt.in | 5 +++-- cmake/libs3_CMakeLists.txt.in | 1 + cmake/libuv_CMakeLists.txt.in | 5 +++-- cmake/lz4_CMakeLists.txt.in | 3 ++- cmake/lzma_CMakeLists.txt.in | 7 ++++--- cmake/msvcregex_CMakeLists.txt.in | 3 ++- cmake/mxml_CMakeLists.txt.in | 1 + cmake/pcre2_CMakeLists.txt.in | 1 + cmake/pthread_CMakeLists.txt.in | 1 + cmake/rocksdb_CMakeLists.txt.in | 2 ++ cmake/sqlite_CMakeLists.txt.in | 3 ++- cmake/ssl_CMakeLists.txt.in | 1 + cmake/stub_CMakeLists.txt.in | 1 + cmake/wcwidth_CMakeLists.txt.in | 3 ++- cmake/wingetopt_CMakeLists.txt.in | 3 ++- cmake/xml2_CMakeLists.txt.in | 1 + cmake/xz_CMakeLists.txt.in | 9 +++++---- cmake/zlib_CMakeLists.txt.in | 1 + cmake/zstd_CMakeLists.txt.in | 11 ++++++----- 30 files changed, 60 insertions(+), 29 deletions(-) diff --git a/cmake/addr2line_CMakeLists.txt.in b/cmake/addr2line_CMakeLists.txt.in index e514764af8..93fb9bb96c 100644 --- a/cmake/addr2line_CMakeLists.txt.in +++ b/cmake/addr2line_CMakeLists.txt.in @@ -5,8 +5,9 @@ ExternalProject_Add(addr2line GIT_TAG master SOURCE_DIR "${TD_CONTRIB_DIR}/addr2line" BINARY_DIR "${TD_CONTRIB_DIR}/addr2line" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/apr-util_CMakeLists.txt.in b/cmake/apr-util_CMakeLists.txt.in index a39100897b..8e14295ad6 100644 --- a/cmake/apr-util_CMakeLists.txt.in +++ b/cmake/apr-util_CMakeLists.txt.in @@ -11,9 +11,10 @@ ExternalProject_Add(aprutil-1 BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 #UPDATE_COMMAND "" - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1/ --with-apr=$ENV{HOME}/.cos-local.1 + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1/ --with-apr=$ENV{HOME}/.cos-local.1 #CONFIGURE_COMMAND ./configure --with-apr=/usr/local/apr BUILD_COMMAND make INSTALL_COMMAND make install TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/apr_CMakeLists.txt.in b/cmake/apr_CMakeLists.txt.in index 18c4eb62a1..af755c729f 100644 --- a/cmake/apr_CMakeLists.txt.in +++ b/cmake/apr_CMakeLists.txt.in @@ -16,4 +16,5 @@ ExternalProject_Add(apr-1 BUILD_COMMAND make INSTALL_COMMAND make install TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/cjson_CMakeLists.txt.in b/cmake/cjson_CMakeLists.txt.in index 32cea7baa1..941c5e786b 100644 --- a/cmake/cjson_CMakeLists.txt.in +++ b/cmake/cjson_CMakeLists.txt.in @@ -9,4 +9,5 @@ ExternalProject_Add(cjson BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - ) \ No newline at end of file + GIT_SHALLOW true + ) diff --git a/cmake/cos_CMakeLists.txt.in b/cmake/cos_CMakeLists.txt.in index ee1e58b50f..0a0a5d4d62 100644 --- a/cmake/cos_CMakeLists.txt.in +++ b/cmake/cos_CMakeLists.txt.in @@ -9,4 +9,5 @@ ExternalProject_Add(cos BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/crashdump_CMakeLists.txt.in b/cmake/crashdump_CMakeLists.txt.in index af4b551159..3bc35a0f58 100644 --- a/cmake/crashdump_CMakeLists.txt.in +++ b/cmake/crashdump_CMakeLists.txt.in @@ -5,8 +5,9 @@ ExternalProject_Add(crashdump GIT_TAG master SOURCE_DIR "${TD_CONTRIB_DIR}/crashdump" BINARY_DIR "${TD_CONTRIB_DIR}/crashdump" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 8e06569f4d..6494177faf 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -16,4 +16,5 @@ ExternalProject_Add(curl2 BUILD_COMMAND make -j INSTALL_COMMAND make install TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/geos_CMakeLists.txt.in b/cmake/geos_CMakeLists.txt.in index f939ccead0..e2b6ec5d4f 100644 --- a/cmake/geos_CMakeLists.txt.in +++ b/cmake/geos_CMakeLists.txt.in @@ -9,4 +9,5 @@ ExternalProject_Add(geos BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/gnuregex_CMakeLists.txt.in b/cmake/gnuregex_CMakeLists.txt.in index 817c22093a..cf748a3a54 100644 --- a/cmake/gnuregex_CMakeLists.txt.in +++ b/cmake/gnuregex_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(gnuregex BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/gtest_CMakeLists.txt.in b/cmake/gtest_CMakeLists.txt.in index fe07f2318b..22498ecf8b 100644 --- a/cmake/gtest_CMakeLists.txt.in +++ b/cmake/gtest_CMakeLists.txt.in @@ -9,4 +9,5 @@ ExternalProject_Add(googletest BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - ) \ No newline at end of file + GIT_SHALLOW true + ) diff --git a/cmake/iconv_CMakeLists.txt.in b/cmake/iconv_CMakeLists.txt.in index 5eb277ce0d..b9b3b084ca 100644 --- a/cmake/iconv_CMakeLists.txt.in +++ b/cmake/iconv_CMakeLists.txt.in @@ -9,4 +9,5 @@ ExternalProject_Add(iconv BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - ) \ No newline at end of file + GIT_SHALLOW true + ) diff --git a/cmake/libdwarf_CMakeLists.txt.in b/cmake/libdwarf_CMakeLists.txt.in index 7de34cfbaa..30383afd89 100644 --- a/cmake/libdwarf_CMakeLists.txt.in +++ b/cmake/libdwarf_CMakeLists.txt.in @@ -5,8 +5,9 @@ ExternalProject_Add(libdwarf GIT_TAG libdwarf-0.3.1 SOURCE_DIR "${TD_CONTRIB_DIR}/libdwarf" BINARY_DIR "${TD_CONTRIB_DIR}/libdwarf" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index f2b6cac953..4884089a2b 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -13,4 +13,5 @@ ExternalProject_Add(libs3 BUILD_COMMAND make build/lib/libs3.a INSTALL_COMMAND make install_static TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in index 673c771fb0..3bfb52fe9b 100644 --- a/cmake/libuv_CMakeLists.txt.in +++ b/cmake/libuv_CMakeLists.txt.in @@ -5,8 +5,9 @@ ExternalProject_Add(libuv GIT_TAG v1.48.0 SOURCE_DIR "${TD_CONTRIB_DIR}/libuv" BINARY_DIR "${TD_CONTRIB_DIR}/libuv" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/lz4_CMakeLists.txt.in b/cmake/lz4_CMakeLists.txt.in index 381f8f5c5b..57420495ee 100644 --- a/cmake/lz4_CMakeLists.txt.in +++ b/cmake/lz4_CMakeLists.txt.in @@ -9,4 +9,5 @@ ExternalProject_Add(lz4 BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - ) \ No newline at end of file + GIT_SHALLOW true + ) diff --git a/cmake/lzma_CMakeLists.txt.in b/cmake/lzma_CMakeLists.txt.in index fb16f89034..b6e96fb89f 100644 --- a/cmake/lzma_CMakeLists.txt.in +++ b/cmake/lzma_CMakeLists.txt.in @@ -1,9 +1,9 @@ -# xz +# xz if (${TD_LINUX}) ExternalProject_Add(lzma2 - GIT_REPOSITORY https://github.com/conor42/fast-lzma2.git + GIT_REPOSITORY https://github.com/conor42/fast-lzma2.git SOURCE_DIR "${TD_CONTRIB_DIR}/lzma2" #BINARY_DIR "" BUILD_IN_SOURCE TRUE @@ -11,5 +11,6 @@ ExternalProject_Add(lzma2 BUILD_COMMAND make INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) -endif() \ No newline at end of file +endif() diff --git a/cmake/msvcregex_CMakeLists.txt.in b/cmake/msvcregex_CMakeLists.txt.in index 686ac4cb5d..5c79c7982b 100644 --- a/cmake/msvcregex_CMakeLists.txt.in +++ b/cmake/msvcregex_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(msvcregex BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" -) \ No newline at end of file + GIT_SHALLOW true +) diff --git a/cmake/mxml_CMakeLists.txt.in b/cmake/mxml_CMakeLists.txt.in index 762df40e10..464ec8c89b 100644 --- a/cmake/mxml_CMakeLists.txt.in +++ b/cmake/mxml_CMakeLists.txt.in @@ -11,4 +11,5 @@ ExternalProject_Add(mxml BUILD_COMMAND make INSTALL_COMMAND make install TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/pcre2_CMakeLists.txt.in b/cmake/pcre2_CMakeLists.txt.in index c3115d8201..b46dedef09 100644 --- a/cmake/pcre2_CMakeLists.txt.in +++ b/cmake/pcre2_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(pcre2 BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/pthread_CMakeLists.txt.in b/cmake/pthread_CMakeLists.txt.in index 774f5dd972..2ec3362d3f 100644 --- a/cmake/pthread_CMakeLists.txt.in +++ b/cmake/pthread_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(pthread BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/rocksdb_CMakeLists.txt.in b/cmake/rocksdb_CMakeLists.txt.in index 45599d82e3..71e59cefd8 100644 --- a/cmake/rocksdb_CMakeLists.txt.in +++ b/cmake/rocksdb_CMakeLists.txt.in @@ -11,6 +11,7 @@ if (${BUILD_CONTRIB}) BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) else() if (NOT ${TD_LINUX}) @@ -24,6 +25,7 @@ else() BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) endif() endif() diff --git a/cmake/sqlite_CMakeLists.txt.in b/cmake/sqlite_CMakeLists.txt.in index 445c49487a..6b77a82af9 100644 --- a/cmake/sqlite_CMakeLists.txt.in +++ b/cmake/sqlite_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(sqlite BUILD_COMMAND "$(MAKE)" INSTALL_COMMAND "" TEST_COMMAND "" -) \ No newline at end of file + GIT_SHALLOW true +) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 4778ea76e0..1098593943 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -12,4 +12,5 @@ ExternalProject_Add(openssl BUILD_COMMAND make -j INSTALL_COMMAND make install_sw -j TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/stub_CMakeLists.txt.in b/cmake/stub_CMakeLists.txt.in index 96cc924d81..6c54d33be7 100644 --- a/cmake/stub_CMakeLists.txt.in +++ b/cmake/stub_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(stub BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/wcwidth_CMakeLists.txt.in b/cmake/wcwidth_CMakeLists.txt.in index 234ccbf915..3fd4e75243 100644 --- a/cmake/wcwidth_CMakeLists.txt.in +++ b/cmake/wcwidth_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(wcwidth BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" -) \ No newline at end of file + GIT_SHALLOW true +) diff --git a/cmake/wingetopt_CMakeLists.txt.in b/cmake/wingetopt_CMakeLists.txt.in index b93925c170..115e91888b 100644 --- a/cmake/wingetopt_CMakeLists.txt.in +++ b/cmake/wingetopt_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(wingetopt BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" -) \ No newline at end of file + GIT_SHALLOW true +) diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in index be14a11c36..0e7492aea7 100644 --- a/cmake/xml2_CMakeLists.txt.in +++ b/cmake/xml2_CMakeLists.txt.in @@ -15,4 +15,5 @@ ExternalProject_Add(xml2 BUILD_COMMAND make -j INSTALL_COMMAND make install && ln -sf $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/xz_CMakeLists.txt.in b/cmake/xz_CMakeLists.txt.in index e704fcbbbf..f3582e38df 100644 --- a/cmake/xz_CMakeLists.txt.in +++ b/cmake/xz_CMakeLists.txt.in @@ -1,14 +1,15 @@ # xz ExternalProject_Add(xz - GIT_REPOSITORY https://github.com/xz-mirror/xz.git - GIT_TAG v5.4.4 + GIT_REPOSITORY https://github.com/xz-mirror/xz.git + GIT_TAG v5.4.4 SOURCE_DIR "${TD_CONTRIB_DIR}/xz" BINARY_DIR "" #BUILD_IN_SOURCE TRUE - CMAKE_ARGS + CMAKE_ARGS CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" -) \ No newline at end of file + GIT_SHALLOW true +) diff --git a/cmake/zlib_CMakeLists.txt.in b/cmake/zlib_CMakeLists.txt.in index 87a17c7377..00e6d7dfce 100644 --- a/cmake/zlib_CMakeLists.txt.in +++ b/cmake/zlib_CMakeLists.txt.in @@ -12,4 +12,5 @@ ExternalProject_Add(zlib BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/zstd_CMakeLists.txt.in b/cmake/zstd_CMakeLists.txt.in index f8604f9db6..d7850a5c7e 100644 --- a/cmake/zstd_CMakeLists.txt.in +++ b/cmake/zstd_CMakeLists.txt.in @@ -1,15 +1,16 @@ -# zstb +# zstb #ExternalProject_Add(zstd - #GIT_REPOSITORY https://github.com/facebook/zstd.git + #GIT_REPOSITORY https://github.com/facebook/zstd.git #GIT_TAG v1.5.5 #SOURCE_DIR "${TD_CONTRIB_DIR}/zstd" #DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" #BINARY_DIR "" - #CMAKE_COMMAND + #CMAKE_COMMAND #CMAKE_ARGS ${TD_CONTRIB_DIR}/zstd/build/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${CMAKE_BINARY_DIR}/build/lib - #BUILD_COMMAND make -j4 + #BUILD_COMMAND make -j4 #INSTALL_COMMAND "" #TEST_COMMAND "" - #) \ No newline at end of file + #GIT_SHALLOW true + #) From bc96802a10a33a3fce3f6db15cf7c162c78ba505 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 18 Sep 2024 11:19:45 +0800 Subject: [PATCH 90/94] set pointer null after free --- source/util/src/tpagedbuf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index e8303b563e..ffb11b156d 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -547,6 +547,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { int32_t code = lruListPushFront(pBuf->lruList, *pi); if (TSDB_CODE_SUCCESS != code) { taosMemoryFree((*pi)->pData); + (*pi)->pData = NULL; terrno = code; return NULL; } @@ -557,7 +558,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { int32_t code = loadPageFromDisk(pBuf, *pi); if (code != 0) { taosMemoryFree((*pi)->pData); - + (*pi)->pData = NULL; terrno = code; return NULL; } From 6be4e32156c7248d2a3f5a57757909f1d8d2ba91 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 18 Sep 2024 13:32:16 +0800 Subject: [PATCH 91/94] fix: error code return in TDB --- source/libs/tdb/src/db/tdbBtree.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index ff40616d70..93c214545f 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -2479,7 +2479,10 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { if (c > 0) { pBtc->idx += 1; } - (void)tdbBtcMoveDownward(pBtc); + if (tdbBtcMoveDownward(pBtc) < 0) { + tdbError("tdb/btc-move-to: btc move downward failed."); + return TSDB_CODE_FAILED; + } } } From 20a7c5d53047f1da4752a844da68a33f6e545a10 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 18 Sep 2024 14:34:11 +0800 Subject: [PATCH 92/94] docs: add queryTableNotExistAsEmpty description --- docs/zh/14-reference/01-components/02-taosc.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 docs/zh/14-reference/01-components/02-taosc.md diff --git a/docs/zh/14-reference/01-components/02-taosc.md b/docs/zh/14-reference/01-components/02-taosc.md old mode 100644 new mode 100755 index 2563fa6811..32baac9a3b --- a/docs/zh/14-reference/01-components/02-taosc.md +++ b/docs/zh/14-reference/01-components/02-taosc.md @@ -38,7 +38,8 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在 |enableCoreFile | crash 时是否生成 core 文件，0: 不生成， 1： 生成；缺省值：1 | |enableScience | 是否开启科学计数法显示浮点数; 0: 不开始, 1: 开启；缺省值：1 | |compressMsgSize | 是否对 RPC 消息进行压缩; -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩; 缺省值 -1| +|queryTableNotExistAsEmpty | 查询表不存在时是否返回空结果集; false: 返回错误; true: 返回空结果集; 缺省值 false| ## API -请参考[连接器](../../connector) \ No newline at end of file +请参考[连接器](../../connector) From f6174d0b095c2a1c6b6d8e3e7b1529085ce3817c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 18 Sep 2024 17:00:38 +0800 Subject: [PATCH 93/94] adjust stream slice res --- source/libs/stream/src/tstreamFileState.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 7cd9fb7ea3..923aaa945b 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -1013,10 +1013,11 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) { SStreamStateCur* pCur = streamStateFillSeekToLast_rocksdb(pFileState->pFileStore); if (pCur == NULL) { - return -1; + return code; } int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); - while (code == TSDB_CODE_SUCCESS) { + int32_t winRes = TSDB_CODE_SUCCESS; + while (winRes == TSDB_CODE_SUCCESS) { if (pFileState->curRowCount >= recoverNum) { break; } @@ -1024,8 +1025,8 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) { void* pVal = NULL; int32_t vlen = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); - code = streamStateFillGetKVByCur_rocksdb(pCur, pNewPos->pKey, (const void**)&pVal, &vlen); - if (code != TSDB_CODE_SUCCESS || isFlushedState(pFileState, pFileState->getTs(pNewPos->pKey), 0)) { + winRes = streamStateFillGetKVByCur_rocksdb(pCur, pNewPos->pKey, (const void**)&pVal, &vlen); + if (winRes != TSDB_CODE_SUCCESS || isFlushedState(pFileState, pFileState->getTs(pNewPos->pKey), 0)) { destroyRowBuffPos(pNewPos); SListNode* pNode = tdListPopTail(pFileState->usedBuffs); taosMemoryFreeClear(pNode); @@ -1036,8 +1037,8 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) { memcpy(pNewPos->pRowBuff, pVal, vlen); taosMemoryFreeClear(pVal); pNewPos->beFlushed = true; - code = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES); - if (code != TSDB_CODE_SUCCESS) { + winRes = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES); + if (winRes != TSDB_CODE_SUCCESS) { destroyRowBuffPos(pNewPos); break; } @@ -1045,7 +1046,7 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) { } streamStateFreeCur(pCur); - return TSDB_CODE_SUCCESS; + return code; } int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, From 442bf96b87e6682e5e68237034c1a344a1220a64 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 18 Sep 2024 17:24:00 +0800 Subject: [PATCH 94/94] add check --- source/libs/executor/src/streamfilloperator.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index 3fcba5fb6c..c178f22784 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -112,6 +112,9 @@ void destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { } void destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { + if (pFillInfo == NULL) { + return; + } if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_SET_VALUE_F || pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { taosMemoryFreeClear(pFillInfo->pResRow->pRowVal);