From b7788aca04f161e539f49b04fae5349e62948013 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 2 Mar 2022 16:22:43 +0800 Subject: [PATCH 1/8] fix --- source/client/src/tmq.c | 54 +++++++++++++---------- source/dnode/vnode/inc/meta.h | 12 ++--- source/dnode/vnode/inc/vnode.h | 21 +++++++++ source/dnode/vnode/src/inc/metaDef.h | 2 +- source/dnode/vnode/src/meta/metaBDBImpl.c | 34 +++++++------- source/dnode/vnode/src/tq/tq.c | 4 +- source/libs/executor/src/executor.c | 8 ++-- source/libs/wal/src/walWrite.c | 13 +++--- source/util/src/terror.c | 3 ++ 9 files changed, 94 insertions(+), 57 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index d9ab23b9fa..ab955be491 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -66,6 +66,7 @@ struct tmq_t { STscObj* pTscObj; tmq_commit_cb* commit_cb; int32_t nextTopicIdx; + int32_t waitingRequest; SArray* clientTopics; // SArray STaosQueue* mqueue; // queue of tmq_message_t STaosQall* qall; @@ -209,12 +210,7 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { if (pParam->tmq->commit_cb) { pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); } - if (!pParam->async) - tsem_post(&pParam->rspSem); - else { - tsem_destroy(&pParam->rspSem); - free(param); - } + if (!pParam->async) tsem_post(&pParam->rspSem); return 0; } @@ -243,6 +239,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs pTmq->status = 0; pTmq->pollCnt = 0; pTmq->epoch = 0; + pTmq->waitingRequest = 0; // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); @@ -315,6 +312,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in } pParam->tmq = tmq; tsem_init(&pParam->rspSem, 0, 0); + pParam->async = async; pRequest->body.requestMsg = (SDataBuf){ .pData = buf, @@ -335,6 +333,9 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in resp = pParam->rspErr; } + tsem_destroy(&pParam->rspSem); + free(pParam); + if (pArray) { taosArrayDestroy(pArray); } @@ -576,7 +577,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { if (tmq_message == NULL) return 0; - SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; + SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; return pRsp->skipLogNum; } @@ -625,13 +626,14 @@ void tmqShowMsg(tmq_message_t* tmq_message) { } int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { - printf("recv poll\n"); + /*printf("recv poll\n");*/ SMqPollCbParam* pParam = (SMqPollCbParam*)param; SMqClientVg* pVg = pParam->pVg; tmq_t* tmq = pParam->tmq; if (code != 0) { printf("msg discard\n"); if (pParam->epoch == tmq->epoch) { + atomic_sub_fetch_32(&tmq->waitingRequest, 1); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } return 0; @@ -646,6 +648,8 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { if (msgEpoch != tmqEpoch) { printf("mismatch rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch); + } else { + atomic_sub_fetch_32(&tmq->waitingRequest, 1); } /*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/ @@ -658,7 +662,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ if (pRsp->consumeRsp.numOfTopics == 0) { - printf("no data\n"); + /*printf("no data\n");*/ if (pParam->epoch == tmq->epoch) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } @@ -667,7 +671,8 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } pRsp->extra = pParam->pVg; taosWriteQitem(tmq->mqueue, pRsp); - printf("poll in queue\n"); + + /*printf("poll in queue\n");*/ /*pParam->rspMsg = (tmq_message_t*)pRsp;*/ /*pVg->currentOffset = pRsp->consumeRsp.rspOffset;*/ @@ -743,7 +748,6 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { } int32_t tmqAskEp(tmq_t* tmq, bool sync) { - printf("ask ep sync %d\n", sync); int32_t tlen = sizeof(SMqCMGetSubEpReq); SMqCMGetSubEpReq* buf = malloc(tlen); if (buf == NULL) { @@ -812,7 +816,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { return TMQ_RESP_ERR__FAIL; } -SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) { +SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { int64_t reqOffset; if (pVg->currentOffset >= 0) { reqOffset = pVg->currentOffset; @@ -832,7 +836,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie strcpy(pReq->topic, pTopic->topicName); strcpy(pReq->cgroup, tmq->groupId); - pReq->blockingTime = blocking_time; + pReq->blockingTime = blockingTime; pReq->consumerId = tmq->consumerId; pReq->epoch = tmq->epoch; pReq->currentOffset = reqOffset; @@ -863,7 +867,7 @@ void tmqClearUnhandleMsg(tmq_t* tmq) { } int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { - printf("call poll\n"); + /*printf("call poll\n");*/ for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { @@ -900,7 +904,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { sendInfo->fp = tmqPollCb; int64_t transporterId = 0; - printf("send poll\n"); + /*printf("send poll\n");*/ + atomic_add_fetch_32(&tmq->waitingRequest, 1); asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; tmq->pollCnt++; @@ -912,7 +917,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { // return int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) { if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__EP_RSP) { - printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch); + /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/ if (rspMsg->head.epoch > atomic_load_32(&tmq->epoch)) { tmqUpdateEp(tmq, rspMsg->head.epoch, &rspMsg->getEpRsp); tmqClearUnhandleMsg(tmq); @@ -935,9 +940,9 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese } if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { - printf("handle poll rsp %d\n", rspMsg->head.mqMsgType); + /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/ if (rspMsg->head.epoch == atomic_load_32(&tmq->epoch)) { - printf("epoch match\n"); + /*printf("epoch match\n");*/ SMqClientVg* pVg = rspMsg->extra; pVg->currentOffset = rspMsg->consumeRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); @@ -947,7 +952,7 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese taosFreeQitem(rspMsg); } } else { - printf("handle ep rsp %d\n", rspMsg->head.mqMsgType); + /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/ bool reset = false; tmqHandleRes(tmq, rspMsg, &reset); taosFreeQitem(rspMsg); @@ -972,12 +977,16 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { if (rspMsg == NULL) { taosReadAllQitems(tmq->mqueue, tmq->qall); } - tmqHandleAllRsp(tmq, blocking_time, false); - - tmqPollImpl(tmq, blocking_time); + rspMsg = tmqHandleAllRsp(tmq, blocking_time, false); + if (rspMsg) { + return rspMsg; + } while (1) { /*printf("cycle\n");*/ + if (atomic_load_32(&tmq->waitingRequest) == 0) { + tmqPollImpl(tmq, blocking_time); + } taosReadAllQitems(tmq->mqueue, tmq->qall); rspMsg = tmqHandleAllRsp(tmq, blocking_time, true); if (rspMsg) { @@ -986,7 +995,6 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { if (blocking_time != 0) { int64_t endTime = taosGetTimestampMs(); if (endTime - startTime > blocking_time) { - printf("normal exit\n"); return NULL; } } diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index c98a97d0dc..fd079b8f32 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -24,8 +24,8 @@ extern "C" { #endif -#define META_SUPER_TABLE TD_SUPER_TABLE -#define META_CHILD_TABLE TD_CHILD_TABLE +#define META_SUPER_TABLE TD_SUPER_TABLE +#define META_CHILD_TABLE TD_CHILD_TABLE #define META_NORMAL_TABLE TD_NORMAL_TABLE // Types exported @@ -50,14 +50,14 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaCommit(SMeta *pMeta); // For Query -STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); -STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); +STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); +STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); -STSchema * metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); +STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); -char * metaTbCursorNext(SMTbCursor *pTbCur); +char *metaTbCursorNext(SMTbCursor *pTbCur); SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid); void metaCloseCtbCurosr(SMCtbCursor *pCtbCur); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 31f04e840a..2c6bfd879a 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -214,6 +214,10 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SA //} static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { + if (pHandle->tbIdHash) { + taosHashClear(pHandle->tbIdHash); + } + pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (pHandle->tbIdHash == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -228,6 +232,23 @@ static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const S return 0; } +static FORCE_INLINE int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { + if (pHandle->tbIdHash == NULL) { + pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (pHandle->tbIdHash == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + + for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { + int64_t *pKey = (int64_t *)taosArrayGet(tbUidList, i); + taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0); + } + + return 0; +} + int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo); diff --git a/source/dnode/vnode/src/inc/metaDef.h b/source/dnode/vnode/src/inc/metaDef.h index f4128f7170..71bfd91356 100644 --- a/source/dnode/vnode/src/inc/metaDef.h +++ b/source/dnode/vnode/src/inc/metaDef.h @@ -76,4 +76,4 @@ struct SMeta { } #endif -#endif /*_TD_META_DEF_H_*/ \ No newline at end of file +#endif /*_TD_META_DEF_H_*/ diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index 3c464391d2..f75d9d3db0 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -62,10 +62,10 @@ static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT * static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); -static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); +static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static void metaClearTbCfg(STbCfg *pTbCfg); static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); -static void * metaDecodeSchema(void *buf, SSchemaWrapper *pSW); +static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW); static void metaDBWLock(SMetaDB *pDB); static void metaDBRLock(SMetaDB *pDB); static void metaDBULock(SMetaDB *pDB); @@ -142,7 +142,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { tb_uid_t uid; char buf[512]; char buf1[512]; - void * pBuf; + void *pBuf; DBT key1, value1; DBT key2, value2; SSchema *pSchema = NULL; @@ -394,7 +394,7 @@ static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { STbCfg *pTbCfg = (STbCfg *)(pValue->app_data); - DBT * pDbt; + DBT *pDbt; if (pTbCfg->type == META_CHILD_TABLE) { // pDbt = calloc(2, sizeof(DBT)); @@ -479,7 +479,7 @@ static void metaClearTbCfg(STbCfg *pTbCfg) { /* ------------------------ FOR QUERY ------------------------ */ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { - STbCfg * pTbCfg = NULL; + STbCfg *pTbCfg = NULL; SMetaDB *pDB = pMeta->pDB; DBT key = {0}; DBT value = {0}; @@ -509,7 +509,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { } STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { - STbCfg * pTbCfg = NULL; + STbCfg *pTbCfg = NULL; SMetaDB *pDB = pMeta->pDB; DBT key = {0}; DBT pkey = {0}; @@ -543,10 +543,10 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { uint32_t nCols; SSchemaWrapper *pSW = NULL; - SMetaDB * pDB = pMeta->pDB; + SMetaDB *pDB = pMeta->pDB; int ret; - void * pBuf; - SSchema * pSchema; + void *pBuf; + SSchema *pSchema; SSchemaKey schemaKey = {uid, sver, 0}; DBT key = {0}; DBT value = {0}; @@ -578,7 +578,7 @@ struct SMTbCursor { SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { SMTbCursor *pTbCur = NULL; - SMetaDB * pDB = pMeta->pDB; + SMetaDB *pDB = pMeta->pDB; pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur)); if (pTbCur == NULL) { @@ -609,7 +609,7 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { DBT key = {0}; DBT value = {0}; STbCfg tbCfg; - void * pBuf; + void *pBuf; for (;;) { if (pTbCur->pCur->get(pTbCur->pCur, &key, &value, DB_NEXT) == 0) { @@ -631,10 +631,10 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { STSchemaBuilder sb; - STSchema * pTSchema = NULL; - SSchema * pSchema; + STSchema *pTSchema = NULL; + SSchema *pSchema; SSchemaWrapper *pSW; - STbCfg * pTbCfg; + STbCfg *pTbCfg; tb_uid_t quid; pTbCfg = metaGetTbInfoByUid(pMeta, uid); @@ -662,13 +662,13 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { } struct SMCtbCursor { - DBC * pCur; + DBC *pCur; tb_uid_t suid; }; SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { SMCtbCursor *pCtbCur = NULL; - SMetaDB * pDB = pMeta->pDB; + SMetaDB *pDB = pMeta->pDB; int ret; pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur)); @@ -700,7 +700,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { DBT skey = {0}; DBT pkey = {0}; DBT pval = {0}; - void * pBuf; + void *pBuf; STbCfg tbCfg; // Set key diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index aa198d0806..ccad006657 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -72,6 +72,8 @@ void tqClose(STQ* pTq) { } int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { + // if waiting + // memcpy and send msg to fetch thread // TODO: add reference // if handle waiting, launch query and response to consumer // @@ -210,7 +212,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; int64_t fetchOffset; - /*int64_t blockingTime = pReq->blockingTime;*/ + int64_t blockingTime = pReq->blockingTime; if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { fetchOffset = 0; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b90ce275d2..19a3874f75 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -95,17 +95,17 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { } int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo* )tinfo; + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; // traverse to the streamscan node to add this table id SOperatorInfo* pInfo = pTaskInfo->pRoot; - while(pInfo->operatorType != OP_StreamScan) { + while (pInfo->operatorType != OP_StreamScan) { pInfo = pInfo->pDownstream[0]; } SStreamBlockScanInfo* pScanInfo = pInfo->info; if (isAdd) { - int32_t code = tqReadHandleSetTbUidList(pScanInfo->readerHandle, tableIdList); + int32_t code = tqReadHandleAddTbUidList(pScanInfo->readerHandle, tableIdList); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -114,4 +114,4 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA } return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 4b1f0ba306..4cc7040119 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -162,8 +162,8 @@ int32_t walEndSnapshot(SWal *pWal) { } // iterate files, until the searched result for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { - if ((pWal->cfg.retentionSize != -1 && pWal->totSize > pWal->cfg.retentionSize) - || (pWal->cfg.retentionPeriod != -1 && iter->closeTs + pWal->cfg.retentionPeriod > ts)) { + if ((pWal->cfg.retentionSize != -1 && pWal->totSize > pWal->cfg.retentionSize) || + (pWal->cfg.retentionPeriod != -1 && iter->closeTs + pWal->cfg.retentionPeriod > ts)) { // delete according to file size or close time deleteCnt++; newTotSize -= iter->fileSize; @@ -279,6 +279,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in } else { // reject skip log or rewrite log // must truncate explicitly first + terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } /*if (!tfValid(pWal->pWriteLogTFile)) return -1;*/ @@ -303,16 +304,18 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { // ftruncate - code = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + return -1; } if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) { // ftruncate - code = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + return -1; } code = walWriteIndex(pWal, index, offset); @@ -329,7 +332,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in pthread_mutex_unlock(&pWal->mutex); - return code; + return 0; } void walFsync(SWal *pWal, bool forceFsync) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c8cafb93dd..f587c988ad 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +// clang-format off + #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" @@ -407,6 +409,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGTYPE, "Invalid msg type") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Unexpected generic error in wal") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") // tfs TAOS_DEFINE_ERROR(TSDB_CODE_FS_APP_ERROR, "tfs out of memory") From bc360c075266da1bda51c528cf039b640ceaef1e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 3 Mar 2022 11:41:49 +0800 Subject: [PATCH 2/8] perf test --- include/common/tcommon.h | 5 +- source/client/src/tmq.c | 153 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 145 insertions(+), 13 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a04e2afc94..e26262a8eb 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -50,7 +50,8 @@ enum { }; enum { - TMQ_MSG_TYPE__POLL_RSP = 0, + TMQ_MSG_TYPE__DUMMY = 0, + TMQ_MSG_TYPE__POLL_RSP, TMQ_MSG_TYPE__EP_RSP, }; @@ -285,4 +286,4 @@ typedef struct SSessionWindow { } #endif -#endif /*_TD_COMMON_DEF_H_*/ +#endif /*_TD_COMMON_DEF_H_*/ diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index ab955be491..0e84371199 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -28,6 +28,11 @@ #include "tqueue.h" #include "tref.h" +static int64_t perfWrite; +static int64_t perfRead; +static int64_t perfRead2; +static int64_t perfRead3; + struct tmq_list_t { int32_t cnt; int32_t tot; @@ -67,6 +72,7 @@ struct tmq_t { tmq_commit_cb* commit_cb; int32_t nextTopicIdx; int32_t waitingRequest; + int32_t readyRequest; SArray* clientTopics; // SArray STaosQueue* mqueue; // queue of tmq_message_t STaosQall* qall; @@ -118,10 +124,12 @@ typedef struct { } SMqAskEpCbParam; typedef struct { - tmq_t* tmq; - SMqClientVg* pVg; - int32_t epoch; - tsem_t rspSem; + tmq_t* tmq; + SMqClientVg* pVg; + int32_t epoch; + tsem_t rspSem; + tmq_message_t** msg; + int32_t sync; } SMqPollCbParam; typedef struct { @@ -240,6 +248,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs pTmq->pollCnt = 0; pTmq->epoch = 0; pTmq->waitingRequest = 0; + pTmq->readyRequest = 0; // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); @@ -651,6 +660,24 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } else { atomic_sub_fetch_32(&tmq->waitingRequest, 1); } + if (pParam->sync == 1) { + /**pParam->msg = malloc(sizeof(tmq_message_t));*/ + *pParam->msg = taosAllocateQitem(sizeof(tmq_message_t)); + if (*pParam->msg) { + memcpy(*pParam->msg, pMsg->pData, sizeof(SMqRspHead)); + tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &((*pParam->msg)->consumeRsp)); + if ((*pParam->msg)->consumeRsp.numOfTopics != 0) { + pVg->currentOffset = (*pParam->msg)->consumeRsp.rspOffset; + } + int64_t begin = clock(); + taosWriteQitem(tmq->mqueue, *pParam->msg); + perfWrite += clock() - begin; + tsem_post(&pParam->rspSem); + return 0; + } + tsem_post(&pParam->rspSem); + return -1; + } /*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/ tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); @@ -671,6 +698,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } pRsp->extra = pParam->pVg; taosWriteQitem(tmq->mqueue, pRsp); + atomic_add_fetch_32(&tmq->readyRequest, 1); /*printf("poll in queue\n");*/ /*pParam->rspMsg = (tmq_message_t*)pRsp;*/ @@ -742,7 +770,8 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->getEpRsp); - taosWriteQitem(tmq->mqueue, pRsp); + + /*taosWriteQitem(tmq->mqueue, pRsp);*/ } return 0; } @@ -866,6 +895,73 @@ void tmqClearUnhandleMsg(tmq_t* tmq) { } } +tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { + tmq_message_t* msg = NULL; + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); + /*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/ + /*continue;*/ + /*}*/ + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); + if (pReq == NULL) { + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + // TODO: out of mem + return NULL; + } + SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam)); + if (param == NULL) { + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + // TODO: out of mem + return NULL; + } + param->tmq = tmq; + param->pVg = pVg; + param->epoch = tmq->epoch; + param->sync = 1; + param->msg = &msg; + tsem_init(¶m->rspSem, 0, 0); + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); + pRequest->body.requestMsg = (SDataBuf){ + .pData = pReq, + .len = sizeof(SMqConsumeReq), + .handle = NULL, + }; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = param; + sendInfo->fp = tmqPollCb; + + int64_t transporterId = 0; + /*printf("send poll\n");*/ + atomic_add_fetch_32(&tmq->waitingRequest, 1); + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); + pVg->pollCnt++; + tmq->pollCnt++; + + int64_t begin = clock(); + tsem_wait(¶m->rspSem); + perfRead3 += clock() - begin; + tmq_message_t* nmsg = NULL; + while (1) { + int64_t begin1 = clock(); + taosReadQitem(tmq->mqueue, (void**)&nmsg); + perfRead2 += clock() - begin1; + if (nmsg == NULL) continue; + /*while (nmsg->head.mqMsgType != TMQ_MSG_TYPE__POLL_RSP) {*/ + /*taosReadQitem(tmq->mqueue, (void**)&nmsg);*/ + /*}*/ + perfRead += clock() - begin; + return nmsg; + } + } + } + return NULL; +} + int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { /*printf("call poll\n");*/ for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { @@ -891,6 +987,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { param->tmq = tmq; param->pVg = pVg; param->epoch = tmq->epoch; + param->sync = 0; SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pRequest->body.requestMsg = (SDataBuf){ .pData = pReq, @@ -940,6 +1037,7 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese } if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { + atomic_sub_fetch_32(&tmq->readyRequest, 1); /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/ if (rspMsg->head.epoch == atomic_load_32(&tmq->epoch)) { /*printf("epoch match\n");*/ @@ -969,10 +1067,38 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* rspMsg = NULL; int64_t startTime = taosGetTimestampMs(); + int64_t status = atomic_load_64(&tmq->status); + tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT); + + while (1) { + rspMsg = tmqSyncPollImpl(tmq, blocking_time); + if (rspMsg && rspMsg->consumeRsp.numOfTopics) { + return rspMsg; + } + + if (blocking_time != 0) { + int64_t endTime = taosGetTimestampMs(); + if (endTime - startTime > blocking_time) { + printf("perf write %f\n", (double)perfWrite / CLOCKS_PER_SEC); + printf("perf read %f\n", (double)perfRead / CLOCKS_PER_SEC); + printf("perf read2 %f\n", (double)perfRead2 / CLOCKS_PER_SEC); + printf("perf read3 %f\n", (double)perfRead3 / CLOCKS_PER_SEC); + return NULL; + } + } else + return NULL; + } +} + +tmq_message_t* tmq_consumer_poll_v0(tmq_t* tmq, int64_t blocking_time) { + tmq_message_t* rspMsg = NULL; + int64_t startTime = taosGetTimestampMs(); + // TODO: put into another thread or delayed queue int64_t status = atomic_load_64(&tmq->status); tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT); +#if 0 taosGetQitem(tmq->qall, (void**)&rspMsg); if (rspMsg == NULL) { taosReadAllQitems(tmq->mqueue, tmq->qall); @@ -981,23 +1107,27 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { if (rspMsg) { return rspMsg; } +#endif while (1) { /*printf("cycle\n");*/ if (atomic_load_32(&tmq->waitingRequest) == 0) { tmqPollImpl(tmq, blocking_time); } + while (atomic_load_32(&tmq->readyRequest) == 0) { + sched_yield(); + if (blocking_time != 0) { + int64_t endTime = taosGetTimestampMs(); + if (endTime - startTime > blocking_time) { + return NULL; + } + } + } taosReadAllQitems(tmq->mqueue, tmq->qall); rspMsg = tmqHandleAllRsp(tmq, blocking_time, true); if (rspMsg) { return rspMsg; } - if (blocking_time != 0) { - int64_t endTime = taosGetTimestampMs(); - if (endTime - startTime > blocking_time) { - return NULL; - } - } } } @@ -1135,6 +1265,7 @@ void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; tDeleteSMqConsumeRsp(pRsp); + /*free(tmq_message);*/ taosFreeQitem(tmq_message); } From 92da01a8ba0872f64f460b756cbaff41ea0487d7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Mar 2022 13:25:39 +0800 Subject: [PATCH 3/8] [td-13039] refactor. --- include/common/tcommon.h | 18 ------- include/common/{tep.h => tdatablock.h} | 0 include/libs/scalar/filter.h | 5 +- include/libs/sync/sync.h | 2 +- include/util/thash.h | 15 ++---- source/client/inc/clientInt.h | 4 +- source/client/src/clientImpl.c | 2 +- source/client/src/tmq.c | 2 +- source/common/src/{tep.c => tdatablock.c} | 2 +- source/common/src/tglobal.c | 2 +- source/common/test/commonTests.cpp | 4 +- source/dnode/mgmt/impl/inc/dndInt.h | 6 +-- .../dnode/mgmt/impl/test/sut/src/client.cpp | 2 +- source/dnode/mnode/impl/inc/mndInt.h | 2 +- source/libs/catalog/test/catalogTests.cpp | 4 +- source/libs/executor/src/executorimpl.c | 10 ++-- source/libs/executor/src/tsort.c | 4 +- source/libs/executor/test/executorTests.cpp | 12 ++--- source/libs/executor/test/sortTests.cpp | 2 +- source/libs/function/src/taggfunction.c | 2 +- .../libs/parser/test/mockCatalogService.cpp | 2 +- source/libs/qworker/test/qworkerTests.cpp | 17 +++--- source/libs/scalar/inc/filterInt.h | 10 ++-- source/libs/scalar/src/filter.c | 6 +-- source/libs/scalar/src/scalar.c | 10 ++-- source/libs/scalar/src/sclvector.c | 12 ++--- .../libs/scalar/test/filter/filterTests.cpp | 16 +++--- .../libs/scalar/test/scalar/scalarTests.cpp | 12 ++--- source/libs/scheduler/test/schedulerTests.cpp | 8 +-- source/libs/sync/src/syncIO.c | 2 +- source/libs/transport/test/pushClient.c | 2 +- source/libs/transport/test/rclient.c | 2 +- source/libs/transport/test/syncClient.c | 2 +- source/libs/transport/test/transUT.cc | 4 +- source/util/src/thash.c | 52 ++++++++++--------- 35 files changed, 116 insertions(+), 141 deletions(-) rename include/common/{tep.h => tdatablock.h} (100%) rename source/common/src/{tep.c => tdatablock.c} (99%) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a04e2afc94..18c1bd5e9a 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -25,24 +25,6 @@ extern "C" { #endif -// typedef struct STimeWindow { -// TSKEY skey; -// TSKEY ekey; -// } STimeWindow; - -// typedef struct { -// int32_t dataLen; -// char name[TSDB_TABLE_FNAME_LEN]; -// char *data; -// } STagData; - -// typedef struct SSchema { -// uint8_t type; -// char name[TSDB_COL_NAME_LEN]; -// int16_t colId; -// int16_t bytes; -// } SSchema; - enum { TMQ_CONF__RESET_OFFSET__LATEST = -1, TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, diff --git a/include/common/tep.h b/include/common/tdatablock.h similarity index 100% rename from include/common/tep.h rename to include/common/tdatablock.h diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index fedb487931..a93180800e 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -19,10 +19,12 @@ extern "C" { #endif +#include "tcommon.h" +#include "nodes.h" + typedef struct SFilterInfo SFilterInfo; typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); - enum { FLT_OPTION_NO_REWRITE = 1, FLT_OPTION_TIMESTAMP = 2, @@ -34,7 +36,6 @@ typedef struct SFilterColumnParam{ SArray* pDataBlock; } SFilterColumnParam; - extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options); extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 53fad4607a..23e3ef7525 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -21,7 +21,7 @@ extern "C" { #endif #include -#include +#include #include "taosdef.h" #include "trpc.h" diff --git a/include/util/thash.h b/include/util/thash.h index 017cc8696f..57b20c65ee 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -28,7 +28,8 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len); typedef void (*_hash_before_fn_t)(void *); typedef void (*_hash_free_fn_t)(void *); -#define HASH_NODE_EXIST(code) (code == -2) +#define HASH_KEY_ALREADY_EXISTS (-2) +#define HASH_NODE_EXIST(code) (code == HASH_KEY_ALREADY_EXISTS) /** * murmur hash algorithm @@ -49,24 +50,14 @@ uint32_t taosIntHash_32(const char *key, uint32_t len); uint32_t taosIntHash_64(const char *key, uint32_t len); _hash_fn_t taosGetDefaultHashFunction(int32_t type); - _equal_fn_t taosGetDefaultEqualFunction(int32_t type); -typedef struct SHashNode { - struct SHashNode *next; - uint32_t hashVal; // the hash value of key - uint32_t dataLen; // length of data - uint32_t keyLen; // length of the key - uint16_t refCount; // reference count - int8_t removed; // flag to indicate removed - char data[]; -} SHashNode; - typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, } SHashLockTypeE; +typedef struct SHashNode SHashNode; typedef struct SHashObj SHashObj; /** diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index c93ea1aabe..158bccf8cc 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,12 +20,12 @@ extern "C" { #endif -#include "tcommon.h" #include "parser.h" #include "query.h" #include "taos.h" +#include "tcommon.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "thash.h" #include "tlist.h" #include "tmsg.h" diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index a0ba668f8a..259d0e5799 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -4,8 +4,8 @@ #include "parser.h" #include "planner.h" #include "scheduler.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "tglobal.h" #include "tmsgtype.h" #include "tpagedbuf.h" diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index d9ab23b9fa..490c352e8e 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -20,8 +20,8 @@ #include "parser.h" #include "planner.h" #include "scheduler.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "tglobal.h" #include "tmsgtype.h" #include "tpagedbuf.h" diff --git a/source/common/src/tep.c b/source/common/src/tdatablock.c similarity index 99% rename from source/common/src/tep.c rename to source/common/src/tdatablock.c index e2880441be..5b7557b749 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tdatablock.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "tep.h" +#include "tdatablock.h" #include "tcompare.h" #include "tglobal.h" diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5998b7c9ce..91fab939cf 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -17,7 +17,7 @@ #include "tglobal.h" #include "tcompare.h" #include "tconfig.h" -#include "tep.h" +#include "tdatablock.h" #include "tlog.h" SConfig *tsCfg = NULL; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 4821d60875..7a7dd3f24b 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -1,7 +1,7 @@ -#include "tcommon.h" #include -#include +#include #include +#include "tcommon.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 9fabe40186..db90baff78 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -23,9 +23,11 @@ extern "C" { #include "os.h" #include "cJSON.h" +#include "monitor.h" #include "tcache.h" #include "tcrc32c.h" -#include "tep.h" +#include "tdatablock.h" +#include "tglobal.h" #include "thash.h" #include "tlockfree.h" #include "tlog.h" @@ -35,8 +37,6 @@ extern "C" { #include "tthread.h" #include "ttime.h" #include "tworker.h" -#include "tglobal.h" -#include "monitor.h" #include "dnode.h" diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index b89cb02834..f22bc9d276 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -14,7 +14,7 @@ */ #include "sut.h" -#include "tep.h" +#include "tdatablock.h" static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { TestClient* client = (TestClient*)parent; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 929d47184b..28bf85e2c0 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -20,7 +20,7 @@ #include "sdb.h" #include "tcache.h" -#include "tep.h" +#include "tdatablock.h" #include "tglobal.h" #include "tqueue.h" #include "ttime.h" diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index b417a645be..ef8da0f897 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -27,13 +27,13 @@ #include "addr_any.h" #include "catalog.h" +#include "catalogInt.h" #include "stub.h" #include "taos.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "trpc.h" #include "tvariant.h" -#include "catalogInt.h" namespace { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 209bd643e5..e037211f1e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -15,12 +15,12 @@ #include "os.h" -#include "tep.h" -#include "tsort.h" -#include "texception.h" #include "parser.h" +#include "tdatablock.h" +#include "texception.h" #include "tglobal.h" #include "tmsg.h" +#include "tsort.h" #include "ttime.h" #include "executorimpl.h" @@ -8730,10 +8730,8 @@ static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type static int64_t getQuerySupportBufSize(size_t numOfTables) { size_t s1 = sizeof(STableQueryInfo); - size_t s2 = sizeof(SHashNode); - // size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb - return (int64_t)((s1 + s2) * 1.5 * numOfTables); + return (int64_t)(s1* 1.5 * numOfTables); } int32_t checkForQueryBuf(size_t numOfTables) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index d042dc0eff..34dd248ba7 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -16,11 +16,11 @@ #include "tcommon.h" #include "query.h" -#include "tsort.h" -#include "tep.h" +#include "tdatablock.h" #include "tdef.h" #include "tlosertree.h" #include "tpagedbuf.h" +#include "tsort.h" #include "tutil.h" typedef struct STupleHandle { diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index b5d8bb8019..3e7cbc83ea 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -26,15 +26,15 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" -#include "tep.h" -#include "trpc.h" -#include "stub.h" #include "executor.h" +#include "stub.h" +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" #include "tmsg.h" #include "tname.h" +#include "trpc.h" +#include "tvariant.h" namespace { diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index fc366e4cc8..32d148be1d 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -29,8 +29,8 @@ #include "executor.h" #include "stub.h" #include "taos.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "trpc.h" #include "tvariant.h" diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index c36fc9c659..7fb63f5910 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -28,8 +28,8 @@ #include "tbuffer.h" #include "tcompression.h" //#include "queryLog.h" +#include "tdatablock.h" #include "tudf.h" -#include "tep.h" #define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) #define GET_INPUT_DATA(x, y) ((char*) colDataGetData((x)->pInput, (y))) diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 00d64bd12a..cdb547ee1b 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -15,10 +15,10 @@ #include "mockCatalogService.h" -#include "tep.h" #include #include #include +#include "tdatablock.h" #include "tname.h" #include "ttypes.h" diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 8ad5a76388..e599bbede1 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -29,18 +29,17 @@ #include "os.h" -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" -#include "tep.h" -#include "trpc.h" +#include "addr_any.h" +#include "dataSinkMgt.h" +#include "executor.h" #include "planner.h" #include "qworker.h" #include "stub.h" -#include "addr_any.h" -#include "executor.h" -#include "dataSinkMgt.h" - +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" +#include "trpc.h" +#include "tvariant.h" namespace { diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index b4fe9a67ca..e1ffb2efd1 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -20,13 +20,13 @@ extern "C" { #endif +#include "query.h" +#include "querynodes.h" +#include "scalar.h" +#include "tcommon.h" +#include "tdatablock.h" #include "thash.h" #include "tname.h" -#include "tcommon.h" -#include "scalar.h" -#include "querynodes.h" -#include "query.h" -#include "tep.h" #define FILTER_DEFAULT_GROUP_SIZE 4 #define FILTER_DEFAULT_UNIT_SIZE 4 diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index e5be22c0a8..a7aea5a7e5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -16,11 +16,11 @@ #include #include "thash.h" //#include "queryLog.h" -#include "tcompare.h" +#include "filter.h" #include "filterInt.h" #include "sclInt.h" -#include "filter.h" -#include "tep.h" +#include "tcompare.h" +#include "tdatablock.h" OptrStr gOptrStr[] = { {0, "invalid"}, diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 0245226dfc..b8cdda9ed1 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1,11 +1,11 @@ -#include "nodes.h" -#include "tcommon.h" -#include "querynodes.h" #include "function.h" #include "functionMgt.h" -#include "sclvector.h" +#include "nodes.h" +#include "querynodes.h" #include "sclInt.h" -#include "tep.h" +#include "sclvector.h" +#include "tcommon.h" +#include "tdatablock.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 3c431ff33f..b066dd2e77 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -15,15 +15,15 @@ #include "os.h" -#include "ttypes.h" -#include "sclvector.h" -#include "tcompare.h" -#include "querynodes.h" +#include "filter.h" #include "filterInt.h" #include "query.h" +#include "querynodes.h" #include "sclInt.h" -#include "tep.h" -#include "filter.h" +#include "sclvector.h" +#include "tcompare.h" +#include "tdatablock.h" +#include "ttypes.h" //GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index fafc1ea42e..cf04f9b334 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -29,16 +29,16 @@ #include "os.h" -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" -#include "tep.h" -#include "stub.h" #include "addr_any.h" -#include "scalar.h" -#include "nodes.h" -#include "tlog.h" #include "filter.h" +#include "nodes.h" +#include "scalar.h" +#include "stub.h" +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" +#include "tlog.h" +#include "tvariant.h" namespace { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index b9aef99088..2e8188e8a9 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -29,15 +29,15 @@ #include "os.h" -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" -#include "tep.h" -#include "stub.h" #include "addr_any.h" -#include "scalar.h" #include "nodes.h" +#include "scalar.h" +#include "stub.h" +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" #include "tlog.h" +#include "tvariant.h" namespace { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 11ed3335e6..27384332e5 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -19,13 +19,13 @@ #include "os.h" -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" #include "catalog.h" #include "scheduler.h" -#include "tep.h" +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" #include "trpc.h" +#include "tvariant.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 3ba145a96b..757718282a 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -14,7 +14,7 @@ */ #include "syncIO.h" -#include +#include #include "syncOnMessage.h" #include "tglobal.h" #include "ttimer.h" diff --git a/source/libs/transport/test/pushClient.c b/source/libs/transport/test/pushClient.c index 4842a0c800..f4babc9980 100644 --- a/source/libs/transport/test/pushClient.c +++ b/source/libs/transport/test/pushClient.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include "os.h" #include "rpcLog.h" #include "taoserror.h" diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index bcdf32bf6a..6fc935cb61 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include "os.h" #include "rpcLog.h" #include "taoserror.h" diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index b7ef296b9d..f43fa7aae6 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include "os.h" #include "rpcLog.h" #include "taoserror.h" diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index 6db709da51..d5e0b65d41 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -15,10 +15,10 @@ #include #include #include -#include "tep.h" +#include "tdatablock.h" #include "tglobal.h" -#include "trpc.h" #include "tlog.h" +#include "trpc.h" using namespace std; const char *label = "APP"; diff --git a/source/util/src/thash.c b/source/util/src/thash.c index efbd9adddf..8b437b9797 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -36,25 +36,35 @@ tfree(_n); \ } while (0); +struct SHashNode { + SHashNode *next; + uint32_t hashVal; // the hash value of key + uint32_t dataLen; // length of data + uint32_t keyLen; // length of the key + uint16_t refCount; // reference count + int8_t removed; // flag to indicate removed + char data[]; +}; + typedef struct SHashEntry { - int32_t num; // number of elements in current entry - SRWLatch latch; // entry latch - SHashNode *next; + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch + SHashNode *next; } SHashEntry; -typedef struct SHashObj { - SHashEntry **hashList; - size_t capacity; // number of slots - size_t size; // number of elements in hash table - _hash_fn_t hashFp; // hash function - _equal_fn_t equalFp; // equal function - _hash_free_fn_t freeFp; // hash node free callback function - SRWLatch lock; // read-write spin lock - SHashLockTypeE type; // lock type - bool enableUpdate; // enable update - SArray *pMemBlock; // memory block allocated for SHashEntry - _hash_before_fn_t callbackFp; // function invoked before return the value to caller -} SHashObj; +struct SHashObj { + SHashEntry ** hashList; + size_t capacity; // number of slots + size_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _equal_fn_t equalFp; // equal function + _hash_free_fn_t freeFp; // hash node free callback function + SRWLatch lock; // read-write spin lock + SHashLockTypeE type; // lock type + bool enableUpdate; // enable update + SArray * pMemBlock; // memory block allocated for SHashEntry + _hash_before_fn_t callbackFp; // function invoked before return the value to caller +}; /* * Function definition @@ -367,7 +377,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da // enable resize taosHashRUnlock(pHashObj); - return pHashObj->enableUpdate ? 0 : -1; + return pHashObj->enableUpdate ? 0 : -2; } } @@ -464,7 +474,7 @@ void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** return data; } -int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t dsize) { +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL || keyLen == 0) { return -1; } @@ -507,8 +517,6 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe prevNode->next = pNode->next; } - if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); - pe->num--; atomic_sub_fetch_64(&pHashObj->size, 1); FREE_HASH_NODE(pNode); @@ -525,10 +533,6 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe return code; } -int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { - return taosHashRemoveWithData(pHashObj, key, keyLen, NULL, 0); -} - void taosHashClear(SHashObj *pHashObj) { if (pHashObj == NULL) { return; From 71e4c4194165a9e2b718156eaa7bf9e71ea6822d Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 4 Mar 2022 16:45:20 +0800 Subject: [PATCH 4/8] [TD-13756]: file system remove func. --- include/os/osFile.h | 8 ++++++-- source/common/src/ttszip.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 8 ++++---- source/libs/index/test/fstTest.cc | 2 +- source/libs/tfs/src/tfs.c | 2 +- source/libs/transport/test/pushServer.c | 2 +- source/libs/transport/test/rserver.c | 2 +- source/libs/wal/src/walMeta.c | 2 +- source/libs/wal/src/walWrite.c | 8 ++++---- source/libs/wal/test/walMetaTest.cpp | 4 ++-- source/os/src/osDir.c | 4 ++-- source/os/src/osFile.c | 4 +++- source/util/src/tlog.c | 4 ++-- source/util/src/tpagedbuf.c | 2 +- 15 files changed, 32 insertions(+), 26 deletions(-) diff --git a/include/os/osFile.h b/include/os/osFile.h index 7e3a5277c8..6ddf1e33c6 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -22,6 +22,7 @@ extern "C" { #include "osSocket.h" +// If the error is in a third-party library, place this header file under the third-party library header file. #ifndef ALLOW_FORBID_FUNC #define open OPEN_FUNC_TAOS_FORBID #define fopen FOPEN_FUNC_TAOS_FORBID @@ -31,6 +32,8 @@ extern "C" { #define fstat FSTAT_FUNC_TAOS_FORBID #define close CLOSE_FUNC_TAOS_FORBID #define fclose FCLOSE_FUNC_TAOS_FORBID + #define fsync FSYNC_FUNC_TAOS_FORBID + // #define fflush FFLUSH_FUNC_TAOS_FORBID #endif #ifndef PATH_MAX @@ -47,13 +50,13 @@ typedef struct TdFile *TdFilePtr; #define TD_FILE_TEXT 0x0020 #define TD_FILE_AUTO_DEL 0x0040 #define TD_FILE_EXCL 0x0080 -#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosGetLineFile, taosEOFFile +#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosEOFFile TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); #define TD_FILE_ACCESS_EXIST_OK 0x1 #define TD_FILE_ACCESS_READ_OK 0x2 #define TD_FILE_ACCESS_WRITE_OK 0x4 -bool taosCheckAccessFile(const char *pathname, int mode); +bool taosCheckAccessFile(const char *pathname, int mode); int32_t taosLockFile(TdFilePtr pFile); int32_t taosUnLockFile(TdFilePtr pFile); @@ -80,6 +83,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile); int32_t taosRenameFile(const char *oldName, const char *newName); int64_t taosCopyFile(const char *from, const char *to); +int32_t taosRemoveFile(const char *path); void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index f20e911681..464c29d287 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -46,7 +46,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { } if (!autoDelete) { - remove(pTSBuf->path); + taosRemoveFile(pTSBuf->path); } if (NULL == allocResForTSBuf(pTSBuf)) { @@ -178,7 +178,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { if (pTSBuf->autoDelete) { // ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); - remove(pTSBuf->path); + taosRemoveFile(pTSBuf->path); } else { // tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 7a95820115..37403f1a11 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1054,7 +1054,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // pfs->metaCacheComp = NULL; // } else { // // remove meta.tmp file -// remove(mf.f.aname); +// taosRemoveFile(mf.f.aname); // taosHashCleanup(pfs->metaCacheComp); // pfs->metaCacheComp = NULL; // } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 411a166caa..a03739c90f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -439,7 +439,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(&pFile); - remove(tfname); + taosRemoveFile(tfname); return -1; } @@ -447,7 +447,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { if (fsheader.len > 0) { if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { taosCloseFile(&pFile); - remove(tfname); + taosRemoveFile(tfname); return -1; } @@ -458,7 +458,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(&pFile); - (void)remove(tfname); + (void)taosRemoveFile(tfname); taosTZfree(pBuf); return -1; } @@ -468,7 +468,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { if (taosFsyncFile(pFile) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(&pFile); - remove(tfname); + taosRemoveFile(tfname); taosTZfree(pBuf); return -1; } diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index cb3206a611..618e20bc4b 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -19,7 +19,7 @@ static std::string fileName = "/tmp/tindex.tindex"; class FstWriter { public: FstWriter() { - remove(fileName.c_str()); + taosRemoveFile(fileName.c_str()); _wc = writerCtxCreate(TFile, fileName.c_str(), false, 64 * 1024 * 1024); _b = fstBuilderCreate(_wc, 0); } diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 26fa90bdef..2579490791 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -202,7 +202,7 @@ void tfsDirname(const STfsFile *pFile, char *dest) { tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN); } -int32_t tfsRemoveFile(const STfsFile *pFile) { return remove(pFile->aname); } +int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); } int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2) { return taosCopyFile(pFile1->aname, pFile2->aname); diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index a1c181ac98..c763b7bd8a 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -192,7 +192,7 @@ int main(int argc, char *argv[]) { if (pDataFile != NULL) { taosCloseFile(&pDataFile); - remove(dataName); + taosRemoveFile(dataName); } return 0; diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 5432a07649..da5284b5c5 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -181,7 +181,7 @@ int main(int argc, char *argv[]) { if (pDataFile != NULL) { taosCloseFile(&pDataFile); - remove(dataName); + taosRemoveFile(dataName); } return 0; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index e64260c541..2a4f3497e4 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -379,7 +379,7 @@ int walSaveMeta(SWal* pWal) { // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); } free(serialized); return 0; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 4b1f0ba306..4ed3c63c18 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -55,9 +55,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) { int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); for (int i = pWal->writeCur; i < fileSetSize; i++) { walBuildLogName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); walBuildIdxName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); } // pop from fileInfoSet taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1); @@ -174,9 +174,9 @@ int32_t walEndSnapshot(SWal *pWal) { for (int i = 0; i < deleteCnt; i++) { SWalFileInfo *pInfo = taosArrayGet(pWal->fileInfoSet, i); walBuildLogName(pWal, pInfo->firstVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); walBuildIdxName(pWal, pInfo->firstVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); } // make new array, remove files diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index b65a200ca1..230555e016 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -339,9 +339,9 @@ TEST_F(WalRetentionEnv, repairMeta1) { //getchar(); char buf[100]; sprintf(buf, "%s/meta-ver%d", pathName, 0); - remove(buf); + taosRemoveFile(buf); sprintf(buf, "%s/meta-ver%d", pathName, 1); - remove(buf); + taosRemoveFile(buf); SetUp(); //getchar(); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index e0f5f80dab..7d7382d83f 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -49,7 +49,7 @@ void taosRemoveDir(const char *dirname) { if (de->d_type & DT_DIR) { taosRemoveDir(filename); } else { - (void)remove(filename); + (void)taosRemoveFile(filename); //printf("file:%s is removed\n", filename); } } @@ -102,7 +102,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { if (fileSec <= 100) continue; int32_t days = (int32_t)(TABS(sec - fileSec) / 86400 + 1); if (days > keepDays) { - (void)remove(filename); + (void)taosRemoveFile(filename); //printf("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); } else { //printf("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 37004e9d70..652e0b5182 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -142,11 +142,13 @@ int64_t taosCopyFile(const char *from, const char *to) { _err: if (pFileFrom != NULL) taosCloseFile(&pFileFrom); if (pFileTo != NULL) taosCloseFile(&pFileTo); - remove(to); + taosRemoveFile(to); return -1; #endif } +int32_t taosRemoveFile(const char *path) { return remove(path); } + int32_t taosRenameFile(const char *oldName, const char *newName) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 1e43c85b87..1c297851c3 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -180,7 +180,7 @@ static void taosKeepOldLog(char *oldName) { char compressFileName[LOG_FILE_NAME_LEN + 20]; snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); if (taosCompressFile(fileName, compressFileName) == 0) { - (void)remove(fileName); + (void)taosRemoveFile(fileName); } } @@ -251,7 +251,7 @@ void taosResetLog() { tsLogObj.lines = tsLogObj.maxLines + 10; taosOpenNewLogFile(); - (void)remove(lastName); + (void)taosRemoveFile(lastName); uInfo("=================================="); uInfo(" reset log file "); diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 5bc4b81be7..8c2fd7809d 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -563,7 +563,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { ps->loadPages, ps->loadBytes / (1024.0 * ps->loadPages)); } - remove(pBuf->path); + taosRemoveFile(pBuf->path); tfree(pBuf->path); SArray** p = taosHashIterate(pBuf->groupSet, NULL); From 0f7829c08e334611b427d582c7f76b331edd2392 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 4 Mar 2022 17:08:46 +0800 Subject: [PATCH 5/8] fix multiple polling --- include/common/tmsg.h | 2 +- source/client/inc/clientInt.h | 28 ++-- source/client/src/clientHb.c | 37 +++--- source/client/src/clientImpl.c | 14 +- source/client/src/clientMsgHandler.c | 71 +++++----- source/client/src/tmq.c | 186 +++++++++++++-------------- source/libs/qcom/src/queryUtil.c | 66 ++++++---- 7 files changed, 212 insertions(+), 192 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b60f09bdf4..ae594e90ca 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -70,7 +70,7 @@ typedef uint16_t tmsg_t; typedef enum { HEARTBEAT_TYPE_MQ = 0, - HEARTBEAT_TYPE_QUERY = 1, + HEARTBEAT_TYPE_QUERY, // types can be added here // HEARTBEAT_TYPE_MAX diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 79909a5696..ad018c6832 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,10 +20,10 @@ extern "C" { #endif -#include "tcommon.h" #include "parser.h" #include "query.h" #include "taos.h" +#include "tcommon.h" #include "tdef.h" #include "tep.h" #include "thash.h" @@ -47,12 +47,12 @@ extern "C" { typedef struct SAppInstInfo SAppInstInfo; -typedef struct SHbConnInfo { +typedef struct { void* param; SClientHbReq* req; } SHbConnInfo; -typedef struct SAppHbMgr { +typedef struct { char* key; // statistics int32_t reportCnt; @@ -68,11 +68,11 @@ typedef struct SAppHbMgr { SHashObj* connInfo; // hash } SAppHbMgr; -typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); +typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); -typedef struct SClientHbMgr { +typedef struct { int8_t inited; // ctl int8_t threadStop; @@ -108,13 +108,13 @@ typedef struct SHeartBeatInfo { } SHeartBeatInfo; struct SAppInstInfo { - int64_t numOfConns; - SCorEpSet mgmtEp; - SInstanceSummary summary; - SList* pConnList; // STscObj linked list - int64_t clusterId; - void* pTransporter; - struct SAppHbMgr* pAppHbMgr; + int64_t numOfConns; + SCorEpSet mgmtEp; + SInstanceSummary summary; + SList* pConnList; // STscObj linked list + int64_t clusterId; + void* pTransporter; + SAppHbMgr* pAppHbMgr; }; typedef struct SAppInfo { @@ -141,10 +141,6 @@ typedef struct STscObj { SAppInstInfo* pAppInfo; } STscObj; -typedef struct SMqConsumer { - STscObj* pTscObj; -} SMqConsumer; - typedef struct SReqResultInfo { const char* pRspMsg; const char* pData; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index e9e64a78b8..354e82b622 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -23,7 +23,7 @@ static SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); static void hbStopThread(); -static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } +static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; @@ -104,7 +104,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo return TSDB_CODE_SUCCESS; } -static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { +static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); if (NULL == info) { tscWarn("fail to get connInfo, may be dropped, connId:%d, type:%d", pRsp->connKey.connId, pRsp->connKey.hbType); @@ -163,7 +163,7 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRs return TSDB_CODE_SUCCESS; } -static int32_t hbMqAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) { +static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) { static int32_t emptyRspNum = 0; if (code != 0) { tfree(param); @@ -226,7 +226,11 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl db->vgVersion = htonl(db->vgVersion); } - SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = sizeof(SDbVgVersion) * dbNum, .value = dbs}; + SKv kv = { + .key = HEARTBEAT_KEY_DBINFO, + .valueLen = sizeof(SDbVgVersion) * dbNum, + .value = dbs, + }; tscDebug("hb got %d expired db, valueLen:%d", dbNum, kv.valueLen); @@ -256,7 +260,11 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC stb->tversion = htons(stb->tversion); } - SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = sizeof(SSTableMetaVersion) * stbNum, .value = stbs}; + SKv kv = { + .key = HEARTBEAT_KEY_STBINFO, + .valueLen = sizeof(SSTableMetaVersion) * stbNum, + .value = stbs, + }; tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen); @@ -288,7 +296,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req return TSDB_CODE_SUCCESS; } -int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) {} +int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; } void hbMgrInitMqHbHandle() { clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle; @@ -396,7 +404,7 @@ static void *hbThreadFunc(void *param) { free(buf); break; } - pInfo->fp = hbMqAsyncCallBack; + pInfo->fp = hbAsyncCallBack; pInfo->msgInfo.pData = buf; pInfo->msgInfo.len = tlen; pInfo->msgType = TDMT_MND_HEARTBEAT; @@ -448,7 +456,6 @@ static void hbStopThread() { } SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { - /*return NULL;*/ hbMgrInit(); SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { @@ -506,7 +513,6 @@ void appHbMgrCleanup(void) { } int hbMgrInit() { - /*return 0;*/ // init once int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1); if (old == 1) return 0; @@ -524,7 +530,7 @@ int hbMgrInit() { } void hbMgrCleanUp() { - return; +#if 0 hbStopThread(); // destroy all appHbMgr @@ -537,6 +543,7 @@ void hbMgrCleanUp() { pthread_mutex_unlock(&clientHbMgr.lock); clientHbMgr.appHbMgrs = NULL; +#endif } int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) { @@ -563,9 +570,11 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo * } int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) { - /*return 0;*/ - SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY}; - SHbConnInfo info = {0}; + SClientHbKey connKey = { + .connId = connId, + .hbType = HEARTBEAT_TYPE_QUERY, + }; + SHbConnInfo info = {0}; switch (hbType) { case HEARTBEAT_TYPE_QUERY: { @@ -586,7 +595,6 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int3 } void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { - /*return;*/ int32_t code = 0; code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); @@ -598,7 +606,6 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *key, void *value, int32_t keyLen, int32_t valueLen) { - return 0; // find req by connection id SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); ASSERT(pReq != NULL); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 3c6a49c3db..e9049ff751 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -225,9 +225,9 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t } int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) { - void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; + void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob, pRequest->sqlstr, &res); + int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob, pRequest->sqlstr, &res); if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.pQueryJob != NULL) { schedulerFreeJob(pRequest->body.pQueryJob); @@ -239,12 +239,12 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { pRequest->body.resInfo.numOfRows = res.numOfRows; - + if (pRequest->body.pQueryJob != NULL) { schedulerFreeJob(pRequest->body.pQueryJob); } } - + pRequest->code = res.code; return pRequest->code; } @@ -435,7 +435,11 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { taosReleaseRef(clientReqRefPool, pSendInfo->requestObjRefId); } - SDataBuf buf = {.len = pMsg->contLen, .pData = NULL, .handle = pMsg->handle}; + SDataBuf buf = { + .len = pMsg->contLen, + .pData = NULL, + .handle = pMsg->handle, + }; if (pMsg->contLen > 0) { buf.pData = calloc(1, pMsg->contLen); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 454fb7f1fe..4f18a08c95 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -13,12 +13,12 @@ * along with this program. If not, see . */ +#include "catalog.h" +#include "clientInt.h" +#include "clientLog.h" #include "os.h" #include "tdef.h" #include "tname.h" -#include "clientInt.h" -#include "clientLog.h" -#include "catalog.h" int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); @@ -81,13 +81,13 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } -SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { +SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) { SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); pMsgSendInfo->requestObjRefId = pRequest->self; - pMsgSendInfo->requestId = pRequest->requestId; - pMsgSendInfo->param = pRequest; - pMsgSendInfo->msgType = pRequest->type; + pMsgSendInfo->requestId = pRequest->requestId; + pMsgSendInfo->param = pRequest; + pMsgSendInfo->msgType = pRequest->type; if (pRequest->type == TDMT_MND_SHOW_RETRIEVE || pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { @@ -103,6 +103,7 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { } else { SVShowTablesFetchReq* pFetchMsg = calloc(1, sizeof(SVShowTablesFetchReq)); if (pFetchMsg == NULL) { + free(pMsgSendInfo); return NULL; } @@ -118,7 +119,9 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { pMsgSendInfo->msgInfo = pRequest->body.requestMsg; } - pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)]; + pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL) + ? genericRspCallback + : handleRequestRspFp[TMSG_INDEX(pRequest->type)]; return pMsgSendInfo; } @@ -132,7 +135,7 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { SShowRsp showRsp = {0}; tDeserializeSShowRsp(pMsg->pData, pMsg->len, &showRsp); - STableMetaRsp *pMetaMsg = &showRsp.tableMeta; + STableMetaRsp* pMetaMsg = &showRsp.tableMeta; tfree(pRequest->body.resInfo.pRspMsg); pRequest->body.resInfo.pRspMsg = pMsg->pData; @@ -158,7 +161,7 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { if (pRequest->type == TDMT_VND_SHOW_TABLES) { SShowReqInfo* pShowInfo = &pRequest->body.showInfo; - int32_t index = pShowInfo->currentIndex; + int32_t index = pShowInfo->currentIndex; SVgroupInfo* pInfo = taosArrayGet(pShowInfo->pArray, index); pShowInfo->vgId = pInfo->vgId; } @@ -168,8 +171,8 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { } int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) { - SRequestObj *pRequest = param; - SReqResultInfo *pResInfo = &pRequest->body.resInfo; + SRequestObj* pRequest = param; + SReqResultInfo* pResInfo = &pRequest->body.resInfo; tfree(pResInfo->pRspMsg); if (code != TSDB_CODE_SUCCESS) { @@ -180,19 +183,19 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) assert(pMsg->len >= sizeof(SRetrieveTableRsp)); - SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg->pData; - pRetrieve->numOfRows = htonl(pRetrieve->numOfRows); - pRetrieve->precision = htons(pRetrieve->precision); + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)pMsg->pData; + pRetrieve->numOfRows = htonl(pRetrieve->numOfRows); + pRetrieve->precision = htons(pRetrieve->precision); - pResInfo->pRspMsg = pMsg->pData; + pResInfo->pRspMsg = pMsg->pData; pResInfo->numOfRows = pRetrieve->numOfRows; - pResInfo->pData = pRetrieve->data; + pResInfo->pData = pRetrieve->data; pResInfo->completed = pRetrieve->completed; pResInfo->current = 0; setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); - tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pRetrieve->numOfRows, + tscDebug("0x%" PRIx64 " numOfRows:%d, complete:%d, qId:0x%" PRIx64, pRequest->self, pRetrieve->numOfRows, pRetrieve->completed, pRequest->body.showInfo.execId); tsem_post(&pRequest->body.rspSem); @@ -213,20 +216,20 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { assert(pMsg->len >= sizeof(SRetrieveTableRsp)); - pResInfo->pRspMsg = pMsg->pData; + pResInfo->pRspMsg = pMsg->pData; - SVShowTablesFetchRsp *pFetchRsp = (SVShowTablesFetchRsp *) pMsg->pData; - pFetchRsp->numOfRows = htonl(pFetchRsp->numOfRows); - pFetchRsp->precision = htons(pFetchRsp->precision); + SVShowTablesFetchRsp* pFetchRsp = (SVShowTablesFetchRsp*)pMsg->pData; + pFetchRsp->numOfRows = htonl(pFetchRsp->numOfRows); + pFetchRsp->precision = htons(pFetchRsp->precision); - pResInfo->pRspMsg = pMsg->pData; + pResInfo->pRspMsg = pMsg->pData; pResInfo->numOfRows = pFetchRsp->numOfRows; - pResInfo->pData = pFetchRsp->data; + pResInfo->pData = pFetchRsp->data; pResInfo->current = 0; setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); - tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pFetchRsp->numOfRows, + tscDebug("0x%" PRIx64 " numOfRows:%d, complete:%d, qId:0x%" PRIx64, pRequest->self, pFetchRsp->numOfRows, pFetchRsp->completed, pRequest->body.showInfo.execId); tsem_post(&pRequest->body.rspSem); @@ -254,7 +257,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); SName name = {0}; - tNameFromString(&name, usedbRsp.db, T_NAME_ACCT|T_NAME_DB); + tNameFromString(&name, usedbRsp.db, T_NAME_ACCT | T_NAME_DB); tFreeSUsedbRsp(&usedbRsp); @@ -377,14 +380,14 @@ void initMsgHandleFp() { tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; #endif - handleRequestRspFp[TMSG_INDEX(TDMT_MND_CONNECT)] = processConnectRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_SHOW)] = processShowRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_MND_CONNECT)] = processConnectRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_MND_SHOW)] = processShowRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = processRetrieveMnodeRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = processCreateDbRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = processCreateTableRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = processCreateDbRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = processCreateTableRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = processShowRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = processRetrieveVndRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = processShowRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = processRetrieveVndRsp; } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 0e84371199..8aca3d6f66 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -28,11 +28,6 @@ #include "tqueue.h" #include "tref.h" -static int64_t perfWrite; -static int64_t perfRead; -static int64_t perfRead2; -static int64_t perfRead3; - struct tmq_list_t { int32_t cnt; int32_t tot; @@ -64,6 +59,7 @@ struct tmq_t { char groupId[256]; char clientId[256]; int8_t autoCommit; + int8_t inWaiting; int64_t consumerId; int32_t epoch; int32_t resetOffsetCfg; @@ -76,6 +72,7 @@ struct tmq_t { SArray* clientTopics; // SArray STaosQueue* mqueue; // queue of tmq_message_t STaosQall* qall; + tsem_t rspSem; // stat int64_t pollCnt; }; @@ -214,9 +211,9 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; - tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; + pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; if (pParam->tmq->commit_cb) { - pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); + pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL, NULL); } if (!pParam->async) tsem_post(&pParam->rspSem); return 0; @@ -244,6 +241,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs return NULL; } pTmq->pTscObj = (STscObj*)conn; + pTmq->inWaiting = 0; pTmq->status = 0; pTmq->pollCnt = 0; pTmq->epoch = 0; @@ -256,6 +254,8 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs pTmq->commit_cb = conf->commit_cb; pTmq->resetOffsetCfg = conf->resetOffset; + tsem_init(&pTmq->rspSem, 0, 0); + pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); @@ -641,16 +641,13 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { tmq_t* tmq = pParam->tmq; if (code != 0) { printf("msg discard\n"); - if (pParam->epoch == tmq->epoch) { - atomic_sub_fetch_32(&tmq->waitingRequest, 1); - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - } - return 0; + goto WRITE_QUEUE_FAIL; } int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; int32_t tmqEpoch = atomic_load_32(&tmq->epoch); if (msgEpoch < tmqEpoch) { + tsem_post(&tmq->rspSem); printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch); return 0; } @@ -660,6 +657,8 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } else { atomic_sub_fetch_32(&tmq->waitingRequest, 1); } + +#if 0 if (pParam->sync == 1) { /**pParam->msg = malloc(sizeof(tmq_message_t));*/ *pParam->msg = taosAllocateQitem(sizeof(tmq_message_t)); @@ -669,45 +668,41 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { if ((*pParam->msg)->consumeRsp.numOfTopics != 0) { pVg->currentOffset = (*pParam->msg)->consumeRsp.rspOffset; } - int64_t begin = clock(); taosWriteQitem(tmq->mqueue, *pParam->msg); - perfWrite += clock() - begin; tsem_post(&pParam->rspSem); return 0; } tsem_post(&pParam->rspSem); return -1; } +#endif /*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/ tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); if (pRsp == NULL) { - printf("fail\n"); - return -1; + goto WRITE_QUEUE_FAIL; } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ if (pRsp->consumeRsp.numOfTopics == 0) { /*printf("no data\n");*/ - if (pParam->epoch == tmq->epoch) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - } taosFreeQitem(pRsp); - return 0; + goto WRITE_QUEUE_FAIL; } + pRsp->extra = pParam->pVg; taosWriteQitem(tmq->mqueue, pRsp); atomic_add_fetch_32(&tmq->readyRequest, 1); - - /*printf("poll in queue\n");*/ - /*pParam->rspMsg = (tmq_message_t*)pRsp;*/ - /*pVg->currentOffset = pRsp->consumeRsp.rspOffset;*/ - - /*printf("rsp offset: %ld\n", rsp.rspOffset);*/ - /*printf("-----msg begin----\n");*/ - /*printf("\n-----msg end------\n");*/ + tsem_post(&tmq->rspSem); return 0; + +WRITE_QUEUE_FAIL: + if (pParam->epoch == tmq->epoch) { + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + } + tsem_post(&tmq->rspSem); + return code; } bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { @@ -744,81 +739,94 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { tmq_t* tmq = pParam->tmq; if (code != 0) { printf("get topic endpoint error, not ready, wait:%d\n", pParam->sync); - if (pParam->sync) { - tsem_post(&pParam->rspSem); - } - return 0; + goto END; } - tscDebug("tmq ask ep cb called"); + + // tmq's epoch is monotomically increase, + // so it's safe to discard any old epoch msg. + // epoch will only increase when received newer epoch ep msg + SMqRspHead* head = pMsg->pData; + int32_t epoch = atomic_load_32(&tmq->epoch); + if (head->epoch <= epoch) { + goto END; + } + if (pParam->sync) { - SMqRspHead* head = pMsg->pData; SMqCMGetSubEpRsp rsp; tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ - int32_t epoch = atomic_load_32(&tmq->epoch); - if (head->epoch > epoch && tmqUpdateEp(tmq, head->epoch, &rsp)) { + if (tmqUpdateEp(tmq, head->epoch, &rsp)) { atomic_store_64(&tmq->status, TMQ_CONSUMER_STATUS__READY); } - tsem_post(&pParam->rspSem); tDeleteSMqCMGetSubEpRsp(&rsp); } else { tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = -1; + goto END; } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->getEpRsp); - /*taosWriteQitem(tmq->mqueue, pRsp);*/ + taosWriteQitem(tmq->mqueue, pRsp); + tsem_post(&tmq->rspSem); } - return 0; + +END: + if (pParam->sync) { + tsem_post(&pParam->rspSem); + } + return code; } int32_t tmqAskEp(tmq_t* tmq, bool sync) { int32_t tlen = sizeof(SMqCMGetSubEpReq); - SMqCMGetSubEpReq* buf = malloc(tlen); - if (buf == NULL) { + SMqCMGetSubEpReq* req = malloc(tlen); + if (req == NULL) { tscError("failed to malloc get subscribe ep buf"); - goto END; + return -1; } - buf->consumerId = htobe64(tmq->consumerId); - buf->epoch = htonl(tmq->epoch); - strcpy(buf->cgroup, tmq->groupId); - - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); - if (pRequest == NULL) { - tscError("failed to malloc subscribe ep request"); - goto END; - } - - pRequest->body.requestMsg = (SDataBuf){ - .pData = buf, - .len = tlen, - .handle = NULL, - }; + req->consumerId = htobe64(tmq->consumerId); + req->epoch = htonl(tmq->epoch); + strcpy(req->cgroup, tmq->groupId); SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam)); if (pParam == NULL) { tscError("failed to malloc subscribe param"); - goto END; + free(req); + return -1; } pParam->tmq = tmq; pParam->sync = sync; tsem_init(&pParam->rspSem, 0, 0); - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + tsem_destroy(&pParam->rspSem); + free(pParam); + free(req); + return -1; + } + + sendInfo->msgInfo = (SDataBuf){ + .pData = req, + .len = tlen, + .handle = NULL, + }; + + sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; sendInfo->param = pParam; sendInfo->fp = tmqAskEpCb; + sendInfo->msgType = TDMT_MND_GET_SUB_EP; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); -END: if (sync) tsem_wait(&pParam->rspSem); return 0; } @@ -942,19 +950,14 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { pVg->pollCnt++; tmq->pollCnt++; - int64_t begin = clock(); tsem_wait(¶m->rspSem); - perfRead3 += clock() - begin; tmq_message_t* nmsg = NULL; while (1) { - int64_t begin1 = clock(); taosReadQitem(tmq->mqueue, (void**)&nmsg); - perfRead2 += clock() - begin1; if (nmsg == NULL) continue; - /*while (nmsg->head.mqMsgType != TMQ_MSG_TYPE__POLL_RSP) {*/ - /*taosReadQitem(tmq->mqueue, (void**)&nmsg);*/ - /*}*/ - perfRead += clock() - begin; + while (nmsg->head.mqMsgType != TMQ_MSG_TYPE__POLL_RSP) { + taosReadQitem(tmq->mqueue, (void**)&nmsg); + } return nmsg; } } @@ -976,12 +979,14 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // TODO: out of mem + tsem_post(&tmq->rspSem); return -1; } SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam)); if (param == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // TODO: out of mem + tsem_post(&tmq->rspSem); return -1; } param->tmq = tmq; @@ -1033,7 +1038,9 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese tmq_message_t* rspMsg = NULL; taosGetQitem(tmq->qall, (void**)&rspMsg); if (rspMsg == NULL) { - break; + taosReadAllQitems(tmq->mqueue, tmq->qall); + taosGetQitem(tmq->qall, (void**)&rspMsg); + if (rspMsg == NULL) return NULL; } if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { @@ -1060,10 +1067,9 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese } } } - return NULL; } -tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { +tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* rspMsg = NULL; int64_t startTime = taosGetTimestampMs(); @@ -1079,10 +1085,6 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { if (blocking_time != 0) { int64_t endTime = taosGetTimestampMs(); if (endTime - startTime > blocking_time) { - printf("perf write %f\n", (double)perfWrite / CLOCKS_PER_SEC); - printf("perf read %f\n", (double)perfRead / CLOCKS_PER_SEC); - printf("perf read2 %f\n", (double)perfRead2 / CLOCKS_PER_SEC); - printf("perf read3 %f\n", (double)perfRead3 / CLOCKS_PER_SEC); return NULL; } } else @@ -1090,44 +1092,37 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { } } -tmq_message_t* tmq_consumer_poll_v0(tmq_t* tmq, int64_t blocking_time) { - tmq_message_t* rspMsg = NULL; +tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { + tmq_message_t* rspMsg; int64_t startTime = taosGetTimestampMs(); // TODO: put into another thread or delayed queue int64_t status = atomic_load_64(&tmq->status); tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT); -#if 0 - taosGetQitem(tmq->qall, (void**)&rspMsg); - if (rspMsg == NULL) { - taosReadAllQitems(tmq->mqueue, tmq->qall); - } rspMsg = tmqHandleAllRsp(tmq, blocking_time, false); if (rspMsg) { return rspMsg; } -#endif while (1) { /*printf("cycle\n");*/ if (atomic_load_32(&tmq->waitingRequest) == 0) { tmqPollImpl(tmq, blocking_time); } - while (atomic_load_32(&tmq->readyRequest) == 0) { - sched_yield(); - if (blocking_time != 0) { - int64_t endTime = taosGetTimestampMs(); - if (endTime - startTime > blocking_time) { - return NULL; - } - } - } - taosReadAllQitems(tmq->mqueue, tmq->qall); - rspMsg = tmqHandleAllRsp(tmq, blocking_time, true); + + tsem_wait(&tmq->rspSem); + + rspMsg = tmqHandleAllRsp(tmq, blocking_time, false); if (rspMsg) { return rspMsg; } + if (blocking_time != 0) { + int64_t endTime = taosGetTimestampMs(); + if (endTime - startTime > blocking_time) { + return NULL; + } + } } } @@ -1277,6 +1272,7 @@ const char* tmq_err2str(tmq_resp_err_t err) { } return "fail"; } + #if 0 tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { tmq_t* pTmq = malloc(sizeof(tmq_t)); diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index a2165453d5..63fbf59c06 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -1,23 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include "os.h" -#include "tmsg.h" #include "query.h" #include "tglobal.h" -#include "tsched.h" +#include "tmsg.h" #include "trpc.h" +#include "tsched.h" -#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) -#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) +#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) +#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) static struct SSchema _s = { .colId = TSDB_TBNAME_COLUMN_INDEX, - .type = TSDB_DATA_TYPE_BINARY, + .type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .name = "tbname", }; -const SSchema* tGetTbnameColumnSchema() { - return &_s; -} +const SSchema* tGetTbnameColumnSchema() { return &_s; } static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) { int32_t rowLen = 0; @@ -87,7 +100,7 @@ int32_t initTaskQueue() { double factor = 4.0; int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); - + int32_t queueSize = tsMaxConnections * 2; pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); if (NULL == pTaskQueue) { @@ -96,19 +109,21 @@ int32_t initTaskQueue() { } qDebug("task queue is initialized, numOfThreads: %d", numOfThreads); + return 0; } int32_t cleanupTaskQueue() { taosCleanUpScheduler(pTaskQueue); + return 0; } static void execHelper(struct SSchedMsg* pSchedMsg) { assert(pSchedMsg != NULL && pSchedMsg->ahandle != NULL); - __async_exec_fn_t execFn = (__async_exec_fn_t) pSchedMsg->ahandle; - int32_t code = execFn(pSchedMsg->thandle); + __async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle; + int32_t code = execFn(pSchedMsg->thandle); if (code != 0 && pSchedMsg->msg != NULL) { - *(int32_t*) pSchedMsg->msg = code; + *(int32_t*)pSchedMsg->msg = code; } } @@ -116,34 +131,33 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) assert(execFn != NULL); SSchedMsg schedMsg = {0}; - schedMsg.fp = execHelper; + schedMsg.fp = execHelper; schedMsg.ahandle = execFn; schedMsg.thandle = execParam; - schedMsg.msg = code; + schedMsg.msg = code; taosScheduleTask(pTaskQueue, &schedMsg); + return 0; } -int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { - char *pMsg = rpcMallocCont(pInfo->msgInfo.len); +int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { + char* pMsg = rpcMallocCont(pInfo->msgInfo.len); if (NULL == pMsg) { - qError("0x%"PRIx64" msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); + qError("0x%" PRIx64 " msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return terrno; } memcpy(pMsg, pInfo->msgInfo.pData, pInfo->msgInfo.len); - SRpcMsg rpcMsg = { - .msgType = pInfo->msgType, - .pCont = pMsg, - .contLen = pInfo->msgInfo.len, - .ahandle = (void*) pInfo, - .handle = pInfo->msgInfo.handle, - .code = 0 - }; + SRpcMsg rpcMsg = {.msgType = pInfo->msgType, + .pCont = pMsg, + .contLen = pInfo->msgInfo.len, + .ahandle = (void*)pInfo, + .handle = pInfo->msgInfo.handle, + .code = 0}; assert(pInfo->fp != NULL); rpcSendRequest(pTransporter, epSet, &rpcMsg, pTransporterId); return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} From 7510a0e7a5e68c0a36a82176b5d42f2f8cd2465c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 4 Mar 2022 18:00:26 +0800 Subject: [PATCH 6/8] remove createRequest in poll --- source/client/src/tmq.c | 115 ++++++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 51 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index f3bafd73fc..35fc557eea 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -202,6 +202,26 @@ int32_t tmq_list_append(tmq_list_t* ptr, const char* src) { return 0; } +void tmqClearUnhandleMsg(tmq_t* tmq) { + tmq_message_t* msg; + while (1) { + taosGetQitem(tmq->qall, (void**)&msg); + if (msg) + taosFreeQitem(msg); + else + break; + } + + taosReadAllQitems(tmq->mqueue, tmq->qall); + while (1) { + taosGetQitem(tmq->qall, (void**)&msg); + if (msg) + taosFreeQitem(msg); + else + break; + } +} + int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; pParam->rspErr = code; @@ -845,6 +865,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j); if (pVg->vgId == pOffset->vgId) { pVg->currentOffset = pOffset->offset; + tmqClearUnhandleMsg(tmq); return TMQ_RESP_ERR__SUCCESS; } } @@ -883,26 +904,6 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClien return pReq; } -void tmqClearUnhandleMsg(tmq_t* tmq) { - tmq_message_t* msg; - while (1) { - taosGetQitem(tmq->qall, (void**)&msg); - if (msg) - taosFreeQitem(msg); - else - break; - } - - taosReadAllQitems(tmq->mqueue, tmq->qall); - while (1) { - taosGetQitem(tmq->qall, (void**)&msg); - if (msg) - taosFreeQitem(msg); - else - break; - } -} - tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { tmq_message_t* msg = NULL; for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { @@ -919,29 +920,35 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { // TODO: out of mem return NULL; } - SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam)); - if (param == NULL) { + + SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam)); + if (pParam == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // TODO: out of mem return NULL; } - param->tmq = tmq; - param->pVg = pVg; - param->epoch = tmq->epoch; - param->sync = 1; - param->msg = &msg; - tsem_init(¶m->rspSem, 0, 0); - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){ + pParam->tmq = tmq; + pParam->pVg = pVg; + pParam->epoch = tmq->epoch; + pParam->sync = 1; + pParam->msg = &msg; + tsem_init(&pParam->rspSem, 0, 0); + + SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + return NULL; + } + + sendInfo->msgInfo = (SDataBuf){ .pData = pReq, .len = sizeof(SMqConsumeReq), .handle = NULL, }; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; - sendInfo->param = param; + sendInfo->param = pParam; sendInfo->fp = tmqPollCb; + sendInfo->msgType = TDMT_VND_CONSUME; int64_t transporterId = 0; /*printf("send poll\n");*/ @@ -950,7 +957,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { pVg->pollCnt++; tmq->pollCnt++; - tsem_wait(¶m->rspSem); + tsem_wait(&pParam->rspSem); tmq_message_t* nmsg = NULL; while (1) { taosReadQitem(tmq->mqueue, (void**)&nmsg); @@ -978,32 +985,40 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - // TODO: out of mem tsem_post(&tmq->rspSem); return -1; } - SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam)); - if (param == NULL) { + SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam)); + if (pParam == NULL) { + free(pReq); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - // TODO: out of mem tsem_post(&tmq->rspSem); return -1; } - param->tmq = tmq; - param->pVg = pVg; - param->epoch = tmq->epoch; - param->sync = 0; - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){ + pParam->tmq = tmq; + pParam->pVg = pVg; + pParam->epoch = tmq->epoch; + pParam->sync = 0; + + SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + free(pReq); + free(pParam); + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + tsem_post(&tmq->rspSem); + return -1; + } + + sendInfo->msgInfo = (SDataBuf){ .pData = pReq, .len = sizeof(SMqConsumeReq), .handle = NULL, }; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; - sendInfo->param = param; + sendInfo->param = pParam; sendInfo->fp = tmqPollCb; + sendInfo->msgType = TDMT_VND_CONSUME; int64_t transporterId = 0; /*printf("send poll\n");*/ @@ -1053,7 +1068,7 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); return rspMsg; } else { - printf("epoch mismatch\n"); + /*printf("epoch mismatch\n");*/ taosFreeQitem(rspMsg); } } else { @@ -1107,9 +1122,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { while (1) { /*printf("cycle\n");*/ - if (atomic_load_32(&tmq->waitingRequest) == 0) { - tmqPollImpl(tmq, blocking_time); - } + tmqPollImpl(tmq, blocking_time); tsem_wait(&tmq->rspSem); From e00ae4d86fabac82fc0a9339cb8016c954dd851d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 4 Mar 2022 19:29:18 +0800 Subject: [PATCH 7/8] merge add/delete/update opera on index --- source/libs/index/inc/indexInt.h | 2 +- source/libs/index/inc/index_util.h | 15 ++++ source/libs/index/src/index.c | 123 +++++++++++++++++++++++----- source/libs/index/src/index_cache.c | 13 +-- source/libs/index/src/index_tfile.c | 1 + source/libs/index/src/index_util.c | 23 +++++- source/libs/index/test/utilUT.cc | 81 ++++++++++++++++++ 7 files changed, 226 insertions(+), 32 deletions(-) diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 0fcaf11087..716c9c2369 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -96,7 +96,7 @@ typedef struct SIndexTermQuery { typedef struct Iterate Iterate; typedef struct IterateValue { - int8_t type; + int8_t type; // opera type, ADD_VALUE/DELETE_VALUE char* colVal; SArray* val; } IterateValue; diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index 985dd657d0..313839bf1d 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -54,7 +54,22 @@ extern "C" { * output:[4, 5] */ void iIntersection(SArray *interResults, SArray *finalResult); + +/* multi sorted result intersection + * input: [1, 2, 4, 5] + * [2, 3, 4, 5] + * [1, 4, 5] + * output:[1, 2, 3, 4, 5] + */ void iUnion(SArray *interResults, SArray *finalResult); + +/* sorted array + * total: [1, 2, 4, 5, 7, 8] + * except: [4, 5] + * return: [1, 2, 7, 8] + */ + +void iExcept(SArray *total, SArray *except); #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 168e819073..57e965a753 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -32,6 +32,18 @@ void* indexQhandle = NULL; static char JSON_COLUMN[] = "JSON"; +#define INDEX_MERGE_ADD_DEL(src, dst, tgt) \ + { \ + bool f = false; \ + for (int i = 0; i < taosArrayGetSize(src); i++) { \ + if (*(uint64_t*)taosArrayGet(src, i) == tgt) { \ + f = true; \ + } \ + } \ + if (f == false) { \ + taosArrayPush(dst, &tgt); \ + } \ + } void indexInit() { // refactor later indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index"); @@ -52,6 +64,14 @@ typedef struct SIdxColInfo { int cVersion; } SIdxColInfo; +typedef struct SIdxMergeHelper { + char* colVal; + SArray* total; + SArray* added; + SArray* deled; + bool reset; +} SIdxMergeHelper; + static pthread_once_t isInit = PTHREAD_ONCE_INIT; // static void indexInit(); static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); @@ -62,8 +82,8 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oTyp static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); // merge cache and tfile by opera type -static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv); -static void indexMergeSameKey(SArray* result, TFileValue* tv); +static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxMergeHelper* helper); +static void indexMergeSameKey(SArray* result, TFileValue* tv, SIdxMergeHelper* helper); // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); @@ -398,7 +418,32 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType return 0; } -static void indexMergeSameKey(SArray* result, TFileValue* tv) { +SIdxMergeHelper* sIdxMergeHelperCreate() { + SIdxMergeHelper* hp = calloc(1, sizeof(SIdxMergeHelper)); + hp->total = taosArrayInit(4, sizeof(uint64_t)); + hp->added = taosArrayInit(4, sizeof(uint64_t)); + hp->deled = taosArrayInit(4, sizeof(uint64_t)); + hp->reset = false; + return hp; +} +void sIdxMergeHelperClear(SIdxMergeHelper* hp) { + if (hp == NULL) { + return; + } + hp->reset = false; + taosArrayClear(hp->total); + taosArrayClear(hp->added); + taosArrayClear(hp->deled); +} +void sIdxMergeHelperDestroy(SIdxMergeHelper* hp) { + if (hp == NULL) { + return; + } + taosArrayDestroy(hp->total); + taosArrayDestroy(hp->added); + taosArrayDestroy(hp->deled); +} +static void indexMergeSameKey(SArray* result, TFileValue* tv, SIdxMergeHelper* helper) { int32_t sz = result ? taosArrayGetSize(result) : 0; if (sz > 0) { // TODO(yihao): remove duplicate tableid @@ -414,26 +459,55 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) { taosArrayPush(result, &tv); } } -static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv) { - // opt - char* colVal = (cv != NULL) ? cv->colVal : tv->colVal; - // design merge-algorithm later, too complicated to handle all kind of situation - TFileValue* tfv = tfileValueCreate(colVal); - if (cv != NULL) { - if (cv->type == ADD_VALUE) { - taosArrayAddAll(tfv->tableId, cv->val); - } else if (cv->type == DEL_VALUE) { - } else if (cv->type == UPDATE_VALUE) { +static void sIdxMergeResult(SArray* result, SIdxMergeHelper* mh) { + taosArraySort(mh->total, uidCompare); + taosArraySort(mh->added, uidCompare); + taosArraySort(mh->deled, uidCompare); + + SArray* arrs = taosArrayInit(2, sizeof(void*)); + taosArrayPush(arrs, &mh->total); + taosArrayPush(arrs, &mh->added); + + iUnion(arrs, result); + taosArrayDestroy(arrs); + + iExcept(result, mh->deled); +} +static void indexMayMergeToFinalResult(SArray* result, TFileValue* tfv, SIdxMergeHelper* help) { + int32_t sz = taosArrayGetSize(result); + if (sz > 0) { + TFileValue* lv = taosArrayGetP(result, sz - 1); + if (tfv != NULL && strcmp(lv->colVal, tfv->colVal) != 0) { + sIdxMergeResult(lv->tableId, help); + sIdxMergeHelperClear(help); + + taosArrayPush(result, &tfv); + } else if (tfv == NULL) { + sIdxMergeResult(lv->tableId, help); } else { - // do nothing + tfileValueDestroy(tfv); + } + } else { + taosArrayPush(result, &tfv); + } +} +static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxMergeHelper* mh) { + char* colVal = (cv != NULL) ? cv->colVal : tv->colVal; + TFileValue* tfv = tfileValueCreate(colVal); + + indexMayMergeToFinalResult(result, tfv, mh); + + if (cv != NULL) { + uint64_t id = *(uint64_t*)taosArrayGet(cv->val, 0); + if (cv->type == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(mh->deled, mh->added, id) + } else if (cv->type == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(mh->added, mh->deled, id) } } if (tv != NULL) { - // opt later - taosArrayAddAll(tfv->tableId, tv->val); + taosArrayAddAll(mh->total, tv->val); } - - indexMergeSameKey(result, tfv); } static void indexDestroyTempResult(SArray* result) { int32_t sz = result ? taosArrayGetSize(result) : 0; @@ -443,6 +517,7 @@ static void indexDestroyTempResult(SArray* result) { } taosArrayDestroy(result); } + int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { if (sIdx == NULL) { return -1; @@ -467,6 +542,8 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { bool cn = cacheIter ? cacheIter->next(cacheIter) : false; bool tn = tfileIter ? tfileIter->next(tfileIter) : false; + + SIdxMergeHelper* help = sIdxMergeHelperCreate(); while (cn == true || tn == true) { IterateValue* cv = (cn == true) ? cacheIter->getValue(cacheIter) : NULL; IterateValue* tv = (tn == true) ? tfileIter->getValue(tfileIter) : NULL; @@ -480,17 +557,19 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { comp = 1; } if (comp == 0) { - indexMergeCacheAndTFile(result, cv, tv); + indexMergeCacheAndTFile(result, cv, tv, help); cn = cacheIter->next(cacheIter); tn = tfileIter->next(tfileIter); } else if (comp < 0) { - indexMergeCacheAndTFile(result, cv, NULL); + indexMergeCacheAndTFile(result, cv, NULL, help); cn = cacheIter->next(cacheIter); } else { - indexMergeCacheAndTFile(result, NULL, tv); + indexMergeCacheAndTFile(result, NULL, tv, help); tn = tfileIter->next(tfileIter); } } + indexMayMergeToFinalResult(result, NULL, help); + int ret = indexGenTFile(sIdx, pCache, result); indexDestroyTempResult(result); @@ -502,6 +581,8 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { tfileReaderUnRef(pReader); indexCacheUnRef(pCache); + sIdxMergeHelperDestroy(help); + int64_t cost = taosGetTimestampUs() - st; if (ret != 0) { indexError("failed to merge, time cost: %" PRId64 "ms", cost / 1000); diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 599bac3fe6..1ac72e10a9 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -267,6 +267,10 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SA SSkipListNode* node = tSkipListIterGet(iter); if (node != NULL) { CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + // if (c->operaType == ADD_VALUE) { + //} else if (c->operaType == DEL_VALUE) { + //} + if (c->operaType == ADD_VALUE || qtype == QUERY_TERM) { if (strcmp(c->colVal, ct->colVal) == 0) { taosArrayPush(result, &c->uid); @@ -411,17 +415,8 @@ static bool indexCacheIteratorNext(Iterate* itera) { SSkipListNode* node = tSkipListIterGet(iter); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); - // equal func - // if (iv->colVal != NULL && ct->colVal != NULL) { - // if (0 == strcmp(iv->colVal, ct->colVal)) { if (iv->type == ADD_VALUE) } - //} else { - // tIterVal.colVal = calloc(1, strlen(ct->colVal) + 1); - // tIterval.colVal = tstrdup(ct->colVal); - //} iv->type = ct->operaType; iv->colVal = tstrdup(ct->colVal); - // iv->colVal = calloc(1, strlen(ct->colVal) + 1); - // memcpy(iv->colVal, ct->colVal, strlen(ct->colVal)); taosArrayPush(iv->val, &ct->uid); } diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 0947c796b2..6cfc199afd 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -429,6 +429,7 @@ static bool tfileIteratorNext(Iterate* iiter) { return false; } + iv->type = ADD_VALUE; // value in tfile always ADD_VALUE iv->colVal = colVal; return true; // std::string key(ch, sz); diff --git a/source/libs/index/src/index_util.c b/source/libs/index/src/index_util.c index fcaab968c2..fc28484de9 100644 --- a/source/libs/index/src/index_util.c +++ b/source/libs/index/src/index_util.c @@ -14,6 +14,7 @@ */ #include "index_util.h" #include "index.h" + typedef struct MergeIndex { int idx; int len; @@ -111,6 +112,26 @@ void iUnion(SArray *inters, SArray *final) { break; } } - tfree(mi); } + +void iExcept(SArray *total, SArray *except) { + int32_t tsz = taosArrayGetSize(total); + int32_t esz = taosArrayGetSize(except); + if (esz == 0 || tsz == 0) { + return; + } + + int vIdx = 0; + for (int i = 0; i < tsz; i++) { + uint64_t val = *(uint64_t *)taosArrayGet(total, i); + int idx = iBinarySearch(except, 0, esz - 1, val); + if (idx >= 0 && idx < esz && *(uint64_t *)taosArrayGet(except, idx) == val) { + continue; + } + taosArraySet(total, vIdx, &val); + vIdx += 1; + } + + taosArrayPopTailBatch(total, tsz - vIdx); +} diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc index aeff20d488..6ba8cc9525 100644 --- a/source/libs/index/test/utilUT.cc +++ b/source/libs/index/test/utilUT.cc @@ -224,3 +224,84 @@ TEST_F(UtilEnv, 04union) { iUnion(src, rslt); assert(taosArrayGetSize(rslt) == 12); } +TEST_F(UtilEnv, 01Except) { + SArray *total = taosArrayInit(4, sizeof(uint64_t)); + { + uint64_t arr1[] = {1, 4, 5, 6}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(total, &arr1[i]); + } + } + + SArray *except = taosArrayInit(4, sizeof(uint64_t)); + { + uint64_t arr1[] = {1, 4, 5, 6}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(except, &arr1[i]); + } + } + iExcept(total, except); + ASSERT_EQ(taosArrayGetSize(total), 0); + + taosArrayClear(total); + taosArrayClear(except); + + { + uint64_t arr1[] = {1, 4, 5, 6, 7, 8}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(total, &arr1[i]); + } + } + + { + uint64_t arr1[] = {2, 4, 5, 6}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(except, &arr1[i]); + } + } + iExcept(total, except); + ASSERT_EQ(taosArrayGetSize(total), 3); + + taosArrayClear(total); + taosArrayClear(except); + { + uint64_t arr1[] = {1, 4, 5, 6, 7, 8, 10, 100}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(total, &arr1[i]); + } + } + + { + uint64_t arr1[] = {2, 4, 5, 6}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(except, &arr1[i]); + } + } + iExcept(total, except); + ASSERT_EQ(taosArrayGetSize(total), 5); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 0), 1); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 1), 7); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 2), 8); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 3), 10); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 4), 100); + + taosArrayClear(total); + taosArrayClear(except); + { + uint64_t arr1[] = {1, 100}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(total, &arr1[i]); + } + } + + { + uint64_t arr1[] = {2, 4, 5, 6}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(except, &arr1[i]); + } + } + iExcept(total, except); + ASSERT_EQ(taosArrayGetSize(total), 2); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 0), 1); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 1), 100); +} From c368e596891489871c105eb80ceb97d542f70dde Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 5 Mar 2022 11:08:15 +0800 Subject: [PATCH 8/8] add update on index --- source/libs/index/src/index.c | 114 +++++++++++----------------- source/libs/index/src/index_tfile.c | 1 - source/libs/index/test/jsonUT.cc | 16 ++++ 3 files changed, 62 insertions(+), 69 deletions(-) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 57e965a753..75dc05fc5b 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -16,6 +16,7 @@ #include "index.h" #include "indexInt.h" #include "index_cache.h" +#include "index_comm.h" #include "index_tfile.h" #include "index_util.h" #include "tdef.h" @@ -30,8 +31,6 @@ void* indexQhandle = NULL; -static char JSON_COLUMN[] = "JSON"; - #define INDEX_MERGE_ADD_DEL(src, dst, tgt) \ { \ bool f = false; \ @@ -64,13 +63,11 @@ typedef struct SIdxColInfo { int cVersion; } SIdxColInfo; -typedef struct SIdxMergeHelper { - char* colVal; +typedef struct SIdxTempResult { SArray* total; SArray* added; SArray* deled; - bool reset; -} SIdxMergeHelper; +} SIdxTempResult; static pthread_once_t isInit = PTHREAD_ONCE_INIT; // static void indexInit(); @@ -82,8 +79,7 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oTyp static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); // merge cache and tfile by opera type -static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxMergeHelper* helper); -static void indexMergeSameKey(SArray* result, TFileValue* tv, SIdxMergeHelper* helper); +static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTempResult* helper); // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); @@ -399,7 +395,6 @@ static void indexInterResultsDestroy(SArray* results) { static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { // refactor, merge interResults into fResults by oType - for (int i = 0; i < taosArrayGetSize(interResults); i--) { SArray* t = taosArrayGetP(interResults, i); taosArraySort(t, uidCompare); @@ -418,98 +413,82 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType return 0; } -SIdxMergeHelper* sIdxMergeHelperCreate() { - SIdxMergeHelper* hp = calloc(1, sizeof(SIdxMergeHelper)); - hp->total = taosArrayInit(4, sizeof(uint64_t)); - hp->added = taosArrayInit(4, sizeof(uint64_t)); - hp->deled = taosArrayInit(4, sizeof(uint64_t)); - hp->reset = false; - return hp; +SIdxTempResult* sIdxTempResultCreate() { + SIdxTempResult* tr = calloc(1, sizeof(SIdxTempResult)); + tr->total = taosArrayInit(4, sizeof(uint64_t)); + tr->added = taosArrayInit(4, sizeof(uint64_t)); + tr->deled = taosArrayInit(4, sizeof(uint64_t)); + return tr; } -void sIdxMergeHelperClear(SIdxMergeHelper* hp) { - if (hp == NULL) { +void sIdxTempResultClear(SIdxTempResult* tr) { + if (tr == NULL) { return; } - hp->reset = false; - taosArrayClear(hp->total); - taosArrayClear(hp->added); - taosArrayClear(hp->deled); + taosArrayClear(tr->total); + taosArrayClear(tr->added); + taosArrayClear(tr->deled); } -void sIdxMergeHelperDestroy(SIdxMergeHelper* hp) { - if (hp == NULL) { +void sIdxTempResultDestroy(SIdxTempResult* tr) { + if (tr == NULL) { return; } - taosArrayDestroy(hp->total); - taosArrayDestroy(hp->added); - taosArrayDestroy(hp->deled); + taosArrayDestroy(tr->total); + taosArrayDestroy(tr->added); + taosArrayDestroy(tr->deled); } -static void indexMergeSameKey(SArray* result, TFileValue* tv, SIdxMergeHelper* helper) { - int32_t sz = result ? taosArrayGetSize(result) : 0; - if (sz > 0) { - // TODO(yihao): remove duplicate tableid - TFileValue* lv = taosArrayGetP(result, sz - 1); - // indexError("merge colVal: %s", lv->colVal); - if (strcmp(lv->colVal, tv->colVal) == 0) { - taosArrayAddAll(lv->tableId, tv->tableId); - tfileValueDestroy(tv); - } else { - taosArrayPush(result, &tv); - } - } else { - taosArrayPush(result, &tv); - } -} -static void sIdxMergeResult(SArray* result, SIdxMergeHelper* mh) { - taosArraySort(mh->total, uidCompare); - taosArraySort(mh->added, uidCompare); - taosArraySort(mh->deled, uidCompare); +static void sIdxTempResultMergeTo(SArray* result, SIdxTempResult* tr) { + taosArraySort(tr->total, uidCompare); + taosArraySort(tr->added, uidCompare); + taosArraySort(tr->deled, uidCompare); SArray* arrs = taosArrayInit(2, sizeof(void*)); - taosArrayPush(arrs, &mh->total); - taosArrayPush(arrs, &mh->added); + taosArrayPush(arrs, &tr->total); + taosArrayPush(arrs, &tr->added); iUnion(arrs, result); taosArrayDestroy(arrs); - iExcept(result, mh->deled); + iExcept(result, tr->deled); } -static void indexMayMergeToFinalResult(SArray* result, TFileValue* tfv, SIdxMergeHelper* help) { +static void indexMayMergeTempToFinalResult(SArray* result, TFileValue* tfv, SIdxTempResult* tr) { int32_t sz = taosArrayGetSize(result); if (sz > 0) { TFileValue* lv = taosArrayGetP(result, sz - 1); if (tfv != NULL && strcmp(lv->colVal, tfv->colVal) != 0) { - sIdxMergeResult(lv->tableId, help); - sIdxMergeHelperClear(help); + sIdxTempResultMergeTo(lv->tableId, tr); + sIdxTempResultClear(tr); taosArrayPush(result, &tfv); } else if (tfv == NULL) { - sIdxMergeResult(lv->tableId, help); + // handle last iterator + sIdxTempResultMergeTo(lv->tableId, tr); } else { + // temp result saved in help tfileValueDestroy(tfv); } } else { taosArrayPush(result, &tfv); } } -static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxMergeHelper* mh) { +static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTempResult* tr) { char* colVal = (cv != NULL) ? cv->colVal : tv->colVal; TFileValue* tfv = tfileValueCreate(colVal); - indexMayMergeToFinalResult(result, tfv, mh); + indexMayMergeTempToFinalResult(result, tfv, tr); if (cv != NULL) { uint64_t id = *(uint64_t*)taosArrayGet(cv->val, 0); if (cv->type == ADD_VALUE) { - INDEX_MERGE_ADD_DEL(mh->deled, mh->added, id) + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, id) } else if (cv->type == DEL_VALUE) { - INDEX_MERGE_ADD_DEL(mh->added, mh->deled, id) + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, id) } } if (tv != NULL) { - taosArrayAddAll(mh->total, tv->val); + taosArrayAddAll(tr->total, tv->val); } } -static void indexDestroyTempResult(SArray* result) { +static void indexDestroyFinalResult(SArray* result) { int32_t sz = result ? taosArrayGetSize(result) : 0; for (size_t i = 0; i < sz; i++) { TFileValue* tv = taosArrayGetP(result, i); @@ -543,7 +522,7 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { bool cn = cacheIter ? cacheIter->next(cacheIter) : false; bool tn = tfileIter ? tfileIter->next(tfileIter) : false; - SIdxMergeHelper* help = sIdxMergeHelperCreate(); + SIdxTempResult* tr = sIdxTempResultCreate(); while (cn == true || tn == true) { IterateValue* cv = (cn == true) ? cacheIter->getValue(cacheIter) : NULL; IterateValue* tv = (tn == true) ? tfileIter->getValue(tfileIter) : NULL; @@ -557,21 +536,22 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { comp = 1; } if (comp == 0) { - indexMergeCacheAndTFile(result, cv, tv, help); + indexMergeCacheAndTFile(result, cv, tv, tr); cn = cacheIter->next(cacheIter); tn = tfileIter->next(tfileIter); } else if (comp < 0) { - indexMergeCacheAndTFile(result, cv, NULL, help); + indexMergeCacheAndTFile(result, cv, NULL, tr); cn = cacheIter->next(cacheIter); } else { - indexMergeCacheAndTFile(result, NULL, tv, help); + indexMergeCacheAndTFile(result, NULL, tv, tr); tn = tfileIter->next(tfileIter); } } - indexMayMergeToFinalResult(result, NULL, help); + indexMayMergeTempToFinalResult(result, NULL, tr); + sIdxTempResultDestroy(tr); int ret = indexGenTFile(sIdx, pCache, result); - indexDestroyTempResult(result); + indexDestroyFinalResult(result); indexCacheDestroyImm(pCache); @@ -581,8 +561,6 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { tfileReaderUnRef(pReader); indexCacheUnRef(pCache); - sIdxMergeHelperDestroy(help); - int64_t cost = taosGetTimestampUs() - st; if (ret != 0) { indexError("failed to merge, time cost: %" PRId64 "ms", cost / 1000); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 6cfc199afd..f5f46b0617 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -102,7 +102,6 @@ void tfileCacheDestroy(TFileCache* tcache) { if (tcache == NULL) { return; } - // free table cache TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); while (reader) { diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index e5c79d137f..df9f8b8439 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -105,6 +105,22 @@ TEST_F(JsonEnv, testWriteMillonData) { } indexMultiTermDestroy(terms); } + { + std::string colName("voltagefdadfa"); + std::string colVal("abxxxxxxxxxxxx"); + for (uint i = 0; i < 1000; i++) { + colVal[i % colVal.size()] = '0' + i % 128; + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 1000; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } + } { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx");