From 0a1476763ab66e3882b6a8f61efdb0e671936217 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 4 Apr 2023 20:01:03 +0800 Subject: [PATCH 1/8] enh: refactor some func names of syncLogReplMgr --- source/libs/sync/inc/syncPipeline.h | 40 ++++---- source/libs/sync/src/syncAppendEntriesReply.c | 2 +- source/libs/sync/src/syncMain.c | 8 +- source/libs/sync/src/syncPipeline.c | 92 +++++++++---------- source/libs/sync/src/syncReplication.c | 4 +- source/libs/sync/src/syncUtil.c | 4 +- 6 files changed, 75 insertions(+), 75 deletions(-) diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index a1de2ee71a..68db811b12 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -59,36 +59,36 @@ typedef struct SSyncLogBuffer { } SSyncLogBuffer; // SSyncLogRepMgr -SSyncLogReplMgr* syncLogReplMgrCreate(); -void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr); -void syncLogReplMgrReset(SSyncLogReplMgr* pMgr); +SSyncLogReplMgr* syncLogReplCreate(); +void syncLogReplDestroy(SSyncLogReplMgr* pMgr); +void syncLogReplReset(SSyncLogReplMgr* pMgr); -int32_t syncNodeLogReplMgrInit(SSyncNode* pNode); -void syncNodeLogReplMgrDestroy(SSyncNode* pNode); +int32_t syncNodeLogReplInit(SSyncNode* pNode); +void syncNodeLogReplDestroy(SSyncNode* pNode); // access -static FORCE_INLINE int64_t syncLogGetRetryBackoffTimeMs(SSyncLogReplMgr* pMgr) { +static FORCE_INLINE int64_t syncLogReplGetRetryBackoffTimeMs(SSyncLogReplMgr* pMgr) { return ((int64_t)1 << pMgr->retryBackoff) * SYNC_LOG_REPL_RETRY_WAIT_MS; } -static FORCE_INLINE int32_t syncLogGetNextRetryBackoff(SSyncLogReplMgr* pMgr) { +static FORCE_INLINE int32_t syncLogReplGetNextRetryBackoff(SSyncLogReplMgr* pMgr) { return TMIN(pMgr->retryBackoff + 1, SYNC_MAX_RETRY_BACKOFF); } -SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); +SyncTerm syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); -int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier); -int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); +int32_t syncLogReplReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, + SRaftId* pDestId, bool* pBarrier); +int32_t syncLogReplReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); -int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); -int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); +int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); // SSyncLogBuffer SSyncLogBuffer* syncLogBufferCreate(); @@ -111,8 +111,8 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf); int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex); -int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, - int32_t applyCode); +int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, + int32_t applyCode); #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index a60f43cd5e..7c343c0e5d 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -85,7 +85,7 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { sError("vgId:%d, failed to get log repl mgr for src addr: 0x%016" PRIx64, ths->vgId, pMsg->srcId.addr); return -1; } - (void)syncLogReplMgrProcessReply(pMgr, ths, pMsg); + (void)syncLogReplProcessReply(pMgr, ths, pMsg); } return 0; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index a617a5d293..c25ea24249 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -949,7 +949,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->changing = false; // replication mgr - if (syncNodeLogReplMgrInit(pSyncNode) < 0) { + if (syncNodeLogReplInit(pSyncNode) < 0) { sError("vgId:%d, failed to init repl mgr since %s.", pSyncNode->vgId, terrstr()); goto _error; } @@ -1122,7 +1122,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { syncNodeStopPingTimer(pSyncNode); syncNodeStopElectTimer(pSyncNode); syncNodeStopHeartbeatTimer(pSyncNode); - syncNodeLogReplMgrDestroy(pSyncNode); + syncNodeLogReplDestroy(pSyncNode); syncRespMgrDestroy(pSyncNode->pSyncRespMgr); pSyncNode->pSyncRespMgr = NULL; @@ -2164,7 +2164,7 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { if (syncLogBufferAppend(ths->pLogBuf, ths, pEntry) < 0) { sError("vgId:%d, failed to enqueue sync log buffer, index:%" PRId64, ths->vgId, pEntry->index); ASSERT(terrno != 0); - (void)syncLogFsmExecute(ths, ths->pFsm, ths->state, raftStoreGetTerm(ths), pEntry, terrno); + (void)syncFsmExecute(ths, ths->pFsm, ths->state, raftStoreGetTerm(ths), pEntry, terrno); syncEntryDestroy(pEntry); return -1; } @@ -2374,7 +2374,7 @@ int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs); - return syncLogReplMgrProcessHeartbeatReply(pMgr, ths, pMsg); + return syncLogReplProcessHeartbeatReply(pMgr, ths, pMsg); } int32_t syncNodeOnHeartbeatReplyOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) { diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 6600b505c1..04e52b3f49 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -86,7 +86,7 @@ _err: return -1; } -SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { +SyncTerm syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { SSyncLogBuffer* pBuf = pNode->pLogBuf; SSyncRaftEntry* pEntry = NULL; SyncIndex prevIndex = index - 1; @@ -316,7 +316,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt " %" PRId64 ", %" PRId64 ")", pNode->vgId, pEntry->index, pEntry->term, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); - SyncTerm term = syncLogReplMgrGetPrevLogTerm(NULL, pNode, index + 1); + SyncTerm term = syncLogReplGetPrevLogTerm(NULL, pNode, index + 1); ASSERT(pEntry->term >= 0); if (term == pEntry->term) { ret = 0; @@ -351,7 +351,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, pEntry->index, pEntry->term, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); - SyncTerm existPrevTerm = syncLogReplMgrGetPrevLogTerm(NULL, pNode, index); + SyncTerm existPrevTerm = syncLogReplGetPrevLogTerm(NULL, pNode, index); ASSERT(pEntry->term == pExist->term && (pEntry->index > pBuf->matchIndex || prevTerm == existPrevTerm)); ret = 0; goto _out; @@ -482,8 +482,8 @@ _out: return matchIndex; } -int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, - int32_t applyCode) { +int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, + int32_t applyCode) { if (pNode->replicaNum == 1 && pNode->restoreFinish && pNode->vgId != 1) { return 0; } @@ -564,7 +564,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm pEntry->term, TMSG_INFO(pEntry->originalRpcType)); } - if (syncLogFsmExecute(pNode, pFsm, role, currentTerm, pEntry, 0) != 0) { + if (syncFsmExecute(pNode, pFsm, role, currentTerm, pEntry, 0) != 0) { sError("vgId:%d, failed to execute sync log entry. index:%" PRId64 ", term:%" PRId64 ", role:%d, current term:%" PRId64, vgId, pEntry->index, pEntry->term, role, currentTerm); @@ -611,7 +611,7 @@ _out: return ret; } -void syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { +void syncLogReplReset(SSyncLogReplMgr* pMgr) { if (pMgr == NULL) return; ASSERT(pMgr->startIndex >= 0); @@ -625,14 +625,14 @@ void syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { pMgr->retryBackoff = 0; } -int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->endIndex <= pMgr->startIndex) { return 0; } SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; if (pMgr->retryBackoff == SYNC_MAX_RETRY_BACKOFF) { - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); sWarn("vgId:%d, reset sync log repl mgr since retry backoff exceeding limit. peer:%" PRIx64, pNode->vgId, pDestId->addr); return -1; @@ -640,7 +640,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { int32_t ret = -1; bool retried = false; - int64_t retryWaitMs = syncLogGetRetryBackoffTimeMs(pMgr); + int64_t retryWaitMs = syncLogReplGetRetryBackoffTimeMs(pMgr); int64_t nowMs = taosGetMonoTimestampMs(); int count = 0; int64_t firstIndex = -1; @@ -657,7 +657,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->states[pos].acked) { if (pMgr->matchIndex < index && pMgr->states[pos].timeMs + (syncGetRetryMaxWaitMs() << 3) < nowMs) { - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); sWarn("vgId:%d, reset sync log repl mgr since stagnation. index:%" PRId64 ", peer:%" PRIx64, pNode->vgId, index, pDestId->addr); goto _out; @@ -666,7 +666,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { } bool barrier = false; - if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate sync log entry since %s. index:%" PRId64 ", dest:%" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); goto _out; @@ -687,7 +687,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { ret = 0; _out: if (retried) { - pMgr->retryBackoff = syncLogGetNextRetryBackoff(pMgr); + pMgr->retryBackoff = syncLogReplGetNextRetryBackoff(pMgr); SSyncLogBuffer* pBuf = pNode->pLogBuf; sInfo("vgId:%d, resend %d sync log entries. dest:%" PRIx64 ", indexes:%" PRId64 " ..., terms: ... %" PRId64 ", retryWaitMs:%" PRId64 ", mgr: [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 @@ -698,7 +698,7 @@ _out: return ret; } -int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; SRaftId destId = pMsg->srcId; ASSERT(pMgr->restored == false); @@ -716,7 +716,7 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p } } else { if (pMsg->lastSendIndex < pMgr->startIndex || pMsg->lastSendIndex >= pMgr->endIndex) { - syncLogReplMgrRetryOnNeed(pMgr, pNode); + syncLogReplRetryOnNeed(pMgr, pNode); return 0; } @@ -750,7 +750,7 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p SyncIndex index = TMIN(pMsg->matchIndex, pNode->pLogBuf->matchIndex); if (pMsg->matchIndex < pNode->pLogBuf->matchIndex) { - term = syncLogReplMgrGetPrevLogTerm(pMgr, pNode, index + 1); + term = syncLogReplGetPrevLogTerm(pMgr, pNode, index + 1); if ((index + 1 < firstVer) || (term < 0) || (term != pMsg->lastMatchTerm && (index + 1 == firstVer || index == firstVer))) { ASSERT(term >= 0 || terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); @@ -773,53 +773,53 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p } // attempt to replicate the raft log at index - (void)syncLogReplMgrReset(pMgr); - return syncLogReplMgrReplicateProbe(pMgr, pNode, index); + (void)syncLogReplReset(pMgr); + return syncLogReplReplicateProbe(pMgr, pNode, index); } -int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg) { +int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); if (pMsg->startTime != 0 && pMsg->startTime != pMgr->peerStartTime) { sInfo("vgId:%d, reset sync log repl mgr in heartbeat. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64 "", pNode->vgId, pMsg->srcId.addr, pMsg->startTime, pMgr->peerStartTime); - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); pMgr->peerStartTime = pMsg->startTime; } taosThreadMutexUnlock(&pBuf->mutex); return 0; } -int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); if (pMsg->startTime != pMgr->peerStartTime) { sInfo("vgId:%d, reset sync log repl mgr in appendlog reply. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64, pNode->vgId, pMsg->srcId.addr, pMsg->startTime, pMgr->peerStartTime); - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); pMgr->peerStartTime = pMsg->startTime; } if (pMgr->restored) { - (void)syncLogReplMgrProcessReplyAsNormal(pMgr, pNode, pMsg); + (void)syncLogReplProcessReplyAsNormal(pMgr, pNode, pMsg); } else { - (void)syncLogReplMgrProcessReplyAsRecovery(pMgr, pNode, pMsg); + (void)syncLogReplProcessReplyAsRecovery(pMgr, pNode, pMsg); } taosThreadMutexUnlock(&pBuf->mutex); return 0; } -int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->restored) { - (void)syncLogReplMgrReplicateAttempt(pMgr, pNode); + (void)syncLogReplReplicateAttempt(pMgr, pNode); } else { - (void)syncLogReplMgrReplicateProbe(pMgr, pNode, pNode->pLogBuf->matchIndex); + (void)syncLogReplReplicateProbe(pMgr, pNode, pNode->pLogBuf->matchIndex); } return 0; } -int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { +int32_t syncLogReplReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { ASSERT(!pMgr->restored); ASSERT(pMgr->startIndex >= 0); int64_t retryMaxWaitMs = syncGetRetryMaxWaitMs(); @@ -829,12 +829,12 @@ int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sy nowMs < pMgr->states[pMgr->startIndex % pMgr->size].timeMs + retryMaxWaitMs) { return 0; } - (void)syncLogReplMgrReset(pMgr); + (void)syncLogReplReset(pMgr); SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; bool barrier = false; SyncTerm term = -1; - if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index:%" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -857,7 +857,7 @@ int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sy return 0; } -int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { ASSERT(pMgr->restored); SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; @@ -879,7 +879,7 @@ int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; bool barrier = false; SyncTerm term = -1; - if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index:%" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -902,7 +902,7 @@ int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) } } - syncLogReplMgrRetryOnNeed(pMgr, pNode); + syncLogReplRetryOnNeed(pMgr, pNode); SSyncLogBuffer* pBuf = pNode->pLogBuf; sTrace("vgId:%d, replicated %d msgs to peer:%" PRIx64 ". indexes:%" PRId64 "..., terms: ...%" PRId64 @@ -913,7 +913,7 @@ int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) return 0; } -int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { ASSERT(pMgr->restored == true); if (pMgr->startIndex <= pMsg->lastSendIndex && pMsg->lastSendIndex < pMgr->endIndex) { if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) { @@ -932,10 +932,10 @@ int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNo pMgr->startIndex = pMgr->matchIndex; } - return syncLogReplMgrReplicateAttempt(pMgr, pNode); + return syncLogReplReplicateAttempt(pMgr, pNode); } -SSyncLogReplMgr* syncLogReplMgrCreate() { +SSyncLogReplMgr* syncLogReplCreate() { SSyncLogReplMgr* pMgr = taosMemoryCalloc(1, sizeof(SSyncLogReplMgr)); if (pMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -949,7 +949,7 @@ SSyncLogReplMgr* syncLogReplMgrCreate() { return pMgr; } -void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr) { +void syncLogReplDestroy(SSyncLogReplMgr* pMgr) { if (pMgr == NULL) { return; } @@ -957,10 +957,10 @@ void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr) { return; } -int32_t syncNodeLogReplMgrInit(SSyncNode* pNode) { +int32_t syncNodeLogReplInit(SSyncNode* pNode) { for (int i = 0; i < TSDB_MAX_REPLICA; i++) { ASSERT(pNode->logReplMgrs[i] == NULL); - pNode->logReplMgrs[i] = syncLogReplMgrCreate(); + pNode->logReplMgrs[i] = syncLogReplCreate(); if (pNode->logReplMgrs[i] == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -970,9 +970,9 @@ int32_t syncNodeLogReplMgrInit(SSyncNode* pNode) { return 0; } -void syncNodeLogReplMgrDestroy(SSyncNode* pNode) { +void syncNodeLogReplDestroy(SSyncNode* pNode) { for (int i = 0; i < TSDB_MAX_REPLICA; i++) { - syncLogReplMgrDestroy(pNode->logReplMgrs[i]); + syncLogReplDestroy(pNode->logReplMgrs[i]); pNode->logReplMgrs[i] = NULL; } } @@ -1103,7 +1103,7 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { // reset repl mgr for (int i = 0; i < pNode->replicaNum; i++) { SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); } syncLogBufferValidate(pBuf); taosThreadMutexUnlock(&pBuf->mutex); @@ -1127,8 +1127,8 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, return pEntry; } -int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier) { +int32_t syncLogReplReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, + SRaftId* pDestId, bool* pBarrier) { SSyncRaftEntry* pEntry = NULL; SRpcMsg msgOut = {0}; bool inBuf = false; @@ -1143,14 +1143,14 @@ int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sy if (pMgr) { sInfo("vgId:%d, reset sync log repl mgr of peer:%" PRIx64 " since %s. index:%" PRId64, pNode->vgId, pDestId->addr, terrstr(), index); - (void)syncLogReplMgrReset(pMgr); + (void)syncLogReplReset(pMgr); } } goto _err; } *pBarrier = syncLogIsReplicationBarrier(pEntry); - prevLogTerm = syncLogReplMgrGetPrevLogTerm(pMgr, pNode, index); + prevLogTerm = syncLogReplGetPrevLogTerm(pMgr, pNode, index); if (prevLogTerm < 0) { sError("vgId:%d, failed to get prev log term since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); goto _err; diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 1d94b288d3..43d2bc839b 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -52,7 +52,7 @@ int32_t syncNodeReplicateReset(SSyncNode* pNode, SRaftId* pDestId) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId); - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); taosThreadMutexUnlock(&pBuf->mutex); return 0; } @@ -74,7 +74,7 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) { continue; } SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; - (void)syncLogReplMgrReplicateOnce(pMgr, pNode); + (void)syncLogReplReplicateOnce(pMgr, pNode); } return 0; } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index a519c76cda..056a597777 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -125,7 +125,7 @@ static void syncLogBufferStates2Str(SSyncNode* pSyncNode, char* buf, int32_t buf pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); } -static void syncLogReplMgrStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { +static void syncLogReplStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { int len = 0; len += snprintf(buf + len, bufLen - len, "%s", "{"); for (int32_t i = 0; i < pSyncNode->replicaNum; i++) { @@ -178,7 +178,7 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo syncCfg2SimpleStr(&pNode->raftCfg.cfg, cfgStr, sizeof(cfgStr)); char replMgrStatesStr[1024] = ""; - syncLogReplMgrStates2Str(pNode, replMgrStatesStr, sizeof(replMgrStatesStr)); + syncLogReplStates2Str(pNode, replMgrStatesStr, sizeof(replMgrStatesStr)); char bufferStatesStr[256] = ""; syncLogBufferStates2Str(pNode, bufferStatesStr, sizeof(bufferStatesStr)); From 360113115f2311601e4d9763e7bb271cba1b6dca Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 7 Apr 2023 09:49:48 +0800 Subject: [PATCH 2/8] fix(tsdb/row merger): remove unused code --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index c323ae1532..e4592d2758 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -727,23 +727,6 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); if (key.version > pMerger->version) { -#if 0 - if (!COL_VAL_IS_NONE(pColVal)) { - if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { - SColVal *tColVal = taosArrayGet(pMerger->pArray, iCol); - code = tRealloc(&tColVal->value.pData, pColVal->value.nData); - if (code) return code; - - tColVal->value.nData = pColVal->value.nData; - if (pColVal->value.nData) { - memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); - } - tColVal->flag = 0; - } else { - taosArraySet(pMerger->pArray, iCol, pColVal); - } - } -#endif if (!COL_VAL_IS_NONE(pColVal)) { if (IS_VAR_DATA_TYPE(pColVal->type)) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); @@ -758,7 +741,6 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) pTColVal->flag = 0; } else { tFree(pTColVal->value.pData); - pTColVal->value.pData = NULL; taosArraySet(pMerger->pArray, iCol, pColVal); } } else { From 50ced8a057951152c85e2f417a9291473b24e20e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 7 Apr 2023 13:22:42 +0800 Subject: [PATCH 3/8] fix: taosbenchmark from/to neg value for main (#20805) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 8be698e9c9..b427177e5b 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 01195d6 + GIT_TAG 63635fc SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 55f7e41f3b1a49ed185de436bc57ff9bb5784e7d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 7 Apr 2023 13:31:33 +0800 Subject: [PATCH 4/8] tsdb/row merger: remove tsdbRowMerge --- source/dnode/vnode/src/inc/tsdb.h | 4 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 146 ++++++++++++------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 7 +- 3 files changed, 79 insertions(+), 78 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0c4ada2cb1..0dd767990e 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -128,7 +128,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); void tsdbRowMergerClear(SRowMerger *pMerger); -int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); +// int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); // TABLEID int32_t tTABLEIDCmprFn(const void *p1, const void *p2); @@ -224,7 +224,7 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, void *tsdbTbDataIterDestroy(STbDataIter *pIter); void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter); -void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum); +void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj *pTableMap, int64_t *rowsNum); // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 1ac327de7f..338231057c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -959,14 +959,15 @@ static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int } static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, - SBlockLoadSuppInfo* pSup) { + SBlockLoadSuppInfo* pSup) { if (IS_VAR_DATA_TYPE(pColVal->type)) { if (!COL_VAL_IS_VALUE(pColVal)) { colDataSetNULL(pColInfoData, rowIndex); } else { varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); if (pColVal->value.nData > pColInfoData->info.bytes) { - tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData, pColInfoData->info.bytes); + tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData, + pColInfoData->info.bytes); return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; } if (pColVal->value.nData > 0) { // pData may be null, if nData is 0 @@ -1794,7 +1795,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* } static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, int64_t key, - SFileBlockDumpInfo* pDumpInfo, bool *copied) { + SFileBlockDumpInfo* pDumpInfo, bool* copied) { // opt version // 1. it is not a border point // 2. the direct next point is not an duplicated timestamp @@ -1843,7 +1844,8 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc } static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader, - STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader, bool *copied) { + STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader, + bool* copied) { int32_t code = TSDB_CODE_SUCCESS; *copied = false; @@ -1856,7 +1858,7 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLas if (code) { return code; } - + *copied = true; return code; } @@ -1865,7 +1867,7 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLas if (code) { return code; } - + *copied = true; return code; } @@ -1977,7 +1979,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); @@ -2025,7 +2027,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); @@ -2038,7 +2040,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { if (init) { - tsdbRowMerge(&merge, &fRow); + tsdbRowMergerAdd(&merge, &fRow, NULL); } else { init = true; int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); @@ -2068,11 +2070,11 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, bool mergeBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); - bool copied = false; - int32_t code = TSDB_CODE_SUCCESS; - SRow* pTSRow = NULL; - SRowMerger merge = {0}; - TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + bool copied = false; + int32_t code = TSDB_CODE_SUCCESS; + SRow* pTSRow = NULL; + SRowMerger merge = {0}; + TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); // only last block exists @@ -2081,7 +2083,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, if (code) { return code; } - + if (copied) { pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; @@ -2092,7 +2094,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange, pReader->idStr); code = tsdbRowMergerGetRow(&merge, &pTSRow); @@ -2108,7 +2110,6 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, if (code != TSDB_CODE_SUCCESS) { return code; } - } } else { // not merge block data int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); @@ -2171,7 +2172,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange, pReader->idStr); @@ -2273,7 +2274,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); @@ -2287,7 +2288,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tsdbRowMerge(&merge, piRow); + tsdbRowMergerAdd(&merge, piRow, NULL); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); @@ -2314,7 +2315,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - tsdbRowMerge(&merge, pRow); + tsdbRowMergerAdd(&merge, pRow, NULL); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); code = tsdbRowMergerInit(&merge, pRow, pSchema); @@ -2346,7 +2347,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tsdbRowMerge(&merge, piRow); + tsdbRowMergerAdd(&merge, piRow, NULL); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); @@ -2365,7 +2366,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); @@ -2387,7 +2388,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (merge.pTSchema == NULL) { return code; } - tsdbRowMerge(&merge, &fRow); + tsdbRowMergerAdd(&merge, &fRow, NULL); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -2557,12 +2558,12 @@ bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - bool copied = false; - int32_t code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo, &copied); + bool copied = false; + int32_t code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo, &copied); if (code) { return code; } - + if (copied) { pBlockScanInfo->lastKey = key; return TSDB_CODE_SUCCESS; @@ -2758,7 +2759,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { if (code) { goto _end; } - + // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); @@ -2776,8 +2777,8 @@ _end: updateComposedBlockInfo(pReader, el, pBlockScanInfo); if (pResBlock->info.rows > 0) { - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%" PRId64 ", elapsed time:%.2f ms %s", + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); } @@ -3018,7 +3019,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { if (code) { return code; } - + if (pResBlock->info.rows >= pReader->capacity) { break; } @@ -3028,8 +3029,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { updateComposedBlockInfo(pReader, el, pScanInfo); if (pResBlock->info.rows > 0) { - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%" PRId64 ", elapsed time:%.2f ms %s", + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); return TSDB_CODE_SUCCESS; @@ -3102,7 +3103,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (code) { return code; } - + if (pResBlock->info.rows >= pReader->capacity) { break; } @@ -3112,8 +3113,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { updateComposedBlockInfo(pReader, el, pScanInfo); if (pResBlock->info.rows > 0) { - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%" PRId64 ", elapsed time:%.2f ms %s", + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); } @@ -3139,7 +3140,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return code; } - static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { int64_t st = taosGetTimestampUs(); LRUHandle* handle = NULL; @@ -3157,8 +3157,8 @@ static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReade return TSDB_CODE_SUCCESS; } - SBlockIdx* pBlockIdx = NULL; - int32_t i = 0; + SBlockIdx* pBlockIdx = NULL; + int32_t i = 0; for (int32_t i = 0; i < num; ++i) { pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i); if (pBlockIdx->suid != pReader->suid) { @@ -3170,7 +3170,7 @@ static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReade continue; } - STableBlockScanInfo *pScanInfo = *p; + STableBlockScanInfo* pScanInfo = *p; tMapDataReset(&pScanInfo->mapData); tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); @@ -3186,15 +3186,14 @@ _end: return code; } - static int32_t doSumSttBlockRows(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; - SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - SSttBlockLoadInfo* pBlockLoadInfo = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + SSttBlockLoadInfo* pBlockLoadInfo = NULL; for (int32_t i = 0; i < pReader->pFileReader->pSet->nSttF; ++i) { // open all last file pBlockLoadInfo = &pLastBlockReader->pInfo[i]; - + code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk); if (code) { return code; @@ -3202,9 +3201,9 @@ static int32_t doSumSttBlockRows(STsdbReader* pReader) { size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); if (size >= 1) { - SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); - SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); - + SSttBlk* pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); + SSttBlk* pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); + // all identical if (pStart->suid == pEnd->suid) { if (pStart->suid != pReader->suid) { @@ -3213,17 +3212,17 @@ static int32_t doSumSttBlockRows(STsdbReader* pReader) { continue; } for (int32_t i = 0; i < size; ++i) { - SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); pReader->rowsNum += p->nRow; } } else { for (int32_t i = 0; i < size; ++i) { - SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); uint64_t s = p->suid; if (s < pReader->suid) { continue; } - + if (s == pReader->suid) { pReader->rowsNum += p->nRow; } else if (s > pReader->suid) { @@ -3238,7 +3237,7 @@ static int32_t doSumSttBlockRows(STsdbReader* pReader) { } static int32_t readRowsCountFromFiles(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; while (1) { bool hasNext = false; @@ -3259,7 +3258,7 @@ static int32_t readRowsCountFromFiles(STsdbReader* pReader) { code = doSumSttBlockRows(pReader); if (code != TSDB_CODE_SUCCESS) { return code; - } + } } pReader->status.loadFromFile = false; @@ -3268,8 +3267,8 @@ static int32_t readRowsCountFromFiles(STsdbReader* pReader) { } static int32_t readRowsCountFromMem(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; - int64_t memNum = 0, imemNum = 0; + int32_t code = TSDB_CODE_SUCCESS; + int64_t memNum = 0, imemNum = 0; if (pReader->pReadSnap->pMem != NULL) { tsdbMemTableCountRows(pReader->pReadSnap->pMem, pReader->status.pTableMap, &memNum); } @@ -3283,7 +3282,6 @@ static int32_t readRowsCountFromMem(STsdbReader* pReader) { return code; } - static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; STableUidList* pUidList = &pStatus->uidList; @@ -3696,7 +3694,7 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe tsdbRowMergerAdd(pMerger, pRow, pTSchema); } else { // column format - tsdbRowMerge(pMerger, pRow); + tsdbRowMergerAdd(pMerger, pRow, NULL); } } @@ -3712,7 +3710,7 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowInd } TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - tsdbRowMerge(pMerger, &fRow); + tsdbRowMergerAdd(pMerger, &fRow, NULL); rowIndex += step; } @@ -3790,7 +3788,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMerge(pMerger, &fRow1); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, pScanInfo->lastBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt, @@ -3863,7 +3861,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, return code; } - tsdbRowMerge(&merge, pNextRow); + tsdbRowMergerAdd(&merge, pNextRow, NULL); } code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, &merge, pReader); @@ -3926,7 +3924,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tsdbRowMerge(&merge, piRow); + tsdbRowMergerAdd(&merge, piRow, NULL); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -4000,7 +3998,7 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) { int32_t outputRowIndex = pBlock->info.rows; int64_t uid = pScanInfo->uid; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); @@ -4106,7 +4104,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader) { SSDataBlock* pBlock = pReader->pResBlock; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; do { // SRow* pTSRow = NULL; @@ -4342,7 +4340,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL if (countOnly) { pReader->readMode = READ_MODE_COUNT_ONLY; } - + tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; @@ -4644,7 +4642,7 @@ _err: } static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SSDataBlock* pBlock = pReader->pResBlock; if (pReader->status.loadFromFile == false) { @@ -4664,15 +4662,15 @@ static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { pBlock->info.rows = pReader->rowsNum; pBlock->info.id.uid = 0; pBlock->info.dataLoad = 0; - + pReader->rowsNum = 0; - + return pBlock->info.rows > 0; } -static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { +static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { int32_t code = TSDB_CODE_SUCCESS; - + // cleanup the data that belongs to the previous data block SSDataBlock* pBlock = pReader->pResBlock; blockDataCleanup(pBlock); @@ -4707,11 +4705,11 @@ static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { return code; } -int32_t tsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { +int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { int32_t code = TSDB_CODE_SUCCESS; *hasNext = false; - + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT) { return code; } @@ -4731,7 +4729,7 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { tsdbReleaseReader(pReader); return code; } - + pReader->step = EXTERNAL_ROWS_PREV; if (*hasNext) { pStatus = &pReader->innerReader[0]->status; @@ -4762,7 +4760,7 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { tsdbReleaseReader(pReader); return code; } - + if (*hasNext) { if (pStatus->composedDataBlock) { qTrace("tsdb/read: %p, unlock read mutex", pReader); @@ -4786,7 +4784,7 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { tsdbReleaseReader(pReader); return code; } - + pReader->step = EXTERNAL_ROWS_NEXT; if (*hasNext) { pStatus = &pReader->innerReader[1]->status; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index e4592d2758..9aeea22b99 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -712,6 +712,9 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) STColumn *pTColumn; int32_t iCol, jCol = 1; + if (NULL == pTSchema) { + pTSchema = pMerger->pTSchema; + } ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { @@ -833,7 +836,7 @@ void tsdbRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); } - +/* int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); @@ -898,7 +901,7 @@ int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { _exit: return code; } - +*/ int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) { return tRowBuild(pMerger->pArray, pMerger->pTSchema, ppRow); } From 1ff8511eb3af8b500a139571bbf6099b14abaf40 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 7 Apr 2023 13:43:07 +0800 Subject: [PATCH 5/8] tsdb/row merger: comment init out --- source/dnode/vnode/src/inc/tsdb.h | 6 ++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 44 +++++++++++++------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 10 ++++-- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0dd767990e..2a85b191a4 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -123,11 +123,11 @@ int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) void tsdbRowClose(STSDBRowIter *pIter); SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger -int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -void tsdbRowMergerClear(SRowMerger *pMerger); +// int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowMergerClear(SRowMerger *pMerger); // int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); // TABLEID diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 338231057c..55cbf2c847 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1969,7 +1969,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1982,7 +1982,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1999,7 +1999,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, pRow, pSchema); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2013,7 +2013,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2030,7 +2030,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2043,7 +2043,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2088,7 +2088,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2112,7 +2112,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } } } else { // not merge block data - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2164,7 +2164,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2263,7 +2263,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2277,7 +2277,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2296,7 +2296,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = tsdbRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, NULL, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2318,7 +2318,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, pRow, NULL); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2333,7 +2333,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2351,7 +2351,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, NULL, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2369,7 +2369,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2380,7 +2380,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2573,7 +2573,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3844,7 +3844,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, pReader->pSchema = pTSchema; } - code = tsdbRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + code = tsdbRowMergerInit(&merge, pReader->pSchema, ¤t, pTSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3856,7 +3856,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); } else { // let's merge rows in file block - code = tsdbRowMergerInit(&merge, ¤t, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, ¤t, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3892,7 +3892,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit2(&merge, pSchema, piRow, piSchema); + int32_t code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3913,7 +3913,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 9aeea22b99..dd11134bd0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -638,13 +638,17 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { // SRowMerger ====================================================== -int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; STColumn *pTColumn; int32_t iCol, jCol = 0; + if (NULL == pResTSchema) { + pResTSchema = pTSchema; + } + pMerger->pTSchema = pResTSchema; pMerger->version = key.version; @@ -774,7 +778,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) pMerger->version = key.version; return code; } - +/* int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); @@ -825,7 +829,7 @@ int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema _exit: return code; } - +*/ void tsdbRowMergerClear(SRowMerger *pMerger) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); From 05ec2b09287bb465d19d10dbc35cf94244d31821 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 7 Apr 2023 14:24:29 +0800 Subject: [PATCH 6/8] fix:[TD-23517] set offset type to wal if only meta & check if tag is the same type in schemaless --- source/dnode/vnode/src/tq/tqScan.c | 2 +- source/libs/executor/src/executor.c | 4 +++- source/libs/parser/src/parInsertSml.c | 6 ++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index b4e50312fd..a26f59f72b 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -170,7 +170,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta qStreamExtractOffset(task, &tmp->rspOffset); *pMetaRsp = *tmp; - tqDebug("tmqsnap task get data"); + tqDebug("tmqsnap task get meta"); break; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b4431a7c3b..1670eb3c59 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1200,7 +1200,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tableListClear(pTableListInfo); if (mtInfo.uid == 0) { - return 0; // no data + goto end; // no data } initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); @@ -1234,6 +1234,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); } } + +end: pTaskInfo->streamInfo.currentOffset = *pOffset; return 0; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 0bcd777d2a..1c921b2d7c 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -125,6 +125,12 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; SSmlKv* kv = taosArrayGet(cols, i); + if(kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type){ + code = TSDB_CODE_SML_INVALID_DATA; + uError("SML smlBuildCol error col not same %s", pTagSchema->name); + goto end; + } + taosArrayPush(*tagName, pTagSchema->name); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; // strcpy(val.colName, pTagSchema->name); From f262ac82308394f67080290a9df65a16c0dd08f7 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 7 Apr 2023 14:37:56 +0800 Subject: [PATCH 7/8] fix(tsdb/read): use irow's schema instead of NULL --- source/dnode/vnode/src/tsdb/tsdbRead.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 55cbf2c847..0cf28c1ed1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3885,13 +3885,12 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p SRow** pTSRow) { SRowMerger merge = {0}; - TSDBKEY k = TSDBROW_KEY(pRow); - TSDBKEY ik = TSDBROW_KEY(piRow); + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3911,8 +3910,6 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } } else { - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; @@ -3924,7 +3921,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tsdbRowMergerAdd(&merge, piRow, NULL); + tsdbRowMergerAdd(&merge, piRow, piSchema); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { From 4fb5390c6fecd8a4670ee3bbdb225db4deee8e0d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 7 Apr 2023 17:10:39 +0800 Subject: [PATCH 8/8] chore: release script community main (#20808) * chore: cus name support in script * chore: merge with 3.0 --- include/os/osDir.h | 28 +++++ include/util/cus_name.h | 31 +++++ packaging/cfg/taos.cfg | 15 ++- packaging/cfg/taosd.service | 2 +- packaging/tools/install.sh | 194 +++++++++++++++---------------- packaging/tools/makeclient.sh | 16 ++- packaging/tools/makepkg.sh | 28 +++-- packaging/tools/remove.sh | 41 ++++--- packaging/tools/remove_client.sh | 18 +-- source/client/src/clientEnv.c | 14 ++- source/common/src/tglobal.c | 4 + source/dnode/mgmt/exe/dmMain.c | 17 ++- source/os/src/osSysinfo.c | 12 ++ tools/shell/CMakeLists.txt | 4 - tools/shell/src/shellArguments.c | 31 +++-- 15 files changed, 291 insertions(+), 164 deletions(-) create mode 100644 include/util/cus_name.h diff --git a/include/os/osDir.h b/include/os/osDir.h index 73871602c5..55c7a15764 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -31,21 +31,49 @@ extern "C" { #endif +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #ifdef WINDOWS + #define TD_TMP_DIR_PATH "C:\\Windows\\Temp\\" +#ifdef CUS_NAME +#define TD_CFG_DIR_PATH "C:\\"CUS_NAME"\\cfg\\" +#define TD_DATA_DIR_PATH "C:\\"CUS_NAME"\\data\\" +#define TD_LOG_DIR_PATH "C:\\"CUS_NAME"\\log\\" +#else #define TD_CFG_DIR_PATH "C:\\TDengine\\cfg\\" #define TD_DATA_DIR_PATH "C:\\TDengine\\data\\" #define TD_LOG_DIR_PATH "C:\\TDengine\\log\\" +#endif // CUS_NAME + #elif defined(_TD_DARWIN_64) + +#ifdef CUS_PROMPT +#define TD_TMP_DIR_PATH "/tmp/"CUS_PROMPT"d/" +#define TD_CFG_DIR_PATH "/etc/"CUS_PROMPT"/" +#define TD_DATA_DIR_PATH "/var/lib/"CUS_PROMPT"/" +#define TD_LOG_DIR_PATH "/var/log/"CUS_PROMPT"/" +#else #define TD_TMP_DIR_PATH "/tmp/taosd/" #define TD_CFG_DIR_PATH "/etc/taos/" #define TD_DATA_DIR_PATH "/var/lib/taos/" #define TD_LOG_DIR_PATH "/var/log/taos/" +#endif // CUS_PROMPT + #else + #define TD_TMP_DIR_PATH "/tmp/" +#ifdef CUS_PROMPT +#define TD_CFG_DIR_PATH "/etc/"CUS_PROMPT"/" +#define TD_DATA_DIR_PATH "/var/lib/"CUS_PROMPT"/" +#define TD_LOG_DIR_PATH "/var/log/"CUS_PROMPT"/" +#else #define TD_CFG_DIR_PATH "/etc/taos/" #define TD_DATA_DIR_PATH "/var/lib/taos/" #define TD_LOG_DIR_PATH "/var/log/taos/" +#endif // CUS_PROMPT #endif typedef struct TdDir *TdDirPtr; diff --git a/include/util/cus_name.h b/include/util/cus_name.h new file mode 100644 index 0000000000..16f677f855 --- /dev/null +++ b/include/util/cus_name.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _CUS_NAME_H_ +#define _CUS_NAME_H_ + +#ifndef CUS_NAME +#define CUS_NAME "TDengine" +#endif + +#ifndef CUS_PROMPT +#define CUS_PROMPT "taos" +#endif + +#ifndef CUS_EMAIL +#define CUS_EMAIL "" +#endif + +#endif // _CUS_NAME_H_ diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index a98dc5a236..2159899aa2 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -1,7 +1,6 @@ ######################################################## # # # Configuration # -# Any questions, please email support@taosdata.com # # # ######################################################## @@ -13,7 +12,7 @@ ############### 1. Cluster End point ############################ -# The end point of the first dnode in the cluster to be connected to when this dnode or a CLI `taos` is started +# The end point of the first dnode in the cluster to be connected to when this dnode or the CLI utility is started # firstEp hostname:6030 # The end point of the second dnode to be connected to if the firstEp is not available @@ -25,7 +24,7 @@ # The FQDN of the host on which this dnode will be started. It can be IP address # fqdn hostname -# The port for external access after this dnode is started +# The port for external access after this dnode is started # serverPort 6030 # The maximum number of connections a dnode can accept @@ -96,7 +95,7 @@ # if free disk space is less than this value, this dnode will fail to start # minimalDataDirGB 2.0 -# enable/disable system monitor +# enable/disable system monitor # monitor 1 # The following parameter is used to limit the maximum number of lines in log files. @@ -114,8 +113,8 @@ # The following parameters are used for debug purpose only by this dnode. # debugFlag is a 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR -# Available debug levels are: -# 131: output warning and error +# Available debug levels are: +# 131: output warning and error # 135: output debug, warning and error # 143: output trace, debug, warning and error to log # 199: output debug, warning and error to both screen and file @@ -130,7 +129,7 @@ # debug flag for util # uDebugFlag 131 -# debug flag for rpc +# debug flag for rpc # rpcDebugFlag 131 # debug flag for jni @@ -139,7 +138,7 @@ # debug flag for query # qDebugFlag 131 -# debug flag for taosc driver +# debug flag for client driver # cDebugFlag 131 # debug flag for dnode messages diff --git a/packaging/cfg/taosd.service b/packaging/cfg/taosd.service index fff4b74e62..52c4b1d1e2 100644 --- a/packaging/cfg/taosd.service +++ b/packaging/cfg/taosd.service @@ -1,5 +1,5 @@ [Unit] -Description=TDengine server service +Description=server service After=network-online.target Wants=network-online.target diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 3ff59498ba..858a6ac668 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -4,7 +4,7 @@ # is required to use systemd to manage services at boot set -e -#set -x +# set -x verMode=edge pagMode=full @@ -34,21 +34,25 @@ benchmarkName="taosBenchmark" dumpName="taosdump" demoName="taosdemo" xname="taosx" -explorerName="${clientName}-explorer" clientName2="taos" -serverName2="taosd" +serverName2="${clientName2}d" +configFile2="${clientName2}.cfg" productName2="TDengine" emailName2="taosdata.com" +xname2="${clientName2}x" +adapterName2="${clientName2}adapter" +explorerName="${clientName2}-explorer" benchmarkName2="${clientName2}Benchmark" +demoName2="${clientName2}demo" dumpName2="${clientName2}dump" uninstallScript2="rm${clientName2}" historyFile="${clientName2}_history" logDir="/var/log/${clientName2}" configDir="/etc/${clientName2}" -installDir="/usr/local/${clientName}" +installDir="/usr/local/${clientName2}" data_dir=${dataDir} log_dir=${logDir} @@ -79,13 +83,6 @@ if command -v sudo >/dev/null; then csudo="sudo " fi -if command -v install >/dev/null; then - echo "" > /dev/null -else - echo -e "${RED} Not found install utility, exit! ${NC}" - exit 1 -fi - update_flag=0 prompt_force=0 @@ -213,15 +210,15 @@ function install_main_path() { function install_bin() { # Remove links - ${csudo}rm -f ${bin_link_dir}/${clientName} || : - ${csudo}rm -f ${bin_link_dir}/${serverName} || : + ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${serverName2} || : ${csudo}rm -f ${bin_link_dir}/${udfdName} || : ${csudo}rm -f ${bin_link_dir}/${adapterName} || : - ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : - ${csudo}rm -f ${bin_link_dir}/${demoName} || : - ${csudo}rm -f ${bin_link_dir}/${benchmarkName} || : - ${csudo}rm -f ${bin_link_dir}/${dumpName} || : - ${csudo}rm -f ${bin_link_dir}/${xname} || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : + ${csudo}rm -f ${bin_link_dir}/${xname2} || : ${csudo}rm -f ${bin_link_dir}/${explorerName} || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : @@ -229,24 +226,23 @@ function install_bin() { ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : - [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : + [ -x ${install_main_dir}/bin/${clientName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName2} ${bin_link_dir}/${clientName2} || : + [ -x ${install_main_dir}/bin/${serverName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${serverName2} ${bin_link_dir}/${serverName2} || : [ -x ${install_main_dir}/bin/${udfdName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${udfdName} ${bin_link_dir}/${udfdName} || : - [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || : - [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : - [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : - [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : - [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname} ${bin_link_dir}/${xname} || : + [ -x ${install_main_dir}/bin/${adapterName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${adapterName2} ${bin_link_dir}/${adapterName2} || : + [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${demoName2} || : + [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || : + [ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2} || : + [ -x ${install_main_dir}/bin/${xname2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname2} ${bin_link_dir}/${xname2} || : [ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : - [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + if [ "$clientName2" == "${clientName}" ]; then + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + fi + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || : - [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName2} || : - [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName2} || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || : fi } @@ -406,7 +402,7 @@ function set_hostname() { ${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network || : fi - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile2} serverFqdn=$newHostname if [[ -e /etc/hosts ]]; then @@ -440,7 +436,7 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" # Write the local FQDN to configuration file - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2} serverFqdn=$localFqdn echo return @@ -462,7 +458,7 @@ function set_ipAsFqdn() { read -p "Please choose an IP from local IP list:" localFqdn else # Write the local FQDN to configuration file - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2} serverFqdn=$localFqdn break fi @@ -526,15 +522,15 @@ function install_adapter_config() { function install_config() { - if [ ! -f "${cfg_install_dir}/${configFile}" ]; then + if [ ! -f "${cfg_install_dir}/${configFile2}" ]; then ${csudo}mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir} + [ -f ${script_dir}/cfg/${configFile2} ] && ${csudo}cp ${script_dir}/cfg/${configFile2} ${cfg_install_dir} ${csudo}chmod 644 ${cfg_install_dir}/* else - ${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new + ${csudo}cp -f ${script_dir}/cfg/${configFile2} ${cfg_install_dir}/${configFile2}.new fi - ${csudo}ln -sf ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg + ${csudo}ln -sf ${cfg_install_dir}/${configFile2} ${install_main_dir}/cfg [ ! -z $1 ] && return 0 || : # only install client @@ -555,7 +551,7 @@ function install_config() { read firstEp while true; do if [ ! -z "$firstEp" ]; then - ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile2} break else break @@ -607,8 +603,8 @@ function install_web() { function clean_service_on_sysvinit() { - if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then - ${csudo}service ${serverName} stop || : + if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then + ${csudo}service ${serverName2} stop || : fi if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then @@ -616,30 +612,30 @@ function clean_service_on_sysvinit() { fi if ((${initd_mod} == 1)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}chkconfig --del ${serverName} || : + if [ -e ${service_config_dir}/${serverName2} ]; then + ${csudo}chkconfig --del ${serverName2} || : fi if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo}chkconfig --del tarbitratord || : fi elif ((${initd_mod} == 2)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}insserv -r ${serverName} || : + if [ -e ${service_config_dir}/${serverName2} ]; then + ${csudo}insserv -r ${serverName2} || : fi if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo}insserv -r tarbitratord || : fi elif ((${initd_mod} == 3)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}update-rc.d -f ${serverName} remove || : + if [ -e ${service_config_dir}/${serverName2} ]; then + ${csudo}update-rc.d -f ${serverName2} remove || : fi if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo}update-rc.d -f tarbitratord remove || : fi fi - ${csudo}rm -f ${service_config_dir}/${serverName} || : + ${csudo}rm -f ${service_config_dir}/${serverName2} || : ${csudo}rm -f ${service_config_dir}/tarbitratord || : if $(which init &>/dev/null); then @@ -660,24 +656,24 @@ function install_service_on_sysvinit() { fi if ((${initd_mod} == 1)); then - ${csudo}chkconfig --add ${serverName} || : - ${csudo}chkconfig --level 2345 ${serverName} on || : + ${csudo}chkconfig --add ${serverName2} || : + ${csudo}chkconfig --level 2345 ${serverName2} on || : elif ((${initd_mod} == 2)); then - ${csudo}insserv ${serverName} || : - ${csudo}insserv -d ${serverName} || : + ${csudo}insserv ${serverName2} || : + ${csudo}insserv -d ${serverName2} || : elif ((${initd_mod} == 3)); then - ${csudo}update-rc.d ${serverName} defaults || : + ${csudo}update-rc.d ${serverName2} defaults || : fi } function clean_service_on_systemd() { - taosd_service_config="${service_config_dir}/${serverName}.service" - if systemctl is-active --quiet ${serverName}; then + service_config="${service_config_dir}/${serverName2}.service" + if systemctl is-active --quiet ${serverName2}; then echo "${productName} is running, stopping it..." - ${csudo}systemctl stop ${serverName} &>/dev/null || echo &>/dev/null + ${csudo}systemctl stop ${serverName2} &>/dev/null || echo &>/dev/null fi - ${csudo}systemctl disable ${serverName} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${taosd_service_config} + ${csudo}systemctl disable ${serverName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${service_config} tarbitratord_service_config="${service_config_dir}/tarbitratord.service" if systemctl is-active --quiet tarbitratord; then @@ -694,19 +690,19 @@ function clean_service_on_systemd() { function install_service_on_systemd() { clean_service_on_systemd - [ -f ${script_dir}/cfg/${serverName}.service ] && - ${csudo}cp ${script_dir}/cfg/${serverName}.service \ + [ -f ${script_dir}/cfg/${serverName2}.service ] && + ${csudo}cp ${script_dir}/cfg/${serverName2}.service \ ${service_config_dir}/ || : # if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then - # [ -f ${script_dir}/cfg/${serverName}.service ] && - # ${csudo}cp ${script_dir}/cfg/${serverName}.service \ + # [ -f ${script_dir}/cfg/${serverName2}.service ] && + # ${csudo}cp ${script_dir}/cfg/${serverName2}.service \ # ${service_config_dir}/${serverName2}.service || : # fi ${csudo}systemctl daemon-reload - ${csudo}systemctl enable ${serverName} + ${csudo}systemctl enable ${serverName2} ${csudo}systemctl daemon-reload } @@ -726,7 +722,7 @@ function install_service() { elif ((${service_mod} == 1)); then install_service_on_sysvinit else - kill_process ${serverName} + kill_process ${serverName2} fi } @@ -763,10 +759,10 @@ function is_version_compatible() { if [ -f ${script_dir}/driver/vercomp.txt ]; then min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt) else - min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5) + min_compatible_version=$(${script_dir}/bin/${serverName2} -V | head -1 | cut -d ' ' -f 5) fi - exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3) + exist_version=$(${installDir}/bin/${serverName2} -V | head -1 | cut -d ' ' -f 3) vercomp $exist_version "3.0.0.0" case $? in 2) @@ -836,13 +832,13 @@ function updateProduct() { echo -e "${GREEN}Start to update ${productName2}...${NC}" # Stop the service if running - if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then if ((${service_mod} == 0)); then - ${csudo}systemctl stop ${serverName} || : + ${csudo}systemctl stop ${serverName2} || : elif ((${service_mod} == 1)); then - ${csudo}service ${serverName} stop || : + ${csudo}service ${serverName2} stop || : else - kill_process ${serverName} + kill_process ${serverName2} fi sleep 1 fi @@ -869,21 +865,21 @@ function updateProduct() { openresty_work=false echo - echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile}" - [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/taosadapter.toml" + echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}" + [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName}${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName} start${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service taosadapter start${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ./${serverName}${NC}" - [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: taosadapter &${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ./${serverName2}${NC}" + [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}" fi if [ ${openresty_work} = 'true' ]; then @@ -894,7 +890,7 @@ function updateProduct() { if ((${prompt_force} == 1)); then echo "" - echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName2} $exist_version!${NC}" + echo -e "${RED}Please run '${serverName2} --force-keep-file' at first time for the exist ${productName2} $exist_version!${NC}" fi echo echo -e "\033[44;32;1m${productName2} is updated successfully!${NC}" @@ -906,7 +902,7 @@ function updateProduct() { echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}" fi - rm -rf $(tar -tf ${tarName} | grep -v "^\./$") + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } function installProduct() { @@ -951,21 +947,21 @@ function installProduct() { # Ask if to start the service echo - echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile}" - [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/taosadapter.toml" + echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}" + [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName}${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName} start${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service taosadapter start${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${serverName}${NC}" - [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: taosadapter &${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${serverName2}${NC}" + [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}" fi if [ ! -z "$firstEp" ]; then @@ -998,7 +994,7 @@ function installProduct() { fi touch ~/.${historyFile} - rm -rf $(tar -tf ${tarName} | grep -v "^\./$") + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } ## ==============================Main program starts from here============================ @@ -1009,7 +1005,7 @@ if [ "$verType" == "server" ]; then echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of ${productName2} 2.x, please clear it before installing!\033[0m" else # Install server and client - if [ -x ${bin_dir}/${serverName} ]; then + if [ -x ${bin_dir}/${serverName2} ]; then update_flag=1 updateProduct else @@ -1019,7 +1015,7 @@ if [ "$verType" == "server" ]; then elif [ "$verType" == "client" ]; then interactiveFqdn=no # Only install client - if [ -x ${bin_dir}/${clientName} ]; then + if [ -x ${bin_dir}/${clientName2} ]; then update_flag=1 updateProduct client else diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index f46de0f94b..9413f55d51 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -23,9 +23,12 @@ clientName2="${12}" productName="TDengine" clientName="taos" +benchmarkName="taosBenchmark" configFile="taos.cfg" tarName="package.tar.gz" +benchmarkName2="${clientName2}Benchmark" + if [ "$osType" != "Darwin" ]; then script_dir="$(dirname $(readlink -f $0))" top_dir="$(readlink -f ${script_dir}/../..)" @@ -53,11 +56,12 @@ fi # Directories and files. -if [ "$verMode" == "cluster" ]; then - sed -i 's/verMode=edge/verMode=cluster/g' ${script_dir}/remove_client.sh - sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh - sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" ${script_dir}/remove_client.sh -fi +#if [ "$verMode" == "cluster" ]; then +# sed -i 's/verMode=edge/verMode=cluster/g' ${script_dir}/remove_client.sh +# sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh +# sed -i "s/configFile2=\"taos\"/configFile2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh +# sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" ${script_dir}/remove_client.sh +#fi if [ "$osType" != "Darwin" ]; then if [ "$pagMode" == "lite" ]; then @@ -66,6 +70,7 @@ if [ "$osType" != "Darwin" ]; then ${script_dir}/remove_client.sh" else bin_files="${build_dir}/bin/${clientName} \ + ${build_dir}/bin/${benchmarkName} \ ${script_dir}/remove_client.sh \ ${script_dir}/set_core.sh \ ${script_dir}/get_client.sh" @@ -153,6 +158,7 @@ if [ "$verMode" == "cluster" ]; then sed -i 's/verMode=edge/verMode=cluster/g' install_client_temp.sh sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_client_temp.sh sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_client_temp.sh + sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" install_client_temp.sh sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_client_temp.sh sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_client_temp.sh diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index a2dec155e8..8a41b13ccb 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -51,9 +51,9 @@ fi if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then cd ${top_dir}/tools/taos-tools/packaging/deb - + taostools_ver=$(git for-each-ref --sort=taggerdate --format '%(tag)' refs/tags|grep -v taos | tail -1) - [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" + [ -z "$taostools_ver" ] && taostools_ver="0.1.0" taostools_install_dir="${release_dir}/${clientName2}Tools-${taostools_ver}" cd ${curr_dir} @@ -96,7 +96,7 @@ else ${taostools_bin_files} \ ${taosx_bin} \ ${explorer_bin_files} \ - ${build_dir}/bin/taosadapter \ + ${build_dir}/bin/${clientName}adapter \ ${build_dir}/bin/udfd \ ${script_dir}/remove.sh \ ${script_dir}/set_core.sh \ @@ -135,12 +135,12 @@ mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} -if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then - cp ${compile_dir}/test/cfg/taosadapter.toml ${install_dir}/cfg || : +if [ -f "${compile_dir}/test/cfg/${clientName}adapter.toml" ]; then + cp ${compile_dir}/test/cfg/${clientName}adapter.toml ${install_dir}/cfg || : fi -if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then - cp ${compile_dir}/test/cfg/taosadapter.service ${install_dir}/cfg || : +if [ -f "${compile_dir}/test/cfg/${clientName}adapter.service" ]; then + cp ${compile_dir}/test/cfg/${clientName}adapter.service ${install_dir}/cfg || : fi if [ -f "${cfg_dir}/${serverName}.service" ]; then @@ -152,16 +152,16 @@ mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${se mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm if [ $adapterName != "taosadapter" ]; then - mv ${install_dir}/cfg/taosadapter.toml ${install_dir}/cfg/$adapterName.toml + mv ${install_dir}/cfg/${clientName2}adapter.toml ${install_dir}/cfg/$adapterName.toml sed -i "s/path = \"\/var\/log\/taos\"/path = \"\/var\/log\/${productName}\"/g" ${install_dir}/cfg/$adapterName.toml sed -i "s/password = \"taosdata\"/password = \"${defaultPasswd}\"/g" ${install_dir}/cfg/$adapterName.toml - mv ${install_dir}/cfg/taosadapter.service ${install_dir}/cfg/$adapterName.service + mv ${install_dir}/cfg/${clientName2}adapter.service ${install_dir}/cfg/$adapterName.service sed -i "s/TDengine/${productName}/g" ${install_dir}/cfg/$adapterName.service sed -i "s/taosAdapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service sed -i "s/taosadapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service - mv ${install_dir}/bin/taosadapter ${install_dir}/bin/${adapterName} + mv ${install_dir}/bin/${clientName2}adapter ${install_dir}/bin/${adapterName} mv ${install_dir}/bin/taosd-dump-cfg.gdb ${install_dir}/bin/${serverName}-dump-cfg.gdb fi @@ -233,8 +233,10 @@ if [ "$verMode" == "cluster" ]; then sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" remove_temp.sh sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" remove_temp.sh + sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" remove_temp.sh sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" remove_temp.sh - sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" remove_temp.sh + cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'` + sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusDomain}\"/g" remove_temp.sh mv remove_temp.sh ${install_dir}/bin/remove.sh fi if [ "$verMode" == "cloud" ]; then @@ -262,8 +264,10 @@ if [ "$verMode" == "cluster" ]; then sed -i 's/verMode=edge/verMode=cluster/g' install_temp.sh sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_temp.sh sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_temp.sh + sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" install_temp.sh sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_temp.sh - sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_temp.sh + cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'` + sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusDomain}\"/g" install_temp.sh mv install_temp.sh ${install_dir}/install.sh fi if [ "$verMode" == "cloud" ]; then diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 2479e48670..8ed3bd74b9 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -40,11 +40,16 @@ serverName2="taosd" clientName2="taos" productName2="TDengine" +adapterName2="${clientName2}adapter" +demoName2="${clientName2}demo" benchmarkName2="${clientName2}Benchmark" dumpName2="${clientName2}dump" +keeperName2="${clientName2}keeper" +xName2="${clientName2}x" +explorerName2="${clientName2}-explorer" uninstallScript2="rm${clientName2}" -installDir="/usr/local/${clientName}" +installDir="/usr/local/${clientName2}" #install main path install_main_dir=${installDir} @@ -55,8 +60,8 @@ local_bin_link_dir="/usr/local/bin" service_config_dir="/etc/systemd/system" -taos_service_name=${serverName} -taosadapter_service_name="taosadapter" +taos_service_name=${serverName2} +taosadapter_service_name="${clientName2}adapter" tarbitrator_service_name="tarbitratord" csudo="" if command -v sudo >/dev/null; then @@ -84,14 +89,14 @@ else fi function kill_taosadapter() { - pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}') + pid=$(ps -ef | grep "${adapterName2}" | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi } function kill_taosd() { - pid=$(ps -ef | grep ${serverName} | grep -v "grep" | awk '{print $2}') + pid=$(ps -ef | grep ${serverName2} | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi @@ -109,17 +114,17 @@ function clean_bin() { ${csudo}rm -f ${bin_link_dir}/${clientName} || : ${csudo}rm -f ${bin_link_dir}/${serverName} || : ${csudo}rm -f ${bin_link_dir}/udfd || : - ${csudo}rm -f ${bin_link_dir}/taosadapter || : - ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : - ${csudo}rm -f ${bin_link_dir}/taosdemo || : - ${csudo}rm -f ${bin_link_dir}/taosdump || : - ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : + ${csudo}rm -f ${bin_link_dir}/${adapterName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}rm -f ${bin_link_dir}/tarbitrator || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : - ${csudo}rm -f ${bin_link_dir}/taoskeeper || : - ${csudo}rm -f ${bin_link_dir}/taosx || : - ${csudo}rm -f ${bin_link_dir}/taos-explorer || : + ${csudo}rm -f ${bin_link_dir}/${keeperName2} || : + ${csudo}rm -f ${bin_link_dir}/${xName2} || : + ${csudo}rm -f ${bin_link_dir}/${explorerName2} || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then ${csudo}rm -f ${bin_link_dir}/${clientName2} || : @@ -130,8 +135,8 @@ function clean_bin() { } function clean_local_bin() { - ${csudo}rm -f ${local_bin_link_dir}/taosBenchmark || : - ${csudo}rm -f ${local_bin_link_dir}/taosdemo || : + ${csudo}rm -f ${local_bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${local_bin_link_dir}/${demoName2} || : } function clean_lib() { @@ -173,7 +178,7 @@ function clean_service_on_systemd() { ${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null ${csudo}rm -f ${taosd_service_config} - taosadapter_service_config="${service_config_dir}/taosadapter.service" + taosadapter_service_config="${service_config_dir}/${clientName2}adapter.service" if systemctl is-active --quiet ${taosadapter_service_name}; then echo "${productName2} ${clientName2}Adapter is running, stopping it..." ${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null @@ -235,8 +240,8 @@ function clean_service_on_sysvinit() { function clean_service_on_launchctl() { ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : - ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : - ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : + ${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : } function clean_service() { diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 10a0fb5e02..2bdb56fac2 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -15,11 +15,12 @@ uninstallScript="rmtaos" clientName2="taos" productName2="TDengine" -benchmarkName2="${clientName}Benchmark" -dumpName2="${clientName}dump" -uninstallScript2="rm${clientName}" +benchmarkName2="${clientName2}Benchmark" +demoName2="${clientName2}demo" +dumpName2="${clientName2}dump" +uninstallScript2="rm${clientName2}" -installDir="/usr/local/${clientName}" +installDir="/usr/local/${clientName2}" #install main path install_main_dir=${installDir} @@ -44,14 +45,17 @@ function kill_client() { function clean_bin() { # Remove link - ${csudo}rm -f ${bin_link_dir}/${clientName} || : - ${csudo}rm -f ${bin_link_dir}/taosdemo || : - ${csudo}rm -f ${bin_link_dir}/taosdump || : + ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}rm -f ${bin_link_dir}/set_core || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || : fi diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index dba1dbcf9a..418103f2a6 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -30,6 +30,10 @@ #include "tsched.h" #include "ttime.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -542,9 +546,15 @@ void taos_init_imp(void) { deltaToUtcInitOnce(); - if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { + char logDirName[64] = {0}; +#ifdef CUS_PROMPT + snprintf(logDirName, 64, "%slog", CUS_PROMPT); +#else + snprintf(logDirName, 64, "taoslog"); +#endif + if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { // ignore create log failed, only print - printf(" WARING: Create taoslog failed:%s. configDir=%s\n", strerror(errno), configDir); + printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir); } if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index aeeec1d61c..1c2d533977 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -228,7 +228,11 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); if (taosIsDir(cfgDir)) { +#ifdef CUS_PROMPT + snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "%s.cfg", CUS_PROMPT, cfgDir); +#else snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); +#endif } else { tstrncpy(cfgFile, cfgDir, sizeof(cfgDir)); } diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 9a092e2df5..f0e020edfe 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -20,6 +20,21 @@ #include "tglobal.h" #include "version.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#else +#ifndef CUS_NAME + #define CUS_NAME "TDengine" +#endif + +#ifndef CUS_PROMPT + #define CUS_PROMPT "taos" +#endif + +#ifndef CUS_EMAIL + #define CUS_EMAIL "" +#endif +#endif // clang-format off #define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'." #define DM_CFG_DIR "Configuration directory." @@ -232,7 +247,7 @@ static void dmDumpCfg() { } static int32_t dmInitLog() { - return taosCreateLog("taosdlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); + return taosCreateLog(CUS_PROMPT"dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); } static void taosCleanupArgs() { diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 52309a7b35..84004ed3c1 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -17,6 +17,10 @@ #include "os.h" #include "taoserror.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #define PROCESS_ITEM 12 #define UUIDLEN37 37 @@ -252,7 +256,11 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { #ifdef WINDOWS // ASSERT(0); #elif defined(_TD_DARWIN_64) +#ifdef CUS_PROMPT + const char *filepath = "/usr/local/"CUS_PROMPT"/email"; +#else const char *filepath = "/usr/local/taos/email"; +#endif // CUS_PROMPT TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); if (pFile == NULL) return false; @@ -264,8 +272,12 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { taosCloseFile(&pFile); return 0; +#else +#ifdef CUS_PROMPT + const char *filepath = "/usr/local/"CUS_PROMPT"/email"; #else const char *filepath = "/usr/local/taos/email"; +#endif // CUS_PROMPT TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); if (pFile == NULL) return false; diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 600e33feab..7b1dc3a541 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -26,10 +26,6 @@ ELSE () SET(LINK_WEBSOCKET "") ENDIF () -IF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) - ADD_DEFINITIONS(-I${CMAKE_CURRENT_SOURCE_DIR}/../../../enterprise/packaging) -ENDIF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) - IF (TD_LINUX AND TD_ALPINE) SET(LINK_ARGP "/usr/lib/libargp.a") ELSE () diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 8111ccc0b1..0f5585991e 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -59,8 +59,8 @@ char cusEmail[] = ""; #define SHELL_VERSION "Print program version." #ifdef WEBSOCKET -#define SHELL_DSN "The dsn to use when connecting to cloud server." -#define SHELL_REST "Use restful mode when connecting." +#define SHELL_DSN "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection." +#define SHELL_REST "Use RESTful mode when connecting." #define SHELL_TIMEOUT "Set the timeout for websocket query in seconds, default is 30." #endif @@ -93,7 +93,11 @@ void shellPrintHelp() { #endif printf("%s%s%s%s\r\n", indent, "-w,", indent, SHELL_WIDTH); printf("%s%s%s%s\r\n", indent, "-V,", indent, SHELL_VERSION); - printf("\r\n\r\nReport bugs to %s.\r\n", cusEmail); +#ifdef CUS_EMAIL + printf("\r\n\r\nReport bugs to %s.\r\n", CUS_EMAIL); +#else + printf("\r\n\r\nReport bugs to %s.\r\n", "support@taosdata.com"); +#endif } #ifdef LINUX @@ -105,7 +109,11 @@ void shellPrintHelp() { #endif const char *argp_program_version = version; -const char *argp_program_bug_address = cusEmail; +#ifdef CUS_EMAIL +const char *argp_program_bug_address = CUS_EMAIL; +#else +const char *argp_program_bug_address = "support@taosdata.com"; +#endif static struct argp_option shellOptions[] = { {"host", 'h', "HOST", 0, SHELL_HOST}, @@ -411,10 +419,19 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { shell.info.clientVersion = "Welcome to the %s Command Line Interface, Client Version:%s\r\n" "Copyright (c) 2022 by %s, all rights reserved.\r\n\r\n"; - strcpy(shell.info.cusName, cusName); - sprintf(shell.info.promptHeader, "%s> ", cusPrompt); +#ifdef CUS_NAME + strcpy(shell.info.cusName, CUS_NAME); +#else + strcpy(shell.info.cusName, "TDengine"); +#endif char promptContinueFormat[32] = {0}; - sprintf(promptContinueFormat, "%%%zus> ", strlen(cusPrompt)); +#ifdef CUS_PROMPT + sprintf(shell.info.promptHeader, "%s> ", CUS_PROMPT); + sprintf(promptContinueFormat, "%%%zus> ", strlen(CUS_PROMPT)); +#else + sprintf(shell.info.promptHeader, "taos> "); + sprintf(promptContinueFormat, "%%%zus> ", strlen("taos")); +#endif sprintf(shell.info.promptContinue, promptContinueFormat, " "); shell.info.promptSize = strlen(shell.info.promptHeader); #ifdef TD_ENTERPRISE