From e34da43e38220d01d70faf09c4db0490fe4e8be5 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 23 Oct 2023 19:32:44 +0800 Subject: [PATCH 01/37] feat: support pipelining of snap replication --- include/util/tdef.h | 2 + source/libs/sync/inc/syncInt.h | 4 +- source/libs/sync/inc/syncSnapshot.h | 19 ++- source/libs/sync/src/syncSnapshot.c | 251 +++++++++++++++++++++------- 4 files changed, 215 insertions(+), 61 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 7f8fe22340..9ea6240b1f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -294,6 +294,8 @@ typedef enum ELogicConditionType { #define TSDB_SYNC_APPLYQ_SIZE_LIMIT 512 #define TSDB_SYNC_NEGOTIATION_WIN 512 +#define TSDB_SYNC_SNAP_BUFFER_SIZE 2048 + #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index cec1a12024..637c18e97d 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -249,8 +249,8 @@ int32_t syncNodeOnRequestVote(SSyncNode* pNode, const SRpcMsg* pMsg); int32_t syncNodeOnRequestVoteReply(SSyncNode* pNode, const SRpcMsg* pMsg); int32_t syncNodeOnAppendEntries(SSyncNode* pNode, const SRpcMsg* pMsg); int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pMsg); -int32_t syncNodeOnSnapshot(SSyncNode* ths, const SRpcMsg* pMsg); -int32_t syncNodeOnSnapshotRsp(SSyncNode* ths, const SRpcMsg* pMsg); +int32_t syncNodeOnSnapshot(SSyncNode* ths, SRpcMsg* pMsg); +int32_t syncNodeOnSnapshotRsp(SSyncNode* ths, SRpcMsg* pMsg); int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pMsg); int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pMsg); int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pMsg); diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 95382132b5..0332204769 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -30,6 +30,15 @@ extern "C" { #define SYNC_SNAPSHOT_RETRY_MS 5000 +typedef struct SSyncSnapBuffer { + void *entries[TSDB_SYNC_SNAP_BUFFER_SIZE]; + int64_t start; + int64_t cursor; + int64_t end; + int64_t size; + TdThreadMutex mutex; +} SSyncSnapBuffer; + typedef struct SSyncSnapshotSender { int8_t start; int32_t seq; @@ -47,6 +56,9 @@ typedef struct SSyncSnapshotSender { int64_t lastSendTime; bool finish; + // buffer + SSyncSnapBuffer *pSndBuf; + // init when create SSyncNode *pSyncNode; int32_t replicaIndex; @@ -72,6 +84,9 @@ typedef struct SSyncSnapshotReceiver { SSnapshotParam snapshotParam; SSnapshot snapshot; + // buffer + SSyncSnapBuffer *pRcvBuf; + // init when create SSyncNode *pSyncNode; } SSyncSnapshotReceiver; @@ -83,8 +98,8 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); // on message -int32_t syncNodeOnSnapshot(SSyncNode *ths, const SRpcMsg *pMsg); -int32_t syncNodeOnSnapshotRsp(SSyncNode *ths, const SRpcMsg *pMsg); +// int32_t syncNodeOnSnapshot(SSyncNode *ths, const SRpcMsg *pMsg); +// int32_t syncNodeOnSnapshotRsp(SSyncNode *ths, const SRpcMsg *pMsg); SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode *pSyncNode, SyncIndex snapshotLastApplyIndex); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 924813eb98..43726307e6 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -23,6 +23,42 @@ #include "syncReplication.h" #include "syncUtil.h" +static void syncSnapBufferReset(SSyncSnapBuffer *pBuf) { + taosThreadMutexLock(&pBuf->mutex); + for (int64_t i = pBuf->start; i < pBuf->end; ++i) { + rpcFreeCont(pBuf->entries[i % pBuf->size]); + pBuf->entries[i % pBuf->size] = NULL; + } + pBuf->start = 1; + pBuf->end = 1; + pBuf->cursor = 0; + taosThreadMutexUnlock(&pBuf->mutex); +} + +static void syncSnapBufferDestroy(SSyncSnapBuffer **ppBuf) { + if (ppBuf == NULL || ppBuf[0] == NULL) return; + SSyncSnapBuffer *pBuf = ppBuf[0]; + + syncSnapBufferReset(pBuf); + + taosThreadMutexDestroy(&pBuf->mutex); + taosMemoryFree(ppBuf[0]); + ppBuf[0] = NULL; + return; +} + +static SSyncSnapBuffer *syncSnapBufferCreate() { + SSyncSnapBuffer *pBuf = taosMemoryCalloc(1, sizeof(SSyncSnapBuffer)); + if (pBuf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + pBuf->size = sizeof(pBuf->entries) / sizeof(void *); + ASSERT(pBuf->size == TSDB_SYNC_SNAP_BUFFER_SIZE); + taosThreadMutexInit(&pBuf->mutex, NULL); + return pBuf; +} + SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { bool condition = (pSyncNode->pFsm->FpSnapshotStartRead != NULL) && (pSyncNode->pFsm->FpSnapshotStopRead != NULL) && (pSyncNode->pFsm->FpSnapshotDoRead != NULL); @@ -49,6 +85,14 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &pSender->snapshot); pSender->finish = false; + pSender->pSndBuf = syncSnapBufferCreate(); + if (pSender->pSndBuf == NULL) { + taosMemoryFree(pSender); + pSender = NULL; + return NULL; + } + syncSnapBufferReset(pSender->pSndBuf); + return pSender; } @@ -67,6 +111,10 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { pSender->pReader = NULL; } + // free snap buffer + if (pSender->pSndBuf) { + syncSnapBufferDestroy(&pSender->pSndBuf); + } // free sender taosMemoryFree(pSender); } @@ -181,6 +229,8 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { pSender->pCurrentBlock = NULL; pSender->blockLen = 0; } + + syncSnapBufferReset(pSender->pSndBuf); } // when sender receive ack, call this function to send msg from seq @@ -193,6 +243,8 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { pSender->blockLen = 0; } + pSender->seq++; + // read data int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader, &pSender->pCurrentBlock, &pSender->blockLen); @@ -362,6 +414,14 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from pReceiver->snapshot.lastApplyTerm = 0; pReceiver->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; + pReceiver->pRcvBuf = syncSnapBufferCreate(); + if (pReceiver->pRcvBuf == NULL) { + taosMemoryFree(pReceiver); + pReceiver = NULL; + return NULL; + } + + syncSnapBufferReset(pReceiver->pRcvBuf); return pReceiver; } @@ -389,6 +449,11 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { pReceiver->snapshot.data = NULL; } + // free snap buf + if (pReceiver->pRcvBuf) { + syncSnapBufferDestroy(&pReceiver->pRcvBuf); + } + // free receiver taosMemoryFree(pReceiver); } @@ -472,6 +537,8 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { } else { sRInfo(pReceiver, "snapshot receiver stop, writer is null"); } + + syncSnapBufferReset(pReceiver->pRcvBuf); } // when recv last snapshot block, apply data into snapshot @@ -765,29 +832,8 @@ _SEND_REPLY: return code; } -static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { - // condition 4 - // transfering - SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; - int64_t timeNow = taosGetTimestampMs(); - int32_t code = 0; - - if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) { - terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; - sRError(pReceiver, "failed to receive snapshot data since %s.", terrstr()); - code = terrno; - goto _SEND_REPLY; - } - - if (snapshotReceiverGotData(pReceiver, pMsg) != 0) { - code = terrno; - if (code >= SYNC_SNAPSHOT_SEQ_INVALID) { - code = TSDB_CODE_SYN_INTERNAL_ERROR; - } - } - -_SEND_REPLY:; - +static int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, int32_t code) { + SSyncNode *pSyncNode = pReceiver->pSyncNode; // build msg SRpcMsg rpcMsg = {0}; if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId)) { @@ -811,10 +857,76 @@ _SEND_REPLY:; sRError(pReceiver, "failed to send snapshot receiver resp since %s", terrstr()); return -1; } + return 0; +} +static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend **ppMsg) { + int32_t code = 0; + SSyncSnapBuffer *pRcvBuf = pReceiver->pRcvBuf; + SyncSnapshotSend *pMsg = ppMsg[0]; + terrno = TSDB_CODE_SUCCESS; + + taosThreadMutexLock(&pRcvBuf->mutex); + + if (pMsg->seq - pRcvBuf->start >= pRcvBuf->size) { + terrno = TSDB_CODE_SYN_BUFFER_FULL; + code = terrno; + goto _out; + } + + ASSERT(pRcvBuf->start <= pRcvBuf->cursor + 1 && pRcvBuf->cursor < pRcvBuf->end); + + if (pMsg->seq > pRcvBuf->cursor) { + pRcvBuf->entries[pMsg->seq % pRcvBuf->size] = pMsg; + ppMsg[0] = NULL; + pRcvBuf->end = TMAX(pMsg->seq + 1, pRcvBuf->end); + } + + for (int64_t seq = pRcvBuf->cursor + 1; seq < pRcvBuf->end; ++seq) { + if (pRcvBuf->entries[seq]) { + pRcvBuf->cursor = seq; + } else { + break; + } + } + + for (int64_t seq = pRcvBuf->start; seq <= pRcvBuf->cursor; ++seq) { + if (snapshotReceiverGotData(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size]) != 0) { + code = terrno; + if (code >= SYNC_SNAPSHOT_SEQ_INVALID) { + code = TSDB_CODE_SYN_INTERNAL_ERROR; + } + } + pRcvBuf->start = seq + 1; + syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], code); + rpcFreeCont(pRcvBuf->entries[seq % pRcvBuf->size]); + pRcvBuf->entries[seq % pRcvBuf->size] = NULL; + if (code) goto _out; + } + +_out: + taosThreadMutexUnlock(&pRcvBuf->mutex); return code; } +static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend **ppMsg) { + // condition 4 + // transfering + SyncSnapshotSend *pMsg = ppMsg[0]; + ASSERT(pMsg); + SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; + int64_t timeNow = taosGetTimestampMs(); + int32_t code = 0; + + if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) { + terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; + sRError(pReceiver, "failed to receive snapshot data since %s.", terrstr()); + return syncSnapSendRsp(pReceiver, pMsg, terrno); + } + + return syncSnapBufferRecv(pReceiver, ppMsg); +} + static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // condition 2 // end, finish FSM @@ -885,8 +997,10 @@ _SEND_REPLY:; // // condition 5, got data, update ack // -int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { - SyncSnapshotSend *pMsg = pRpcMsg->pCont; +int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { + SyncSnapshotSend **ppMsg = (SyncSnapshotSend **)&pRpcMsg->pCont; + SyncSnapshotSend *pMsg = ppMsg[0]; + ASSERT(pMsg); SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; // if already drop replica, do not process @@ -935,9 +1049,9 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { // force close, no response syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process force stop"); snapshotReceiverStop(pReceiver); - } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { + } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq <= SYNC_SNAPSHOT_SEQ_END) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq data"); - code = syncNodeOnSnapshotReceive(pSyncNode, pMsg); + code = syncNodeOnSnapshotReceive(pSyncNode, ppMsg); } else { // error log sRError(pReceiver, "snapshot receiver recv error seq:%d, my ack:%d", pMsg->seq, pReceiver->ack); @@ -1038,14 +1152,62 @@ static int32_t snapshotSenderSignatureCmp(SSyncSnapshotSender *pSender, SyncSnap return 0; } +static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp **ppMsg) { + int32_t code = 0; + SSyncSnapBuffer *pSndBuf = pSender->pSndBuf; + SyncSnapshotRsp *pMsg = ppMsg[0]; + + taosThreadMutexLock(&pSndBuf->mutex); + + if (pMsg->ack - pSndBuf->start >= pSndBuf->size) { + terrno = TSDB_CODE_SYN_BUFFER_FULL; + code = terrno; + goto _out; + } + + ASSERT(pSndBuf->start <= pSndBuf->cursor + 1 && pSndBuf->cursor < pSndBuf->end); + + if (pMsg->ack > pSndBuf->cursor && pSndBuf->entries[pMsg->ack % pSndBuf->size] == NULL) { + pSndBuf->entries[pMsg->ack % pSndBuf->size] = pMsg; + ppMsg[0] = NULL; + pSndBuf->end = TMAX(pMsg->ack + 1, pSndBuf->end); + } + + for (int64_t ack = pSndBuf->cursor + 1; ack < pSndBuf->end; ++ack) { + if (pSndBuf->entries[ack % pSndBuf->size]) { + pSndBuf->cursor = ack; + } else { + break; + } + } + + for (int64_t ack = pSndBuf->start; ack < pSndBuf->cursor; ++ack) { + rpcFreeCont(pSndBuf->entries[ack % pSndBuf->size]); + pSndBuf->entries[ack % pSndBuf->size] = NULL; + pSndBuf->start = ack + 1; + } + + while (pSender->seq - pSndBuf->start < (pSndBuf->size >> 2)) { + if (snapshotSend(pSender) != 0) { + code = terrno; + goto _out; + } + } + +_out: + taosThreadMutexUnlock(&pSndBuf->mutex); + return code; +} + // sender on message // // condition 1 sender receives SYNC_SNAPSHOT_SEQ_END, close sender // condition 2 sender receives ack, set seq = ack + 1, send msg from seq // condition 3 sender receives error msg, just print error log // -int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { - SyncSnapshotRsp *pMsg = pRpcMsg->pCont; +int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { + SyncSnapshotRsp **ppMsg = (SyncSnapshotRsp **)&pRpcMsg->pCont; + SyncSnapshotRsp *pMsg = ppMsg[0]; // if already drop replica, do not process if (!syncNodeInRaftGroup(pSyncNode, &pMsg->srcId)) { @@ -1123,12 +1285,8 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { if (pMsg->ack == SYNC_SNAPSHOT_SEQ_BEGIN) { syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq begin"); - if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) { - return -1; - } - if (snapshotSend(pSender) != 0) { - return -1; + goto _ERROR; } return 0; } @@ -1142,30 +1300,9 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { } // send next msg - if (pMsg->ack == pSender->seq) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq data"); - // update sender ack - if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) { - return -1; - } - if (snapshotSend(pSender) != 0) { - return -1; - } - } else if (pMsg->ack == pSender->seq - 1) { - // maybe resend - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq and resend"); - if (snapshotReSend(pSender) != 0) { - return -1; - } - } else { - // error log - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "receive error ack"); - sSError(pSender, "snapshot sender receive error ack:%d, my seq:%d", pMsg->ack, pSender->seq); - snapshotSenderStop(pSender, true); - syncNodeReplicateReset(pSyncNode, &pMsg->srcId); - return -1; + if (syncSnapBufferSend(pSender, ppMsg) != 0) { + goto _ERROR; } - return 0; _ERROR: From f299a2810944c47d71188ec7f27fe89de841018c Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Thu, 19 Oct 2023 11:34:04 +0800 Subject: [PATCH 02/37] feat: support to_timestamp/to_char --- docs/en/12-taos-sql/10-function.md | 83 ++ docs/zh/12-taos-sql/10-function.md | 83 ++ include/common/ttime.h | 21 + include/libs/function/functionMgt.h | 2 + include/libs/scalar/scalar.h | 2 + include/util/taoserror.h | 1 + include/util/tdef.h | 9 + source/common/src/ttime.c | 877 +++++++++++++++++- source/common/test/commonTests.cpp | 225 ++++- source/libs/function/src/builtins.c | 48 + source/libs/scalar/src/sclfunc.c | 56 ++ source/util/src/terror.c | 1 + tests/parallel_test/cases.task | 4 + .../2-query/func_to_char_timestamp.py | 160 ++++ 14 files changed, 1566 insertions(+), 6 deletions(-) create mode 100644 tests/system-test/2-query/func_to_char_timestamp.py diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 340a3e917b..266cdb4958 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -483,6 +483,89 @@ return_timestamp: { - The precision of the returned timestamp is same as the precision set for the current data base in use - return_timestamp indicates whether the returned value type is TIMESTAMP or not. If this parameter set to 1, function will return TIMESTAMP type. Otherwise function will return BIGINT type. If parameter is omitted, default return value type is BIGINT. +#### TO_CHAR + +```sql +TO_CHAR(ts, str_literal) +``` + +**Description**: Convert a ts column to string as the format specified + +**Return value type**: VARCHAR + +**Applicable column types**: TIMESTAMP + +**Nested query**: It can be used in both the outer query and inner query in a nested query. + +**Applicable table types**: standard tables and supertables + +**Supported Formats** + +| **Format** | **Comment**| **example** | +| --- | --- | --- | +|AM,am,PM,pm| Meridiem indicator(without periods) | 07:00:00am| +|A.M.,a.m.,P.M.,p.m.| Meridiem indicator(with periods)| 07:00:00a.m.| +|YYYY,yyyy|year, 4 or more digits| 2023-10-10| +|YYY,yyy| year, last 3 digits| 023-10-10| +|YY,yy| year, last 2 digits| 23-10-10| +|Y,y| year, last digit| 3-10-10| +|MONTH|full uppercase of month| 2023-JANUARY-01| +|Month|full capitalized month| 2023-January-01| +|month|full lowercase of month| 2023-january-01| +|MON| abbreviated uppercase of month(3 char)| JAN, SEP| +|Mon| abbreviated capitalized month| Jan, Sep| +|mon|abbreviated lowercase of month| jan, sep| +|MM,mm|month number 01-12|2023-01-01| +|DD,dd|month day, 01-31|| +|DAY|full uppercase of week day|MONDAY| +|Day|full capitalized week day|Monday| +|day|full lowercase of week day|monday| +|DY|abbreviated uppercase of week day|MON| +|Dy|abbreviated capitalized week day|Mon| +|dy|abbreviated lowercase of week day|mon| +|DDD|year day, 001-366|| +|D,d|week day number, 1-7, Sunday(1) to Saturday(7)|| +|HH24,hh24|hour of day, 00-23|2023-01-30 23:59:59| +|hh12,HH12, hh, HH| hour of day, 01-12|2023-01-30 12:59:59PM| +|MI,mi|minute, 00-59|| +|SS,ss|second, 00-59|| +|MS,ms|milli second, 000-999|| +|US,us|micro second, 000000-999999|| +|NS,ns|nano second, 000000000-999999999|| +|TZH,tzh|time zone hour|2023-01-30 11:59:59PM +08| + +**More explanations**: +- The output format of `Month`, `Day` are left aligined, like`2023-OCTOBER -01`, `2023-SEPTEMBER-01`, `September` is the longest, no paddings. Week days are slimilar. +- When `ms`,`us`,`ns` are used in `to_char`, like `to_char(ts, 'yyyy-mm-dd hh:mi:ss.ms.us.ns')`, The time of `ms`,`us`,`ns` corresponds to the same fraction seconds. When ts is `1697182085123`, the output of `ms` is `123`, `us` is `123000`, `ns` is `123000000`. +- If we want to output some characters of format without converting, surround it with double quotes. `to_char(ts, 'yyyy-mm-dd "is formated by yyyy-mm-dd"')`. If want to output double quotes, add a back slash before double quote, like `to_char(ts, '\"yyyy-mm-dd\"')` will output `"2023-10-10"`. +- For formats that output digits, the uppercase and lowercase formats are the same. + +#### TO_TIMESTAMP + +```sql +TO_TIMESTAMP(str_literal, str_literal) +``` + +**Description**: Convert a formated timestamp string to a timestamp + +**Return value type**: TIMESTAMP + +**Applicable column types**: VARCHAR + +**Nested query**: It can be used in both the outer query and inner query in a nested query. + +**Applicable table types**: standard tables and supertables + +**Supported Formats**: The same as `TO_CHAR`. + +**More explanations**: +- When `ms`, `us`, `ns` are used in `to_timestamp`, if multi of them are specified, the results are accumulated. For example, `to_timestamp('2023-10-10 10:10:10.123.000456.000000789', 'yyyy-mm-dd hh:mi:ss.ms.us.ns')` will output the timestamp of `2023-10-10 10:10:10.123456789`. +- The uppercase or lowercase of `MONTH`, `MON`, `DAY`, `DY` and formtas that output digits have same effect when used in `to_timestamp`, like `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month` can be replaced by `MONTH`, or `month`. The cases are ignored. +- If multi times are specified for one component, the previous will be overwritten. Like `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, the output year will be `2022`. +- The default timetsamp if some components are not specified will be: `1970-01-01 00:00:00` with your local timezone. +- If `AM` or `PM` is specified in formats, the Hour must between `1-12`. +- In some cases, `to_timestamp` can convert correctly even the format and the timestamp string are not totally matched. Like `to_timetamp('200101/2', 'yyyyMM1/dd')`, the digit `1` in format string are ignored, and the output timestsamp is `2001-01-02 00:00:00`. Spaces and tabs in formats and tiemstamp string are also ignored automatically. + ### Time and Date Functions diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 8b87a18e54..806ff3c6a8 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -483,6 +483,89 @@ return_timestamp: { - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 - return_timestamp 指定函数返回值是否为时间戳类型，设置为1时返回 TIMESTAMP 类型，设置为0时返回 BIGINT 类型。如不指定缺省返回 BIGINT 类型。 +#### TO_CHAR + +```sql +TO_CHAR(ts, str_literal) +``` + +**功能说明**: 将timestamp类型按照指定格式转换为字符串 + +**返回结果数据类型**: VARCHAR + +**应用字段**: TIMESTAMP + +**嵌套子查询支持**: 适用于内层查询和外层查询 + +**适用于**: 表和超级表 + +**支持的格式** + +| **格式** | **说明**| **例子** | +| --- | --- | --- | +|AM,am,PM,pm| 无点分隔的上午下午 | 07:00:00am| +|A.M.,a.m.,P.M.,p.m.| 有点分割的上午下午| 07:00:00a.m.| +|YYYY,yyyy|年, 4个及以上数字| 2023-10-10| +|YYY,yyy| 年, 最后3位数字| 023-10-10| +|YY,yy| 年, 最后2位数字| 23-10-10| +|Y,y|年, 最后一位数字| 3-10-10| +|MONTH|月, 全大写| 2023-JANUARY-01| +|Month|月, 首字母大写| 2023-January-01| +|month|月, 全小写| 2023-january-01| +|MON| 月, 缩写, 全大写(三个字符)| JAN, SEP| +|Mon| 月, 缩写, 首字母大写| Jan, Sep| +|mon|月, 缩写, 全小写| jan, sep| +|MM,mm|月, 数字 01-12|2023-01-01| +|DD,dd|月日, 01-31|| +|DAY|周日, 全大写|MONDAY| +|Day|周日, 首字符大写|Monday| +|day|周日, 全小写|monday| +|DY|周日, 缩写, 全大写|MON| +|Dy|周日, 缩写, 首字符大写|Mon| +|dy|周日, 缩写, 全小写|mon| +|DDD|年日, 001-366|| +|D,d|周日, 数字, 1-7, Sunday(1) to Saturday(7)|| +|HH24,hh24|小时, 00-23|2023-01-30 23:59:59| +|hh12,HH12, hh, HH| 小时, 01-12|2023-01-30 12:59:59PM| +|MI,mi|分钟, 00-59|| +|SS,ss|秒, 00-59|| +|MS,ms|毫秒, 000-999|| +|US,us|微秒, 000000-999999|| +|NS,ns|纳秒, 000000000-999999999|| +|TZH,tzh|时区小时|2023-01-30 11:59:59PM +08| + +**使用说明**: +- `Month`, `Day`等的输出格式是左对齐的, 右侧添加空格, 如`2023-OCTOBER -01`, `2023-SEPTEMBER-01`, 9月是月份中英文字母数最长的, 因此9月没有空格. 星期类似. +- 使用`ms`, `us`, `ns`时, 以上三种格式的输出只在精度上不同, 比如ts为 `1697182085123`, `ms` 的输出为 `123`, `us` 的输出为 `123000`, `ns` 的输出为 `123000000`. +- 如果想要在格式串中指定某些部分不做转换, 可以使用双引号, 如`to_char(ts, 'yyyy-mm-dd "is formated by yyyy-mm-dd"')`. 如果想要输出双引号, 那么在双引号之前加一个反斜杠, 如 `to_char(ts, '\"yyyy-mm-dd\"')` 将会输出 `"2023-10-10"`. +- 那些输出是数字的格式, 如`YYYY`, `DD`, 大写与小写意义相同, 即`yyyy` 和 `YYYY` 可以互换. + +#### TO_TIMESTAMP + +```sql +TO_TIMESTAMP(str_literal, str_literal) +``` + +**功能说明**: 将字符串按照指定格式转化为时间戳. + +**返回结果数据类型**: TIMESTAMP + +**应用字段**: VARCHAR + +**嵌套子查询支持**: 适用于内层查询和外层查询 + +**适用于**: 表和超级表 + +**支持的格式**: 与`to_char`相同 + +**使用说明**: +- 若`ms`, `us`, `ns`同时指定, 那么结果时间戳包含上述三个字段的和. 如 `to_timestamp('2023-10-10 10:10:10.123.000456.000000789', 'yyyy-mm-dd hh:mi:ss.ms.us.ns')` 输出是 `2023-10-10 10:10:10.123456789`. +- `MONTH`, `MON`, `DAY`, `DY` 以及其他输出为数字的格式的大小写意义相同, 如 `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month`可以被替换为`MONTH` 或者`Month`. +- 如果同一字段被指定了多次, 那么前面的指定将会被覆盖. 如 `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, 输出年份是`2022`. +- 如果某些部分没有指定 那么默认时间为本地时区的 `1970-01-01 00:00:00`, 未指定部分为对应默认值. +- 如果格式串中有`AM`, `PM`等, 那么小时必须是12小时制, 范围必须是01-12. +- `to_timestamp`转换具有一定的容错机制, 在格式串和时间戳串不完全对应时, 有时也可转换, 如: `to_timestamp('200101/2', 'yyyyMM1/dd')`, 格式串中多出来的1会被丢弃. 格式串与时间戳串中多余的空格字符(空格, tab等)也会被 自动忽略. 如`to_timestamp(' 23 年 - 1 月 - 01 日 ', 'yy 年-MM月-dd日')` 可以被成功转换. 虽然`MM`等字段需要两个数字对应(只有一位时前面补0), 在`to_timestamp`时, 一个数字也可以成功转换. + ### 时间和日期函数 diff --git a/include/common/ttime.h b/include/common/ttime.h index 37e3045817..75bbcddd0e 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -90,6 +90,27 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision); +struct STm { + struct tm tm; + int64_t fsec; // in NANOSECOND +}; + +int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm); +int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision); + +/// @brief convert a timestamp to a formatted string +/// @param format the timestamp format, must null terminated +void taosTs2Char(const char* format, int64_t ts, int32_t precision, char* out); +/// @brief convert a formatted timestamp string to a timestamp +/// @param format must null terminated +/// @param tsStr must null terminated +/// @retval 0 for success, otherwise error occured +int32_t taosChar2Ts(const char* format, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, + int32_t errMsgLen); + +void TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out); +int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const char* tsStr); + #ifdef __cplusplus } #endif diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 48c2210f46..865f1b2295 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -94,6 +94,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_TO_ISO8601, FUNCTION_TYPE_TO_UNIXTIMESTAMP, FUNCTION_TYPE_TO_JSON, + FUNCTION_TYPE_TO_TIMESTAMP, + FUNCTION_TYPE_TO_CHAR, // date and time function FUNCTION_TYPE_NOW = 2500, diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 2e6652f860..789ba554e2 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -80,6 +80,8 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t toTimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t toCharFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 39ae3fb97a..10784bdb0c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -739,6 +739,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803) #define TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2804) #define TSDB_CODE_FUNC_DUP_TIMESTAMP TAOS_DEF_ERROR_CODE(0, 0x2805) +#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED TAOS_DEF_ERROR_CODE(0, 0x2806) //udf #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) diff --git a/include/util/tdef.h b/include/util/tdef.h index 287617970c..14c507b9a2 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -109,6 +109,15 @@ extern const int32_t TYPE_BYTES[21]; #define TSDB_INS_USER_STABLES_DBNAME_COLID 2 +static const int64_t TICK_PER_SECOND[] = { + 1000LL, // MILLISECOND + 1000000LL, // MICROSECOND + 1000000000LL, // NANOSECOND + 0LL, // HOUR + 0LL, // MINUTE + 1LL // SECOND +}; + #define TSDB_TICK_PER_SECOND(precision) \ ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI \ ? 1000LL \ diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 425218f0e1..3450e32f4a 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -25,7 +25,6 @@ #include "tlog.h" - // ==== mktime() kernel code =================// static int64_t m_deltaUtc = 0; @@ -679,7 +678,7 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { } // The following code handles the y/n time duration - int64_t numOfMonth = (unit == 'y')? duration*12:duration; + int64_t numOfMonth = (unit == 'y') ? duration * 12 : duration; int64_t fraction = t % TSDB_TICK_PER_SECOND(precision); struct tm tm; @@ -722,7 +721,7 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { * Total num of windows is ret + 1(the first window) */ int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision, - int32_t order) { + int32_t order) { if (ekey < skey) { int64_t tmp = ekey; ekey = skey; @@ -765,7 +764,6 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { int32_t precision = pInterval->precision; if (IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { - start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; time_t tt = (time_t)start; @@ -796,7 +794,7 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { int64_t newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; if (newe < ts) { // move towards the greater endpoint - while(newe < ts && news < ts) { + while (newe < ts && news < ts) { news += pInterval->sliding; newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; } @@ -975,3 +973,872 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) tstrncpy(buf, ts, bufLen); } + +int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) { + tm->fsec = ts % TICK_PER_SECOND[precision] * (TICK_PER_SECOND[TSDB_TIME_PRECISION_NANO] / TICK_PER_SECOND[precision]); + time_t t = ts / TICK_PER_SECOND[precision]; + taosLocalTime(&t, &tm->tm, NULL); + return TSDB_CODE_SUCCESS; +} + +int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision) { + *ts = taosMktime(&tm->tm); + *ts *= TICK_PER_SECOND[precision]; + *ts += tm->fsec / (TICK_PER_SECOND[TSDB_TIME_PRECISION_NANO] / TICK_PER_SECOND[precision]); + return TSDB_CODE_SUCCESS; +} + +typedef struct { + const char* name; + int len; + int id; + bool isDigit; +} TSFormatKeyWord; + +typedef enum { + // TSFKW_AD, // BC AD + // TSFKW_A_D, // A.D. B.C. + TSFKW_AM, // AM, PM + TSFKW_A_M, // A.M., P.M. + // TSFKW_BC, // BC AD + // TSFKW_B_C, // B.C. A.D. + TSFKW_DAY, // MONDAY, TUESDAY ... + TSFKW_DDD, // Day of year 001-366 + TSFKW_DD, // Day of month 01-31 + TSFKW_Day, // Sunday, Monday + TSFKW_DY, // MON, TUE + TSFKW_Dy, // Mon, Tue + TSFKW_dy, // mon, tue + TSFKW_D, // 1-7 -> Sunday(1) -> Saturday(7) + TSFKW_HH24, + TSFKW_HH12, + TSFKW_HH, + TSFKW_MI, // minute + TSFKW_MM, + TSFKW_MONTH, // JANUARY, FEBRUARY + TSFKW_MON, + TSFKW_Month, + TSFKW_Mon, + TSFKW_MS, + TSFKW_NS, + TSFKW_OF, + TSFKW_PM, + TSFKW_P_M, + TSFKW_SS, + // TSFKW_TZM, + TSFKW_TZH, + // TSFKW_TZ, + TSFKW_US, + TSFKW_YYYY, + TSFKW_YYY, + TSFKW_YY, + TSFKW_Y, + // TSFKW_a_d, + // TSFKW_ad, + TSFKW_am, + TSFKW_a_m, + // TSFKW_b_c, + // TSFKW_bc, + TSFKW_d, + TSFKW_day, + TSFKW_ddd, + TSFKW_dd, + TSFKW_hh24, + TSFKW_hh12, + TSFKW_hh, + TSFKW_mm, + TSFKW_month, + TSFKW_mon, + TSFKW_ms, + TSFKW_ns, + TSFKW_pm, + TSFKW_p_m, + TSFKW_ss, + TSFKW_tzh, + // TSFKW_tzm, + // TSFKW_tz, + TSFKW_us, + TSFKW_yyyy, + TSFKW_yyy, + TSFKW_yy, + TSFKW_y, + TSFKW_last_ +} TSFormatKeywordId; + +// clang-format off +static const TSFormatKeyWord formatKeyWords[] = { + //{"A.D.", 4, TSFKW_A_D}, + {"A.M.", 4, TSFKW_A_M, false}, + //{"AD", 2, TSFKW_AD, false}, + {"AM", 2, TSFKW_AM, false}, + //{"B.C.", 4, TSFKW_B_C, false}, + //{"BC", 2, TSFKW_BC, false}, + {"DAY", 3, TSFKW_DAY, false}, + {"DDD", 3, TSFKW_DDD, true}, + {"DD", 2, TSFKW_DD, true}, + {"DY", 2, TSFKW_DY, false}, + {"Day", 3, TSFKW_Day, false}, + {"Dy", 2, TSFKW_Dy, false}, + {"D", 1, TSFKW_D, true}, + {"HH24", 4, TSFKW_HH24, true}, + {"HH12", 4, TSFKW_HH12, true}, + {"HH", 2, TSFKW_HH, true}, + {"MI", 2, TSFKW_MI, true}, + {"MM", 2, TSFKW_MM, true}, + {"MONTH", 5, TSFKW_MONTH, false}, + {"MON", 3, TSFKW_MON, false}, + {"MS", 2, TSFKW_MS, true}, + {"Month", 5, TSFKW_Month, false}, + {"Mon", 3, TSFKW_Mon, false}, + {"NS", 2, TSFKW_NS, true}, + //{"OF", 2, TSFKW_OF, false}, + {"P.M.", 4, TSFKW_P_M, false}, + {"PM", 2, TSFKW_PM, false}, + {"SS", 2, TSFKW_SS, true}, + {"TZH", 3, TSFKW_TZH, false}, + //{"TZM", 3, TSFKW_TZM}, + //{"TZ", 2, TSFKW_TZ}, + {"US", 2, TSFKW_US, true}, + {"YYYY", 4, TSFKW_YYYY, true}, + {"YYY", 3, TSFKW_YYY, true}, + {"YY", 2, TSFKW_YY, true}, + {"Y", 1, TSFKW_Y, true}, + //{"a.d.", 4, TSFKW_a_d, false}, + {"a.m.", 4, TSFKW_a_m, false}, + //{"ad", 2, TSFKW_ad, false}, + {"am", 2, TSFKW_am, false}, + //{"b.c.", 4, TSFKW_b_c, false}, + //{"bc", 2, TSFKW_bc, false}, + {"day", 3, TSFKW_day, false}, + {"ddd", 3, TSFKW_DDD, true}, + {"dd", 2, TSFKW_DD, true}, + {"dy", 2, TSFKW_dy, false}, + {"d", 1, TSFKW_D, true}, + {"hh24", 4, TSFKW_HH24, true}, + {"hh12", 4, TSFKW_HH12, true}, + {"hh", 2, TSFKW_HH, true}, + {"mi", 2, TSFKW_MI, true}, + {"mm", 2, TSFKW_MM, true}, + {"month", 5, TSFKW_month, false}, + {"mon", 3, TSFKW_mon, false}, + {"ms", 2, TSFKW_MS, true}, + {"ns", 2, TSFKW_NS, true}, + //{"of", 2, TSFKW_OF, false}, + {"p.m.", 4, TSFKW_p_m, false}, + {"pm", 2, TSFKW_pm, false}, + {"ss", 2, TSFKW_SS, true}, + {"tzh", 3, TSFKW_TZH, false}, + //{"tzm", 3, TSFKW_TZM}, + //{"tz", 2, TSFKW_tz}, + {"us", 2, TSFKW_US, true}, + {"yyyy", 4, TSFKW_YYYY, true}, + {"yyy", 3, TSFKW_YYY, true}, + {"yy", 2, TSFKW_YY, true}, + {"y", 1, TSFKW_Y, true}, + {NULL, 0, 0} +}; +// clang-format on + +typedef struct { + uint8_t type; + char c[2]; + const TSFormatKeyWord* key; +} TSFormatNode; + +static const char* const weekDays[] = {"Sunday", "Monday", "Tuesday", "Wednesday", + "Thursday", "Friday", "Saturday", "NULL"}; +static const char* const shortWeekDays[] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "NULL"}; +static const char* const fullMonths[] = {"January", "February", "March", "April", "May", "June", "July", + "August", "September", "October", "November", "December", NULL}; +static const char* const months[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", + "Aug", "Sep", "Oct", "Nov", "Dec", NULL}; +#define A_M_STR "A.M." +#define a_m_str "a.m." +#define AM_STR "AM" +#define am_str "am" +#define P_M_STR "P.M." +#define p_m_str "p.m." +#define PM_STR "PM" +#define pm_str "pm" +static const char* const apms[] = {AM_STR, PM_STR, am_str, pm_str, NULL}; +static const char* const long_apms[] = {A_M_STR, P_M_STR, a_m_str, p_m_str, NULL}; + +#define TS_FORMAT_NODE_TYPE_KEYWORD 1 +#define TS_FORMAT_NODE_TYPE_SEPARATOR 2 +#define TS_FORMAT_NODE_TYPE_CHAR 3 + +static const TSFormatKeyWord* keywordSearch(const char* str) { + if (*str < 'A' || *str > 'z' || (*str > 'Z' && *str < 'a')) return NULL; + int32_t idx = 0; + const TSFormatKeyWord* key = &formatKeyWords[idx++]; + while (key->name) { + if (0 == strncmp(key->name, str, key->len)) { + return key; + } + key = &formatKeyWords[idx++]; + } + return NULL; +} + +static bool isSeperatorChar(char c) { + return (c > 0x20 && c < 0x7F && !(c >= 'A' && c <= 'Z') && !(c >= 'a' && c <= 'z') && !(c >= '0' && c <= '9')); +} + +static void parseTsFormat(const char* format_str, SArray* formats) { + while (*format_str) { + const TSFormatKeyWord* key = keywordSearch(format_str); + if (key) { + TSFormatNode format = {.key = key, .type = TS_FORMAT_NODE_TYPE_KEYWORD}; + taosArrayPush(formats, &format); + format_str += key->len; + } else { + if (*format_str == '"') { + // for double quoted string + format_str++; + while (*format_str) { + if (*format_str == '"') { + format_str++; + break; + } + if (*format_str == '\\' && *(format_str + 1)) format_str++; + TSFormatNode format = {.type = TS_FORMAT_NODE_TYPE_CHAR, .key = NULL}; + format.c[0] = *format_str; + format.c[1] = '\0'; + taosArrayPush(formats, &format); + format_str++; + } + } else { + // for other strings + if (*format_str == '\\' && *(format_str + 1)) format_str++; + TSFormatNode format = { + .type = isSeperatorChar(*format_str) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR, + .key = NULL}; + format.c[0] = *format_str; + format.c[1] = '\0'; + taosArrayPush(formats, &format); + format_str++; + } + } + } +} + +static void tm2char(const SArray* formats, const struct STm* tm, char* s) { + int32_t size = taosArrayGetSize(formats); + for (int32_t i = 0; i < size; ++i) { + TSFormatNode* format = taosArrayGet(formats, i); + if (format->type != TS_FORMAT_NODE_TYPE_KEYWORD) { + strcpy(s, format->c); + s += strlen(s); + continue; + } + + switch (format->key->id) { + case TSFKW_AM: + case TSFKW_PM: + sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "PM" : "AM"); + s += strlen(s); + break; + case TSFKW_A_M: + case TSFKW_P_M: + sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "P.M." : "A.M."); + s += strlen(s); + break; + case TSFKW_am: + case TSFKW_pm: + sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "pm" : "am"); + s += strlen(s); + break; + case TSFKW_a_m: + case TSFKW_p_m: + sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "p.m." : "a.m."); + s += strlen(s); + break; + case TSFKW_DDD: + sprintf(s, "%d", tm->tm.tm_yday); + s += strlen(s); + break; + case TSFKW_DD: + sprintf(s, "%02d", tm->tm.tm_mday); + s += strlen(s); + break; + case TSFKW_D: + sprintf(s, "%d", tm->tm.tm_wday + 1); + s += strlen(s); + break; + case TSFKW_DAY: { + // MONDAY, TUESDAY... + const char* wd = weekDays[tm->tm.tm_wday]; + char buf[10] = {0}; + for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = toupper(wd[i]); + sprintf(s, "%-9s", buf); + s += strlen(s); + break; + } + case TSFKW_Day: + // Monday, TuesDay... + sprintf(s, "%-9s", weekDays[tm->tm.tm_wday]); + s += strlen(s); + break; + case TSFKW_day: { + const char* wd = weekDays[tm->tm.tm_wday]; + char buf[10] = {0}; + for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = tolower(wd[i]); + sprintf(s, "%-9s", buf); + s += strlen(s); + break; + } + case TSFKW_DY: { + // MON, TUE + const char* wd = shortWeekDays[tm->tm.tm_wday]; + char buf[8] = {0}; + for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = toupper(wd[i]); + sprintf(s, "%3s", buf); + s += strlen(s); + break; + } + case TSFKW_Dy: + // Mon, Tue + sprintf(s, "%3s", shortWeekDays[tm->tm.tm_wday]); + s += strlen(s); + break; + case TSFKW_dy: { + // mon, tue + const char* wd = shortWeekDays[tm->tm.tm_wday]; + char buf[8] = {0}; + for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = tolower(wd[i]); + sprintf(s, "%3s", buf); + s += strlen(s); + break; + } + case TSFKW_HH24: + sprintf(s, "%02d", tm->tm.tm_hour); + s += strlen(s); + break; + case TSFKW_HH: + case TSFKW_HH12: + // 0 or 12 o'clock in 24H coresponds to 12 o'clock (AM/PM) in 12H + sprintf(s, "%02d", tm->tm.tm_hour % 12 == 0 ? 12 : tm->tm.tm_hour % 12); + s += strlen(s); + break; + case TSFKW_MI: + sprintf(s, "%02d", tm->tm.tm_min); + s += strlen(s); + break; + case TSFKW_MM: + sprintf(s, "%02d", tm->tm.tm_mon + 1); + s += strlen(s); + break; + case TSFKW_MONTH: { + const char* mon = fullMonths[tm->tm.tm_mon]; + char buf[10] = {0}; + for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = toupper(mon[i]); + sprintf(s, "%-9s", buf); + s += strlen(s); + break; + } + case TSFKW_MON: { + const char* mon = months[tm->tm.tm_mon]; + char buf[10] = {0}; + for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = toupper(mon[i]); + sprintf(s, "%s", buf); + s += strlen(s); + break; + } + case TSFKW_Month: + sprintf(s, "%-9s", fullMonths[tm->tm.tm_mon]); + s += strlen(s); + break; + case TSFKW_month: { + const char* mon = fullMonths[tm->tm.tm_mon]; + char buf[10] = {0}; + for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = tolower(mon[i]); + sprintf(s, "%-9s", buf); + s += strlen(s); + break; + } + case TSFKW_Mon: + sprintf(s, "%s", months[tm->tm.tm_mon]); + s += strlen(s); + break; + case TSFKW_mon: { + const char* mon = months[tm->tm.tm_mon]; + char buf[10] = {0}; + for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = tolower(mon[i]); + sprintf(s, "%s", buf); + s += strlen(s); + break; + } + case TSFKW_SS: + sprintf(s, "%02d", tm->tm.tm_sec); + s += strlen(s); + break; + case TSFKW_MS: + sprintf(s, "%03" PRId64, tm->fsec / 1000000L); + s += strlen(s); + break; + case TSFKW_US: + sprintf(s, "%06" PRId64, tm->fsec / 1000L); + s += strlen(s); + break; + case TSFKW_NS: + sprintf(s, "%09" PRId64, tm->fsec); + s += strlen(s); + break; + case TSFKW_TZH: + sprintf(s, "%s%02d", tsTimezone < 0 ? "-" : "+", tsTimezone); + s += strlen(s); + break; + case TSFKW_YYYY: + sprintf(s, "%04d", tm->tm.tm_year + 1900); + s += strlen(s); + break; + case TSFKW_YYY: + sprintf(s, "%03d", (tm->tm.tm_year + 1900) % 1000); + s += strlen(s); + break; + case TSFKW_YY: + sprintf(s, "%02d", (tm->tm.tm_year + 1900) % 100); + s += strlen(s); + break; + case TSFKW_Y: + sprintf(s, "%01d", (tm->tm.tm_year + 1900) % 10); + s += strlen(s); + break; + default: + break; + } + } +} + +/// @brief find s in arr case insensitively +/// @retval the index in arr if found, -1 if not found +static int32_t strArrayCaseSearch(const char* const* arr, const char* s) { + if (!*s) return -1; + const char* const* fmt = arr; + for (; *fmt; ++fmt) { + const char *l, *r; + for (l = fmt[0], r = s;; l++, r++) { + if (*l == '\0') return fmt - arr; + if (*r == '\0' || tolower(*l) != tolower(*r)) break; + } + } + return -1; +} + +static const char* tsFormatStr2Int32(int32_t* dest, const char* str, int32_t len, bool needMoreDigit) { + char* last; + int64_t res; + const char* s = str; + if (len <= 0) { + res = taosStr2Int64(s, &last, 10); + s = last; + } else { + char buf[16] = {0}; + strncpy(buf, s, len); + int32_t copiedLen = strlen(buf); + if (copiedLen < len) { + if (!needMoreDigit) { + // digits not enough, that's ok, cause we do not need more digits + // '2023-1' 'YYYY-MM' + // '202a' 'YYYY' -> 202 + res = taosStr2Int64(s, &last, 10); + s += copiedLen; + } else { + // bytes not enough, and there are other digit formats to match + // '2023-1' 'YYYY-MMDD' + return NULL; + } + } else { + if (needMoreDigit) { + res = taosStr2Int64(buf, &last, 10); + // bytes enough, but digits not enough, like '202a12' 'YYYYMM', YYYY needs four digits + if (last - buf < len) return NULL; + s += last - buf; + } else { + res = taosStr2Int64(s, &last, 10); + s = last; + } + } + } + if (s == str) { + // no integers found + return NULL; + } + if (errno == ERANGE || res > INT32_MAX || res < INT32_MIN) { + // out of range + return NULL; + } + *dest = res; + return s; +} + +static int32_t adjustYearTo2020(int32_t year) { + if (year < 70) return year + 2000; // 2000 - 2069 + if (year < 100) return year + 1900; // 1970 - 1999 + if (year < 520) return year + 2000; // 2100 - 2519 + if (year < 1000) return year + 1000; // 1520 - 1999 + return year; +} + +static bool checkTm(const struct tm* tm) { + if (tm->tm_mon < 0 || tm->tm_mon > 11) return false; + if (tm->tm_wday < 0 || tm->tm_wday > 6) return false; + if (tm->tm_yday < 0 || tm->tm_yday > 365) return false; + if (tm->tm_mday < 0 || tm->tm_mday > 31) return false; + if (tm->tm_hour < 0 || tm->tm_hour > 23) return false; + if (tm->tm_min < 0 || tm->tm_min > 59) return false; + if (tm->tm_sec < 0 || tm->tm_sec > 60) return false; + return true; +} + +static bool needMoreDigits(SArray* formats, int32_t curIdx) { + if (curIdx == taosArrayGetSize(formats) - 1) return false; + TSFormatNode* pNextNode = taosArrayGet(formats, curIdx + 1); + if (pNextNode->type == TS_FORMAT_NODE_TYPE_SEPARATOR) { + return false; + } else if (pNextNode->type == TS_FORMAT_NODE_TYPE_KEYWORD) { + return pNextNode->key->isDigit; + } else { + return isdigit(pNextNode->c[0]); + } +} + +/// @brief convert a formatted time str to timestamp +/// @param[in] s the formatted timestamp str +/// @param[in] formats array of TSFormatNode, output of parseTsFormat +/// @param[out] ts output timestamp +/// @param precision the timestamp precision to convert to, sec/milli/micro/nano +/// @param[out] sErrPos if not NULL, when err occured, points to the failed position of s, only set when ret is -1 +/// @param[out] fErrIdx if not NULL, when err occured, the idx of the failed format idx, only set when ret is -1 +/// @retval 0 for success +/// @retval -1 for format and s mismatch error +/// @retval -2 if datetime err, like 2023-13-32 25:61:69 +static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t precision, const char** sErrPos, + int32_t* fErrIdx) { + int32_t size = taosArrayGetSize(formats); + int32_t pm = 0; // default am + int32_t hour12 = 0; // default HH24 + int32_t year = 0, mon = 0, yd = 0, md = 1, wd = 0; + int32_t hour = 0, min = 0, sec = 0, us = 0, ms = 0, ns = 0; + int32_t tzSign = 1, tz = tsTimezone; + int32_t err = 0; + + for (int32_t i = 0; i < size; ++i) { + while (isspace(*s)) { + s++; + } + TSFormatNode* node = taosArrayGet(formats, i); + if (node->type == TS_FORMAT_NODE_TYPE_SEPARATOR) { + // separator matches any character + if (isSeperatorChar(s[0])) s += strlen(node->c); + continue; + } + if (node->type == TS_FORMAT_NODE_TYPE_CHAR) { + if (!isspace(node->c[0])) s += strlen(node->c); + continue; + } + assert(node->type == TS_FORMAT_NODE_TYPE_KEYWORD); + switch (node->key->id) { + case TSFKW_A_M: + case TSFKW_P_M: + case TSFKW_a_m: + case TSFKW_p_m: { + int32_t idx = strArrayCaseSearch(long_apms, s); + if (idx >= 0) { + s += strlen(long_apms[idx]); + pm = idx % 2; + hour12 = 1; + } else { + err = -1; + } + } break; + case TSFKW_AM: + case TSFKW_PM: + case TSFKW_am: + case TSFKW_pm: { + int32_t idx = strArrayCaseSearch(apms, s); + if (idx >= 0) { + s += strlen(apms[idx]); + pm = idx % 2; + hour12 = 1; + } else { + err = -1; + } + } break; + case TSFKW_HH: + case TSFKW_HH12: { + const char* newPos = tsFormatStr2Int32(&hour, s, 2, needMoreDigits(formats, i)); + if (NULL == newPos || hour > 12 || hour <= 0) { + err = -1; + } else { + hour12 = 1; + s = newPos; + } + } break; + case TSFKW_HH24: { + const char* newPos = tsFormatStr2Int32(&hour, s, 2, needMoreDigits(formats, i)); + if (NULL == newPos) { + err = -1; + } else { + hour12 = 0; + s = newPos; + } + } break; + case TSFKW_MI: { + const char* newPos = tsFormatStr2Int32(&min, s, 2, needMoreDigits(formats, i)); + if (NULL == newPos) { + err = -1; + } else { + s = newPos; + } + } break; + case TSFKW_SS: { + const char* newPos = tsFormatStr2Int32(&sec, s, 2, needMoreDigits(formats, i)); + if (NULL == newPos) + err = -1; + else + s = newPos; + } break; + case TSFKW_MS: { + const char* newPos = tsFormatStr2Int32(&ms, s, 3, needMoreDigits(formats, i)); + if (NULL == newPos) + err = -1; + else { + int32_t len = newPos - s; + ms *= len == 1 ? 100 : len == 2 ? 10 : 1; + s = newPos; + } + } break; + case TSFKW_US: { + const char* newPos = tsFormatStr2Int32(&us, s, 6, needMoreDigits(formats, i)); + if (NULL == newPos) + err = -1; + else { + int32_t len = newPos - s; + us *= len == 1 ? 100000 : len == 2 ? 10000 : len == 3 ? 1000 : len == 4 ? 100 : len == 5 ? 10 : 1; + s = newPos; + } + } break; + case TSFKW_NS: { + const char* newPos = tsFormatStr2Int32(&ns, s, 9, needMoreDigits(formats, i)); + if (NULL == newPos) + err = -1; + else { + int32_t len = newPos - s; + ns *= len == 1 ? 100000000 + : len == 2 ? 10000000 + : len == 3 ? 1000000 + : len == 4 ? 100000 + : len == 5 ? 10000 + : len == 6 ? 1000 + : len == 7 ? 100 + : len == 8 ? 10 + : 1; + s = newPos; + } + } break; + case TSFKW_TZH: { + tzSign = *s == '-' ? -1 : 1; + const char* newPos = tsFormatStr2Int32(&tz, s, -1, needMoreDigits(formats, i)); + if (NULL == newPos) + err = -1; + else { + s = newPos; + } + } break; + case TSFKW_MONTH: + case TSFKW_Month: + case TSFKW_month: { + int32_t idx = strArrayCaseSearch(fullMonths, s); + if (idx >= 0) { + s += strlen(fullMonths[idx]); + mon = idx; + } else { + err = -1; + } + } break; + case TSFKW_MON: + case TSFKW_Mon: + case TSFKW_mon: { + int32_t idx = strArrayCaseSearch(months, s); + if (idx >= 0) { + s += strlen(months[idx]); + mon = idx; + } else { + err = -1; + } + } break; + case TSFKW_MM: { + const char* newPos = tsFormatStr2Int32(&mon, s, 2, needMoreDigits(formats, i)); + if (NULL == newPos) { + err = -1; + } else { + s = newPos; + mon -= 1; + } + } break; + case TSFKW_DAY: + case TSFKW_Day: + case TSFKW_day: { + int32_t idx = strArrayCaseSearch(weekDays, s); + if (idx >= 0) { + s += strlen(weekDays[idx]); + wd = idx; + } else { + err = -1; + } + } break; + case TSFKW_DY: + case TSFKW_Dy: + case TSFKW_dy: { + int32_t idx = strArrayCaseSearch(shortWeekDays, s); + if (idx >= 0) { + s += strlen(shortWeekDays[idx]); + wd = idx; + } else { + err = -1; + } + } break; + case TSFKW_DDD: { + const char* newPos = tsFormatStr2Int32(&yd, s, 3, needMoreDigits(formats, i)); + if (NULL == newPos) { + err = -1; + } else { + s = newPos; + } + } break; + case TSFKW_DD: { + const char* newPos = tsFormatStr2Int32(&md, s, 2, needMoreDigits(formats, i)); + if (NULL == newPos) { + err = -1; + } else { + s = newPos; + } + } break; + case TSFKW_D: { + const char* newPos = tsFormatStr2Int32(&wd, s, 1, needMoreDigits(formats, i)); + if (NULL == newPos) { + err = -1; + } else { + s = newPos; + } + } break; + case TSFKW_YYYY: { + const char* newPos = tsFormatStr2Int32(&year, s, 4, needMoreDigits(formats, i)); + if (NULL == newPos) { + err = -1; + } else { + s = newPos; + } + } break; + case TSFKW_YYY: { + const char* newPos = tsFormatStr2Int32(&year, s, 3, needMoreDigits(formats, i)); + if (NULL == newPos) { + err = -1; + } else { + year = adjustYearTo2020(year); + s = newPos; + } + } break; + case TSFKW_YY: { + const char* newPos = tsFormatStr2Int32(&year, s, 2, needMoreDigits(formats, i)); + if (NULL == newPos) { + err = -1; + } else { + year = adjustYearTo2020(year); + s = newPos; + } + } break; + case TSFKW_Y: { + const char* newPos = tsFormatStr2Int32(&year, s, 1, needMoreDigits(formats, i)); + if (NULL == newPos) { + err = -1; + } else { + year = adjustYearTo2020(year); + s = newPos; + } + } break; + default: + break; + } + if (err) { + if (sErrPos) *sErrPos = s; + if (fErrIdx) *fErrIdx = i; + return err; + } + } + struct STm tm = {0}; + tm.tm.tm_year = year - 1900; + tm.tm.tm_mon = mon; + tm.tm.tm_yday = yd; + tm.tm.tm_mday = md; + tm.tm.tm_wday = wd; + if (hour12) { + if (pm && hour < 12) + tm.tm.tm_hour = hour + 12; + else if (!pm && hour == 12) + tm.tm.tm_hour = 0; + else + tm.tm.tm_hour = hour; + } else { + tm.tm.tm_hour = hour; + } + tm.tm.tm_min = min; + tm.tm.tm_sec = sec; + if (!checkTm(&tm.tm)) return -2; + if (tz < -12 || tz > 12) return -2; + tm.fsec = ms * 1000000 + us * 1000 + ns; + int32_t ret = taosTm2Ts(&tm, ts, precision); + *ts += 60 * 60 * (tsTimezone - tz) * TICK_PER_SECOND[precision]; + return ret; +} + +void taosTs2Char(const char* format, int64_t ts, int32_t precision, char* out) { + SArray* formats = taosArrayInit(8, sizeof(TSFormatNode)); + parseTsFormat(format, formats); + struct STm tm; + taosTs2Tm(ts, precision, &tm); + tm2char(formats, &tm, out); + taosArrayDestroy(formats); +} + +int32_t taosChar2Ts(const char* format, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, + int32_t errMsgLen) { + const char* sErrPos; + int32_t fErrIdx; + SArray* formats = taosArrayInit(4, sizeof(TSFormatNode)); + parseTsFormat(format, formats); + int32_t code = char2ts(tsStr, formats, ts, precision, &sErrPos, &fErrIdx); + if (code == -1) { + TSFormatNode* fNode = (taosArrayGet(formats, fErrIdx)); + snprintf(errMsg, errMsgLen, "mismatch format for: %s and %s", sErrPos, + fErrIdx < taosArrayGetSize(formats) ? ((TSFormatNode*)taosArrayGet(formats, fErrIdx))->key->name : ""); + } else if (code == -2) { + snprintf(errMsg, errMsgLen, "timestamp format error: %s -> %s", tsStr, format); + } + taosArrayDestroy(formats); + return code; +} + +void TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out) { + SArray* formats = taosArrayInit(4, sizeof(TSFormatNode)); + parseTsFormat(format, formats); + struct STm tm; + taosTs2Tm(ts, precision, &tm); + tm2char(formats, &tm, out); + taosArrayDestroy(formats); +} + +int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const char* tsStr) { + const char* sErrPos; + int32_t fErrIdx; + SArray* formats = taosArrayInit(4, sizeof(TSFormatNode)); + parseTsFormat(format, formats); + int32_t code = char2ts(tsStr, formats, ts, precision, &sErrPos, &fErrIdx); + if (code == -1) { + printf("failed position: %s\n", sErrPos); + printf("failed format: %s\n", ((TSFormatNode*)taosArrayGet(formats, fErrIdx))->key->name); + } + taosArrayDestroy(formats); + return code; +} diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 8a77087d23..49a16351ca 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -13,6 +13,7 @@ #include "tdatablock.h" #include "tdef.h" #include "tvariant.h" +#include "ttime.h" namespace { // @@ -260,4 +261,226 @@ TEST(testCase, var_dataBlock_split_test) { } } -#pragma GCC diagnostic pop \ No newline at end of file +void check_tm(const STm* tm, int32_t y, int32_t mon, int32_t d, int32_t h, int32_t m, int32_t s, int64_t fsec) { + ASSERT_EQ(tm->tm.tm_year, y); + ASSERT_EQ(tm->tm.tm_mon, mon); + ASSERT_EQ(tm->tm.tm_mday, d); + ASSERT_EQ(tm->tm.tm_hour, h); + ASSERT_EQ(tm->tm.tm_min, m); + ASSERT_EQ(tm->tm.tm_sec, s); + ASSERT_EQ(tm->fsec, fsec); +} + +void test_timestamp_tm_conversion(int64_t ts, int32_t precision, int32_t y, int32_t mon, int32_t d, int32_t h, int32_t m, int32_t s, int64_t fsec) { + int64_t ts_tmp; + char buf[128] = {0}; + struct STm tm; + taosFormatUtcTime(buf, 128, ts, precision); + printf("formated ts of %ld, precision: %d is: %s\n", ts, precision, buf); + taosTs2Tm(ts, precision, &tm); + check_tm(&tm, y, mon, d, h, m, s, fsec); + taosTm2Ts(&tm, &ts_tmp, precision); + ASSERT_EQ(ts, ts_tmp); +} + +TEST(timeTest, timestamp2tm) { + const char* ts_str_ns = "2023-10-12T11:29:00.775726171+0800"; + const char* ts_str_us = "2023-10-12T11:29:00.775726+0800"; + const char* ts_str_ms = "2023-10-12T11:29:00.775+0800"; + int64_t ts, tmp_ts = 0; + struct STm tm; + + ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_ns, &ts, strlen(ts_str_ns), TSDB_TIME_PRECISION_NANO, 0)); + test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_NANO, 2023 - 1900, 9 /* mon start from 0*/, 12, 11, 29, 0, + 775726171L); + + ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_us, &ts, strlen(ts_str_us), TSDB_TIME_PRECISION_MICRO, 0)); + test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MICRO, 2023 - 1900, 9 /* mon start from 0*/, 12, 11, 29, 0, + 775726000L); + + ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_ms, &ts, strlen(ts_str_ms), TSDB_TIME_PRECISION_MILLI, 0)); + test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, 2023 - 1900, 9 /* mon start from 0*/, 12, 11, 29, 0, + 775000000L); + + ts = -5364687943000; // milliseconds since epoch, Wednesday, January 1, 1800 1:00:00 AM GMT+08:06 + test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, 1800 - 1900, 0 /* mon start from 0*/, 1, 1, 0, 0, + 000000000L); + + ts = 0; + test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, 1970 - 1900, 0 /* mon start from 0*/, 1, 8, 0, 0, + 000000000L); + + ts = -62198784343000; // milliseconds before epoch, Friday, January 1, -0001 12:00:00 AM GMT+08:06 + test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, -1 - 1900, 0 /* mon start from 0*/, 1, + 0 /* hour start from 0*/, 0, 0, 000000000L); +} + +void test_ts2char(int64_t ts, const char* format, int32_t precison, const char* expected) { + char buf[128] = {0}; + TEST_ts2char(format, ts, precison, buf); + printf("ts: %ld format: %s res: [%s], expected: [%s]\n", ts, format, buf, expected); + ASSERT_STREQ(expected, buf); +} + +TEST(timeTest, ts2char) { + osDefaultInit(); + if (tsTimezone != TdEastZone8) GTEST_SKIP(); + int64_t ts; + const char* format = "YYYY-MM-DD"; + ts = 0; + test_ts2char(ts, format, TSDB_TIME_PRECISION_MILLI, "1970-01-01"); + test_ts2char(ts, format, TSDB_TIME_PRECISION_MICRO, "1970-01-01"); + test_ts2char(ts, format, TSDB_TIME_PRECISION_NANO, "1970-01-01"); + test_ts2char(ts, format, TSDB_TIME_PRECISION_SECONDS, "1970-01-01"); + + ts = 1697163517; + test_ts2char(ts, "YYYY-MM-DD", TSDB_TIME_PRECISION_SECONDS, "2023-10-13"); + ts = 1697163517000; + test_ts2char(ts, "YYYY-MM-DD-Day-DAY", TSDB_TIME_PRECISION_MILLI, "2023-10-13-Friday -FRIDAY "); +#ifndef WINDOWS + // double quoted: year, month, day are not parsed + test_ts2char(ts, + "YYYY-YYY-YY-Y-yyyy-yyy-yy-y-\"年\"-MONTH-MON-Month-Mon-month-mon-\"月\"-DDD-DD-D-ddd-dd-d-DAY-Day-" + "day-\"日\"", + TSDB_TIME_PRECISION_MILLI, + "2023-023-23-3-2023-023-23-3-年-OCTOBER -OCT-October -Oct-october " + "-oct-月-285-13-6-285-13-6-FRIDAY -Friday -friday -日"); +#endif + ts = 1697182085123L; // Friday, October 13, 2023 3:28:05.123 PM GMT+08:00 + test_ts2char(ts, "HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am", TSDB_TIME_PRECISION_MILLI, + "15:15:03:03:03:03:28:28:05:05:123:123:123000:123000:123000000:123000000:PM:PM:pm:pm"); + + // double quotes normal output + test_ts2char(ts, "\\\"HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am\\\"", TSDB_TIME_PRECISION_MILLI, + "\"15:15:03:03:03:03:28:28:05:05:123:123:123000:123000:123000000:123000000:PM:PM:pm:pm\""); + test_ts2char(ts, "\\\"HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am", TSDB_TIME_PRECISION_MILLI, + "\"15:15:03:03:03:03:28:28:05:05:123:123:123000:123000:123000000:123000000:PM:PM:pm:pm"); + // double quoted strings recognized as literal string, parsing skipped + test_ts2char(ts, "\"HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am", TSDB_TIME_PRECISION_MILLI, + "HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am"); + test_ts2char(ts, "yyyy-mm-dd hh24:mi:ss.nsamaaa", TSDB_TIME_PRECISION_MILLI, "2023-10-13 15:28:05.123000000pmaaa"); + test_ts2char(ts, "aaa--yyyy-mm-dd hh24:mi:ss.nsamaaa", TSDB_TIME_PRECISION_MILLI, "aaa--2023-10-13 15:28:05.123000000pmaaa"); + test_ts2char(ts, "add--yyyy-mm-dd hh24:mi:ss.nsamaaa", TSDB_TIME_PRECISION_MILLI, "a13--2023-10-13 15:28:05.123000000pmaaa"); + + ts = 1693946405000; + test_ts2char(ts, "Day, Month dd, YYYY hh24:mi:ss AM TZH:tzh", TSDB_TIME_PRECISION_MILLI, "Wednesday, September 06, 2023 04:40:05 AM +08:+08"); + + ts = -62198784343000; // milliseconds before epoch, Friday, January 1, -0001 12:00:00 AM GMT+08:06 + test_ts2char(ts, "Day, Month dd, YYYY hh12:mi:ss AM", TSDB_TIME_PRECISION_MILLI, "Friday , January 01, -001 12:00:00 AM"); +} + +TEST(timeTest, char2ts) { + osDefaultInit(); + if (tsTimezone != TdEastZone8) GTEST_SKIP(); + int64_t ts; + int32_t code = + TEST_char2ts("YYYY-DD-MM HH12:MI:SS:MSPM", &ts, TSDB_TIME_PRECISION_MILLI, "2023-10-10 12:00:00.000AM"); + ASSERT_EQ(code, 0); + ASSERT_EQ(ts, 1696867200000LL); + + // 2009-1-1 00:00:00 + ASSERT_EQ(0, TEST_char2ts("YYYY-YYY-YY-Y", &ts, TSDB_TIME_PRECISION_MILLI, "2023-123-23-9")); + ASSERT_EQ(1230739200000LL, ts); + // 2023-1-1 + ASSERT_EQ(0, TEST_char2ts("YYYY-YYY-YY", &ts, TSDB_TIME_PRECISION_MILLI, "2023-123-23-9")); + ASSERT_EQ(ts, 1672502400000LL); + + // 2123-1-1, the second year(123) is used, which converted to 2123 + ASSERT_EQ(0, TEST_char2ts("YYYY-YYY", &ts, TSDB_TIME_PRECISION_MILLI, "2023-123-23-9")); + ASSERT_EQ(ts, 4828176000000LL); + // 2023-1-1 12:10:10am + ASSERT_EQ(0, TEST_char2ts("yyyy-mm-dd HH12:MI:SSAM", &ts, TSDB_TIME_PRECISION_MILLI, "2023-1-1 12:10:10am")); + ASSERT_EQ(ts, 1672503010000LL); + + // 2023-1-1 21:10:10.123 + ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH12:MI:ss.msa.m.", &ts, TSDB_TIME_PRECISION_MILLI, "23-1-01 9:10:10.123p.m.")); + ASSERT_EQ(ts, 1672578610123LL); + + // 2023-1-1 21:10:10.123456789 + ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH:MI:ss.ms.us.nsa.m.", &ts, TSDB_TIME_PRECISION_NANO, + "23-1-01 9:10:10.123.000456.000000789p.m.")); + ASSERT_EQ(ts, 1672578610123456789LL); + + // 2023-1-1 21:10:10.120450780 + ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH24:MI:SS.ms.us.ns", &ts, TSDB_TIME_PRECISION_NANO, + " 23 - 1 - 01 \t 21:10:10 . 12 . \t 00045 . 00000078 \t")); + ASSERT_EQ(ts, 1672578610120450780LL); + +#ifndef WINDOWS + // 2023-1-1 21:10:10.120450780 + ASSERT_EQ(0, TEST_char2ts("yy \"年\"-MM 月-dd \"日\" HH24:MI:ss.ms.us.ns TZH", &ts, TSDB_TIME_PRECISION_NANO, + " 23 年 - 1 月 - 01 日 \t 21:10:10 . 12 . \t 00045 . 00000078 \t+08")); + ASSERT_EQ(ts, 1672578610120450780LL); +#endif + + // 2023-1-1 19:10:10.123456789+06 -> 2023-1-1 21:10:10.123456789+08 + ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH:MI:ss.ms.us.nsa.m.TZH", &ts, TSDB_TIME_PRECISION_NANO, + "23-1-01 7:10:10.123.000456.000000789p.m.6")); + ASSERT_EQ(ts, 1672578610123456789LL); + + // 2023-1-1 12:10:10.123456789-01 -> 2023-1-1 21:10:10.123456789+08 + ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH24:MI:ss.ms.us.nsTZH", &ts, TSDB_TIME_PRECISION_NANO, + "23-1-01 12:10:10.123.000456.000000789-1")); + ASSERT_EQ(ts, 1672578610123456789LL); + + // 2100-01-01 11:10:10.124456+08 + ASSERT_EQ( + 0, TEST_char2ts("yyyy-MM-dd HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO, "2100-01-01 11:10:10.124456+08")); + ASSERT_EQ(ts, 4102456210124456LL); + + // 2100-01-01 11:10:10.124456+08 Firday + ASSERT_EQ(0, TEST_char2ts("yyyy/MONTH/dd DAY HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO, + "2100/january/01 friday 11:10:10.124456+08")); + ASSERT_EQ(ts, 4102456210124456LL); + + ASSERT_EQ(0, TEST_char2ts("yyyy/Month/dd Day HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO, + "2100/january/01 FRIDAY 11:10:10.124456+08")); + ASSERT_EQ(ts, 4102456210124456LL); + ASSERT_EQ(0, TEST_char2ts("yyyy/Month/dd Dy HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO, + "2100/january/01 Fri 11:10:10.124456+08")); + ASSERT_EQ(ts, 4102456210124456LL); + + ASSERT_EQ(0, TEST_char2ts("yyyy/month/dd day HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO, + "2100/january/01 Friday 11:10:10.124456+08")); + ASSERT_EQ(ts, 4102456210124456LL); + + // 2100-02-01 11:10:10.124456+08 Firday + ASSERT_EQ(0, TEST_char2ts("yyyy/mon/dd DY HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO, + "2100/Feb/01 Mon 11:10:10.124456+08")); + ASSERT_EQ(ts, 4105134610124456LL); + + // 2100-02-01 11:10:10.124456+08 Firday + ASSERT_EQ(0, TEST_char2ts("yyyy/mon/dd DY DDD-DD-D HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO, + "2100/Feb/01 Mon 100-1-01 11:10:10.124456+08")); + ASSERT_EQ(ts, 4105134610124456LL); + + ASSERT_EQ(0, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "21000101")); + + // What is Fe? + ASSERT_EQ(-1, TEST_char2ts("yyyy/mon/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "2100/Fe/01")); + // '/' cannot convert to MM + ASSERT_EQ(-1, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "2100/2/1")); + // nothing to be converted to dd + ASSERT_EQ(-1, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "210012")); + ASSERT_EQ(-1, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "21001")); + ASSERT_EQ(-1, TEST_char2ts("yyyyMM-dd ", &ts, TSDB_TIME_PRECISION_MICRO, "23a1-1")); + + // 2100-1-2 + ASSERT_EQ(0, TEST_char2ts("yyyyMM/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "21001/2")); + ASSERT_EQ(ts, 4102502400000000LL); + + // default to 1970-1-1 00:00:00+08 -> 1969-12-31 16:00:00+00 + ASSERT_EQ(0, TEST_char2ts("YYYY", &ts, TSDB_TIME_PRECISION_SECONDS, "1970")); + ASSERT_EQ(ts, -1 * tsTimezone * 60 * 60); + + ASSERT_EQ(0, TEST_char2ts("yyyyMM1/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210001/2")); + ASSERT_EQ(ts, 4102502400000000LL); + + ASSERT_EQ(-2, TEST_char2ts("yyyyMM/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210013/2")); + ASSERT_EQ(-2, TEST_char2ts("yyyyMM/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210011/32")); + ASSERT_EQ(-1, TEST_char2ts("HH12:MI:SS", &ts, TSDB_TIME_PRECISION_MICRO, "21:12:12")); + ASSERT_EQ(-1, TEST_char2ts("yyyy/MM1/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "2100111111111/11/2")); + ASSERT_EQ(-2, TEST_char2ts("yyyy/MM1/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "23/11/2-13")); +} + +#pragma GCC diagnostic pop diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 68a83fa662..628a609715 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2083,6 +2083,34 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int return TSDB_CODE_SUCCESS; } +static int32_t translateToTimestamp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (LIST_LENGTH(pFunc->pParameterList) != 2) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + if (!IS_STR_DATA_TYPE(para1Type) || !IS_STR_DATA_TYPE(para2Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + pFunc->node.resType = + (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateToChar(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (LIST_LENGTH(pFunc->pParameterList) != 2) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + // currently only support to_char(timestamp, str) + if (!IS_STR_DATA_TYPE(para2Type) || !IS_TIMESTAMP_TYPE(para1Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_VARCHAR].bytes, .type = TSDB_DATA_TYPE_VARCHAR}; + return TSDB_CODE_SUCCESS; +} + static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); if (2 != numOfParams && 3 != numOfParams) { @@ -3284,6 +3312,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = castFunction, .finalizeFunc = NULL }, + { + .name = "to_timestamp", + .type = FUNCTION_TYPE_TO_TIMESTAMP, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateToTimestamp, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = toTimestampFunction, + .finalizeFunc = NULL + }, + { + .name = "to_char", + .type = FUNCTION_TYPE_TO_CHAR, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateToChar, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = toCharFunction, + .finalizeFunc = NULL + }, { .name = "to_iso8601", .type = FUNCTION_TYPE_TO_ISO8601, diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 4df7454df8..ee2ba47ce8 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1197,6 +1197,62 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu return TSDB_CODE_SUCCESS; } +#define TS_FORMAT_MAX_LEN 4096 +int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput) { + int64_t ts; + char * tsStr = taosMemoryMalloc(TS_FORMAT_MAX_LEN); + char * format = taosMemoryMalloc(TS_FORMAT_MAX_LEN); + int32_t len, code = TSDB_CODE_SUCCESS; + for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { + if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) + colDataSetNULL(pOutput->columnData, i); + + char *tsData = colDataGetData(pInput[0].columnData, i); + char *formatData = colDataGetData(pInput[1].columnData, pInput[1].numOfRows > 1 ? i : 0); + len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(tsData)); + strncpy(tsStr, varDataVal(tsData), len); + tsStr[len] = '\0'; + len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(formatData)); + strncpy(format, varDataVal(formatData), len); + format[len] = '\0'; + int32_t precision = pOutput->columnData->info.precision; + char errMsg[128] = {0}; + code = taosChar2Ts(format, tsStr, &ts, precision, errMsg, 128); + if (code) { + qError("func to_timestamp failed %s", errMsg); + code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED; + break; + } + colDataSetVal(pOutput->columnData, i, (char *)&ts, false); + } + taosMemoryFree(tsStr); + taosMemoryFree(format); + return code; +} + +int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput) { + char * format = taosMemoryMalloc(TS_FORMAT_MAX_LEN); + char * out = taosMemoryMalloc(TS_FORMAT_MAX_LEN * 2); + int32_t len; + for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { + if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) + colDataSetNULL(pOutput->columnData, i); + + char *ts = colDataGetData(pInput[0].columnData, i); + char *formatData = colDataGetData(pInput[1].columnData, pInput[1].numOfRows > 1 ? i : 0); + len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(formatData)); + strncpy(format, varDataVal(formatData), len); + format[len] = '\0'; + int32_t precision = pInput[0].columnData->info.precision; + taosTs2Char(format, *(int64_t *)ts, precision, varDataVal(out)); + varDataSetLen(out, strlen(varDataVal(out))); + colDataSetVal(pOutput->columnData, i, out, false); + } + taosMemoryFree(format); + taosMemoryFree(out); + return TSDB_CODE_SUCCESS; +} + /** Time functions **/ static int64_t offsetFromTz(char *timezone, int64_t factor) { char *minStr = &timezone[3]; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 383e4e9d8a..9e5a50ce85 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -601,6 +601,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_TYPE, "Invalid function par TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid function para value") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION, "Not buildin function") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP, "Duplicate timestamps not allowed in function") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED, "Func to_timestamp failed, check log for detail") //udf TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 8b0451604c..bd20a2745f 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -61,6 +61,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_limit_opt_2.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_limit_opt_2.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_limit_opt_2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/func_to_char_timestamp.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/func_to_char_timestamp.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/func_to_char_timestamp.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/func_to_char_timestamp.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py diff --git a/tests/system-test/2-query/func_to_char_timestamp.py b/tests/system-test/2-query/func_to_char_timestamp.py new file mode 100644 index 0000000000..3d3435d9c7 --- /dev/null +++ b/tests/system-test/2-query/func_to_char_timestamp.py @@ -0,0 +1,160 @@ +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +# from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 4 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + self.duraion = '1h' + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d duration %s"%(dbName, vgroups, replica, duration)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, paraDict): + colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"]) + tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"]) + sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString) + tdLog.debug("%s"%(sqlString)) + tsql.execute(sqlString) + return + + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): + for i in range(ctbNum): + sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % \ + (dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,(i+ctbStartIdx) % 5,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx) + tsql.execute(sqlString) + + tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + for i in range(ctbNum): + rowsBatched = 0 + sql += " %s%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + if (i < ctbNum/2): + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10, j%10, j%10) + else: + sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10) + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsBatched = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(ctbPrefix,i) + else: + sql = "insert into " + if sql != pre_insert: + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'test', + 'dropFlag': 1, + 'vgroups': 2, + 'stbName': 'meters', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'FLOAT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'smallint', 'count':1},{'type': 'tinyint', 'count':1},{'type': 'bool', 'count':1},{'type': 'binary', 'len':10, 'count':1},{'type': 'nchar', 'len':10, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}], + 'ctbPrefix': 't', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 10000, + 'batchNum': 3000, + 'startTs': 1537146000000, + 'tsStep': 600000} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("create database") + self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion) + + tdLog.info("create stb") + self.create_stable(tsql=tdSql, paraDict=paraDict) + + tdLog.info("create child tables") + self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], \ + stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],\ + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"]) + self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],\ + ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],\ + rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],\ + startTs=paraDict["startTs"],tsStep=paraDict["tsStep"]) + return + + def convert_ts_and_check(self, ts_str: str, ts_format: str, expect_ts_char: str, expect_ts: str): + tdSql.query("select to_timestamp('%s', '%s')" % (ts_str, ts_format), queryTimes=1) + tdSql.checkData(0, 0, expect_ts) + tdSql.query("select to_char(to_timestamp('%s', '%s'), '%s')" % (ts_str, ts_format, ts_format), queryTimes=1) + tdSql.checkData(0, 0, expect_ts_char) + + def test_to_timestamp(self): + self.convert_ts_and_check('2023-10-10 12:13:14.123', 'YYYY-MM-DD HH:MI:SS.MS', '2023-10-10 12:13:14.123', '2023-10-10 00:13:14.123000') + self.convert_ts_and_check('2023-10-10 12:00:00.000AM', 'YYYY-DD-MM HH12:MI:SS.MSPM', '2023-10-10 12:00:00.000AM', '2023-10-10 00:00:00.000000') + self.convert_ts_and_check('2023-01-01 12:10:10am', 'yyyy-mm-dd HH12:MI:SSAM', '2023-01-01 12:10:10AM', '2023-1-1 00:10:10.000000') + self.convert_ts_and_check('23-1-01 9:10:10.123p.m.', 'yy-MM-dd HH12:MI:ss.msa.m.', '23-01-01 09:10:10.123p.m.', '2023-1-1 21:10:10.123000') + self.convert_ts_and_check('23-1-01 9:10:10.123.000456.000000789p.m.', 'yy-MM-dd HH12:MI:ss.ms.us.nsa.m.', '23-01-01 09:10:10.123.123000.123000000p.m.', '2023-1-1 21:10:10.123000') + self.convert_ts_and_check(' 23 -1 - 01 \t 21:10:10 . 12 . \t 00045 . 00000078 \t', 'yy-MM-dd HH24:MI:SS.ms.us.ns', '23-01-01 21:10:10.120.120000.120000000', '2023-1-1 21:10:10.120000') + self.convert_ts_and_check(' 23 年 -1 月 - 01 日 \t 21:10:10 . 12 . \t 00045 . 00000078 \t+08', 'yy\"年\"-MM月-dd日 HH24:MI:SS.ms.us.ns TZH', '23年-01月-01日 21:10:10.120.120000.120000000 +08', '2023-1-1 21:10:10.120000') + self.convert_ts_and_check('23-1-01 7:10:10.123p.m.6', 'yy-MM-dd HH:MI:ss.msa.m.TZH', '23-01-01 09:10:10.123p.m.+08', '2023-1-1 21:10:10.123000') + + self.convert_ts_and_check('2023-OCTober-19 10:10:10AM Thu', 'yyyy-month-dd hh24:mi:ssam dy', '2023-october -19 10:10:10am thu', '2023-10-19 10:10:10') + + tdSql.error("select to_timestamp('210013/2', 'yyyyMM/dd')") + tdSql.error("select to_timestamp('2100111111111/13/2', 'yyyyMM/dd')") + + tdSql.error("select to_timestamp('210a12/2', 'yyyyMM/dd')") + + tdSql.query("select to_timestamp(to_char(ts, 'yy-mon-dd hh24:mi:ss dy'), 'yy-mon-dd hh24:mi:ss dy') == ts from meters limit 10") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkRows(10) + + tdSql.query("select to_char(ts, 'yy-mon-dd hh24:mi:ss.msa.m.TZH Day') from meters where to_timestamp(to_char(ts, 'yy-mon-dd hh24:mi:ss dy'), 'yy-mon-dd hh24:mi:ss dy') != ts") + tdSql.checkRows(0) + + def run(self): + self.prepareTestEnv() + self.test_to_timestamp() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From f444cd7a6d7b813c09bfc3f6a910189479457e08 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 24 Oct 2023 10:15:39 +0800 Subject: [PATCH 03/37] enh: log signature of snap sender/receiver while started or stopped --- source/libs/sync/src/syncSnapshot.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 43726307e6..3017f4e33c 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -175,7 +175,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { SyncSnapshotSend *pMsg = rpcMsg.pCont; pMsg->srcId = pSender->pSyncNode->myRaftId; pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - pMsg->term = raftStoreGetTerm(pSender->pSyncNode); + pMsg->term = pSender->term; pMsg->beginIndex = pSender->snapshotParam.start; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; @@ -189,15 +189,15 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { memcpy(pMsg->data, snapInfo.data, dataLen); } - // event log - syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender start"); - // send msg if (syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg) != 0) { sSError(pSender, "snapshot sender send msg failed since %s", terrstr()); goto _out; } + sSInfo(pSender, "snapshot sender started. signature:(%" PRId64 ", %" PRId64 "), to dnode:%d", pSender->term, + pSender->startTime, DID(&pMsg->destId)); + code = 0; _out: if (snapInfo.data) { @@ -231,6 +231,10 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { } syncSnapBufferReset(pSender->pSndBuf); + + SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; + sSInfo(pSender, "snapshot sender stopped. signature:(%" PRId64 ", %" PRId64 "), to dnode:%d", pSender->term, + pSender->startTime, DID(&destId)); } // when sender receive ack, call this function to send msg from seq @@ -515,14 +519,14 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p pReceiver->startTime = pPreMsg->startTime; ASSERT(pReceiver->startTime); - // event log - sRInfo(pReceiver, "snapshot receiver is start"); + sRInfo(pReceiver, "snapshot receiver started. signature:(%" PRId64 ", %" PRId64 "), from dnode:%d", pReceiver->term, + pReceiver->startTime, DID(&pReceiver->fromId)); } // just set start = false // FpSnapshotStopWrite should not be called void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { - sRInfo(pReceiver, "snapshot receiver stop, not apply, writer:%p", pReceiver->pWriter); + sRDebug(pReceiver, "snapshot receiver stop, not apply, writer:%p", pReceiver->pWriter); int8_t stopped = !atomic_val_compare_exchange_8(&pReceiver->start, true, false); if (stopped) return; @@ -539,6 +543,9 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { } syncSnapBufferReset(pReceiver->pRcvBuf); + + sRInfo(pReceiver, "snapshot receiver stopped. signature:(%" PRId64 ", %" PRId64 "), from dnode:%d", pReceiver->term, + pReceiver->startTime, DID(&pReceiver->fromId)); } // when recv last snapshot block, apply data into snapshot From 1c60e67a8380813bb78f1cde4f86dce03d101de8 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 24 Oct 2023 14:16:46 +0800 Subject: [PATCH 04/37] enh: send the END snap msg at last --- source/libs/sync/src/syncSnapshot.c | 52 ++++++++++++++++++----------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 3017f4e33c..7e556b28f5 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -247,25 +247,28 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { pSender->blockLen = 0; } - pSender->seq++; + if (pSender->seq != SYNC_SNAPSHOT_SEQ_END) { + pSender->seq++; - // read data - int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader, - &pSender->pCurrentBlock, &pSender->blockLen); - if (ret != 0) { - sSError(pSender, "snapshot sender read failed since %s", terrstr()); - return -1; - } + // read data + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader, + &pSender->pCurrentBlock, &pSender->blockLen); + if (ret != 0) { + sSError(pSender, "snapshot sender read failed since %s", terrstr()); + return -1; + } - if (pSender->blockLen > 0) { - // has read data - sSDebug(pSender, "vgId:%d, snapshot sender continue to read, blockLen:%d seq:%d", pSender->pSyncNode->vgId, - pSender->blockLen, pSender->seq); - } else { - // read finish, update seq to end - pSender->seq = SYNC_SNAPSHOT_SEQ_END; - sSInfo(pSender, "vgId:%d, snapshot sender read to the end, blockLen:%d seq:%d", pSender->pSyncNode->vgId, - pSender->blockLen, pSender->seq); + if (pSender->blockLen > 0) { + // has read data + sSDebug(pSender, "vgId:%d, snapshot sender continue to read, blockLen:%d seq:%d", pSender->pSyncNode->vgId, + pSender->blockLen, pSender->seq); + } else { + // read finish, update seq to end + pSender->seq = SYNC_SNAPSHOT_SEQ_END; + sSInfo(pSender, "vgId:%d, snapshot sender read to the end, blockLen:%d seq:%d", pSender->pSyncNode->vgId, + pSender->blockLen, pSender->seq); + return 0; + } } // build msg @@ -1188,19 +1191,28 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp } } - for (int64_t ack = pSndBuf->start; ack < pSndBuf->cursor; ++ack) { + for (int64_t ack = pSndBuf->start; ack <= pSndBuf->cursor; ++ack) { rpcFreeCont(pSndBuf->entries[ack % pSndBuf->size]); pSndBuf->entries[ack % pSndBuf->size] = NULL; pSndBuf->start = ack + 1; } - while (pSender->seq - pSndBuf->start < (pSndBuf->size >> 2)) { + while (pSender->seq != SYNC_SNAPSHOT_SEQ_END && pSender->seq - pSndBuf->start < (pSndBuf->size >> 2)) { + if (snapshotSend(pSender) != 0) { + code = terrno; + goto _out; + } + if (pSender->seq != SYNC_SNAPSHOT_SEQ_END) { + pSndBuf->end = TMAX(pSender->seq + 1, pSndBuf->end); + } + } + + if (pSender->seq == SYNC_SNAPSHOT_SEQ_END && pSndBuf->end <= pSndBuf->start) { if (snapshotSend(pSender) != 0) { code = terrno; goto _out; } } - _out: taosThreadMutexUnlock(&pSndBuf->mutex); return code; From 811f1bbbea25bbbd2fc470f7cd80919b71fb8dc2 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 24 Oct 2023 15:47:47 +0800 Subject: [PATCH 05/37] enh: tidy up logging msg in syncNodeOnSnapshotRsp --- source/libs/sync/src/syncSnapshot.c | 30 +++++++++++++---------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 7e556b28f5..195354ac79 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -195,8 +195,8 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { goto _out; } - sSInfo(pSender, "snapshot sender started. signature:(%" PRId64 ", %" PRId64 "), to dnode:%d", pSender->term, - pSender->startTime, DID(&pMsg->destId)); + sSInfo(pSender, "snapshot sender start to dnode:%d. signature:(%" PRId64 ", %" PRId64 ")", DID(&pMsg->destId), + pSender->term, pSender->startTime); code = 0; _out: @@ -233,8 +233,8 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { syncSnapBufferReset(pSender->pSndBuf); SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - sSInfo(pSender, "snapshot sender stopped. signature:(%" PRId64 ", %" PRId64 "), to dnode:%d", pSender->term, - pSender->startTime, DID(&destId)); + sSInfo(pSender, "snapshot sender stop to dnode:%d. signature:(%" PRId64 ", %" PRId64 "), finish:%d", DID(&destId), + pSender->term, pSender->startTime, finish); } // when sender receive ack, call this function to send msg from seq @@ -522,8 +522,8 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p pReceiver->startTime = pPreMsg->startTime; ASSERT(pReceiver->startTime); - sRInfo(pReceiver, "snapshot receiver started. signature:(%" PRId64 ", %" PRId64 "), from dnode:%d", pReceiver->term, - pReceiver->startTime, DID(&pReceiver->fromId)); + sRInfo(pReceiver, "snapshot receiver start from dnode:%d. signature:(%" PRId64 ", %" PRId64 ")", + DID(&pReceiver->fromId), pReceiver->term, pReceiver->startTime); } // just set start = false @@ -547,8 +547,8 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { syncSnapBufferReset(pReceiver->pRcvBuf); - sRInfo(pReceiver, "snapshot receiver stopped. signature:(%" PRId64 ", %" PRId64 "), from dnode:%d", pReceiver->term, - pReceiver->startTime, DID(&pReceiver->fromId)); + sRInfo(pReceiver, "snapshot receiver stop from dnode:%d. signature:(%" PRId64 ", %" PRId64 ")", + DID(&pReceiver->fromId), pReceiver->term, pReceiver->startTime); } // when recv last snapshot block, apply data into snapshot @@ -1266,7 +1266,6 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { // state, term, seq/ack if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender not leader"); sSError(pSender, "snapshot sender not leader"); terrno = TSDB_CODE_SYN_NOT_LEADER; goto _ERROR; @@ -1274,15 +1273,13 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { SyncTerm currentTerm = raftStoreGetTerm(pSyncNode); if (pMsg->term != currentTerm) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender and receiver term not match"); - sSError(pSender, "snapshot sender term not equal, msg term:%" PRId64 " currentTerm:%" PRId64, pMsg->term, + sSError(pSender, "snapshot sender term mismatch, msg term:%" PRId64 " currentTerm:%" PRId64, pMsg->term, currentTerm); terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; goto _ERROR; } if (pMsg->code != 0) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "receive error code"); sSError(pSender, "snapshot sender receive error:%s 0x%x and stop sender", tstrerror(pMsg->code), pMsg->code); terrno = pMsg->code; goto _ERROR; @@ -1290,12 +1287,10 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { // prepare , send begin msg if (pMsg->ack == SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq pre-snapshot"); return syncNodeOnSnapshotPrepRsp(pSyncNode, pSender, pMsg); } if (pSender->pReader == NULL || pSender->finish) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender invalid"); sSError(pSender, "snapshot sender invalid error:%s 0x%x, pReader:%p finish:%d", tstrerror(pMsg->code), pMsg->code, pSender->pReader, pSender->finish); terrno = pMsg->code; @@ -1303,7 +1298,7 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { } if (pMsg->ack == SYNC_SNAPSHOT_SEQ_BEGIN) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq begin"); + sSInfo(pSender, "process seq begin"); if (snapshotSend(pSender) != 0) { goto _ERROR; } @@ -1312,7 +1307,7 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { // receive ack is finish, close sender if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq end"); + sSInfo(pSender, "process seq end"); snapshotSenderStop(pSender, true); syncNodeReplicateReset(pSyncNode, &pMsg->srcId); return 0; @@ -1320,12 +1315,13 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { // send next msg if (syncSnapBufferSend(pSender, ppMsg) != 0) { + sSError(pSender, "failed to send snapshot msg since %s. seq:%d", terrstr(), pSender->seq); goto _ERROR; } return 0; _ERROR: - snapshotSenderStop(pSender, true); + snapshotSenderStop(pSender, false); syncNodeReplicateReset(pSyncNode, &pMsg->srcId); return -1; } From 4d61e87c0f66fddd0ae3946b553772b8530fc044 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 24 Oct 2023 17:11:32 +0800 Subject: [PATCH 06/37] enh: add signature info in logging msgs of snap replication --- source/libs/sync/src/syncSnapshot.c | 56 +++++++++++++---------------- source/libs/sync/src/syncUtil.c | 51 +++++++++++++------------- 2 files changed, 49 insertions(+), 58 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 195354ac79..ad782bd51d 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -195,8 +195,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { goto _out; } - sSInfo(pSender, "snapshot sender start to dnode:%d. signature:(%" PRId64 ", %" PRId64 ")", DID(&pMsg->destId), - pSender->term, pSender->startTime); + sSInfo(pSender, "snapshot sender start, to dnode:%d.", DID(&pMsg->destId)); code = 0; _out: @@ -233,8 +232,7 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { syncSnapBufferReset(pSender->pSndBuf); SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - sSInfo(pSender, "snapshot sender stop to dnode:%d. signature:(%" PRId64 ", %" PRId64 "), finish:%d", DID(&destId), - pSender->term, pSender->startTime, finish); + sSInfo(pSender, "snapshot sender stop, to dnode:%d, finish:%d", DID(&destId), finish); } // when sender receive ack, call this function to send msg from seq @@ -522,8 +520,7 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p pReceiver->startTime = pPreMsg->startTime; ASSERT(pReceiver->startTime); - sRInfo(pReceiver, "snapshot receiver start from dnode:%d. signature:(%" PRId64 ", %" PRId64 ")", - DID(&pReceiver->fromId), pReceiver->term, pReceiver->startTime); + sRInfo(pReceiver, "snapshot receiver start, from dnode:%d.", DID(&pReceiver->fromId)); } // just set start = false @@ -547,8 +544,7 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { syncSnapBufferReset(pReceiver->pRcvBuf); - sRInfo(pReceiver, "snapshot receiver stop from dnode:%d. signature:(%" PRId64 ", %" PRId64 ")", - DID(&pReceiver->fromId), pReceiver->term, pReceiver->startTime); + sRInfo(pReceiver, "snapshot receiver stop, from dnode:%d.", DID(&pReceiver->fromId)); } // when recv last snapshot block, apply data into snapshot @@ -676,22 +672,22 @@ static int32_t syncNodeOnSnapshotPrep(SSyncNode *pSyncNode, SyncSnapshotSend *pM int32_t order = 0; if ((order = snapshotReceiverSignatureCmp(pReceiver, pMsg)) < 0) { sRInfo(pReceiver, - "received a new snapshot preparation. restart receiver" - "receiver signature: (%" PRId64 ", %" PRId64 "), msg signature:(%" PRId64 ", %" PRId64 ")", - pReceiver->term, pReceiver->startTime, pMsg->term, pMsg->startTime); + "received a new snapshot preparation. restart receiver." + " msg signature:(%" PRId64 ", %" PRId64 ")", + pMsg->term, pMsg->startTime); goto _START_RECEIVER; } else if (order == 0) { sRInfo(pReceiver, - "received a duplicate snapshot preparation. send reply" - "receiver signature: (%" PRId64 ", %" PRId64 "), msg signature:(%" PRId64 ", %" PRId64 ")", - pReceiver->term, pReceiver->startTime, pMsg->term, pMsg->startTime); + "received a duplicate snapshot preparation. send reply." + " msg signature:(%" PRId64 ", %" PRId64 ")", + pMsg->term, pMsg->startTime); goto _SEND_REPLY; } else { // ignore sRError(pReceiver, - "received a stale snapshot preparation. ignore" - "receiver signature: (%" PRId64 ", %" PRId64 "), msg signature:(%" PRId64 ", %" PRId64 ")", - pReceiver->term, pReceiver->startTime, pMsg->term, pMsg->startTime); + "received a stale snapshot preparation. ignore." + " msg signature:(%" PRId64 ", %" PRId64 ")", + pMsg->term, pMsg->startTime); terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; code = terrno; goto _SEND_REPLY; @@ -809,6 +805,8 @@ static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *p goto _SEND_REPLY; } + sRInfo(pReceiver, "snapshot begin"); + code = 0; _SEND_REPLY: if (code != 0 && terrno != 0) { @@ -1009,8 +1007,7 @@ _SEND_REPLY:; // int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { SyncSnapshotSend **ppMsg = (SyncSnapshotSend **)&pRpcMsg->pCont; - SyncSnapshotSend *pMsg = ppMsg[0]; - ASSERT(pMsg); + SyncSnapshotSend *pMsg = ppMsg[0]; SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; // if already drop replica, do not process @@ -1040,16 +1037,16 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER || pSyncNode->state == TAOS_SYNC_STATE_LEARNER) { if (pMsg->term == raftStoreGetTerm(pSyncNode)) { if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT) { - sInfo("vgId:%d, receive pre-snapshot msg of snapshot replication. signature:(%" PRId64 ", %" PRId64 ")", + sInfo("vgId:%d, receive prepare msg of snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", pSyncNode->vgId, pMsg->term, pMsg->startTime); code = syncNodeOnSnapshotPrep(pSyncNode, pMsg); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { - sInfo("vgId:%d, receive begin msg of snapshot replication. signature:(%" PRId64 ", %" PRId64 ")", + sInfo("vgId:%d, receive begin msg of snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", pSyncNode->vgId, pMsg->term, pMsg->startTime); code = syncNodeOnSnapshotBegin(pSyncNode, pMsg); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { - sInfo("vgId:%d, receive end msg of snapshot replication. signature: (%" PRId64 ", %" PRId64 ")", - pSyncNode->vgId, pMsg->term, pMsg->startTime); + sInfo("vgId:%d, receive end msg of snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", pSyncNode->vgId, + pMsg->term, pMsg->startTime); code = syncNodeOnSnapshotEnd(pSyncNode, pMsg); if (syncLogBufferReInit(pSyncNode->pLogBuf, pSyncNode) != 0) { sRError(pReceiver, "failed to reinit log buffer since %s", terrstr()); @@ -1059,7 +1056,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { // force close, no response syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process force stop"); snapshotReceiverStop(pReceiver); - } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq <= SYNC_SNAPSHOT_SEQ_END) { + } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq data"); code = syncNodeOnSnapshotReceive(pSyncNode, ppMsg); } else { @@ -1139,10 +1136,7 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend pSendMsg->startTime = pSender->startTime; pSendMsg->seq = SYNC_SNAPSHOT_SEQ_BEGIN; - ASSERT(pSendMsg->startTime); - - sSInfo(pSender, "begin snapshot replication to dnode %d. startTime:%" PRId64, DID(&pSendMsg->destId), - pSendMsg->startTime); + sSInfo(pSender, "begin snapshot replication to dnode %d." PRId64, DID(&pSendMsg->destId)); // send msg syncLogSendSyncSnapshotSend(pSyncNode, pSendMsg, "snapshot sender reply pre"); @@ -1252,10 +1246,8 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { // check signature int32_t order = 0; if ((order = snapshotSenderSignatureCmp(pSender, pMsg)) > 0) { - sSError(pSender, - "received a stale snapshot rsp. ignore it" - "sender signature: (%" PRId64 ", %" PRId64 "), msg signature:(%" PRId64 ", %" PRId64 ")", - pSender->term, pSender->startTime, pMsg->term, pMsg->startTime); + sSError(pSender, "received a stale snapshot rsp, msg signature:(%" PRId64 ", %" PRId64 "), ignore it.", pMsg->term, + pMsg->startTime); terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; return -1; } else if (order < 0) { diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 9acc17e130..dc16b0e958 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -266,22 +266,21 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla int32_t writeLen = vsnprintf(eventLog, sizeof(eventLog), format, argpointer); va_end(argpointer); - taosPrintLog(flags, level, dflag, - "vgId:%d, %s, sync:%s, snap-sender:{%p start:%" PRId64 " end:%" PRId64 " last-index:%" PRId64 - " last-term:%" PRIu64 " last-cfg:%" PRId64 - ", seq:%d ack:%d finish:%d, as:%d dnode:%d}" - ", term:%" PRIu64 ", commit-index:%" PRId64 ", firstver:%" PRId64 ", lastver:%" PRId64 - ", min-match:%" PRId64 ", snap:{last-index:%" PRId64 ", term:%" PRIu64 - "}, standby:%d, batch-sz:%d, replicas:%d, last-cfg:%" PRId64 - ", chging:%d, restore:%d, quorum:%d, lc-timer:{elect:%" PRId64 ", hb:%" PRId64 "}, peer:%s, cfg:%s", - pNode->vgId, eventLog, syncStr(pNode->state), pSender, pSender->snapshotParam.start, - pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, - pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, pSender->finish, pSender->replicaIndex, - DID(&pNode->replicasId[pSender->replicaIndex]), raftStoreGetTerm(pNode), pNode->commitIndex, - logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, - pNode->raftCfg.isStandBy, pNode->raftCfg.batchSize, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, - pNode->changing, pNode->restoreFinish, syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, - pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr); + taosPrintLog( + flags, level, dflag, + "vgId:%d, %s, sync:%s, snap-sender:%p signature:(%" PRId64 ", %" PRId64 "), {start:%" PRId64 " end:%" PRId64 + " last-index:%" PRId64 " last-term:%" PRIu64 " last-cfg:%" PRId64 + ", seq:%d ack:%d finish:%d, as:%d, to-dnode:%d}" + ", term:%" PRIu64 ", commit-index:%" PRId64 ", firstver:%" PRId64 ", lastver:%" PRId64 ", min-match:%" PRId64 + ", snap:{last-index:%" PRId64 ", term:%" PRIu64 "}, standby:%d, batch-sz:%d, replicas:%d, last-cfg:%" PRId64 + ", chging:%d, restore:%d, quorum:%d, peer:%s, cfg:%s", + pNode->vgId, eventLog, syncStr(pNode->state), pSender, pSender->term, pSender->startTime, + pSender->snapshotParam.start, pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex, + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, pSender->finish, + pSender->replicaIndex, DID(&pNode->replicasId[pSender->replicaIndex]), raftStoreGetTerm(pNode), + pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, + snapshot.lastApplyTerm, pNode->raftCfg.isStandBy, pNode->raftCfg.batchSize, pNode->replicaNum, + pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, pNode->quorum, peerStr, cfgStr); } void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t dflag, SSyncSnapshotReceiver* pReceiver, @@ -316,19 +315,19 @@ void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t df taosPrintLog( flags, level, dflag, "vgId:%d, %s, sync:%s," - " snap-receiver:{%p started:%d acked:%d term:%" PRIu64 " start-time:%" PRId64 " from-dnode:%d, start:%" PRId64 - " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRIu64 " last-cfg:%" PRId64 + " snap-receiver:%p signature:(%" PRId64 ", %" PRId64 "), {start:%d ack:%d term:%" PRIu64 " start-time:%" PRId64 + " from-dnode:%d, start:%" PRId64 " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRIu64 " last-cfg:%" PRId64 "}" ", term:%" PRIu64 ", commit-index:%" PRId64 ", firstver:%" PRId64 ", lastver:%" PRId64 ", min-match:%" PRId64 ", snap:{last-index:%" PRId64 ", last-term:%" PRIu64 "}, standby:%d, batch-sz:%d, replicas:%d, last-cfg:%" PRId64 - ", chging:%d, restore:%d, quorum:%d, lc-timers:{elect:%" PRId64 ", hb:%" PRId64 "}, peer:%s, cfg:%s", - pNode->vgId, eventLog, syncStr(pNode->state), pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term, - pReceiver->startTime, DID(&pReceiver->fromId), pReceiver->snapshotParam.start, pReceiver->snapshotParam.end, - pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm, pReceiver->snapshot.lastConfigIndex, - raftStoreGetTerm(pNode), pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, - snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->raftCfg.isStandBy, pNode->raftCfg.batchSize, - pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, - syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr); + ", chging:%d, restore:%d, quorum:%d, peer:%s, cfg:%s", + pNode->vgId, eventLog, syncStr(pNode->state), pReceiver, pReceiver->term, pReceiver->startTime, pReceiver->start, + pReceiver->ack, pReceiver->term, pReceiver->startTime, DID(&pReceiver->fromId), pReceiver->snapshotParam.start, + pReceiver->snapshotParam.end, pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm, + pReceiver->snapshot.lastConfigIndex, raftStoreGetTerm(pNode), pNode->commitIndex, logBeginIndex, logLastIndex, + pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->raftCfg.isStandBy, + pNode->raftCfg.batchSize, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, + pNode->restoreFinish, pNode->quorum, peerStr, cfgStr); } void syncLogRecvTimer(SSyncNode* pSyncNode, const SyncTimeout* pMsg, const char* s) { From 95188ab8c387d10ad54c7a58a8c4264dc3274864 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 24 Oct 2023 17:42:17 +0800 Subject: [PATCH 07/37] enh: adjust some logging msgs for sync log replication mgr --- source/libs/sync/src/syncPipeline.c | 34 ++++++++++++++--------------- source/libs/sync/src/syncSnapshot.c | 8 +++---- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index a7ee37cc3b..f2386797c1 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -797,7 +797,7 @@ _out: pMgr->retryBackoff = syncLogReplGetNextRetryBackoff(pMgr); SSyncLogBuffer* pBuf = pNode->pLogBuf; sInfo("vgId:%d, resend %d sync log entries. dest:%" PRIx64 ", indexes:%" PRId64 " ..., terms: ... %" PRId64 - ", retryWaitMs:%" PRId64 ", mgr: [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 + ", retryWaitMs:%" PRId64 ", repl-mgr:[%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, count, pDestId->addr, firstIndex, term, retryWaitMs, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); @@ -815,9 +815,9 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn ASSERT(pMgr->matchIndex == 0); if (pMsg->matchIndex < 0) { pMgr->restored = true; - sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 - ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", - pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, + sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), repl-mgr:[%" PRId64 " %" PRId64 ", %" PRId64 + "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", + pNode->vgId, DID(&destId), destId.addr, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); return 0; } @@ -832,9 +832,9 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn if (pMsg->success && pMsg->matchIndex == pMsg->lastSendIndex) { pMgr->matchIndex = pMsg->matchIndex; pMgr->restored = true; - sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 - ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", - pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, + sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), repl-mgr:[%" PRId64 " %" PRId64 ", %" PRId64 + "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", + pNode->vgId, DID(&destId), destId.addr, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); return 0; } @@ -958,10 +958,10 @@ int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex inde pMgr->endIndex = index + 1; SSyncLogBuffer* pBuf = pNode->pLogBuf; - sTrace("vgId:%d, probe peer:%" PRIx64 " with msg of index:%" PRId64 " term:%" PRId64 ". mgr (rs:%d): [%" PRId64 - " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", - pNode->vgId, pDestId->addr, index, term, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, - pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); + sTrace("vgId:%d, probe peer:%" PRIx64 " with msg of index:%" PRId64 " term:%" PRId64 ". repl-mgr:[%" PRId64 + " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", + pNode->vgId, pDestId->addr, index, term, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex, + pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); return 0; } @@ -1002,9 +1002,9 @@ int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { pMgr->endIndex = index + 1; if (barrier) { - sInfo("vgId:%d, replicated sync barrier to dnode:%d. index:%" PRId64 ", term:%" PRId64 - ", repl mgr: rs(%d) [%" PRId64 " %" PRId64 ", %" PRId64 ")", - pNode->vgId, DID(pDestId), index, term, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex); + sInfo("vgId:%d, replicated sync barrier to dnode:%d. index:%" PRId64 ", term:%" PRId64 ", repl-mgr:[%" PRId64 + " %" PRId64 ", %" PRId64 ")", + pNode->vgId, DID(pDestId), index, term, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex); break; } } @@ -1013,10 +1013,10 @@ int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { SSyncLogBuffer* pBuf = pNode->pLogBuf; sTrace("vgId:%d, replicated %d msgs to peer:%" PRIx64 ". indexes:%" PRId64 "..., terms: ...%" PRId64 - ", mgr: (rs:%d) [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 + ", repl-mgr:[%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", - pNode->vgId, count, pDestId->addr, firstIndex, term, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, - pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); + pNode->vgId, count, pDestId->addr, firstIndex, term, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, + pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); return 0; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index ad782bd51d..7e431111f2 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -258,13 +258,11 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { if (pSender->blockLen > 0) { // has read data - sSDebug(pSender, "vgId:%d, snapshot sender continue to read, blockLen:%d seq:%d", pSender->pSyncNode->vgId, - pSender->blockLen, pSender->seq); + sSDebug(pSender, "snapshot sender continue to read, blockLen:%d seq:%d", pSender->blockLen, pSender->seq); } else { // read finish, update seq to end pSender->seq = SYNC_SNAPSHOT_SEQ_END; - sSInfo(pSender, "vgId:%d, snapshot sender read to the end, blockLen:%d seq:%d", pSender->pSyncNode->vgId, - pSender->blockLen, pSender->seq); + sSInfo(pSender, "snapshot sender read to the end, blockLen:%d seq:%d", pSender->blockLen, pSender->seq); return 0; } } @@ -1136,7 +1134,7 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend pSendMsg->startTime = pSender->startTime; pSendMsg->seq = SYNC_SNAPSHOT_SEQ_BEGIN; - sSInfo(pSender, "begin snapshot replication to dnode %d." PRId64, DID(&pSendMsg->destId)); + sSInfo(pSender, "begin snapshot replication to dnode %d.", DID(&pSendMsg->destId)); // send msg syncLogSendSyncSnapshotSend(pSyncNode, pSendMsg, "snapshot sender reply pre"); From 374217cefffefcedae3c68a2a52d9a4eeed47b86 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 24 Oct 2023 19:20:53 +0800 Subject: [PATCH 08/37] fixup: adjust a logging msg in syncNodeStartSnapshot --- source/libs/sync/src/syncSnapshot.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 7e431111f2..c2ba30a6bb 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -383,8 +383,6 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { return 0; } - sSInfo(pSender, "snapshot sender start"); - int32_t code = snapshotSenderStart(pSender); if (code != 0) { sSError(pSender, "snapshot sender start error since %s", terrstr()); @@ -803,8 +801,6 @@ static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *p goto _SEND_REPLY; } - sRInfo(pReceiver, "snapshot begin"); - code = 0; _SEND_REPLY: if (code != 0 && terrno != 0) { From 4e3a8b893df4c6ee66eb887758f0da0126e617bc Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 24 Oct 2023 20:03:48 +0800 Subject: [PATCH 09/37] enh: restore wal log from snapshot after finish --- source/libs/sync/src/syncSnapshot.c | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index c2ba30a6bb..8a83891735 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -539,8 +539,6 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { } syncSnapBufferReset(pReceiver->pRcvBuf); - - sRInfo(pReceiver, "snapshot receiver stop, from dnode:%d.", DID(&pReceiver->fromId)); } // when recv last snapshot block, apply data into snapshot @@ -548,7 +546,7 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap int32_t code = 0; if (pReceiver->pWriter != NULL) { // write data - sRInfo(pReceiver, "snapshot receiver write finish, blockLen:%d seq:%d", pMsg->dataLen, pMsg->seq); + sRInfo(pReceiver, "snapshot receiver write about to finish, blockLen:%d seq:%d", pMsg->dataLen, pMsg->seq); if (pMsg->dataLen > 0) { code = pReceiver->pSyncNode->pFsm->FpSnapshotDoWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); @@ -558,15 +556,6 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap } } - // reset wal - sRInfo(pReceiver, "snapshot receiver log restore"); - code = - pReceiver->pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pReceiver->pSyncNode->pLogStore, pMsg->lastIndex); - if (code != 0) { - sRError(pReceiver, "failed to snapshot receiver log restore since %s", terrstr()); - return -1; - } - // update commit index if (pReceiver->snapshot.lastApplyIndex > pReceiver->pSyncNode->commitIndex) { pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex; @@ -578,7 +567,6 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap } // stop writer, apply data - sRInfo(pReceiver, "snapshot receiver apply write"); code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true, &pReceiver->snapshot); if (code != 0) { @@ -586,10 +574,21 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap return -1; } pReceiver->pWriter = NULL; + sRInfo(pReceiver, "snapshot receiver write stopped"); // update progress pReceiver->ack = SYNC_SNAPSHOT_SEQ_END; + // reset wal + code = + pReceiver->pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pReceiver->pSyncNode->pLogStore, pMsg->lastIndex); + if (code != 0) { + sRError(pReceiver, "failed to snapshot receiver log restore since %s", terrstr()); + return -1; + } + sRInfo(pReceiver, "wal log restored from snapshot"); + + // get fsmState SSnapshot snapshot = {0}; pReceiver->pSyncNode->pFsm->FpGetSnapshotInfo(pReceiver->pSyncNode->pFsm, &snapshot); pReceiver->pSyncNode->fsmState = snapshot.state; @@ -599,8 +598,6 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap return -1; } - // event log - sRInfo(pReceiver, "snapshot receiver got last data and apply snapshot finished"); return 0; } From 8b2d57504c50de10130808abc3ada23153790c84 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Oct 2023 13:54:45 +0800 Subject: [PATCH 10/37] enhance: bi mode default tag scan --- source/libs/parser/src/parAstCreater.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 06c098af92..7ad61f4324 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -978,6 +978,9 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr SNodeList* pHint) { CHECK_PARSER_STATUS(pCxt); SNode* select = createSelectStmtImpl(isDistinct, pProjectionList, pTable, pHint); + if (pCxt->pQueryCxt->biMode) { + setSelectStmtTagMode(pCxt, select, true); + } CHECK_OUT_OF_MEM(select); return select; } From 7d9d1c6877a8fee601822d4199c210eac297cb22 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 25 Oct 2023 14:50:53 +0800 Subject: [PATCH 11/37] enh: terminate snap replication on timeout --- include/libs/sync/sync.h | 1 + source/libs/sync/src/syncTimeout.c | 8 +++----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index ad525a2aa7..e3b5f3611c 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -46,6 +46,7 @@ extern "C" { #define SYNC_HEARTBEAT_SLOW_MS 1500 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 #define SYNC_SNAP_RESEND_MS 1000 * 60 +#define SYNC_SNAP_TIMEOUT_MS 1000 * 180 #define SYNC_VND_COMMIT_MIN_MS 3000 diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 37166805ce..91c7494fa4 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -78,11 +78,9 @@ static int32_t syncNodeTimerRoutine(SSyncNode* ths) { SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(ths->peersId[i])); if (pSender != NULL) { if (ths->isStart && ths->state == TAOS_SYNC_STATE_LEADER && pSender->start && - timeNow - pSender->lastSendTime > SYNC_SNAP_RESEND_MS) { - snapshotReSend(pSender); - } else { - sTrace("vgId:%d, do not resend: nstart%d, now:%" PRId64 ", lstsend:%" PRId64 ", diff:%" PRId64, ths->vgId, - ths->isStart, timeNow, pSender->lastSendTime, timeNow - pSender->lastSendTime); + timeNow - pSender->lastSendTime > SYNC_SNAP_TIMEOUT_MS) { + sSError(pSender, "snapshot timeout, terminate. lastSendTime:%d", pSender->lastSendTime); + snapshotSenderStop(pSender, false); } } } From ec5b1f2ec175b33c5aa8f6b40ce98e006787d2fa Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 25 Oct 2023 15:04:14 +0800 Subject: [PATCH 12/37] enh: logging buf info for snapshot reader and sender --- source/libs/sync/inc/syncSnapshot.h | 2 +- source/libs/sync/src/syncSnapshot.c | 16 ++-------- source/libs/sync/src/syncUtil.c | 49 ++++++++++++++++------------- 3 files changed, 30 insertions(+), 37 deletions(-) diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 0332204769..93c62544f2 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -56,7 +56,7 @@ typedef struct SSyncSnapshotSender { int64_t lastSendTime; bool finish; - // buffer + // ring buffer for ack SSyncSnapBuffer *pSndBuf; // init when create diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 8a83891735..751f777b18 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -345,20 +345,6 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { return 0; } -static int32_t snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { - if (pMsg->ack != pSender->seq) { - sSError(pSender, "snapshot sender update seq failed, ack:%d seq:%d", pMsg->ack, pSender->seq); - terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; - return -1; - } - - pSender->ack = pMsg->ack; - pSender->seq++; - - sSDebug(pSender, "snapshot sender update seq:%d", pSender->seq); - return 0; -} - // return 0, start ok // return 1, last snapshot finish ok // return -1, error @@ -1182,6 +1168,8 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp pSndBuf->start = ack + 1; } + pSender->ack = pSndBuf->start - 1; + while (pSender->seq != SYNC_SNAPSHOT_SEQ_END && pSender->seq - pSndBuf->start < (pSndBuf->size >> 2)) { if (snapshotSend(pSender) != 0) { code = terrno; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index dc16b0e958..9e6ea94e78 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -266,21 +266,24 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla int32_t writeLen = vsnprintf(eventLog, sizeof(eventLog), format, argpointer); va_end(argpointer); - taosPrintLog( - flags, level, dflag, - "vgId:%d, %s, sync:%s, snap-sender:%p signature:(%" PRId64 ", %" PRId64 "), {start:%" PRId64 " end:%" PRId64 - " last-index:%" PRId64 " last-term:%" PRIu64 " last-cfg:%" PRId64 - ", seq:%d ack:%d finish:%d, as:%d, to-dnode:%d}" - ", term:%" PRIu64 ", commit-index:%" PRId64 ", firstver:%" PRId64 ", lastver:%" PRId64 ", min-match:%" PRId64 - ", snap:{last-index:%" PRId64 ", term:%" PRIu64 "}, standby:%d, batch-sz:%d, replicas:%d, last-cfg:%" PRId64 - ", chging:%d, restore:%d, quorum:%d, peer:%s, cfg:%s", - pNode->vgId, eventLog, syncStr(pNode->state), pSender, pSender->term, pSender->startTime, - pSender->snapshotParam.start, pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, pSender->finish, - pSender->replicaIndex, DID(&pNode->replicasId[pSender->replicaIndex]), raftStoreGetTerm(pNode), - pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, - snapshot.lastApplyTerm, pNode->raftCfg.isStandBy, pNode->raftCfg.batchSize, pNode->replicaNum, - pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, pNode->quorum, peerStr, cfgStr); + taosPrintLog(flags, level, dflag, + "vgId:%d, %s, sync:%s, snap-sender:%p signature:(%" PRId64 ", %" PRId64 "), {start:%" PRId64 + " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRIu64 " last-cfg:%" PRId64 + ", seq:%d, ack:%d, " + " buf:[%" PRId64 " %" PRId64 ", %" PRId64 + "), finish:%d, as:%d, to-dnode:%d}" + ", term:%" PRIu64 ", commit-index:%" PRId64 ", firstver:%" PRId64 ", lastver:%" PRId64 + ", min-match:%" PRId64 ", snap:{last-index:%" PRId64 ", term:%" PRIu64 + "}, standby:%d, batch-sz:%d, replicas:%d, last-cfg:%" PRId64 + ", chging:%d, restore:%d, quorum:%d, peer:%s, cfg:%s", + pNode->vgId, eventLog, syncStr(pNode->state), pSender, pSender->term, pSender->startTime, + pSender->snapshotParam.start, pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex, + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, + pSender->pSndBuf->start, pSender->pSndBuf->cursor, pSender->pSndBuf->end, pSender->finish, + pSender->replicaIndex, DID(&pNode->replicasId[pSender->replicaIndex]), raftStoreGetTerm(pNode), + pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, + snapshot.lastApplyTerm, pNode->raftCfg.isStandBy, pNode->raftCfg.batchSize, pNode->replicaNum, + pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, pNode->quorum, peerStr, cfgStr); } void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t dflag, SSyncSnapshotReceiver* pReceiver, @@ -315,19 +318,21 @@ void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t df taosPrintLog( flags, level, dflag, "vgId:%d, %s, sync:%s," - " snap-receiver:%p signature:(%" PRId64 ", %" PRId64 "), {start:%d ack:%d term:%" PRIu64 " start-time:%" PRId64 + " snap-receiver:%p signature:(%" PRId64 ", %" PRId64 "), {start:%d ack:%d buf:[%" PRId64 " %" PRId64 ", %" PRId64 + ")" " from-dnode:%d, start:%" PRId64 " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRIu64 " last-cfg:%" PRId64 "}" ", term:%" PRIu64 ", commit-index:%" PRId64 ", firstver:%" PRId64 ", lastver:%" PRId64 ", min-match:%" PRId64 ", snap:{last-index:%" PRId64 ", last-term:%" PRIu64 "}, standby:%d, batch-sz:%d, replicas:%d, last-cfg:%" PRId64 ", chging:%d, restore:%d, quorum:%d, peer:%s, cfg:%s", pNode->vgId, eventLog, syncStr(pNode->state), pReceiver, pReceiver->term, pReceiver->startTime, pReceiver->start, - pReceiver->ack, pReceiver->term, pReceiver->startTime, DID(&pReceiver->fromId), pReceiver->snapshotParam.start, - pReceiver->snapshotParam.end, pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm, - pReceiver->snapshot.lastConfigIndex, raftStoreGetTerm(pNode), pNode->commitIndex, logBeginIndex, logLastIndex, - pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->raftCfg.isStandBy, - pNode->raftCfg.batchSize, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, - pNode->restoreFinish, pNode->quorum, peerStr, cfgStr); + pReceiver->ack, pReceiver->pRcvBuf->start, pReceiver->pRcvBuf->cursor, pReceiver->pRcvBuf->end, + DID(&pReceiver->fromId), pReceiver->snapshotParam.start, pReceiver->snapshotParam.end, + pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm, pReceiver->snapshot.lastConfigIndex, + raftStoreGetTerm(pNode), pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, + snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->raftCfg.isStandBy, pNode->raftCfg.batchSize, + pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, pNode->quorum, peerStr, + cfgStr); } void syncLogRecvTimer(SSyncNode* pSyncNode, const SyncTimeout* pMsg, const char* s) { From 25aade13336149bcbcd3bd5384b554d3b9d368b0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 25 Oct 2023 19:44:21 +0800 Subject: [PATCH 13/37] feat: alloc disk acorrding to avail disk space --- source/libs/tfs/src/tfsTier.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index 1c47182e2a..d4f228a537 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -110,7 +110,9 @@ int32_t tfsAllocDiskOnTier(STfsTier *pTier) { } int32_t retId = -1; + int64_t avail = 0; for (int32_t id = 0; id < TFS_MAX_DISKS_PER_TIER; ++id) { +#if 0 // round-robin int32_t diskId = (pTier->nextid + id) % pTier->ndisk; STfsDisk *pDisk = pTier->disks[diskId]; @@ -126,6 +128,18 @@ int32_t tfsAllocDiskOnTier(STfsTier *pTier) { terrno = 0; pTier->nextid = (diskId + 1) % pTier->ndisk; break; +#else // select the disk with the most available space + STfsDisk *pDisk = pTier->disks[id]; + if (pDisk == NULL) continue; + + if (pDisk->size.avail < tsMinDiskFreeSize) continue; + + if (pDisk->size.avail > avail) { + avail = pDisk->size.avail; + retId = id; + terrno = 0; + } +#endif } tfsUnLockTier(pTier); From fe553352b85812fc4c24934f50fd440e6007f44e Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 25 Oct 2023 20:25:57 +0800 Subject: [PATCH 14/37] enh: cache snap blocks sent in snapshotSend for resending --- source/libs/sync/inc/syncSnapshot.h | 13 ++++ source/libs/sync/src/syncSnapshot.c | 108 +++++++++++++++++----------- 2 files changed, 81 insertions(+), 40 deletions(-) diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 93c62544f2..e68702568a 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -37,8 +37,21 @@ typedef struct SSyncSnapBuffer { int64_t end; int64_t size; TdThreadMutex mutex; + void (*entryDeleteCb)(void *ptr); } SSyncSnapBuffer; +typedef struct SyncSnapBlock { + int32_t seq; + int8_t acked; + int64_t sendTimeMs; + + int16_t blockType; + void *pBlock; + int32_t blockLen; +} SyncSnapBlock; + +void syncSnapBlockDestroy(void *ptr); + typedef struct SSyncSnapshotSender { int8_t start; int32_t seq; diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 751f777b18..eee0ab2cc9 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -26,7 +26,9 @@ static void syncSnapBufferReset(SSyncSnapBuffer *pBuf) { taosThreadMutexLock(&pBuf->mutex); for (int64_t i = pBuf->start; i < pBuf->end; ++i) { - rpcFreeCont(pBuf->entries[i % pBuf->size]); + if (pBuf->entryDeleteCb) { + pBuf->entryDeleteCb(pBuf->entries[i % pBuf->size]); + } pBuf->entries[i % pBuf->size] = NULL; } pBuf->start = 1; @@ -85,17 +87,29 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &pSender->snapshot); pSender->finish = false; - pSender->pSndBuf = syncSnapBufferCreate(); - if (pSender->pSndBuf == NULL) { + SSyncSnapBuffer *pSndBuf = syncSnapBufferCreate(); + if (pSndBuf == NULL) { taosMemoryFree(pSender); pSender = NULL; return NULL; } - syncSnapBufferReset(pSender->pSndBuf); + pSndBuf->entryDeleteCb = syncSnapBlockDestroy; + pSender->pSndBuf = pSndBuf; + syncSnapBufferReset(pSender->pSndBuf); return pSender; } +void syncSnapBlockDestroy(void *ptr) { + SyncSnapBlock *pBlk = ptr; + if (pBlk->pBlock != NULL) { + taosMemoryFree(pBlk->pBlock); + pBlk->pBlock = NULL; + pBlk->blockLen = 0; + } + taosMemoryFree(pBlk); +} + void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { if (pSender == NULL) return; @@ -238,23 +252,26 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { // when sender receive ack, call this function to send msg from seq // seq = ack + 1, already updated static int32_t snapshotSend(SSyncSnapshotSender *pSender) { - // free memory last time (current seq - 1) - if (pSender->pCurrentBlock != NULL) { - taosMemoryFree(pSender->pCurrentBlock); - pSender->pCurrentBlock = NULL; - pSender->blockLen = 0; - } + int32_t code = -1; + SyncSnapBlock *pBlk = NULL; if (pSender->seq != SYNC_SNAPSHOT_SEQ_END) { pSender->seq++; + pBlk = taosMemoryCalloc(1, sizeof(SyncSnapBlock)); + if (pBlk == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OUT; + } + // read data - int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader, - &pSender->pCurrentBlock, &pSender->blockLen); + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader, &pBlk->pBlock, + &pBlk->blockLen); if (ret != 0) { sSError(pSender, "snapshot sender read failed since %s", terrstr()); - return -1; + goto _OUT; } + pBlk->seq = pSender->seq; if (pSender->blockLen > 0) { // has read data @@ -263,7 +280,8 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { // read finish, update seq to end pSender->seq = SYNC_SNAPSHOT_SEQ_END; sSInfo(pSender, "snapshot sender read to the end, blockLen:%d seq:%d", pSender->blockLen, pSender->seq); - return 0; + code = 0; + goto _OUT; } } @@ -271,7 +289,7 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { SRpcMsg rpcMsg = {0}; if (syncBuildSnapshotSend(&rpcMsg, pSender->blockLen, pSender->pSyncNode->vgId) != 0) { sSError(pSender, "vgId:%d, snapshot sender build msg failed since %s", pSender->pSyncNode->vgId, terrstr()); - return -1; + goto _OUT; } SyncSnapshotSend *pMsg = rpcMsg.pCont; @@ -286,25 +304,35 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { pMsg->startTime = pSender->startTime; pMsg->seq = pSender->seq; - if (pSender->pCurrentBlock != NULL) { - memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); - } - - // event log - if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { - syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender finish"); - } else { - syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender sending"); + if (pBlk != NULL && pBlk->pBlock != NULL) { + memcpy(pMsg->data, pBlk->pBlock, pBlk->blockLen); } // send msg if (syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg) != 0) { sSError(pSender, "snapshot sender send msg failed since %s", terrstr()); - return -1; + goto _OUT; } - pSender->lastSendTime = taosGetTimestampMs(); - return 0; + // put in buffer + int64_t nowMs = taosGetTimestampMs(); + if (pBlk) { + ASSERT(pBlk->seq != SYNC_SNAPSHOT_SEQ_END); + pBlk->sendTimeMs = nowMs; + pSender->pSndBuf->entries[pSender->seq % pSender->pSndBuf->size] = pBlk; + pBlk = NULL; + pSender->pSndBuf->end = pSender->seq + 1; + } + + pSender->lastSendTime = nowMs; + code = 0; + +_OUT:; + if (pBlk != NULL) { + syncSnapBlockDestroy(pBlk); + pBlk = NULL; + } + return code; } // send snapshot data from cache @@ -319,7 +347,7 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { SyncSnapshotSend *pMsg = rpcMsg.pCont; pMsg->srcId = pSender->pSyncNode->myRaftId; pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - pMsg->term = raftStoreGetTerm(pSender->pSyncNode); + pMsg->term = pSender->term; pMsg->beginIndex = pSender->snapshotParam.start; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; @@ -332,9 +360,6 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); } - // event log - syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender resend"); - // send msg if (syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg) != 0) { sSError(pSender, "snapshot sender resend msg failed since %s", terrstr()); @@ -401,12 +426,14 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from pReceiver->snapshot.lastApplyTerm = 0; pReceiver->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; - pReceiver->pRcvBuf = syncSnapBufferCreate(); - if (pReceiver->pRcvBuf == NULL) { + SSyncSnapBuffer *pRcvBuf = syncSnapBufferCreate(); + if (pRcvBuf == NULL) { taosMemoryFree(pReceiver); pReceiver = NULL; return NULL; } + pRcvBuf->entryDeleteCb = rpcFreeCont; + pReceiver->pRcvBuf = pRcvBuf; syncSnapBufferReset(pReceiver->pRcvBuf); return pReceiver; @@ -884,7 +911,7 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot } pRcvBuf->start = seq + 1; syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], code); - rpcFreeCont(pRcvBuf->entries[seq % pRcvBuf->size]); + pRcvBuf->entryDeleteCb(pRcvBuf->entries[seq % pRcvBuf->size]); pRcvBuf->entries[seq % pRcvBuf->size] = NULL; if (code) goto _out; } @@ -1148,14 +1175,15 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp ASSERT(pSndBuf->start <= pSndBuf->cursor + 1 && pSndBuf->cursor < pSndBuf->end); - if (pMsg->ack > pSndBuf->cursor && pSndBuf->entries[pMsg->ack % pSndBuf->size] == NULL) { - pSndBuf->entries[pMsg->ack % pSndBuf->size] = pMsg; - ppMsg[0] = NULL; - pSndBuf->end = TMAX(pMsg->ack + 1, pSndBuf->end); + if (pMsg->ack > pSndBuf->cursor && pMsg->ack < pSndBuf->end) { + SyncSnapBlock *pBlk = pSndBuf->entries[pMsg->ack % pSndBuf->size]; + ASSERT(pBlk); + pBlk->acked = 1; } for (int64_t ack = pSndBuf->cursor + 1; ack < pSndBuf->end; ++ack) { - if (pSndBuf->entries[ack % pSndBuf->size]) { + SyncSnapBlock *pBlk = pSndBuf->entries[ack % pSndBuf->size]; + if (pBlk->acked) { pSndBuf->cursor = ack; } else { break; @@ -1163,7 +1191,7 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp } for (int64_t ack = pSndBuf->start; ack <= pSndBuf->cursor; ++ack) { - rpcFreeCont(pSndBuf->entries[ack % pSndBuf->size]); + pSndBuf->entryDeleteCb(pSndBuf->entries[ack % pSndBuf->size]); pSndBuf->entries[ack % pSndBuf->size] = NULL; pSndBuf->start = ack + 1; } From dc03d8cd1a7638b2b5d9b2e99bb12979f98767c6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 26 Oct 2023 20:07:38 +0800 Subject: [PATCH 15/37] disable tfs unit test --- source/libs/tfs/test/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/tfs/test/CMakeLists.txt b/source/libs/tfs/test/CMakeLists.txt index ee28dcf723..2fd0836a1d 100644 --- a/source/libs/tfs/test/CMakeLists.txt +++ b/source/libs/tfs/test/CMakeLists.txt @@ -8,7 +8,7 @@ target_link_libraries( PUBLIC gtest_main ) -add_test( - NAME tfs_test - COMMAND tfs_test -) +# add_test( +# NAME tfs_test +# COMMAND tfs_test +# ) From 70850697a48445e103badb6519e2642ff059e0f8 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 25 Oct 2023 20:01:22 +0800 Subject: [PATCH 16/37] feat: support to_timestamp/to_char fix comments --- docs/en/12-taos-sql/10-function.md | 5 +- docs/zh/12-taos-sql/10-function.md | 9 +- include/common/ttime.h | 13 +- source/common/src/ttime.c | 229 ++++++++++++------ source/common/test/commonTests.cpp | 20 +- source/libs/function/src/builtins.c | 2 +- source/libs/scalar/src/sclfunc.c | 39 ++- .../2-query/func_to_char_timestamp.py | 25 +- 8 files changed, 236 insertions(+), 106 deletions(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 266cdb4958..c986a98e46 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -486,7 +486,7 @@ return_timestamp: { #### TO_CHAR ```sql -TO_CHAR(ts, str_literal) +TO_CHAR(ts, format_str_literal) ``` **Description**: Convert a ts column to string as the format specified @@ -539,11 +539,12 @@ TO_CHAR(ts, str_literal) - When `ms`,`us`,`ns` are used in `to_char`, like `to_char(ts, 'yyyy-mm-dd hh:mi:ss.ms.us.ns')`, The time of `ms`,`us`,`ns` corresponds to the same fraction seconds. When ts is `1697182085123`, the output of `ms` is `123`, `us` is `123000`, `ns` is `123000000`. - If we want to output some characters of format without converting, surround it with double quotes. `to_char(ts, 'yyyy-mm-dd "is formated by yyyy-mm-dd"')`. If want to output double quotes, add a back slash before double quote, like `to_char(ts, '\"yyyy-mm-dd\"')` will output `"2023-10-10"`. - For formats that output digits, the uppercase and lowercase formats are the same. +- The local time zone will be used to convert the timestamp. #### TO_TIMESTAMP ```sql -TO_TIMESTAMP(str_literal, str_literal) +TO_TIMESTAMP(ts_str_literal, format_str_literal) ``` **Description**: Convert a formated timestamp string to a timestamp diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 806ff3c6a8..44ab3d5091 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -486,7 +486,7 @@ return_timestamp: { #### TO_CHAR ```sql -TO_CHAR(ts, str_literal) +TO_CHAR(ts, format_str_literal) ``` **功能说明**: 将timestamp类型按照指定格式转换为字符串 @@ -504,7 +504,7 @@ TO_CHAR(ts, str_literal) | **格式** | **说明**| **例子** | | --- | --- | --- | |AM,am,PM,pm| 无点分隔的上午下午 | 07:00:00am| -|A.M.,a.m.,P.M.,p.m.| 有点分割的上午下午| 07:00:00a.m.| +|A.M.,a.m.,P.M.,p.m.| 有点分隔的上午下午| 07:00:00a.m.| |YYYY,yyyy|年, 4个及以上数字| 2023-10-10| |YYY,yyy| 年, 最后3位数字| 023-10-10| |YY,yy| 年, 最后2位数字| 23-10-10| @@ -537,13 +537,14 @@ TO_CHAR(ts, str_literal) **使用说明**: - `Month`, `Day`等的输出格式是左对齐的, 右侧添加空格, 如`2023-OCTOBER -01`, `2023-SEPTEMBER-01`, 9月是月份中英文字母数最长的, 因此9月没有空格. 星期类似. - 使用`ms`, `us`, `ns`时, 以上三种格式的输出只在精度上不同, 比如ts为 `1697182085123`, `ms` 的输出为 `123`, `us` 的输出为 `123000`, `ns` 的输出为 `123000000`. -- 如果想要在格式串中指定某些部分不做转换, 可以使用双引号, 如`to_char(ts, 'yyyy-mm-dd "is formated by yyyy-mm-dd"')`. 如果想要输出双引号, 那么在双引号之前加一个反斜杠, 如 `to_char(ts, '\"yyyy-mm-dd\"')` 将会输出 `"2023-10-10"`. +- 时间格式中无法匹配规则的内容会直接输出. 如果想要在格式串中指定某些能够匹配规则的部分不做转换, 可以使用双引号, 如`to_char(ts, 'yyyy-mm-dd "is formated by yyyy-mm-dd"')`. 如果想要输出双引号, 那么在双引号之前加一个反斜杠, 如 `to_char(ts, '\"yyyy-mm-dd\"')` 将会输出 `"2023-10-10"`. - 那些输出是数字的格式, 如`YYYY`, `DD`, 大写与小写意义相同, 即`yyyy` 和 `YYYY` 可以互换. +- 默认输出的时间为本地时区的时间. #### TO_TIMESTAMP ```sql -TO_TIMESTAMP(str_literal, str_literal) +TO_TIMESTAMP(ts_str_literal, format_str_literal) ``` **功能说明**: 将字符串按照指定格式转化为时间戳. diff --git a/include/common/ttime.h b/include/common/ttime.h index 75bbcddd0e..306b5105d0 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -100,15 +100,22 @@ int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision); /// @brief convert a timestamp to a formatted string /// @param format the timestamp format, must null terminated -void taosTs2Char(const char* format, int64_t ts, int32_t precision, char* out); +/// @param [in,out] formats the formats array pointer generated. Shouldn't be NULL. +/// If (*formats == NULL), [format] will be used and [formats] will be updated to the new generated +/// formats array; If not NULL, [formats] will be used instead of [format] to skip parse formats again. +/// @param out output buffer, should be initialized by memset +/// @notes remember to free the generated formats +void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen); /// @brief convert a formatted timestamp string to a timestamp /// @param format must null terminated +/// @param [in, out] formats, see taosTs2Char /// @param tsStr must null terminated /// @retval 0 for success, otherwise error occured -int32_t taosChar2Ts(const char* format, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, +/// @notes remember to free the generated formats even when error occured +int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, int32_t errMsgLen); -void TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out); +void TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out, int32_t outLen); int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const char* tsStr); #ifdef __cplusplus diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 3450e32f4a..aad844da88 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -1008,7 +1008,6 @@ typedef enum { TSFKW_Day, // Sunday, Monday TSFKW_DY, // MON, TUE TSFKW_Dy, // Mon, Tue - TSFKW_dy, // mon, tue TSFKW_D, // 1-7 -> Sunday(1) -> Saturday(7) TSFKW_HH24, TSFKW_HH12, @@ -1021,12 +1020,12 @@ typedef enum { TSFKW_Mon, TSFKW_MS, TSFKW_NS, - TSFKW_OF, + //TSFKW_OF, TSFKW_PM, TSFKW_P_M, TSFKW_SS, - // TSFKW_TZM, TSFKW_TZH, + // TSFKW_TZM, // TSFKW_TZ, TSFKW_US, TSFKW_YYYY, @@ -1039,13 +1038,15 @@ typedef enum { TSFKW_a_m, // TSFKW_b_c, // TSFKW_bc, - TSFKW_d, TSFKW_day, TSFKW_ddd, TSFKW_dd, + TSFKW_dy, // mon, tue + TSFKW_d, TSFKW_hh24, TSFKW_hh12, TSFKW_hh, + TSFKW_mi, TSFKW_mm, TSFKW_month, TSFKW_mon, @@ -1067,17 +1068,17 @@ typedef enum { // clang-format off static const TSFormatKeyWord formatKeyWords[] = { - //{"A.D.", 4, TSFKW_A_D}, - {"A.M.", 4, TSFKW_A_M, false}, //{"AD", 2, TSFKW_AD, false}, + //{"A.D.", 4, TSFKW_A_D}, {"AM", 2, TSFKW_AM, false}, - //{"B.C.", 4, TSFKW_B_C, false}, + {"A.M.", 4, TSFKW_A_M, false}, //{"BC", 2, TSFKW_BC, false}, + //{"B.C.", 4, TSFKW_B_C, false}, {"DAY", 3, TSFKW_DAY, false}, {"DDD", 3, TSFKW_DDD, true}, {"DD", 2, TSFKW_DD, true}, - {"DY", 2, TSFKW_DY, false}, {"Day", 3, TSFKW_Day, false}, + {"DY", 2, TSFKW_DY, false}, {"Dy", 2, TSFKW_Dy, false}, {"D", 1, TSFKW_D, true}, {"HH24", 4, TSFKW_HH24, true}, @@ -1087,13 +1088,13 @@ static const TSFormatKeyWord formatKeyWords[] = { {"MM", 2, TSFKW_MM, true}, {"MONTH", 5, TSFKW_MONTH, false}, {"MON", 3, TSFKW_MON, false}, - {"MS", 2, TSFKW_MS, true}, {"Month", 5, TSFKW_Month, false}, {"Mon", 3, TSFKW_Mon, false}, + {"MS", 2, TSFKW_MS, true}, {"NS", 2, TSFKW_NS, true}, //{"OF", 2, TSFKW_OF, false}, - {"P.M.", 4, TSFKW_P_M, false}, {"PM", 2, TSFKW_PM, false}, + {"P.M.", 4, TSFKW_P_M, false}, {"SS", 2, TSFKW_SS, true}, {"TZH", 3, TSFKW_TZH, false}, //{"TZM", 3, TSFKW_TZM}, @@ -1104,9 +1105,9 @@ static const TSFormatKeyWord formatKeyWords[] = { {"YY", 2, TSFKW_YY, true}, {"Y", 1, TSFKW_Y, true}, //{"a.d.", 4, TSFKW_a_d, false}, - {"a.m.", 4, TSFKW_a_m, false}, //{"ad", 2, TSFKW_ad, false}, {"am", 2, TSFKW_am, false}, + {"a.m.", 4, TSFKW_a_m, false}, //{"b.c.", 4, TSFKW_b_c, false}, //{"bc", 2, TSFKW_bc, false}, {"day", 3, TSFKW_day, false}, @@ -1124,8 +1125,8 @@ static const TSFormatKeyWord formatKeyWords[] = { {"ms", 2, TSFKW_MS, true}, {"ns", 2, TSFKW_NS, true}, //{"of", 2, TSFKW_OF, false}, - {"p.m.", 4, TSFKW_p_m, false}, {"pm", 2, TSFKW_pm, false}, + {"p.m.", 4, TSFKW_p_m, false}, {"ss", 2, TSFKW_SS, true}, {"tzh", 3, TSFKW_TZH, false}, //{"tzm", 3, TSFKW_TZM}, @@ -1139,9 +1140,34 @@ static const TSFormatKeyWord formatKeyWords[] = { }; // clang-format on +#define TS_FROMAT_KEYWORD_INDEX_SIZE ('z' - 'A' + 1) +static const int TSFormatKeywordIndex[TS_FROMAT_KEYWORD_INDEX_SIZE] = { + /*A*/ TSFKW_AM, -1, -1, + /*D*/ TSFKW_DAY, -1, -1, -1, + /*H*/ TSFKW_HH24, -1, -1, -1, -1, + /*M*/ TSFKW_MI, + /*N*/ TSFKW_NS, -1, + /*P*/ TSFKW_PM, -1, -1, + /*S*/ TSFKW_SS, + /*T*/ TSFKW_TZH, + /*U*/ TSFKW_US, -1, -1, -1, + /*Y*/ TSFKW_YYYY, -1, + /*[ \ ] ^ _ `*/ -1, -1, -1, -1, -1, -1, + /*a*/ TSFKW_am, -1, -1, + /*d*/ TSFKW_day, -1, -1, -1, + /*h*/ TSFKW_hh24, -1, -1, -1, -1, + /*m*/ TSFKW_mi, + /*n*/ TSFKW_ns, -1, + /*p*/ TSFKW_pm, -1, -1, + /*s*/ TSFKW_ss, + /*t*/ TSFKW_tzh, + /*u*/ TSFKW_us, -1, -1, -1, + /*y*/ TSFKW_yyyy, -1}; + typedef struct { uint8_t type; - char c[2]; + const char* c; + int32_t len; const TSFormatKeyWord* key; } TSFormatNode; @@ -1169,9 +1195,10 @@ static const char* const long_apms[] = {A_M_STR, P_M_STR, a_m_str, p_m_str, NULL static const TSFormatKeyWord* keywordSearch(const char* str) { if (*str < 'A' || *str > 'z' || (*str > 'Z' && *str < 'a')) return NULL; - int32_t idx = 0; + int32_t idx = TSFormatKeywordIndex[str[0] - 'A']; + if (idx < 0) return NULL; const TSFormatKeyWord* key = &formatKeyWords[idx++]; - while (key->name) { + while (key->name && str[0] == key->name[0]) { if (0 == strncmp(key->name, str, key->len)) { return key; } @@ -1184,74 +1211,110 @@ static bool isSeperatorChar(char c) { return (c > 0x20 && c < 0x7F && !(c >= 'A' && c <= 'Z') && !(c >= 'a' && c <= 'z') && !(c >= '0' && c <= '9')); } -static void parseTsFormat(const char* format_str, SArray* formats) { - while (*format_str) { - const TSFormatKeyWord* key = keywordSearch(format_str); +static void parseTsFormat(const char* formatStr, SArray* formats) { + TSFormatNode* lastOtherFormat = NULL; + while (*formatStr) { + const TSFormatKeyWord* key = keywordSearch(formatStr); if (key) { TSFormatNode format = {.key = key, .type = TS_FORMAT_NODE_TYPE_KEYWORD}; taosArrayPush(formats, &format); - format_str += key->len; + formatStr += key->len; + lastOtherFormat = NULL; } else { - if (*format_str == '"') { + if (*formatStr == '"') { + lastOtherFormat = NULL; // for double quoted string - format_str++; - while (*format_str) { - if (*format_str == '"') { - format_str++; + formatStr++; + TSFormatNode* last = NULL; + while (*formatStr) { + if (*formatStr == '"') { + formatStr++; break; } - if (*format_str == '\\' && *(format_str + 1)) format_str++; - TSFormatNode format = {.type = TS_FORMAT_NODE_TYPE_CHAR, .key = NULL}; - format.c[0] = *format_str; - format.c[1] = '\0'; - taosArrayPush(formats, &format); - format_str++; + if (*formatStr == '\\' && *(formatStr + 1)) { + formatStr++; + last = NULL; // stop expanding last format, create new format + } + if (last) { + // expand + assert(last->type == TS_FORMAT_NODE_TYPE_CHAR); + last->len++; + formatStr++; + } else { + // create new + TSFormatNode format = {.type = TS_FORMAT_NODE_TYPE_CHAR, .key = NULL}; + format.c = formatStr; + format.len = 1; + taosArrayPush(formats, &format); + formatStr++; + last = taosArrayGetLast(formats); + } } } else { // for other strings - if (*format_str == '\\' && *(format_str + 1)) format_str++; - TSFormatNode format = { - .type = isSeperatorChar(*format_str) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR, + if (*formatStr == '\\' && *(formatStr + 1)) { + formatStr++; + lastOtherFormat = NULL; // stop expanding + } else { + if (lastOtherFormat && !isSeperatorChar(*formatStr)) { + // expanding + } else { + // create new + lastOtherFormat = NULL; + } + } + if (lastOtherFormat) { + assert(lastOtherFormat->type == TS_FORMAT_NODE_TYPE_CHAR); + lastOtherFormat->len++; + formatStr++; + } else { + TSFormatNode format = { + .type = isSeperatorChar(*formatStr) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR, .key = NULL}; - format.c[0] = *format_str; - format.c[1] = '\0'; - taosArrayPush(formats, &format); - format_str++; + format.c = formatStr; + format.len = 1; + taosArrayPush(formats, &format); + formatStr++; + if (format.type == TS_FORMAT_NODE_TYPE_CHAR) lastOtherFormat = taosArrayGetLast(formats); + } } } } } -static void tm2char(const SArray* formats, const struct STm* tm, char* s) { +static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) { int32_t size = taosArrayGetSize(formats); + const char* start = s; for (int32_t i = 0; i < size; ++i) { TSFormatNode* format = taosArrayGet(formats, i); if (format->type != TS_FORMAT_NODE_TYPE_KEYWORD) { - strcpy(s, format->c); - s += strlen(s); + if (s - start + format->len + 1 > outLen) break; + strncpy(s, format->c, format->len); + s += format->len; continue; } + if (s - start + 16 > outLen) break; switch (format->key->id) { case TSFKW_AM: case TSFKW_PM: sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "PM" : "AM"); - s += strlen(s); + s += 2; break; case TSFKW_A_M: case TSFKW_P_M: sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "P.M." : "A.M."); - s += strlen(s); + s += 4; break; case TSFKW_am: case TSFKW_pm: sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "pm" : "am"); - s += strlen(s); + s += 2; break; case TSFKW_a_m: case TSFKW_p_m: sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "p.m." : "a.m."); - s += strlen(s); + s += 4; break; case TSFKW_DDD: sprintf(s, "%d", tm->tm.tm_yday); @@ -1259,11 +1322,11 @@ static void tm2char(const SArray* formats, const struct STm* tm, char* s) { break; case TSFKW_DD: sprintf(s, "%02d", tm->tm.tm_mday); - s += strlen(s); + s += 2; break; case TSFKW_D: sprintf(s, "%d", tm->tm.tm_wday + 1); - s += strlen(s); + s += 1; break; case TSFKW_DAY: { // MONDAY, TUESDAY... @@ -1293,13 +1356,13 @@ static void tm2char(const SArray* formats, const struct STm* tm, char* s) { char buf[8] = {0}; for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = toupper(wd[i]); sprintf(s, "%3s", buf); - s += strlen(s); + s += 3; break; } case TSFKW_Dy: // Mon, Tue sprintf(s, "%3s", shortWeekDays[tm->tm.tm_wday]); - s += strlen(s); + s += 3; break; case TSFKW_dy: { // mon, tue @@ -1307,26 +1370,26 @@ static void tm2char(const SArray* formats, const struct STm* tm, char* s) { char buf[8] = {0}; for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = tolower(wd[i]); sprintf(s, "%3s", buf); - s += strlen(s); + s += 3; break; } case TSFKW_HH24: sprintf(s, "%02d", tm->tm.tm_hour); - s += strlen(s); + s += 2; break; case TSFKW_HH: case TSFKW_HH12: // 0 or 12 o'clock in 24H coresponds to 12 o'clock (AM/PM) in 12H sprintf(s, "%02d", tm->tm.tm_hour % 12 == 0 ? 12 : tm->tm.tm_hour % 12); - s += strlen(s); + s += 2; break; case TSFKW_MI: sprintf(s, "%02d", tm->tm.tm_min); - s += strlen(s); + s += 2; break; case TSFKW_MM: sprintf(s, "%02d", tm->tm.tm_mon + 1); - s += strlen(s); + s += 2; break; case TSFKW_MONTH: { const char* mon = fullMonths[tm->tm.tm_mon]; @@ -1370,19 +1433,19 @@ static void tm2char(const SArray* formats, const struct STm* tm, char* s) { } case TSFKW_SS: sprintf(s, "%02d", tm->tm.tm_sec); - s += strlen(s); + s += 2; break; case TSFKW_MS: sprintf(s, "%03" PRId64, tm->fsec / 1000000L); - s += strlen(s); + s += 3; break; case TSFKW_US: sprintf(s, "%06" PRId64, tm->fsec / 1000L); - s += strlen(s); + s += 6; break; case TSFKW_NS: sprintf(s, "%09" PRId64, tm->fsec); - s += strlen(s); + s += 9; break; case TSFKW_TZH: sprintf(s, "%s%02d", tsTimezone < 0 ? "-" : "+", tsTimezone); @@ -1429,6 +1492,7 @@ static const char* tsFormatStr2Int32(int32_t* dest, const char* str, int32_t len char* last; int64_t res; const char* s = str; + if ('\0' == str[0]) return NULL; if (len <= 0) { res = taosStr2Int64(s, &last, 10); s = last; @@ -1523,18 +1587,27 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec int32_t tzSign = 1, tz = tsTimezone; int32_t err = 0; - for (int32_t i = 0; i < size; ++i) { - while (isspace(*s)) { + for (int32_t i = 0; i < size && *s != '\0'; ++i) { + while (isspace(*s) && *s != '\0') { s++; } + if (!s) break; TSFormatNode* node = taosArrayGet(formats, i); if (node->type == TS_FORMAT_NODE_TYPE_SEPARATOR) { // separator matches any character - if (isSeperatorChar(s[0])) s += strlen(node->c); + if (isSeperatorChar(s[0])) s += node->len; continue; } if (node->type == TS_FORMAT_NODE_TYPE_CHAR) { - if (!isspace(node->c[0])) s += strlen(node->c); + int32_t pos = 0; + // skip leading spaces + while (isspace(node->c[pos]) && node->len > 0) pos++; + while (pos < node->len && *s != '\0') { + if (!isspace(node->c[pos++])) { + while (isspace(*s) && *s != '\0') s++; + if (*s != '\0') s++; // forward together + } + } continue; } assert(node->type == TS_FORMAT_NODE_TYPE_KEYWORD); @@ -1545,7 +1618,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec case TSFKW_p_m: { int32_t idx = strArrayCaseSearch(long_apms, s); if (idx >= 0) { - s += strlen(long_apms[idx]); + s += 4; pm = idx % 2; hour12 = 1; } else { @@ -1558,7 +1631,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec case TSFKW_pm: { int32_t idx = strArrayCaseSearch(apms, s); if (idx >= 0) { - s += strlen(apms[idx]); + s += 2; pm = idx % 2; hour12 = 1; } else { @@ -1793,39 +1866,41 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec return ret; } -void taosTs2Char(const char* format, int64_t ts, int32_t precision, char* out) { - SArray* formats = taosArrayInit(8, sizeof(TSFormatNode)); - parseTsFormat(format, formats); +void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) { + if (!*formats) { + *formats = taosArrayInit(8, sizeof(TSFormatNode)); + parseTsFormat(format, *formats); + } struct STm tm; taosTs2Tm(ts, precision, &tm); - tm2char(formats, &tm, out); - taosArrayDestroy(formats); + tm2char(*formats, &tm, out, outLen); } -int32_t taosChar2Ts(const char* format, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, +int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, int32_t errMsgLen) { const char* sErrPos; int32_t fErrIdx; - SArray* formats = taosArrayInit(4, sizeof(TSFormatNode)); - parseTsFormat(format, formats); - int32_t code = char2ts(tsStr, formats, ts, precision, &sErrPos, &fErrIdx); + if (!*formats) { + *formats = taosArrayInit(4, sizeof(TSFormatNode)); + parseTsFormat(format, *formats); + } + int32_t code = char2ts(tsStr, *formats, ts, precision, &sErrPos, &fErrIdx); if (code == -1) { - TSFormatNode* fNode = (taosArrayGet(formats, fErrIdx)); + TSFormatNode* fNode = (taosArrayGet(*formats, fErrIdx)); snprintf(errMsg, errMsgLen, "mismatch format for: %s and %s", sErrPos, - fErrIdx < taosArrayGetSize(formats) ? ((TSFormatNode*)taosArrayGet(formats, fErrIdx))->key->name : ""); + fErrIdx < taosArrayGetSize(*formats) ? ((TSFormatNode*)taosArrayGet(*formats, fErrIdx))->key->name : ""); } else if (code == -2) { snprintf(errMsg, errMsgLen, "timestamp format error: %s -> %s", tsStr, format); } - taosArrayDestroy(formats); return code; } -void TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out) { +void TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out, int32_t outLen) { SArray* formats = taosArrayInit(4, sizeof(TSFormatNode)); parseTsFormat(format, formats); struct STm tm; taosTs2Tm(ts, precision, &tm); - tm2char(formats, &tm, out); + tm2char(formats, &tm, out, outLen); taosArrayDestroy(formats); } diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 49a16351ca..dc320ebcb2 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -316,8 +316,8 @@ TEST(timeTest, timestamp2tm) { } void test_ts2char(int64_t ts, const char* format, int32_t precison, const char* expected) { - char buf[128] = {0}; - TEST_ts2char(format, ts, precison, buf); + char buf[256] = {0}; + TEST_ts2char(format, ts, precison, buf, 256); printf("ts: %ld format: %s res: [%s], expected: [%s]\n", ts, format, buf, expected); ASSERT_STREQ(expected, buf); } @@ -408,8 +408,8 @@ TEST(timeTest, char2ts) { #ifndef WINDOWS // 2023-1-1 21:10:10.120450780 - ASSERT_EQ(0, TEST_char2ts("yy \"年\"-MM 月-dd \"日\" HH24:MI:ss.ms.us.ns TZH", &ts, TSDB_TIME_PRECISION_NANO, - " 23 年 - 1 月 - 01 日 \t 21:10:10 . 12 . \t 00045 . 00000078 \t+08")); + ASSERT_EQ(0, TEST_char2ts("yy \"年\"-MM 月-dd \"日 子\" HH24:MI:ss.ms.us.ns TZH", &ts, TSDB_TIME_PRECISION_NANO, + " 23 年 - 1 月 - 01 日 子 \t 21:10:10 . 12 . \t 00045 . 00000078 \t+08")); ASSERT_EQ(ts, 1672578610120450780LL); #endif @@ -437,7 +437,7 @@ TEST(timeTest, char2ts) { "2100/january/01 FRIDAY 11:10:10.124456+08")); ASSERT_EQ(ts, 4102456210124456LL); ASSERT_EQ(0, TEST_char2ts("yyyy/Month/dd Dy HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO, - "2100/january/01 Fri 11:10:10.124456+08")); + "2100/january/01 Fri 11:10:10.124456+08:00")); ASSERT_EQ(ts, 4102456210124456LL); ASSERT_EQ(0, TEST_char2ts("yyyy/month/dd day HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO, @@ -461,7 +461,8 @@ TEST(timeTest, char2ts) { // '/' cannot convert to MM ASSERT_EQ(-1, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "2100/2/1")); // nothing to be converted to dd - ASSERT_EQ(-1, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "210012")); + ASSERT_EQ(0, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "210012")); + ASSERT_EQ(ts, 4131273600000000LL); // 2100-12-1 ASSERT_EQ(-1, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "21001")); ASSERT_EQ(-1, TEST_char2ts("yyyyMM-dd ", &ts, TSDB_TIME_PRECISION_MICRO, "23a1-1")); @@ -481,6 +482,13 @@ TEST(timeTest, char2ts) { ASSERT_EQ(-1, TEST_char2ts("HH12:MI:SS", &ts, TSDB_TIME_PRECISION_MICRO, "21:12:12")); ASSERT_EQ(-1, TEST_char2ts("yyyy/MM1/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "2100111111111/11/2")); ASSERT_EQ(-2, TEST_char2ts("yyyy/MM1/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "23/11/2-13")); + ASSERT_EQ(0, TEST_char2ts("yyyy年 MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年1/1+0")); + ASSERT_EQ(ts, 0); + ASSERT_EQ(-1, TEST_char2ts("yyyy年a MM/dd", &ts, TSDB_TIME_PRECISION_MICRO, "2023年1/2")); + ASSERT_EQ(0, TEST_char2ts("yyyy年 MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 1/1+0")); + ASSERT_EQ(ts, 0); + ASSERT_EQ(0, TEST_char2ts("yyyy年 a a a MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 a a a 1/1+0")); + ASSERT_EQ(0, TEST_char2ts("yyyy年 a a a a a a a a a a a a a a a MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 a ")); } #pragma GCC diagnostic pop diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 628a609715..84aff9fa88 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2107,7 +2107,7 @@ static int32_t translateToChar(SFunctionNode* pFunc, char* pErrBuf, int32_t len) if (!IS_STR_DATA_TYPE(para2Type) || !IS_TIMESTAMP_TYPE(para1Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_VARCHAR].bytes, .type = TSDB_DATA_TYPE_VARCHAR}; + pFunc->node.resType = (SDataType){.bytes = 4096, .type = TSDB_DATA_TYPE_VARCHAR}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index ee2ba47ce8..48886b1eec 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1203,9 +1203,13 @@ int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam char * tsStr = taosMemoryMalloc(TS_FORMAT_MAX_LEN); char * format = taosMemoryMalloc(TS_FORMAT_MAX_LEN); int32_t len, code = TSDB_CODE_SUCCESS; + SArray *formats = NULL; + for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { - if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) + if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) { colDataSetNULL(pOutput->columnData, i); + continue; + } char *tsData = colDataGetData(pInput[0].columnData, i); char *formatData = colDataGetData(pInput[1].columnData, pInput[1].numOfRows > 1 ? i : 0); @@ -1213,11 +1217,17 @@ int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam strncpy(tsStr, varDataVal(tsData), len); tsStr[len] = '\0'; len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(formatData)); - strncpy(format, varDataVal(formatData), len); - format[len] = '\0'; + if (pInput[1].numOfRows > 1 || i == 0) { + strncpy(format, varDataVal(formatData), len); + format[len] = '\0'; + if (formats) { + taosArrayDestroy(formats); + formats = NULL; + } + } int32_t precision = pOutput->columnData->info.precision; char errMsg[128] = {0}; - code = taosChar2Ts(format, tsStr, &ts, precision, errMsg, 128); + code = taosChar2Ts(format, &formats, tsStr, &ts, precision, errMsg, 128); if (code) { qError("func to_timestamp failed %s", errMsg); code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED; @@ -1225,6 +1235,7 @@ int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam } colDataSetVal(pOutput->columnData, i, (char *)&ts, false); } + if (formats) taosArrayDestroy(formats); taosMemoryFree(tsStr); taosMemoryFree(format); return code; @@ -1232,22 +1243,32 @@ int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput) { char * format = taosMemoryMalloc(TS_FORMAT_MAX_LEN); - char * out = taosMemoryMalloc(TS_FORMAT_MAX_LEN * 2); + char * out = taosMemoryCalloc(1, TS_FORMAT_MAX_LEN + VARSTR_HEADER_SIZE); int32_t len; + SArray *formats = NULL; for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { - if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) + if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) { colDataSetNULL(pOutput->columnData, i); + continue; + } char *ts = colDataGetData(pInput[0].columnData, i); char *formatData = colDataGetData(pInput[1].columnData, pInput[1].numOfRows > 1 ? i : 0); len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(formatData)); - strncpy(format, varDataVal(formatData), len); - format[len] = '\0'; + if (pInput[1].numOfRows > 1 || i == 0) { + strncpy(format, varDataVal(formatData), len); + format[len] = '\0'; + if (formats) { + taosArrayDestroy(formats); + formats = NULL; + } + } int32_t precision = pInput[0].columnData->info.precision; - taosTs2Char(format, *(int64_t *)ts, precision, varDataVal(out)); + taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN); varDataSetLen(out, strlen(varDataVal(out))); colDataSetVal(pOutput->columnData, i, out, false); } + if (formats) taosArrayDestroy(formats); taosMemoryFree(format); taosMemoryFree(out); return TSDB_CODE_SUCCESS; diff --git a/tests/system-test/2-query/func_to_char_timestamp.py b/tests/system-test/2-query/func_to_char_timestamp.py index 3d3435d9c7..639811d275 100644 --- a/tests/system-test/2-query/func_to_char_timestamp.py +++ b/tests/system-test/2-query/func_to_char_timestamp.py @@ -60,10 +60,15 @@ class TDTestCase: rowsBatched = 0 sql += " %s%d values "%(ctbPrefix,i) for j in range(rowsPerTbl): - if (i < ctbNum/2): - sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10, j%10, j%10) + if i % 3 == 0: + ts_format = 'NULL' else: - sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10) + ts_format = "'yyyy-mm-dd hh24:mi:ss'" + + if (i < ctbNum/2): + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'2023-11-01 10:10:%d', %s, 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10, j%10, ts_format, j%10) + else: + sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,NULL , %s, 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, ts_format, j%10) rowsBatched += 1 if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): tsql.execute(sql) @@ -85,7 +90,7 @@ class TDTestCase: 'stbName': 'meters', 'colPrefix': 'c', 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'FLOAT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'smallint', 'count':1},{'type': 'tinyint', 'count':1},{'type': 'bool', 'count':1},{'type': 'binary', 'len':10, 'count':1},{'type': 'nchar', 'len':10, 'count':1}], + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'FLOAT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'smallint', 'count':1},{'type': 'tinyint', 'count':1},{'type': 'bool', 'count':1},{'type': 'varchar', 'len':1024, 'count':2},{'type': 'nchar', 'len':10, 'count':1}], 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}], 'ctbPrefix': 't', 'ctbStartIdx': 0, @@ -146,6 +151,18 @@ class TDTestCase: tdSql.query("select to_char(ts, 'yy-mon-dd hh24:mi:ss.msa.m.TZH Day') from meters where to_timestamp(to_char(ts, 'yy-mon-dd hh24:mi:ss dy'), 'yy-mon-dd hh24:mi:ss dy') != ts") tdSql.checkRows(0) + tdSql.query("select to_timestamp(c8, 'YYYY-MM-DD hh24:mi:ss') from meters") + tdSql.query("select to_timestamp(c8, c9) from meters") + + format = "YYYY-MM-DD HH:MI:SS" + for i in range(500): + format = format + "1234567890" + tdSql.query("select to_char(ts, '%s') from meters" % (format), queryTimes=1) + time_str = '2023-11-11 10:10:10' + for i in range(500): + time_str = time_str + "1234567890" + tdSql.query("select to_timestamp('%s', '%s')" % (time_str, format)) + def run(self): self.prepareTestEnv() self.test_to_timestamp() From 418602808312b7ae6d71563c0e715ff2b876decb Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 27 Oct 2023 11:09:18 +0800 Subject: [PATCH 17/37] fix: interval more than 1000 years --- include/util/taoserror.h | 1 + source/libs/parser/src/parTranslater.c | 3 ++ source/libs/parser/src/parUtil.c | 2 + source/os/src/osTime.c | 69 +++++++++++--------------- source/util/src/terror.c | 1 + 5 files changed, 37 insertions(+), 39 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 6fbe4422ac..671d390638 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -657,6 +657,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR TAOS_DEF_ERROR_CODE(0, 0x2618) #define TSDB_CODE_PAR_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x2619) #define TSDB_CODE_PAR_INVALID_TABLE_OPTION TAOS_DEF_ERROR_CODE(0, 0x261A) +#define TSDB_CODE_PAR_INTER_VALUE_TOO_BIG TAOS_DEF_ERROR_CODE(0, 0x261B) #define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST TAOS_DEF_ERROR_CODE(0, 0x2624) #define TSDB_CODE_PAR_AGG_FUNC_NESTING TAOS_DEF_ERROR_CODE(0, 0x2627) #define TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE TAOS_DEF_ERROR_CODE(0, 0x2628) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 72293e2f8c..bff0b46b61 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3512,6 +3512,7 @@ static void convertVarDuration(SValueNode* pOffset, uint8_t precision) { pOffset->unit = units[precision]; } +static const int64_t maxKeepMS = (int64_t)3600 * 1000 * 24 * (365 * 1000 + 250); static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; @@ -3520,6 +3521,8 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime, getPrecisionStr(precision)); + } else if (pInter->datum.i > maxKeepMS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_BIG, 1000, "years"); } if (NULL != pInterval->pOffset) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 0ab8927bd0..825e02f9aa 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -65,6 +65,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "This statement is no longer supported"; case TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL: return "Interval cannot be less than %d %s"; + case TSDB_CODE_PAR_INTER_VALUE_TOO_BIG: + return "Interval cannot be more than %d %s"; case TSDB_CODE_PAR_DB_NOT_SPECIFIED: return "Database not specified"; case TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME: diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 05233065fa..e506ee603b 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -477,47 +477,38 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) { return res; } #ifdef WINDOWS - if (*timep < 0) { - if (*timep < -2208988800LL) { - if (buf != NULL) { - sprintf(buf, "NaN"); - } - return NULL; - } - - SYSTEMTIME s; - FILETIME f; - LARGE_INTEGER offset; - struct tm tm1; - time_t tt = 0; - if (localtime_s(&tm1, &tt) != 0 ) { - if (buf != NULL) { - sprintf(buf, "NaN"); - } - return NULL; - } - offset.QuadPart = TIMEEPOCH1900; - offset.QuadPart += *timep * 10000000; - f.dwLowDateTime = offset.QuadPart & 0xffffffff; - f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff; - FileTimeToSystemTime(&f, &s); - result->tm_sec = s.wSecond; - result->tm_min = s.wMinute; - result->tm_hour = s.wHour; - result->tm_mday = s.wDay; - result->tm_mon = s.wMonth - 1; - result->tm_year = s.wYear - 1900; - result->tm_wday = s.wDayOfWeek; - result->tm_yday = 0; - result->tm_isdst = 0; - } else { - if (localtime_s(result, timep) != 0) { - if (buf != NULL) { - sprintf(buf, "NaN"); - } - return NULL; + if (*timep < -2208988800LL) { + if (buf != NULL) { + sprintf(buf, "NaN"); } + return NULL; } + + SYSTEMTIME s; + FILETIME f; + LARGE_INTEGER offset; + struct tm tm1; + time_t tt = 0; + if (localtime_s(&tm1, &tt) != 0) { + if (buf != NULL) { + sprintf(buf, "NaN"); + } + return NULL; + } + offset.QuadPart = TIMEEPOCH1900; + offset.QuadPart += *timep * 10000000; + f.dwLowDateTime = offset.QuadPart & 0xffffffff; + f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff; + FileTimeToSystemTime(&f, &s); + result->tm_sec = s.wSecond; + result->tm_min = s.wMinute; + result->tm_hour = s.wHour; + result->tm_mday = s.wDay; + result->tm_mon = s.wMonth - 1; + result->tm_year = s.wYear - 1900; + result->tm_wday = s.wDayOfWeek; + result->tm_yday = 0; + result->tm_isdst = 0; #else res = localtime_r(timep, result); if (res == NULL && buf != NULL) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4cc86d51b7..83b0ec7a82 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -518,6 +518,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_PORT, "Port should be an in TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ENDPOINT, "Endpoint should be in the format of 'fqdn:port'") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_EXPRIE_STATEMENT, "This statement is no longer supported") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, "Interval too small") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_VALUE_TOO_BIG, "Interval too big") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_DB_NOT_SPECIFIED, "Database not specified") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "Invalid identifier name") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR, "Corresponding super table not in this db") From b422c29d4e94dfe2a7c6550ea305b596160d83df Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 27 Oct 2023 15:16:03 +0800 Subject: [PATCH 18/37] fix interval limit on us/ns db --- source/libs/parser/src/parTranslater.c | 16 +++++++++++++++- source/os/test/osTimeTests.cpp | 4 ---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bff0b46b61..d65f97dce7 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3499,6 +3499,20 @@ static const char* getPrecisionStr(uint8_t precision) { return "unknown"; } +static int64_t getPrecisionMultiple(uint8_t precision) { + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: + return 1; + case TSDB_TIME_PRECISION_MICRO: + return 1000; + case TSDB_TIME_PRECISION_NANO: + return 1000000; + default: + break; + } + return 1; +} + static void convertVarDuration(SValueNode* pOffset, uint8_t precision) { const int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; const int8_t units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND}; @@ -3521,7 +3535,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime, getPrecisionStr(precision)); - } else if (pInter->datum.i > maxKeepMS) { + } else if (pInter->datum.i / getPrecisionMultiple(precision) > maxKeepMS) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_BIG, 1000, "years"); } diff --git a/source/os/test/osTimeTests.cpp b/source/os/test/osTimeTests.cpp index 5dd15db4ab..90c089e310 100644 --- a/source/os/test/osTimeTests.cpp +++ b/source/os/test/osTimeTests.cpp @@ -114,10 +114,6 @@ TEST(osTimeTests, taosLocalTime) { ASSERT_EQ(local_time->tm_min, 0); ASSERT_EQ(local_time->tm_sec, 0); - time_t over_timep = 6406301441633558; - local_time = taosLocalTime(&over_timep, &result, NULL); - ASSERT_EQ(local_time, nullptr); - time_t neg_timep3 = -78115158887; local_time = taosLocalTime(&neg_timep3, &result, NULL); ASSERT_EQ(local_time, nullptr); From 43e6dec6c218fcc6b3993c247d18ed90d021bf79 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 27 Oct 2023 09:30:30 +0000 Subject: [PATCH 19/37] TD-26971 --- source/dnode/vnode/src/vnd/vnodeSvr.c | 72 +++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index c46ea15111..040ae96972 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -20,6 +20,7 @@ #include "vnode.h" #include "vnodeInt.h" #include "audit.h" +#include "tstrbuild.h" static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -886,6 +887,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, char tbName[TSDB_TABLE_FNAME_LEN]; STbUidStore *pStore = NULL; SArray *tbUids = NULL; + SArray *tbNames = NULL; pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; pRsp->code = TSDB_CODE_SUCCESS; @@ -902,7 +904,8 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp)); tbUids = taosArrayInit(req.nReqs, sizeof(int64_t)); - if (rsp.pArray == NULL || tbUids == NULL) { + tbNames = taosArrayInit(req.nReqs, sizeof(char*)); + if (rsp.pArray == NULL || tbUids == NULL || tbNames == NULL) { rcode = -1; terrno = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -947,14 +950,9 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, taosArrayPush(rsp.pArray, &cRsp); - if(tsEnableAuditCreateTable){ - int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; - - SName name = {0}; - tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); - - auditRecord(NULL, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name)); - } + char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); + strcpy(str, pCreateReq->name); + taosArrayPush(tbNames, str); } vDebug("vgId:%d, add %d new created tables into query table list", TD_VID(pVnode), (int32_t)taosArrayGetSize(tbUids)); @@ -976,6 +974,29 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen); tEncodeSVCreateTbBatchRsp(&encoder, &rsp); + if(tsEnableAuditCreateTable){ + int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; + + SName name = {0}; + tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); + + SStringBuilder sb = {0}; + for(int32_t iReq = 0; iReq < req.nReqs; iReq++){ + char* key = taosArrayGet(tbNames, iReq); + taosStringBuilderAppendStringLen(&sb, key, strlen(key)); + if(iReq < req.nReqs - 1){ + taosStringBuilderAppendChar(&sb, ','); + } + } + + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + + auditRecord(NULL, clusterId, "createTable", name.dbname, "", keyJoined, len); + + taosStringBuilderDestroy(&sb); + } + _exit: for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; @@ -987,6 +1008,7 @@ _exit: taosArrayDestroy(tbUids); tDecoderClear(&decoder); tEncoderClear(&encoder); + taosArrayDestroy(tbNames); return rcode; } @@ -1120,6 +1142,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in int32_t ret; SArray *tbUids = NULL; STbUidStore *pStore = NULL; + SArray *tbNames = NULL; pRsp->msgType = TDMT_VND_DROP_TABLE_RSP; pRsp->pCont = NULL; @@ -1138,7 +1161,8 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in // process req tbUids = taosArrayInit(req.nReqs, sizeof(int64_t)); rsp.pArray = taosArrayInit(req.nReqs, sizeof(SVDropTbRsp)); - if (tbUids == NULL || rsp.pArray == NULL) goto _exit; + tbNames = taosArrayInit(req.nReqs, sizeof(char*)); + if (tbUids == NULL || rsp.pArray == NULL || tbNames == NULL) goto _exit; for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { SVDropTbReq *pDropTbReq = req.pReqs + iReq; @@ -1159,11 +1183,38 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in } taosArrayPush(rsp.pArray, &dropTbRsp); + + char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); + strcpy(str, pDropTbReq->name); + taosArrayPush(tbNames, str); } tqUpdateTbUidList(pVnode->pTq, tbUids, false); tdUpdateTbUidList(pVnode->pSma, pStore, false); + if(tsEnableAuditCreateTable){ + int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; + + SName name = {0}; + tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); + + SStringBuilder sb = {0}; + for(int32_t iReq = 0; iReq < req.nReqs; iReq++){ + char* key = taosArrayGet(tbNames, iReq); + taosStringBuilderAppendStringLen(&sb, key, strlen(key)); + if(iReq < req.nReqs - 1){ + taosStringBuilderAppendChar(&sb, ','); + } + } + + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + + auditRecord(NULL, clusterId, "createTable", name.dbname, "", keyJoined, len); + + taosStringBuilderDestroy(&sb); + } + _exit: taosArrayDestroy(tbUids); tdUidStoreFree(pStore); @@ -1174,6 +1225,7 @@ _exit: tEncodeSVDropTbBatchRsp(&encoder, &rsp); tEncoderClear(&encoder); taosArrayDestroy(rsp.pArray); + taosArrayDestroy(tbNames); return 0; } From cdb1d8296ddd322d8f0de91656ed9a153017d71f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 27 Oct 2023 17:48:19 +0800 Subject: [PATCH 20/37] feat: concurrency on fileset --- source/dnode/vnode/src/inc/tsdb.h | 18 +- source/dnode/vnode/src/sma/smaRollup.c | 2 - source/dnode/vnode/src/tsdb/tsdbCommit.c | 20 +- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 25 +- source/dnode/vnode/src/tsdb/tsdbFS2.c | 319 ++++++++++---------- source/dnode/vnode/src/tsdb/tsdbFS2.h | 50 +-- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 33 +- source/dnode/vnode/src/tsdb/tsdbFSet2.h | 44 ++- source/dnode/vnode/src/tsdb/tsdbMerge.c | 117 ++++--- source/dnode/vnode/src/tsdb/tsdbOpen.c | 9 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 66 ++-- source/dnode/vnode/src/tsdb/tsdbRetention.c | 264 +++++++++------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 18 +- 13 files changed, 542 insertions(+), 443 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 79112babc3..1efcd9417d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -309,7 +309,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STs void tsdbUntakeReadSnap2(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); // tsdbMerge.c ============================================================================================== -int32_t tsdbMerge(void *arg); +int32_t tsdbSchedMerge(STsdb *tsdb, int32_t fid); // tsdbDiskData ============================================================================================== int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder); @@ -371,7 +371,7 @@ struct STsdb { char *path; SVnode *pVnode; STsdbKeepCfg keepCfg; - TdThreadRwlock rwLock; + TdThreadMutex mutex; SMemTable *mem; SMemTable *imem; STsdbFS fs; // old @@ -668,8 +668,8 @@ struct SDelFWriter { }; #include "tarray2.h" -//#include "tsdbFS2.h" -// struct STFileSet; +// #include "tsdbFS2.h" +// struct STFileSet; typedef struct STFileSet STFileSet; typedef TARRAY2(STFileSet *) TFileSetArray; @@ -677,9 +677,9 @@ typedef struct STSnapRange STSnapRange; typedef TARRAY2(STSnapRange *) TSnapRangeArray; // disjoint snap ranges // util -int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); -int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); -void tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap); +int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); +int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); +void tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap); SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges); // snap partition list @@ -873,8 +873,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); -void tMergeTreePinSttBlock(SMergeTree* pMTree); -void tMergeTreeUnpinSttBlock(SMergeTree* pMTree); +void tMergeTreePinSttBlock(SMergeTree *pMTree); +void tMergeTreeUnpinSttBlock(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 8da2fff5a6..14c5baa402 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -31,8 +31,6 @@ SSmaMgmt smaMgmt = { typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; -extern int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now); - static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static void tdUidStoreDestory(STbUidStore *pStore); static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index b4c2c0a979..55ae25aee4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -131,7 +131,7 @@ int32_t tsdbBegin(STsdb *pTsdb) { TSDB_CHECK_CODE(code, lino, _exit); // lock - if ((code = taosThreadRwlockWrlock(&pTsdb->rwLock))) { + if ((code = taosThreadMutexLock(&pTsdb->mutex))) { code = TAOS_SYSTEM_ERROR(code); TSDB_CHECK_CODE(code, lino, _exit); } @@ -139,7 +139,7 @@ int32_t tsdbBegin(STsdb *pTsdb) { pTsdb->mem = pMemTable; // unlock - if ((code = taosThreadRwlockUnlock(&pTsdb->rwLock))) { + if ((code = taosThreadMutexUnlock(&pTsdb->mutex))) { code = TAOS_SYSTEM_ERROR(code); TSDB_CHECK_CODE(code, lino, _exit); } @@ -152,11 +152,11 @@ _exit: } int32_t tsdbPrepareCommit(STsdb *pTsdb) { - taosThreadRwlockWrlock(&pTsdb->rwLock); + taosThreadMutexLock(&pTsdb->mutex); ASSERT(pTsdb->imem == NULL); pTsdb->imem = pTsdb->mem; pTsdb->mem = NULL; - taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadMutexUnlock(&pTsdb->mutex); return 0; } @@ -171,9 +171,9 @@ int32_t tsdbCommit(STsdb *pTsdb, SCommitInfo *pInfo) { // check if (pMemTable->nRow == 0 && pMemTable->nDel == 0) { - taosThreadRwlockWrlock(&pTsdb->rwLock); + taosThreadMutexLock(&pTsdb->mutex); pTsdb->imem = NULL; - taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadMutexUnlock(&pTsdb->mutex); tsdbUnrefMemTable(pMemTable, NULL, true); goto _exit; @@ -501,6 +501,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; SDFileSet *pRSet = NULL; + // memory pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision); pCommitter->expLevel = tsdbFidLevel(pCommitter->commitFid, &pCommitter->pTsdb->keepCfg, taosGetTimestampSec()); @@ -798,6 +799,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; + // commit file data start code = tsdbCommitFileDataStart(pCommitter); TSDB_CHECK_CODE(code, lino, _exit); @@ -1650,18 +1652,18 @@ int32_t tsdbFinishCommit(STsdb *pTsdb) { SMemTable *pMemTable = pTsdb->imem; // lock - taosThreadRwlockWrlock(&pTsdb->rwLock); + taosThreadMutexLock(&pTsdb->mutex); code = tsdbFSCommit(pTsdb); if (code) { - taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadMutexUnlock(&pTsdb->mutex); TSDB_CHECK_CODE(code, lino, _exit); } pTsdb->imem = NULL; // unlock - taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadMutexUnlock(&pTsdb->mutex); if (pMemTable) { tsdbUnrefMemTable(pMemTable, NULL, true); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 6dc492f420..79ecdab15c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -367,7 +367,12 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { int32_t lino = 0; STsdb *tsdb = committer->tsdb; - committer->ctx->fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision); + int32_t fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision); + + // check if can commit + tsdbFSCheckCommit(tsdb, fid); + + committer->ctx->fid = fid; committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now); tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey, &committer->ctx->maxKey); @@ -549,11 +554,11 @@ _exit: } int32_t tsdbPreCommit(STsdb *tsdb) { - taosThreadRwlockWrlock(&tsdb->rwLock); + taosThreadMutexLock(&tsdb->mutex); ASSERT(tsdb->imem == NULL); tsdb->imem = tsdb->mem; tsdb->mem = NULL; - taosThreadRwlockUnlock(&tsdb->rwLock); + taosThreadMutexUnlock(&tsdb->mutex); return 0; } @@ -568,15 +573,13 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) { int64_t nDel = imem->nDel; if (nRow == 0 && nDel == 0) { - taosThreadRwlockWrlock(&tsdb->rwLock); + taosThreadMutexLock(&tsdb->mutex); tsdb->imem = NULL; - taosThreadRwlockUnlock(&tsdb->rwLock); + taosThreadMutexUnlock(&tsdb->mutex); tsdbUnrefMemTable(imem, NULL, true); } else { SCommitter2 committer[1]; - tsdbFSCheckCommit(tsdb->pFS); - code = tsdbOpenCommitter(tsdb, info, committer); TSDB_CHECK_CODE(code, lino, _exit); @@ -605,14 +608,14 @@ int32_t tsdbCommitCommit(STsdb *tsdb) { if (tsdb->imem == NULL) goto _exit; SMemTable *pMemTable = tsdb->imem; - taosThreadRwlockWrlock(&tsdb->rwLock); + taosThreadMutexLock(&tsdb->mutex); code = tsdbFSEditCommit(tsdb->pFS); if (code) { - taosThreadRwlockUnlock(&tsdb->rwLock); + taosThreadMutexUnlock(&tsdb->mutex); TSDB_CHECK_CODE(code, lino, _exit); } tsdb->imem = NULL; - taosThreadRwlockUnlock(&tsdb->rwLock); + taosThreadMutexUnlock(&tsdb->mutex); tsdbUnrefMemTable(pMemTable, NULL, true); _exit: @@ -640,4 +643,4 @@ _exit: tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); } return code; -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 93a16b5502..38d221d978 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -55,25 +55,11 @@ static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) { TARRAY2_INIT(fs[0]->fSetArr); TARRAY2_INIT(fs[0]->fSetArrTmp); - // background task queue - taosThreadMutexInit(fs[0]->mutex, NULL); - fs[0]->bgTaskQueue->next = fs[0]->bgTaskQueue; - fs[0]->bgTaskQueue->prev = fs[0]->bgTaskQueue; - - taosThreadMutexInit(&fs[0]->commitMutex, NULL); - taosThreadCondInit(&fs[0]->canCommit, NULL); - fs[0]->blockCommit = false; - return 0; } static int32_t destroy_fs(STFileSystem **fs) { if (fs[0] == NULL) return 0; - taosThreadMutexDestroy(&fs[0]->commitMutex); - taosThreadCondDestroy(&fs[0]->canCommit); - taosThreadMutexDestroy(fs[0]->mutex); - - ASSERT(fs[0]->bgTaskNum == 0); TARRAY2_DESTROY(fs[0]->fSetArr, NULL); TARRAY2_DESTROY(fs[0]->fSetArrTmp, NULL); @@ -264,10 +250,11 @@ static int32_t apply_commit(STFileSystem *fs) { if (fset1 && fset2) { if (fset1->fid < fset2->fid) { // delete fset1 - TARRAY2_REMOVE(fsetArray1, i1, tsdbTFileSetRemove); + tsdbTFileSetRemove(fset1); + i1++; } else if (fset1->fid > fset2->fid) { // create new file set with fid of fset2->fid - code = tsdbTFileSetInitDup(fs->tsdb, fset2, &fset1); + code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1); if (code) return code; code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn); if (code) return code; @@ -282,10 +269,11 @@ static int32_t apply_commit(STFileSystem *fs) { } } else if (fset1) { // delete fset1 - TARRAY2_REMOVE(fsetArray1, i1, tsdbTFileSetRemove); + tsdbTFileSetRemove(fset1); + i1++; } else { // create new file set with fid of fset2->fid - code = tsdbTFileSetInitDup(fs->tsdb, fset2, &fset1); + code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1); if (code) return code; code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn); if (code) return code; @@ -512,7 +500,8 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { TARRAY2_FOREACH(lvl->fobjArr, fobj) { code = tsdbFSDoScanAndFixFile(fs, fobj); if (code) { - fset->maxVerValid = (fobj->f->minVer <= fobj->f->maxVer) ? TMIN(fset->maxVerValid, fobj->f->minVer - 1) : -1; + fset->maxVerValid = + (fobj->f->minVer <= fobj->f->maxVer) ? TMIN(fset->maxVerValid, fobj->f->minVer - 1) : -1; corrupt = true; } } @@ -592,7 +581,7 @@ static int32_t tsdbFSDupState(STFileSystem *fs) { const STFileSet *fset1; TARRAY2_FOREACH(src, fset1) { STFileSet *fset2; - code = tsdbTFileSetInitDup(fs->tsdb, fset1, &fset2); + code = tsdbTFileSetInitCopy(fs->tsdb, fset1, &fset2); if (code) return code; code = TARRAY2_APPEND(dst, fset2); if (code) return code; @@ -665,12 +654,6 @@ static int32_t close_file_system(STFileSystem *fs) { return 0; } -static int32_t apply_edit(STFileSystem *pFS) { - int32_t code = 0; - ASSERTS(0, "TODO: Not implemented yet"); - return code; -} - static int32_t fset_cmpr_fn(const struct STFileSet *pSet1, const struct STFileSet *pSet2) { if (pSet1->fid < pSet2->fid) { return -1; @@ -710,10 +693,23 @@ static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) { TSDB_CHECK_CODE(code, lino, _exit); } - // remove empty file set + // remove empty empty stt level and empty file set int32_t i = 0; while (i < TARRAY2_SIZE(fsetArray)) { fset = TARRAY2_GET(fsetArray, i); + + SSttLvl *lvl; + int32_t j = 0; + while (j < TARRAY2_SIZE(fset->lvlArr)) { + lvl = TARRAY2_GET(fset->lvlArr, j); + + if (TARRAY2_SIZE(lvl->fobjArr) == 0) { + TARRAY2_REMOVE(fset->lvlArr, j, tsdbSttLvlClear); + } else { + j++; + } + } + if (tsdbTFileSetIsEmpty(fset)) { TARRAY2_REMOVE(fsetArray, i, tsdbTFileSetClear); } else { @@ -753,13 +749,13 @@ _exit: static void tsdbDoWaitBgTask(STFileSystem *fs, STFSBgTask *task) { task->numWait++; - taosThreadCondWait(task->done, fs->mutex); + taosThreadCondWait(task->done, &fs->tsdb->mutex); task->numWait--; if (task->numWait == 0) { taosThreadCondDestroy(task->done); - if (task->free) { - task->free(task->arg); + if (task->destroy) { + task->destroy(task->arg); } taosMemoryFree(task); } @@ -770,8 +766,8 @@ static void tsdbDoDoneBgTask(STFileSystem *fs, STFSBgTask *task) { taosThreadCondBroadcast(task->done); } else { taosThreadCondDestroy(task->done); - if (task->free) { - task->free(task->arg); + if (task->destroy) { + task->destroy(task->arg); } taosMemoryFree(task); } @@ -780,23 +776,16 @@ static void tsdbDoDoneBgTask(STFileSystem *fs, STFSBgTask *task) { int32_t tsdbCloseFS(STFileSystem **fs) { if (fs[0] == NULL) return 0; - taosThreadMutexLock(fs[0]->mutex); - fs[0]->stop = true; - - if (fs[0]->bgTaskRunning) { - tsdbDoWaitBgTask(fs[0], fs[0]->bgTaskRunning); - } - taosThreadMutexUnlock(fs[0]->mutex); - + tsdbFSDisableBgTask(fs[0]); close_file_system(fs[0]); destroy_fs(fs); return 0; } int64_t tsdbFSAllocEid(STFileSystem *fs) { - taosThreadRwlockRdlock(&fs->tsdb->rwLock); + taosThreadMutexLock(&fs->tsdb->mutex); int64_t cid = ++fs->neid; - taosThreadRwlockUnlock(&fs->tsdb->rwLock); + taosThreadMutexUnlock(&fs->tsdb->mutex); return cid; } @@ -837,27 +826,34 @@ _exit: return code; } -static int32_t tsdbFSSetBlockCommit(STFileSystem *fs, bool block) { - taosThreadMutexLock(&fs->commitMutex); +static int32_t tsdbFSSetBlockCommit(STFileSet *fset, bool block) { if (block) { - fs->blockCommit = true; + fset->blockCommit = true; } else { - fs->blockCommit = false; - taosThreadCondSignal(&fs->canCommit); + fset->blockCommit = false; + if (fset->numWaitCommit > 0) { + taosThreadCondSignal(&fset->canCommit); + } } - taosThreadMutexUnlock(&fs->commitMutex); return 0; } -int32_t tsdbFSCheckCommit(STFileSystem *fs) { - taosThreadMutexLock(&fs->commitMutex); - while (fs->blockCommit) { - taosThreadCondWait(&fs->canCommit, &fs->commitMutex); +int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid) { + taosThreadMutexLock(&tsdb->mutex); + STFileSet *fset; + tsdbFSGetFSet(tsdb->pFS, fid, &fset); + if (fset) { + while (fset->blockCommit) { + fset->numWaitCommit++; + taosThreadCondWait(&fset->canCommit, &tsdb->mutex); + fset->numWaitCommit--; + } } - taosThreadMutexUnlock(&fs->commitMutex); + taosThreadMutexUnlock(&tsdb->mutex); return 0; } +// IMPORTANT: the caller must hold fs->tsdb->mutex int32_t tsdbFSEditCommit(STFileSystem *fs) { int32_t code = 0; int32_t lino = 0; @@ -867,36 +863,57 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { TSDB_CHECK_CODE(code, lino, _exit); // schedule merge - if (fs->tsdb->pVnode->config.sttTrigger > 1) { + int32_t sttTrigger = fs->tsdb->pVnode->config.sttTrigger; + if (sttTrigger > 1) { STFileSet *fset; - int32_t sttTrigger = fs->tsdb->pVnode->config.sttTrigger; - bool schedMerge = false; - bool blockCommit = false; - TARRAY2_FOREACH_REVERSE(fs->fSetArr, fset) { - if (TARRAY2_SIZE(fset->lvlArr) == 0) continue; + if (TARRAY2_SIZE(fset->lvlArr) == 0) { + tsdbFSSetBlockCommit(fset, false); + continue; + } SSttLvl *lvl = TARRAY2_FIRST(fset->lvlArr); - if (lvl->level != 0) continue; + if (lvl->level != 0) { + tsdbFSSetBlockCommit(fset, false); + continue; + } int32_t numFile = TARRAY2_SIZE(lvl->fobjArr); if (numFile >= sttTrigger) { - schedMerge = true; + // launch merge + code = tsdbSchedMerge(fs->tsdb, fset->fid); + TSDB_CHECK_CODE(code, lino, _exit); } if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR) { - blockCommit = true; + tsdbFSSetBlockCommit(fset, true); + } else { + tsdbFSSetBlockCommit(fset, false); } + } + } - if (schedMerge && blockCommit) break; + // clear empty level and fset + int32_t i = 0; + while (i < TARRAY2_SIZE(fs->fSetArr)) { + STFileSet *fset = TARRAY2_GET(fs->fSetArr, i); + + int32_t j = 0; + while (j < TARRAY2_SIZE(fset->lvlArr)) { + SSttLvl *lvl = TARRAY2_GET(fset->lvlArr, j); + + if (TARRAY2_SIZE(lvl->fobjArr) == 0) { + TARRAY2_REMOVE(fset->lvlArr, j, tsdbSttLvlClear); + } else { + j++; + } } - if (schedMerge) { - code = tsdbFSScheduleBgTask(fs, TSDB_BG_TASK_MERGER, tsdbMerge, NULL, fs->tsdb, NULL); - TSDB_CHECK_CODE(code, lino, _exit); + if (tsdbTFileSetIsEmpty(fset) && fset->bgTaskRunning == NULL) { + TARRAY2_REMOVE(fs->fSetArr, i, tsdbTFileSetClear); + } else { + i++; } - - tsdbFSSetBlockCommit(fs, blockCommit); } _exit: @@ -933,15 +950,15 @@ int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr) { TARRAY2_INIT(fsetArr[0]); - taosThreadRwlockRdlock(&fs->tsdb->rwLock); + taosThreadMutexLock(&fs->tsdb->mutex); TARRAY2_FOREACH(fs->fSetArr, fset) { - code = tsdbTFileSetInitDup(fs->tsdb, fset, &fset1); + code = tsdbTFileSetInitCopy(fs->tsdb, fset, &fset1); if (code) break; code = TARRAY2_APPEND(fsetArr[0], fset1); if (code) break; } - taosThreadRwlockUnlock(&fs->tsdb->rwLock); + taosThreadMutexUnlock(&fs->tsdb->mutex); if (code) { TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear); @@ -961,9 +978,9 @@ int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr) { } int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr) { - taosThreadRwlockRdlock(&fs->tsdb->rwLock); + taosThreadMutexLock(&fs->tsdb->mutex); int32_t code = tsdbFSCreateRefSnapshotWithoutLock(fs, fsetArr); - taosThreadRwlockUnlock(&fs->tsdb->rwLock); + taosThreadMutexUnlock(&fs->tsdb->mutex); return code; } @@ -1017,7 +1034,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange } } - taosThreadRwlockRdlock(&fs->tsdb->rwLock); + taosThreadMutexLock(&fs->tsdb->mutex); TARRAY2_FOREACH(fs->fSetArr, fset) { int64_t ever = VERSION_MAX; if (pHash) { @@ -1034,7 +1051,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange code = TARRAY2_APPEND(fsetArr[0], fset1); if (code) break; } - taosThreadRwlockUnlock(&fs->tsdb->rwLock); + taosThreadMutexUnlock(&fs->tsdb->mutex); _out: if (code) { @@ -1089,7 +1106,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev } } - taosThreadRwlockRdlock(&fs->tsdb->rwLock); + taosThreadMutexLock(&fs->tsdb->mutex); TARRAY2_FOREACH(fs->fSetArr, fset) { int64_t sver1 = sver; int64_t ever1 = ever; @@ -1118,7 +1135,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev fsr1 = NULL; } - taosThreadRwlockUnlock(&fs->tsdb->rwLock); + taosThreadMutexUnlock(&fs->tsdb->mutex); if (code) { tsdbTSnapRangeClear(&fsr1); @@ -1137,59 +1154,69 @@ _out: const char *gFSBgTaskName[] = {NULL, "MERGE", "RETENTION", "COMPACT"}; static int32_t tsdbFSRunBgTask(void *arg) { - STFileSystem *fs = (STFileSystem *)arg; + STFSBgTask *task = (STFSBgTask *)arg; + STFileSystem *fs = task->fs; + STFileSet *fset; - ASSERT(fs->bgTaskRunning != NULL); + tsdbFSGetFSet(fs, task->fid, &fset); - fs->bgTaskRunning->launchTime = taosGetTimestampMs(); - fs->bgTaskRunning->run(fs->bgTaskRunning->arg); - fs->bgTaskRunning->finishTime = taosGetTimestampMs(); + ASSERT(fset != NULL && fset->bgTaskRunning == task); + + task->launchTime = taosGetTimestampMs(); + task->run(task->arg); + task->finishTime = taosGetTimestampMs(); tsdbDebug("vgId:%d bg task:%s task id:%" PRId64 " finished, schedule time:%" PRId64 " launch time:%" PRId64 " finish time:%" PRId64, - TD_VID(fs->tsdb->pVnode), gFSBgTaskName[fs->bgTaskRunning->type], fs->bgTaskRunning->taskid, - fs->bgTaskRunning->scheduleTime, fs->bgTaskRunning->launchTime, fs->bgTaskRunning->finishTime); + TD_VID(fs->tsdb->pVnode), gFSBgTaskName[task->type], task->taskid, task->scheduleTime, task->launchTime, + task->finishTime); - taosThreadMutexLock(fs->mutex); + taosThreadMutexLock(&fs->tsdb->mutex); // free last - tsdbDoDoneBgTask(fs, fs->bgTaskRunning); - fs->bgTaskRunning = NULL; + tsdbDoDoneBgTask(fs, task); + fset->bgTaskRunning = NULL; // schedule next - if (fs->bgTaskNum > 0) { + if (fset->bgTaskNum > 0) { if (fs->stop) { - while (fs->bgTaskNum > 0) { - STFSBgTask *task = fs->bgTaskQueue->next; - task->prev->next = task->next; - task->next->prev = task->prev; - fs->bgTaskNum--; - tsdbDoDoneBgTask(fs, task); + while (fset->bgTaskNum > 0) { + STFSBgTask *nextTask = fset->bgTaskQueue->next; + nextTask->prev->next = nextTask->next; + nextTask->next->prev = nextTask->prev; + fset->bgTaskNum--; + tsdbDoDoneBgTask(fs, nextTask); } } else { // pop task from head - fs->bgTaskRunning = fs->bgTaskQueue->next; - fs->bgTaskRunning->prev->next = fs->bgTaskRunning->next; - fs->bgTaskRunning->next->prev = fs->bgTaskRunning->prev; - fs->bgTaskNum--; - vnodeScheduleTaskEx(1, tsdbFSRunBgTask, arg); + fset->bgTaskRunning = fset->bgTaskQueue->next; + fset->bgTaskRunning->prev->next = fset->bgTaskRunning->next; + fset->bgTaskRunning->next->prev = fset->bgTaskRunning->prev; + fset->bgTaskNum--; + vnodeScheduleTaskEx(1, tsdbFSRunBgTask, fset->bgTaskRunning); } } - taosThreadMutexUnlock(fs->mutex); + taosThreadMutexUnlock(&fs->tsdb->mutex); return 0; } -static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), - void (*destroy)(void *), void *arg, int64_t *taskid) { +// IMPORTANT: the caller must hold the fs->tsdb->mutex +int32_t tsdbFSScheduleBgTask(STFileSystem *fs, int32_t fid, EFSBgTaskT type, int32_t (*run)(void *), + void (*destroy)(void *), void *arg, int64_t *taskid) { if (fs->stop) { if (destroy) { destroy(arg); } - return 0; // TODO: use a better error code + return 0; } - for (STFSBgTask *task = fs->bgTaskQueue->next; task != fs->bgTaskQueue; task = task->next) { + STFileSet *fset; + tsdbFSGetFSet(fs, fid, &fset); + + ASSERT(fset != NULL); + + for (STFSBgTask *task = fset->bgTaskQueue->next; task != fset->bgTaskQueue; task = task->next) { if (task->type == type) { if (destroy) { destroy(arg); @@ -1203,22 +1230,24 @@ static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT type, int if (task == NULL) return TSDB_CODE_OUT_OF_MEMORY; taosThreadCondInit(task->done, NULL); + task->fs = fs; + task->fid = fid; task->type = type; task->run = run; - task->free = destroy; + task->destroy = destroy; task->arg = arg; task->scheduleTime = taosGetTimestampMs(); task->taskid = ++fs->taskid; - if (fs->bgTaskRunning == NULL && fs->bgTaskNum == 0) { + if (fset->bgTaskRunning == NULL && fset->bgTaskNum == 0) { // launch task directly - fs->bgTaskRunning = task; - vnodeScheduleTaskEx(1, tsdbFSRunBgTask, fs); + fset->bgTaskRunning = task; + vnodeScheduleTaskEx(1, tsdbFSRunBgTask, task); } else { // add to the queue tail - fs->bgTaskNum++; - task->next = fs->bgTaskQueue; - task->prev = fs->bgTaskQueue->prev; + fset->bgTaskNum++; + task->next = fset->bgTaskQueue; + task->prev = fset->bgTaskQueue->prev; task->prev->next = task; task->next->prev = task; } @@ -1227,68 +1256,30 @@ static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT type, int return 0; } -int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void (*free)(void *), void *arg, - int64_t *taskid) { - taosThreadMutexLock(fs->mutex); - int32_t code = tsdbFSScheduleBgTaskImpl(fs, type, run, free, arg, taskid); - taosThreadMutexUnlock(fs->mutex); - return code; -} +int32_t tsdbFSDisableBgTask(STFileSystem *fs) { + taosThreadMutexLock(&fs->tsdb->mutex); + for (;;) { + fs->stop = true; + bool done = true; -int32_t tsdbFSWaitBgTask(STFileSystem *fs, int64_t taskid) { - STFSBgTask *task = NULL; - - taosThreadMutexLock(fs->mutex); - - if (fs->bgTaskRunning && fs->bgTaskRunning->taskid == taskid) { - task = fs->bgTaskRunning; - } else { - for (STFSBgTask *taskt = fs->bgTaskQueue->next; taskt != fs->bgTaskQueue; taskt = taskt->next) { - if (taskt->taskid == taskid) { - task = taskt; + STFileSet *fset; + TARRAY2_FOREACH(fs->fSetArr, fset) { + if (fset->bgTaskRunning) { + tsdbDoWaitBgTask(fs, fset->bgTaskRunning); + done = false; break; } } - } - if (task) { - tsdbDoWaitBgTask(fs, task); + if (done) break; } - - taosThreadMutexUnlock(fs->mutex); + taosThreadMutexUnlock(&fs->tsdb->mutex); return 0; } -int32_t tsdbFSWaitAllBgTask(STFileSystem *fs) { - taosThreadMutexLock(fs->mutex); - - while (fs->bgTaskRunning) { - taosThreadCondWait(fs->bgTaskRunning->done, fs->mutex); - } - - taosThreadMutexUnlock(fs->mutex); - return 0; -} - -static int32_t tsdbFSDoDisableBgTask(STFileSystem *fs) { - fs->stop = true; - - if (fs->bgTaskRunning) { - tsdbDoWaitBgTask(fs, fs->bgTaskRunning); - } - return 0; -} - -int32_t tsdbFSDisableBgTask(STFileSystem *fs) { - taosThreadMutexLock(fs->mutex); - int32_t code = tsdbFSDoDisableBgTask(fs); - taosThreadMutexUnlock(fs->mutex); - return code; -} - int32_t tsdbFSEnableBgTask(STFileSystem *fs) { - taosThreadMutexLock(fs->mutex); + taosThreadMutexLock(&fs->tsdb->mutex); fs->stop = false; - taosThreadMutexUnlock(fs->mutex); + taosThreadMutexUnlock(&fs->tsdb->mutex); return 0; -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 31b98e5656..a3a8e2f575 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -22,22 +22,11 @@ extern "C" { #endif -/* Exposed Handle */ -typedef struct STFileSystem STFileSystem; -typedef struct STFSBgTask STFSBgTask; -// typedef TARRAY2(STFileSet *) TFileSetArray; - typedef enum { TSDB_FEDIT_COMMIT = 1, // TSDB_FEDIT_MERGE } EFEditT; -typedef enum { - TSDB_BG_TASK_MERGER = 1, - TSDB_BG_TASK_RETENTION, - TSDB_BG_TASK_COMPACT, -} EFSBgTaskT; - typedef enum { TSDB_FCURRENT = 1, TSDB_FCURRENT_C, // for commit @@ -67,37 +56,17 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e int32_t tsdbFSEditCommit(STFileSystem *fs); int32_t tsdbFSEditAbort(STFileSystem *fs); // background task -int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void (*free)(void *), void *arg, - int64_t *taskid); -int32_t tsdbFSWaitBgTask(STFileSystem *fs, int64_t taskid); -int32_t tsdbFSWaitAllBgTask(STFileSystem *fs); +int32_t tsdbFSScheduleBgTask(STFileSystem *fs, int32_t fid, EFSBgTaskT type, int32_t (*run)(void *), + void (*destroy)(void *), void *arg, int64_t *taskid); int32_t tsdbFSDisableBgTask(STFileSystem *fs); int32_t tsdbFSEnableBgTask(STFileSystem *fs); // other int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset); -int32_t tsdbFSCheckCommit(STFileSystem *fs); +int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid); // utils int32_t save_fs(const TFileSetArray *arr, const char *fname); int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype); -struct STFSBgTask { - EFSBgTaskT type; - int32_t (*run)(void *arg); - void (*free)(void *arg); - void *arg; - - TdThreadCond done[1]; - int32_t numWait; - - int64_t taskid; - int64_t scheduleTime; - int64_t launchTime; - int64_t finishTime; - - struct STFSBgTask *prev; - struct STFSBgTask *next; -}; - /* Exposed Structs */ struct STFileSystem { STsdb *tsdb; @@ -109,17 +78,8 @@ struct STFileSystem { TFileSetArray fSetArrTmp[1]; // background task queue - TdThreadMutex mutex[1]; - bool stop; - int64_t taskid; - int32_t bgTaskNum; - STFSBgTask bgTaskQueue[1]; - STFSBgTask *bgTaskRunning; - - // block commit variables - TdThreadMutex commitMutex; - TdThreadCond canCommit; - bool blockCommit; + bool stop; + int64_t taskid; }; #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 620fcb3a47..642d555366 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -342,11 +342,6 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) { int32_t idx = TARRAY2_SEARCH_IDX(lvl->fobjArr, &tfobjp, tsdbTFileObjCmpr, TD_EQ); ASSERT(idx >= 0); TARRAY2_REMOVE(lvl->fobjArr, idx, tsdbSttLvlClearFObj); - - if (TARRAY2_SIZE(lvl->fobjArr) == 0) { - // TODO: remove the stt level if no file exists anymore - // TARRAY2_REMOVE(&fset->lvlArr, lvl - fset->lvlArr.data, tsdbSttLvlClear); - } } else { ASSERT(tsdbIsSameTFile(&op->of, fset->farr[op->of.type]->f)); tsdbTFileObjUnref(fset->farr[op->of.type]); @@ -454,10 +449,22 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) { fset[0]->fid = fid; fset[0]->maxVerValid = VERSION_MAX; TARRAY2_INIT(fset[0]->lvlArr); + + // background task queue + fset[0]->bgTaskNum = 0; + fset[0]->bgTaskQueue->next = fset[0]->bgTaskQueue; + fset[0]->bgTaskQueue->prev = fset[0]->bgTaskQueue; + fset[0]->bgTaskRunning = NULL; + + // block commit variables + taosThreadCondInit(&fset[0]->canCommit, NULL); + fset[0]->numWaitCommit = 0; + fset[0]->blockCommit = false; + return 0; } -int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) { +int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) { int32_t code = tsdbTFileSetInit(fset1->fid, fset); if (code) return code; @@ -588,21 +595,23 @@ int32_t tsdbTFileSetClear(STFileSet **fset) { TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlClear); + taosThreadCondDestroy(&fset[0]->canCommit); taosMemoryFree(fset[0]); fset[0] = NULL; return 0; } -int32_t tsdbTFileSetRemove(STFileSet **fset) { +int32_t tsdbTFileSetRemove(STFileSet *fset) { + if (fset == NULL) return 0; + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { - if (fset[0]->farr[ftype] == NULL) continue; - tsdbTFileObjRemove(fset[0]->farr[ftype]); + if (fset->farr[ftype] == NULL) continue; + tsdbTFileObjRemove(fset->farr[ftype]); } - TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlRemove); - taosMemoryFree(fset[0]); - fset[0] = NULL; + TARRAY2_DESTROY(fset->lvlArr, tsdbSttLvlRemove); + return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index ea0f99f68e..34f174ade7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -28,6 +28,8 @@ typedef struct SSttLvl SSttLvl; typedef TARRAY2(STFileObj *) TFileObjArray; typedef TARRAY2(SSttLvl *) TSttLvlArray; typedef TARRAY2(STFileOp) TFileOpArray; +typedef struct STFileSystem STFileSystem; +typedef struct STFSBgTask STFSBgTask; typedef enum { TSDB_FOP_NONE = 0, @@ -41,10 +43,10 @@ typedef enum { // init/clear int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset); -int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset); +int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset); int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset); int32_t tsdbTFileSetClear(STFileSet **fset); -int32_t tsdbTFileSetRemove(STFileSet **fset); +int32_t tsdbTFileSetRemove(STFileSet *fset); int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset, TFileOpArray *fopArr); @@ -58,6 +60,7 @@ int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset); // cmpr int32_t tsdbTFileSetCmprFn(const STFileSet **fset1, const STFileSet **fset2); // edit +int32_t tsdbSttLvlClear(SSttLvl **lvl); int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op); int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *fset); // max commit id @@ -70,6 +73,33 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset); int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl); int32_t tsdbSttLvlClear(SSttLvl **lvl); +typedef enum { + TSDB_BG_TASK_MERGER = 1, + TSDB_BG_TASK_RETENTION, + TSDB_BG_TASK_COMPACT, +} EFSBgTaskT; + +struct STFSBgTask { + STFileSystem *fs; + int32_t fid; + + EFSBgTaskT type; + int32_t (*run)(void *arg); + void (*destroy)(void *arg); + void *arg; + + TdThreadCond done[1]; + int32_t numWait; + + int64_t taskid; + int64_t scheduleTime; + int64_t launchTime; + int64_t finishTime; + + struct STFSBgTask *prev; + struct STFSBgTask *next; +}; + struct STFileOp { tsdb_fop_t optype; int32_t fid; @@ -87,6 +117,16 @@ struct STFileSet { int64_t maxVerValid; STFileObj *farr[TSDB_FTYPE_MAX]; // file array TSttLvlArray lvlArr[1]; // level array + + // background task queue + int32_t bgTaskNum; + STFSBgTask bgTaskQueue[1]; + STFSBgTask *bgTaskRunning; + + // block commit variables + TdThreadCond canCommit; + int32_t numWaitCommit; + bool blockCommit; }; struct STSnapRange { diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index e659cedba3..0c20a342d3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -15,11 +15,17 @@ #include "tsdbMerge.h" -#define TSDB_MAX_LEVEL 6 // means max level is 7 +#define TSDB_MAX_LEVEL 2 // means max level is 3 typedef struct { - STsdb *tsdb; - TFileSetArray *fsetArr; + STsdb *tsdb; + int32_t fid; +} SMergeArg; + +typedef struct { + STsdb *tsdb; + int32_t fid; + STFileSet *fset; int32_t sttTrigger; int32_t maxRow; @@ -313,7 +319,6 @@ static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) { if (merger->ctx->fset->farr[ftype]) { config.files[ftype].exist = true; config.files[ftype].file = merger->ctx->fset->farr[ftype]->f[0]; - } else { config.files[ftype].exist = false; } @@ -397,13 +402,13 @@ static int32_t tsdbMergeFileSetEnd(SMerger *merger) { code = tsdbFSEditBegin(merger->tsdb->pFS, merger->fopArr, TSDB_FEDIT_MERGE); TSDB_CHECK_CODE(code, lino, _exit); - taosThreadRwlockWrlock(&merger->tsdb->rwLock); + taosThreadMutexLock(&merger->tsdb->mutex); code = tsdbFSEditCommit(merger->tsdb->pFS); if (code) { - taosThreadRwlockUnlock(&merger->tsdb->rwLock); + taosThreadMutexUnlock(&merger->tsdb->mutex); TSDB_CHECK_CODE(code, lino, _exit); } - taosThreadRwlockUnlock(&merger->tsdb->rwLock); + taosThreadMutexUnlock(&merger->tsdb->mutex); _exit: if (code) { @@ -478,30 +483,21 @@ _exit: } static int32_t tsdbDoMerge(SMerger *merger) { - int32_t code = 0; - int32_t lino = 0; + int32_t code = 0; + int32_t lino = 0; + SSttLvl *lvl = TARRAY2_FIRST(merger->fset->lvlArr); - STFileSet *fset; - TARRAY2_FOREACH(merger->fsetArr, fset) { - if (TARRAY2_SIZE(fset->lvlArr) == 0) continue; + if (TARRAY2_SIZE(merger->fset->lvlArr) == 0) return 0; + if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) return 0; - SSttLvl *lvl = TARRAY2_FIRST(fset->lvlArr); + code = tsdbMergerOpen(merger); + TSDB_CHECK_CODE(code, lino, _exit); - if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) continue; + code = tsdbMergeFileSet(merger, merger->fset); + TSDB_CHECK_CODE(code, lino, _exit); - if (!merger->ctx->opened) { - code = tsdbMergerOpen(merger); - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tsdbMergeFileSet(merger, fset); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (merger->ctx->opened) { - code = tsdbMergerClose(merger); - TSDB_CHECK_CODE(code, lino, _exit); - } + code = tsdbMergerClose(merger); + TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { @@ -512,36 +508,73 @@ _exit: return code; } -int32_t tsdbMerge(void *arg) { - int32_t code = 0; - int32_t lino = 0; - STsdb *tsdb = (STsdb *)arg; +static int32_t tsdbMergeGetFSet(SMerger *merger) { + STFileSet *fset; - SMerger merger[1] = {{ - .tsdb = tsdb, - .sttTrigger = tsdb->pVnode->config.sttTrigger, - }}; - - if (merger->sttTrigger <= 1) { + taosThreadMutexLock(&merger->tsdb->mutex); + tsdbFSGetFSet(merger->tsdb->pFS, merger->fid, &fset); + if (fset == NULL) { + taosThreadMutexUnlock(&merger->tsdb->mutex); return 0; } - code = tsdbFSCreateCopySnapshot(tsdb->pFS, &merger->fsetArr); + int32_t code = tsdbTFileSetInitCopy(merger->tsdb, fset, &merger->fset); + if (code) { + taosThreadMutexUnlock(&merger->tsdb->mutex); + return code; + } + taosThreadMutexUnlock(&merger->tsdb->mutex); + return 0; +} + +static int32_t tsdbMerge(void *arg) { + int32_t code = 0; + int32_t lino = 0; + SMergeArg *mergeArg = (SMergeArg *)arg; + STsdb *tsdb = mergeArg->tsdb; + + SMerger merger[1] = {{ + .tsdb = tsdb, + .fid = mergeArg->fid, + .sttTrigger = tsdb->pVnode->config.sttTrigger, + }}; + + if (merger->sttTrigger <= 1) return 0; + + // copy snapshot + code = tsdbMergeGetFSet(merger); TSDB_CHECK_CODE(code, lino, _exit); + if (merger->fset == NULL) return 0; + + // do merge + tsdbDebug("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid); code = tsdbDoMerge(merger); + tsdbDebug("vgId:%d merge done, fid:%d", TD_VID(tsdb->pVnode), mergeArg->fid); TSDB_CHECK_CODE(code, lino, _exit); - tsdbFSDestroyCopySnapshot(&merger->fsetArr); - _exit: if (code) { TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbFatal("vgId:%d, failed to merge stt files since %s. code:%d", TD_VID(tsdb->pVnode), terrstr(), code); taosMsleep(100); exit(EXIT_FAILURE); - } else if (merger->ctx->opened) { - tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); } + tsdbTFileSetClear(&merger->fset); return code; } + +int32_t tsdbSchedMerge(STsdb *tsdb, int32_t fid) { + SMergeArg *arg = taosMemoryMalloc(sizeof(*arg)); + if (arg == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + arg->tsdb = tsdb; + arg->fid = fid; + + int32_t code = tsdbFSScheduleBgTask(tsdb->pFS, fid, TSDB_BG_TASK_MERGER, tsdbMerge, taosMemoryFree, arg, NULL); + if (code) taosMemoryFree(arg); + + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 6dd66c7a40..c32b2eedd7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -53,7 +53,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee snprintf(pTsdb->path, TD_PATH_MAX, "%s%s%s", pVnode->path, TD_DIRSEP, dir); // taosRealPath(pTsdb->path, NULL, slen); pTsdb->pVnode = pVnode; - taosThreadRwlockInit(&pTsdb->rwLock, NULL); + taosThreadMutexInit(&pTsdb->mutex, NULL); if (!pKeepCfg) { tsdbSetKeepCfg(pTsdb, &pVnode->config.tsdbCfg); } else { @@ -99,15 +99,14 @@ int tsdbClose(STsdb **pTsdb) { tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d, keepTimeOffset:%d", TD_VID(pdb->pVnode), pdb->path, pdb->keepCfg.days, pdb->keepCfg.keep0, pdb->keepCfg.keep1, pdb->keepCfg.keep2, pdb->keepCfg.keepTimeOffset); - taosThreadRwlockWrlock(&(*pTsdb)->rwLock); + taosThreadMutexLock(&(*pTsdb)->mutex); tsdbMemTableDestroy((*pTsdb)->mem, true); (*pTsdb)->mem = NULL; - taosThreadRwlockUnlock(&(*pTsdb)->rwLock); - - taosThreadRwlockDestroy(&(*pTsdb)->rwLock); + taosThreadMutexUnlock(&(*pTsdb)->mutex); tsdbCloseFS(&(*pTsdb)->pFS); tsdbCloseCache(*pTsdb); + taosThreadMutexDestroy(&(*pTsdb)->mutex); taosMemoryFreeClear(*pTsdb); } return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 65cebf0ca0..41e0bd373e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1105,8 +1105,9 @@ static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* p (pVerRange->maxVer < pBlock->record.maxVer && pVerRange->maxVer >= pBlock->record.minVer); } -static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, - int32_t* nextIndex, int32_t order, SBrinRecord* pRecord) { +static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo, + STableBlockScanInfo* pTableBlockScanInfo, int32_t* nextIndex, int32_t order, + SBrinRecord* pRecord) { bool asc = ASCENDING_TRAVERSE(order); if (asc && pBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockIdxList) - 1) { return false; @@ -1119,7 +1120,8 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo int32_t step = asc ? 1 : -1; // *nextIndex = pBlockInfo->tbBlockIdx + step; // *pBlockIndex = *(SBlockIndex*)taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); - STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); + STableDataBlockIdx* pTableDataBlockIdx = + taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex); memcpy(pRecord, &p->record, sizeof(SBrinRecord)); @@ -1145,7 +1147,8 @@ static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlock return -1; } -static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t index, int32_t step) { +static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t index, + int32_t step) { if (index < 0 || index >= pBlockIter->numOfBlocks) { return -1; } @@ -1153,12 +1156,13 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte SFileDataBlockInfo fblock = *(SFileDataBlockInfo*)taosArrayGet(pBlockIter->blockList, index); pBlockIter->index += step; - if (index != pBlockIter->index) { + if (index != pBlockIter->index) { if (index > pBlockIter->index) { for (int32_t i = index - 1; i >= pBlockIter->index; --i) { SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i); - STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + STableBlockScanInfo* pBlockScanInfo = + getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); pTableDataBlockIdx->globalIndex = i + 1; @@ -1168,13 +1172,13 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte for (int32_t i = index + 1; i <= pBlockIter->index; ++i) { SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i); - STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + STableBlockScanInfo* pBlockScanInfo = + getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); pTableDataBlockIdx->globalIndex = i - 1; taosArraySet(pBlockIter->blockList, i - 1, pBlockInfo); } - } taosArraySet(pBlockIter->blockList, pBlockIter->index, &fblock); @@ -1286,7 +1290,8 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* int32_t neighborIndex = 0; SBrinRecord rec = {0}; - bool hasNeighbor = getNeighborBlockOfSameTable(&pReader->status.blockIter, pBlockInfo, pScanInfo, &neighborIndex, pReader->info.order, &rec); + bool hasNeighbor = getNeighborBlockOfSameTable(&pReader->status.blockIter, pBlockInfo, pScanInfo, &neighborIndex, + pReader->info.order, &rec); // overlap with neighbor if (hasNeighbor) { @@ -1420,9 +1425,7 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc } } -static void doPinSttBlock(SLastBlockReader* pLastBlockReader) { - tMergeTreePinSttBlock(&pLastBlockReader->mergeTree); -} +static void doPinSttBlock(SLastBlockReader* pLastBlockReader) { tMergeTreePinSttBlock(&pLastBlockReader->mergeTree); } static void doUnpinSttBlock(SLastBlockReader* pLastBlockReader) { tMergeTreeUnpinSttBlock(&pLastBlockReader->mergeTree); @@ -1568,7 +1571,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - int32_t code = tsdbRowMergerAdd(pMerger, fRow1, NULL); + int32_t code = tsdbRowMergerAdd(pMerger, fRow1, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1618,7 +1621,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - int32_t code = tsdbRowMergerAdd(pMerger, fRow1, NULL); + int32_t code = tsdbRowMergerAdd(pMerger, fRow1, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1826,8 +1829,8 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* int64_t key = hasDataInFileBlock(pBlockData, pDumpInfo) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; - TSDBKEY k = TSDBROW_KEY(pRow); - TSDBKEY ik = TSDBROW_KEY(piRow); + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); STSchema* pSchema = NULL; if (pRow->type == TSDBROW_ROW_FMT) { @@ -2219,8 +2222,9 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; TSDBROW *pRow = NULL, *piRow = NULL; - int64_t key = (pBlockData->nRow > 0 && (!pDumpInfo->allDumped)) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : - (ASCENDING_TRAVERSE(pReader->info.order) ? INT64_MAX : INT64_MIN); + int64_t key = (pBlockData->nRow > 0 && (!pDumpInfo->allDumped)) + ? pBlockData->aTSKEY[pDumpInfo->rowIndex] + : (ASCENDING_TRAVERSE(pReader->info.order) ? INT64_MAX : INT64_MIN); if (pBlockScanInfo->iter.hasVal) { pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); } @@ -2257,7 +2261,8 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock *loadNeighbor = false; SBrinRecord rec = {0}; - bool hasNeighbor = getNeighborBlockOfSameTable(&pReader->status.blockIter, pBlockInfo, pBlockScanInfo, &nextIndex, pReader->info.order, &rec); + bool hasNeighbor = getNeighborBlockOfSameTable(&pReader->status.blockIter, pBlockInfo, pBlockScanInfo, &nextIndex, + pReader->info.order, &rec); if (!hasNeighbor) { // do nothing return code; } @@ -2268,7 +2273,7 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock // 1. find the next neighbor block in the scan block list STableDataBlockIdx* tableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, nextIndex); - int32_t neighborIndex = tableDataBlockIdx->globalIndex; + int32_t neighborIndex = tableDataBlockIdx->globalIndex; // 2. remove it from the scan block list setFileBlockActiveInBlockIter(pReader, pBlockIter, neighborIndex, step); @@ -2704,7 +2709,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { (ASCENDING_TRAVERSE(pReader->info.order)) ? pBlockInfo->record.firstKey : pBlockInfo->record.lastKey; code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { - bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader); + bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader); int64_t tsLast = bHasDataInLastBlock ? getCurrentKeyInLastBlock(pLastBlockReader) : INT64_MIN; if (!bHasDataInLastBlock || ((asc && pBlockInfo->record.lastKey < tsLast) || (!asc && pBlockInfo->record.firstKey > tsLast))) { @@ -3479,7 +3484,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, // start to merge duplicated rows STSchema* pTSchema = NULL; - if (current.type == TSDBROW_ROW_FMT) { // get the correct schema for row-wise data in memory + if (current.type == TSDBROW_ROW_FMT) { // get the correct schema for row-wise data in memory pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); if (pTSchema == NULL) { return terrno; @@ -3525,8 +3530,8 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p SRow** pTSRow) { SRowMerger* pMerger = &pReader->status.merger; - TSDBKEY k = TSDBROW_KEY(pRow); - TSDBKEY ik = TSDBROW_KEY(piRow); + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); STSchema* pSchema = NULL; if (pRow->type == TSDBROW_ROW_FMT) { @@ -4907,12 +4912,12 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs SVersionRange* pRange = &pReader->info.verRange; // lock - taosThreadRwlockRdlock(&pTsdb->rwLock); + taosThreadMutexLock(&pTsdb->mutex); // alloc STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap)); if (pSnap == NULL) { - taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadMutexUnlock(&pTsdb->mutex); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -4922,7 +4927,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pMem = pTsdb->mem; pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode)); if (pSnap->pNode == NULL) { - taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadMutexUnlock(&pTsdb->mutex); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -4937,7 +4942,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pIMem = pTsdb->imem; pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode)); if (pSnap->pINode == NULL) { - taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadMutexUnlock(&pTsdb->mutex); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -4952,7 +4957,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs code = tsdbFSCreateRefSnapshotWithoutLock(pTsdb->pFS, &pSnap->pfSetArray); // unlock - taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadMutexUnlock(&pTsdb->mutex); if (code == TSDB_CODE_SUCCESS) { tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); @@ -5005,4 +5010,5 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { pReader->status.fileIter.pLastBlockReader->mergeTree.idStr = pReader->idStr; } -void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } +void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ +} diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index f2665dcf26..6c41b46c73 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -25,11 +25,6 @@ typedef struct { TFileSetArray *fsetArr; TFileOpArray fopArr[1]; - - struct { - int32_t fsetArrIdx; - STFileSet *fset; - } ctx[1]; } SRTNer; static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) { @@ -227,8 +222,8 @@ _exit: typedef struct { STsdb *tsdb; - int32_t sync; int64_t now; + int32_t fid; } SRtnArg; static int32_t tsdbDoRetentionBegin(SRtnArg *arg, SRTNer *rtner) { @@ -263,15 +258,15 @@ static int32_t tsdbDoRetentionEnd(SRTNer *rtner) { code = tsdbFSEditBegin(rtner->tsdb->pFS, rtner->fopArr, TSDB_FEDIT_MERGE); TSDB_CHECK_CODE(code, lino, _exit); - taosThreadRwlockWrlock(&rtner->tsdb->rwLock); + taosThreadMutexLock(&rtner->tsdb->mutex); code = tsdbFSEditCommit(rtner->tsdb->pFS); if (code) { - taosThreadRwlockUnlock(&rtner->tsdb->rwLock); + taosThreadMutexUnlock(&rtner->tsdb->mutex); TSDB_CHECK_CODE(code, lino, _exit); } - taosThreadRwlockUnlock(&rtner->tsdb->rwLock); + taosThreadMutexUnlock(&rtner->tsdb->mutex); TARRAY2_DESTROY(rtner->fopArr, NULL); @@ -285,95 +280,83 @@ _exit: return code; } -static int32_t tsdbDoRetention2(void *arg) { - int32_t code = 0; - int32_t lino = 0; - SRTNer rtner[1] = {0}; +static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + STFileObj *fobj = NULL; + int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now); - code = tsdbDoRetentionBegin(arg, rtner); - TSDB_CHECK_CODE(code, lino, _exit); + if (expLevel < 0) { // remove the fileset + for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) { + if (fobj == NULL) continue; - for (rtner->ctx->fsetArrIdx = 0; rtner->ctx->fsetArrIdx < TARRAY2_SIZE(rtner->fsetArr); rtner->ctx->fsetArrIdx++) { - rtner->ctx->fset = TARRAY2_GET(rtner->fsetArr, rtner->ctx->fsetArrIdx); - - STFileObj *fobj; - int32_t expLevel = tsdbFidLevel(rtner->ctx->fset->fid, &rtner->tsdb->keepCfg, rtner->now); - - if (expLevel < 0) { // remove the file set - for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) { - if (fobj == NULL) continue; - - int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs); - if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) { - code = tsdbRemoveFileObjectS3(rtner, fobj); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - code = tsdbDoRemoveFileObject(rtner, fobj); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - SSttLvl *lvl; - TARRAY2_FOREACH(rtner->ctx->fset->lvlArr, lvl) { - TARRAY2_FOREACH(lvl->fobjArr, fobj) { - code = tsdbDoRemoveFileObject(rtner, fobj); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - } else if (expLevel == 0) { - continue; - } else { - SDiskID did; - - if (tfsAllocDisk(rtner->tsdb->pVnode->pTfs, expLevel, &did) < 0) { - code = terrno; + int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs); + if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) { + code = tsdbRemoveFileObjectS3(rtner, fobj); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDoRemoveFileObject(rtner, fobj); TSDB_CHECK_CODE(code, lino, _exit); } - tfsMkdirRecurAt(rtner->tsdb->pVnode->pTfs, rtner->tsdb->path, did); + } - // data - for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) { - if (fobj == NULL) continue; + SSttLvl *lvl; + TARRAY2_FOREACH(fset->lvlArr, lvl) { + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + code = tsdbDoRemoveFileObject(rtner, fobj); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } else if (expLevel == 0) { // only migrate to upper level + return 0; + } else { // migrate + SDiskID did; + if (tfsAllocDisk(rtner->tsdb->pVnode->pTfs, expLevel, &did) < 0) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } + tfsMkdirRecurAt(rtner->tsdb->pVnode->pTfs, rtner->tsdb->path, did); + + // data + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = fset->farr[ftype], 1); ++ftype) { + if (fobj == NULL) continue; + + if (fobj->f->did.level == did.level) continue; + + int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs); + if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) { + code = tsdbMigrateDataFileS3(rtner, fobj, &did); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + if (tsS3Enabled) { + int64_t fsize = 0; + if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(terrno); + tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(rtner->tsdb->pVnode), __func__, + fobj->fname, tstrerror(code)); + TSDB_CHECK_CODE(code, lino, _exit); + } + s3EvictCache(fobj->fname, fsize * 2); + } + + code = tsdbDoMigrateFileObj(rtner, fobj, &did); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + // stt + SSttLvl *lvl; + TARRAY2_FOREACH(fset->lvlArr, lvl) { + TARRAY2_FOREACH(lvl->fobjArr, fobj) { if (fobj->f->did.level == did.level) continue; - int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs); - if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) { - code = tsdbMigrateDataFileS3(rtner, fobj, &did); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - if (tsS3Enabled) { - int64_t fsize = 0; - if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(terrno); - tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(rtner->tsdb->pVnode), __func__, - fobj->fname, tstrerror(code)); - TSDB_CHECK_CODE(code, lino, _exit); - } - s3EvictCache(fobj->fname, fsize * 2); - } - - code = tsdbDoMigrateFileObj(rtner, fobj, &did); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - // stt - SSttLvl *lvl; - TARRAY2_FOREACH(rtner->ctx->fset->lvlArr, lvl) { - TARRAY2_FOREACH(lvl->fobjArr, fobj) { - if (fobj->f->did.level == did.level) continue; - - code = tsdbDoMigrateFileObj(rtner, fobj, &did); - TSDB_CHECK_CODE(code, lino, _exit); - } + code = tsdbDoMigrateFileObj(rtner, fobj, &did); + TSDB_CHECK_CODE(code, lino, _exit); } } } - code = tsdbDoRetentionEnd(rtner); - TSDB_CHECK_CODE(code, lino, _exit); - _exit: if (code) { if (TARRAY2_DATA(rtner->fopArr)) { @@ -389,30 +372,105 @@ _exit: return code; } -static void tsdbFreeRtnArg(void *arg) { - SRtnArg *rArg = (SRtnArg *)arg; - if (rArg->sync) { - tsem_post(&rArg->tsdb->pVnode->canCommit); +static int32_t tsdbDoRetentionSync(void *arg) { + int32_t code = 0; + int32_t lino = 0; + SRTNer rtner[1] = {0}; + + code = tsdbDoRetentionBegin(arg, rtner); + TSDB_CHECK_CODE(code, lino, _exit); + + STFileSet *fset; + TARRAY2_FOREACH(rtner->fsetArr, fset) { + code = tsdbDoRetentionOnFileSet(rtner, fset); + TSDB_CHECK_CODE(code, lino, _exit); } - taosMemoryFree(arg); + + code = tsdbDoRetentionEnd(rtner); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + } + tsem_post(&((SRtnArg *)arg)->tsdb->pVnode->canCommit); + return code; } -int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) { - SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); - if (arg == NULL) return TSDB_CODE_OUT_OF_MEMORY; - arg->tsdb = tsdb; - arg->sync = sync; - arg->now = now; +static int32_t tsdbDoRetentionAsync(void *arg) { + int32_t code = 0; + int32_t lino = 0; + SRTNer rtner[1] = {0}; - if (sync) { - tsem_wait(&tsdb->pVnode->canCommit); + code = tsdbDoRetentionBegin(arg, rtner); + TSDB_CHECK_CODE(code, lino, _exit); + + STFileSet *fset; + TARRAY2_FOREACH(rtner->fsetArr, fset) { + if (fset->fid != ((SRtnArg *)arg)->fid) continue; + + code = tsdbDoRetentionOnFileSet(rtner, fset); + TSDB_CHECK_CODE(code, lino, _exit); } - int64_t taskid; - int32_t code = - tsdbFSScheduleBgTask(tsdb->pFS, TSDB_BG_TASK_RETENTION, tsdbDoRetention2, tsdbFreeRtnArg, arg, &taskid); + code = tsdbDoRetentionEnd(rtner); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: if (code) { - tsdbFreeRtnArg(arg); + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); } return code; } + +static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); } + +int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) { + int32_t code = 0; + + if (sync) { // sync retention + SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); + if (arg == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + arg->tsdb = tsdb; + arg->now = now; + arg->fid = INT32_MAX; + + tsem_wait(&tsdb->pVnode->canCommit); + code = vnodeScheduleTask(tsdbDoRetentionSync, arg); + if (code) { + tsem_post(&tsdb->pVnode->canCommit); + taosMemoryFree(arg); + return code; + } + } else { // async retention + taosThreadMutexLock(&tsdb->mutex); + + STFileSet *fset; + TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { + SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); + if (arg == NULL) { + taosThreadMutexUnlock(&tsdb->mutex); + return TSDB_CODE_OUT_OF_MEMORY; + } + + arg->tsdb = tsdb; + arg->now = now; + arg->fid = fset->fid; + + code = tsdbFSScheduleBgTask(tsdb->pFS, fset->fid, TSDB_BG_TASK_RETENTION, tsdbDoRetentionAsync, tsdbFreeRtnArg, + arg, NULL); + if (code) { + tsdbFreeRtnArg(arg); + taosThreadMutexUnlock(&tsdb->mutex); + return code; + } + } + + taosThreadMutexUnlock(&tsdb->mutex); + } + + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 3b4827a6be..df7154b775 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -38,8 +38,8 @@ struct STsdbSnapReader { struct { int32_t fsrArrIdx; STSnapRange* fsr; - bool isDataDone; - bool isTombDone; + bool isDataDone; + bool isTombDone; } ctx[1]; // reader @@ -1095,17 +1095,17 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) { code = tsdbFSEditAbort(writer[0]->tsdb->pFS); TSDB_CHECK_CODE(code, lino, _exit); } else { - taosThreadRwlockWrlock(&writer[0]->tsdb->rwLock); + taosThreadMutexLock(&writer[0]->tsdb->mutex); code = tsdbFSEditCommit(writer[0]->tsdb->pFS); if (code) { - taosThreadRwlockUnlock(&writer[0]->tsdb->rwLock); + taosThreadMutexUnlock(&writer[0]->tsdb->mutex); TSDB_CHECK_CODE(code, lino, _exit); } writer[0]->tsdb->pFS->fsstate = TSDB_FS_STATE_NORMAL; - taosThreadRwlockUnlock(&writer[0]->tsdb->rwLock); + taosThreadMutexUnlock(&writer[0]->tsdb->mutex); } tsdbFSEnableBgTask(tsdb->pFS); @@ -1236,7 +1236,7 @@ static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP if (fset->farr[ftype] == NULL) continue; typ = tsdbFTypeToSRangeTyp(ftype); ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX); - STFile* f = fset->farr[ftype]->f; + STFile* f = fset->farr[ftype]->f; if (f->maxVer > fset->maxVerValid) { corrupt = true; tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 @@ -1255,7 +1255,7 @@ static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP TARRAY2_FOREACH(fset->lvlArr, lvl) { STFileObj* fobj; TARRAY2_FOREACH(lvl->fobjArr, fobj) { - STFile* f = fobj->f; + STFile* f = fobj->f; if (f->maxVer > fset->maxVerValid) { corrupt = true; tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 @@ -1299,7 +1299,7 @@ static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { } int32_t code = 0; - taosThreadRwlockRdlock(&fs->tsdb->rwLock); + taosThreadMutexLock(&fs->tsdb->mutex); STFileSet* fset; TARRAY2_FOREACH(fs->fSetArr, fset) { STsdbSnapPartition* pItem = NULL; @@ -1311,7 +1311,7 @@ static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn); ASSERT(code == 0); } - taosThreadRwlockUnlock(&fs->tsdb->rwLock); + taosThreadMutexUnlock(&fs->tsdb->mutex); if (code) { TARRAY2_DESTROY(pList, tsdbSnapPartitionClear); From 4163a3be7c8233d9ca32de6d570b766ba8e5c00e Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 27 Oct 2023 15:21:55 +0800 Subject: [PATCH 21/37] feat: resend snap replication of data on timeout --- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 1 + source/dnode/vnode/src/vnd/vnodeSync.c | 5 +- source/libs/sync/inc/syncSnapshot.h | 6 +- source/libs/sync/src/syncMain.c | 3 +- source/libs/sync/src/syncSnapshot.c | 265 +++++++++------------ source/libs/sync/src/syncTimeout.c | 5 +- 6 files changed, 126 insertions(+), 159 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 87b407efcb..91244e321f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -584,6 +584,7 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * // commit json if (!rollback) { + ASSERT(pVnode->config.vgId == pWriter->info.config.vgId); pWriter->info.state.committed = pWriter->ever; pVnode->config = pWriter->info.config; pVnode->state = (SVState){.committed = pWriter->info.state.committed, diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 6c03ed68e9..c9e805d80b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -516,7 +516,10 @@ static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool pVnode->config.vgId, isApply, pSnapshot->lastApplyIndex, pSnapshot->lastApplyTerm, pSnapshot->lastConfigIndex); int32_t code = vnodeSnapWriterClose(pWriter, !isApply, pSnapshot); - vInfo("vgId:%d, apply vnode snapshot finished, code:0x%x", pVnode->config.vgId, code); + if (code != 0) { + vError("vgId:%d, failed to finish applying vnode snapshot since %s, code:0x%x", pVnode->config.vgId, terrstr(), + code); + } return code; } diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index e68702568a..f8ee99e8a0 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -22,9 +22,9 @@ extern "C" { #include "syncInt.h" -#define SYNC_SNAPSHOT_SEQ_INVALID -2 #define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -3 -#define SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT -1 +#define SYNC_SNAPSHOT_SEQ_INVALID -2 +#define SYNC_SNAPSHOT_SEQ_PREP -1 #define SYNC_SNAPSHOT_SEQ_BEGIN 0 #define SYNC_SNAPSHOT_SEQ_END 0x7FFFFFFF @@ -57,8 +57,6 @@ typedef struct SSyncSnapshotSender { int32_t seq; int32_t ack; void *pReader; - void *pCurrentBlock; - int32_t blockLen; SSnapshotParam snapshotParam; SSnapshot snapshot; SSyncCfg lastConfig; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f9dc10da02..199c7a1445 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -818,7 +818,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion) { if (!taosCheckExistFile(pSyncNode->configPath)) { // create a new raft config file - sInfo("vgId:%d, create a new raft config file", pSyncNode->vgId); + sInfo("vgId:%d, create a new raft config file", pSyncInfo->vgId); + pSyncNode->vgId = pSyncInfo->vgId; pSyncNode->raftCfg.isStandBy = pSyncInfo->isStandBy; pSyncNode->raftCfg.snapshotStrategy = pSyncInfo->snapshotStrategy; pSyncNode->raftCfg.lastConfigIndex = pSyncInfo->syncCfg.lastIndex; diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index eee0ab2cc9..92d9571906 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -31,9 +31,9 @@ static void syncSnapBufferReset(SSyncSnapBuffer *pBuf) { } pBuf->entries[i % pBuf->size] = NULL; } - pBuf->start = 1; - pBuf->end = 1; - pBuf->cursor = 0; + pBuf->start = SYNC_SNAPSHOT_SEQ_BEGIN + 1; + pBuf->end = pBuf->start; + pBuf->cursor = pBuf->start - 1; taosThreadMutexUnlock(&pBuf->mutex); } @@ -76,8 +76,6 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; pSender->pReader = NULL; - pSender->pCurrentBlock = NULL; - pSender->blockLen = 0; pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS; pSender->pSyncNode = pSyncNode; pSender->replicaIndex = replicaIndex; @@ -113,12 +111,6 @@ void syncSnapBlockDestroy(void *ptr) { void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { if (pSender == NULL) return; - // free current block - if (pSender->pCurrentBlock != NULL) { - taosMemoryFree(pSender->pCurrentBlock); - pSender->pCurrentBlock = NULL; - } - // close reader if (pSender->pReader != NULL) { pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); @@ -141,11 +133,9 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { int8_t started = atomic_val_compare_exchange_8(&pSender->start, false, true); if (started) return 0; - pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; + pSender->seq = SYNC_SNAPSHOT_SEQ_PREP; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; pSender->pReader = NULL; - pSender->pCurrentBlock = NULL; - pSender->blockLen = 0; pSender->snapshotParam.start = SYNC_INDEX_INVALID; pSender->snapshotParam.end = SYNC_INDEX_INVALID; pSender->snapshot.data = NULL; @@ -196,7 +186,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; pMsg->lastConfig = pSender->lastConfig; pMsg->startTime = pSender->startTime; - pMsg->seq = SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT; + pMsg->seq = pSender->seq; if (dataLen > 0) { pMsg->payloadType = snapInfo.type; @@ -236,13 +226,6 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { pSender->pReader = NULL; } - // free current block - if (pSender->pCurrentBlock != NULL) { - taosMemoryFree(pSender->pCurrentBlock); - pSender->pCurrentBlock = NULL; - pSender->blockLen = 0; - } - syncSnapBufferReset(pSender->pSndBuf); SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; @@ -255,39 +238,45 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { int32_t code = -1; SyncSnapBlock *pBlk = NULL; - if (pSender->seq != SYNC_SNAPSHOT_SEQ_END) { + if (pSender->seq < SYNC_SNAPSHOT_SEQ_END) { pSender->seq++; - pBlk = taosMemoryCalloc(1, sizeof(SyncSnapBlock)); - if (pBlk == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _OUT; - } + if (pSender->seq > SYNC_SNAPSHOT_SEQ_BEGIN) { + pBlk = taosMemoryCalloc(1, sizeof(SyncSnapBlock)); + if (pBlk == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OUT; + } - // read data - int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader, &pBlk->pBlock, - &pBlk->blockLen); - if (ret != 0) { - sSError(pSender, "snapshot sender read failed since %s", terrstr()); - goto _OUT; - } - pBlk->seq = pSender->seq; + pBlk->seq = pSender->seq; - if (pSender->blockLen > 0) { - // has read data - sSDebug(pSender, "snapshot sender continue to read, blockLen:%d seq:%d", pSender->blockLen, pSender->seq); - } else { - // read finish, update seq to end - pSender->seq = SYNC_SNAPSHOT_SEQ_END; - sSInfo(pSender, "snapshot sender read to the end, blockLen:%d seq:%d", pSender->blockLen, pSender->seq); - code = 0; - goto _OUT; + // read data + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader, + &pBlk->pBlock, &pBlk->blockLen); + if (ret != 0) { + sSError(pSender, "snapshot sender read failed since %s", terrstr()); + goto _OUT; + } + + if (pBlk->blockLen > 0) { + // has read data + sSDebug(pSender, "snapshot sender continue to read, blockLen:%d seq:%d", pBlk->blockLen, pBlk->seq); + } else { + // read finish, update seq to end + pSender->seq = SYNC_SNAPSHOT_SEQ_END; + sSInfo(pSender, "snapshot sender read to the end"); + code = 0; + goto _OUT; + } } } + ASSERT(pSender->seq >= SYNC_SNAPSHOT_SEQ_BEGIN && pSender->seq <= SYNC_SNAPSHOT_SEQ_END); + + int32_t blockLen = (pBlk != NULL) ? pBlk->blockLen : 0; // build msg SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSend(&rpcMsg, pSender->blockLen, pSender->pSyncNode->vgId) != 0) { + if (syncBuildSnapshotSend(&rpcMsg, blockLen, pSender->pSyncNode->vgId) != 0) { sSError(pSender, "vgId:%d, snapshot sender build msg failed since %s", pSender->pSyncNode->vgId, terrstr()); goto _OUT; } @@ -304,7 +293,7 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { pMsg->startTime = pSender->startTime; pMsg->seq = pSender->seq; - if (pBlk != NULL && pBlk->pBlock != NULL) { + if (pBlk != NULL && pBlk->pBlock != NULL && pBlk->blockLen > 0) { memcpy(pMsg->data, pBlk->pBlock, pBlk->blockLen); } @@ -317,13 +306,12 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { // put in buffer int64_t nowMs = taosGetTimestampMs(); if (pBlk) { - ASSERT(pBlk->seq != SYNC_SNAPSHOT_SEQ_END); + ASSERT(pBlk->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pBlk->seq < SYNC_SNAPSHOT_SEQ_END); pBlk->sendTimeMs = nowMs; pSender->pSndBuf->entries[pSender->seq % pSender->pSndBuf->size] = pBlk; pBlk = NULL; - pSender->pSndBuf->end = pSender->seq + 1; + pSender->pSndBuf->end = TMAX(pSender->seq + 1, pSender->pSndBuf->end); } - pSender->lastSendTime = nowMs; code = 0; @@ -337,36 +325,52 @@ _OUT:; // send snapshot data from cache int32_t snapshotReSend(SSyncSnapshotSender *pSender) { - // build msg - SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSend(&rpcMsg, pSender->blockLen, pSender->pSyncNode->vgId) != 0) { - sSError(pSender, "snapshot sender build msg failed since %s", terrstr()); - return -1; + SSyncSnapBuffer *pSndBuf = pSender->pSndBuf; + int32_t code = -1; + + taosThreadMutexLock(&pSndBuf->mutex); + + for (int32_t seq = pSndBuf->cursor + 1; seq < pSndBuf->end; ++seq) { + SyncSnapBlock *pBlk = pSndBuf->entries[seq % pSndBuf->size]; + ASSERT(pBlk && !pBlk->acked); + int64_t nowMs = taosGetTimestampMs(); + if (nowMs < pBlk->sendTimeMs + SYNC_SNAP_RESEND_MS) { + continue; + } + // build msg + SRpcMsg rpcMsg = {0}; + if (syncBuildSnapshotSend(&rpcMsg, pBlk->blockLen, pSender->pSyncNode->vgId) != 0) { + sSError(pSender, "snapshot sender build msg failed since %s", terrstr()); + goto _out; + } + + SyncSnapshotSend *pMsg = rpcMsg.pCont; + pMsg->srcId = pSender->pSyncNode->myRaftId; + pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; + pMsg->term = pSender->term; + pMsg->beginIndex = pSender->snapshotParam.start; + pMsg->lastIndex = pSender->snapshot.lastApplyIndex; + pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; + pMsg->lastConfig = pSender->lastConfig; + pMsg->startTime = pSender->startTime; + pMsg->seq = pBlk->seq; + + if (pBlk->pBlock != NULL && pBlk->blockLen > 0) { + memcpy(pMsg->data, pBlk->pBlock, pBlk->blockLen); + } + + // send msg + if (syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg) != 0) { + sSError(pSender, "snapshot sender resend msg failed since %s", terrstr()); + goto _out; + } + pBlk->sendTimeMs = nowMs; + pSender->lastSendTime = nowMs; } - - SyncSnapshotSend *pMsg = rpcMsg.pCont; - pMsg->srcId = pSender->pSyncNode->myRaftId; - pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - pMsg->term = pSender->term; - pMsg->beginIndex = pSender->snapshotParam.start; - pMsg->lastIndex = pSender->snapshot.lastApplyIndex; - pMsg->lastTerm = pSender->snapshot.lastApplyTerm; - pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; - pMsg->lastConfig = pSender->lastConfig; - pMsg->startTime = pSender->startTime; - pMsg->seq = pSender->seq; - - if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) { - memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); - } - - // send msg - if (syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg) != 0) { - sSError(pSender, "snapshot sender resend msg failed since %s", terrstr()); - return -1; - } - - pSender->lastSendTime = taosGetTimestampMs(); + code = 0; +_out:; + taosThreadMutexUnlock(&pSndBuf->mutex); return 0; } @@ -523,7 +527,7 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p int8_t started = atomic_val_compare_exchange_8(&pReceiver->start, false, true); if (started) return; - pReceiver->ack = SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT; + pReceiver->ack = SYNC_SNAPSHOT_SEQ_PREP; pReceiver->term = pPreMsg->term; pReceiver->fromId = pPreMsg->srcId; pReceiver->startTime = pPreMsg->startTime; @@ -592,6 +596,11 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap // update progress pReceiver->ack = SYNC_SNAPSHOT_SEQ_END; + // get fsmState + SSnapshot snapshot = {0}; + pReceiver->pSyncNode->pFsm->FpGetSnapshotInfo(pReceiver->pSyncNode->pFsm, &snapshot); + pReceiver->pSyncNode->fsmState = snapshot.state; + // reset wal code = pReceiver->pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pReceiver->pSyncNode->pLogStore, pMsg->lastIndex); @@ -600,12 +609,6 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap return -1; } sRInfo(pReceiver, "wal log restored from snapshot"); - - // get fsmState - SSnapshot snapshot = {0}; - pReceiver->pSyncNode->pFsm->FpGetSnapshotInfo(pReceiver->pSyncNode->pFsm, &snapshot); - pReceiver->pSyncNode->fsmState = snapshot.state; - } else { sRError(pReceiver, "snapshot receiver finish error since writer is null"); return -1; @@ -892,6 +895,9 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot pRcvBuf->entries[pMsg->seq % pRcvBuf->size] = pMsg; ppMsg[0] = NULL; pRcvBuf->end = TMAX(pMsg->seq + 1, pRcvBuf->end); + } else { + syncSnapSendRsp(pReceiver, pMsg, code); + goto _out; } for (int64_t seq = pRcvBuf->cursor + 1; seq < pRcvBuf->end; ++seq) { @@ -991,7 +997,7 @@ _SEND_REPLY:; // receiver on message // -// condition 1, recv SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT +// condition 1, recv SYNC_SNAPSHOT_SEQ_PREP // if receiver already start // if sender.start-time > receiver.start-time, restart receiver(reply snapshot start) // if sender.start-time = receiver.start-time, maybe duplicate msg @@ -1040,7 +1046,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { int32_t code = 0; if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER || pSyncNode->state == TAOS_SYNC_STATE_LEARNER) { if (pMsg->term == raftStoreGetTerm(pSyncNode)) { - if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT) { + if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PREP) { sInfo("vgId:%d, receive prepare msg of snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", pSyncNode->vgId, pMsg->term, pMsg->startTime); code = syncNodeOnSnapshotPrep(pSyncNode, pMsg); @@ -1052,16 +1058,14 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { sInfo("vgId:%d, receive end msg of snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", pSyncNode->vgId, pMsg->term, pMsg->startTime); code = syncNodeOnSnapshotEnd(pSyncNode, pMsg); - if (syncLogBufferReInit(pSyncNode->pLogBuf, pSyncNode) != 0) { + if (code != 0) { + sRError(pReceiver, "failed to end snapshot."); + code = -1; + } else if (syncLogBufferReInit(pSyncNode->pLogBuf, pSyncNode) != 0) { sRError(pReceiver, "failed to reinit log buffer since %s", terrstr()); code = -1; } - } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { - // force close, no response - syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process force stop"); - snapshotReceiverStop(pReceiver); } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { - syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq data"); code = syncNodeOnSnapshotReceive(pSyncNode, ppMsg); } else { // error log @@ -1118,38 +1122,7 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend // update next index syncIndexMgrSetIndex(pSyncNode->pNextIndex, &pMsg->srcId, snapshot.lastApplyIndex + 1); - // update seq - pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; - - // build begin msg - SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSend(&rpcMsg, 0, pSender->pSyncNode->vgId) != 0) { - sSError(pSender, "prepare snapshot failed since build msg error"); - return -1; - } - - SyncSnapshotSend *pSendMsg = rpcMsg.pCont; - pSendMsg->srcId = pSender->pSyncNode->myRaftId; - pSendMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - pSendMsg->term = raftStoreGetTerm(pSender->pSyncNode); - pSendMsg->beginIndex = pSender->snapshotParam.start; - pSendMsg->lastIndex = pSender->snapshot.lastApplyIndex; - pSendMsg->lastTerm = pSender->snapshot.lastApplyTerm; - pSendMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; - pSendMsg->lastConfig = pSender->lastConfig; - pSendMsg->startTime = pSender->startTime; - pSendMsg->seq = SYNC_SNAPSHOT_SEQ_BEGIN; - - sSInfo(pSender, "begin snapshot replication to dnode %d.", DID(&pSendMsg->destId)); - - // send msg - syncLogSendSyncSnapshotSend(pSyncNode, pSendMsg, "snapshot sender reply pre"); - if (syncNodeSendMsgById(&pSendMsg->destId, pSender->pSyncNode, &rpcMsg) != 0) { - sSError(pSender, "prepare snapshot failed since send msg error"); - return -1; - } - - return 0; + return snapshotSend(pSender); } static int32_t snapshotSenderSignatureCmp(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { @@ -1166,10 +1139,18 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp SyncSnapshotRsp *pMsg = ppMsg[0]; taosThreadMutexLock(&pSndBuf->mutex); + if (snapshotSenderSignatureCmp(pSender, pMsg) != 0) { + code = terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; + goto _out; + } + + if (pSender->pReader == NULL || pSender->finish) { + code = terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _out; + } if (pMsg->ack - pSndBuf->start >= pSndBuf->size) { - terrno = TSDB_CODE_SYN_BUFFER_FULL; - code = terrno; + code = terrno = TSDB_CODE_SYN_BUFFER_FULL; goto _out; } @@ -1196,16 +1177,11 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp pSndBuf->start = ack + 1; } - pSender->ack = pSndBuf->start - 1; - while (pSender->seq != SYNC_SNAPSHOT_SEQ_END && pSender->seq - pSndBuf->start < (pSndBuf->size >> 2)) { if (snapshotSend(pSender) != 0) { code = terrno; goto _out; } - if (pSender->seq != SYNC_SNAPSHOT_SEQ_END) { - pSndBuf->end = TMAX(pSender->seq + 1, pSndBuf->end); - } } if (pSender->seq == SYNC_SNAPSHOT_SEQ_END && pSndBuf->end <= pSndBuf->start) { @@ -1285,23 +1261,17 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { } // prepare , send begin msg - if (pMsg->ack == SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT) { + if (pMsg->ack == SYNC_SNAPSHOT_SEQ_PREP) { return syncNodeOnSnapshotPrepRsp(pSyncNode, pSender, pMsg); } - if (pSender->pReader == NULL || pSender->finish) { - sSError(pSender, "snapshot sender invalid error:%s 0x%x, pReader:%p finish:%d", tstrerror(pMsg->code), pMsg->code, - pSender->pReader, pSender->finish); - terrno = pMsg->code; - goto _ERROR; - } - - if (pMsg->ack == SYNC_SNAPSHOT_SEQ_BEGIN) { - sSInfo(pSender, "process seq begin"); - if (snapshotSend(pSender) != 0) { + // send next msg + if (pMsg->ack >= SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->ack < SYNC_SNAPSHOT_SEQ_END) { + if (syncSnapBufferSend(pSender, ppMsg) != 0) { + sSError(pSender, "failed to replicate snap since %s. seq:%d, pReader:%p, finish:%d", terrstr(), pSender->seq, + pSender->pReader, pSender->finish); goto _ERROR; } - return 0; } // receive ack is finish, close sender @@ -1312,11 +1282,6 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { return 0; } - // send next msg - if (syncSnapBufferSend(pSender, ppMsg) != 0) { - sSError(pSender, "failed to send snapshot msg since %s. seq:%d", terrstr(), pSender->seq); - goto _ERROR; - } return 0; _ERROR: diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 91c7494fa4..5837308e59 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -78,9 +78,8 @@ static int32_t syncNodeTimerRoutine(SSyncNode* ths) { SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(ths->peersId[i])); if (pSender != NULL) { if (ths->isStart && ths->state == TAOS_SYNC_STATE_LEADER && pSender->start && - timeNow - pSender->lastSendTime > SYNC_SNAP_TIMEOUT_MS) { - sSError(pSender, "snapshot timeout, terminate. lastSendTime:%d", pSender->lastSendTime); - snapshotSenderStop(pSender, false); + timeNow - pSender->lastSendTime > SYNC_SNAP_RESEND_MS) { + snapshotReSend(pSender); } } } From a7f3041ff3e28179d7916a103e45967608fad1db Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 27 Oct 2023 17:59:27 +0800 Subject: [PATCH 22/37] more fix --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index ee3abf7559..cc77474e79 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -191,7 +191,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pMemTable->nDel++; pMemTable->minVer = TMIN(pMemTable->minVer, version); - pMemTable->maxVer = TMIN(pMemTable->maxVer, version); + pMemTable->maxVer = TMAX(pMemTable->maxVer, version); /* if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && tsdbKeyCmprFn(&lastKey, &pTbData->maxKey) >= 0) { tsdbCacheDeleteLastrow(pTsdb->lruCache, pTbData->uid, eKey); From c3f9cae36bfa9fa8ce7df630834259999b14e3f6 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 27 Oct 2023 18:55:30 +0800 Subject: [PATCH 23/37] refact: improve code of syncNodeOnSnapshot --- source/libs/sync/src/syncSnapshot.c | 106 +++++++++++++++------------- 1 file changed, 58 insertions(+), 48 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 92d9571906..56735d479e 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -1019,6 +1019,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { SyncSnapshotSend **ppMsg = (SyncSnapshotSend **)&pRpcMsg->pCont; SyncSnapshotSend *pMsg = ppMsg[0]; SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; + int32_t code = 0; // if already drop replica, do not process if (!syncNodeInRaftGroup(pSyncNode, &pMsg->srcId)) { @@ -1042,47 +1043,56 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { syncNodeUpdateTermWithoutStepDown(pSyncNode, pMsg->term); } - // state, term, seq/ack - int32_t code = 0; - if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER || pSyncNode->state == TAOS_SYNC_STATE_LEARNER) { - if (pMsg->term == raftStoreGetTerm(pSyncNode)) { - if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PREP) { - sInfo("vgId:%d, receive prepare msg of snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", - pSyncNode->vgId, pMsg->term, pMsg->startTime); - code = syncNodeOnSnapshotPrep(pSyncNode, pMsg); - } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { - sInfo("vgId:%d, receive begin msg of snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", - pSyncNode->vgId, pMsg->term, pMsg->startTime); - code = syncNodeOnSnapshotBegin(pSyncNode, pMsg); - } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { - sInfo("vgId:%d, receive end msg of snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", pSyncNode->vgId, - pMsg->term, pMsg->startTime); - code = syncNodeOnSnapshotEnd(pSyncNode, pMsg); - if (code != 0) { - sRError(pReceiver, "failed to end snapshot."); - code = -1; - } else if (syncLogBufferReInit(pSyncNode->pLogBuf, pSyncNode) != 0) { - sRError(pReceiver, "failed to reinit log buffer since %s", terrstr()); - code = -1; - } - } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { - code = syncNodeOnSnapshotReceive(pSyncNode, ppMsg); - } else { - // error log - sRError(pReceiver, "snapshot receiver recv error seq:%d, my ack:%d", pMsg->seq, pReceiver->ack); - code = -1; - } - } else { - // error log - sRError(pReceiver, "snapshot receiver term not equal"); - code = -1; - } - } else { - // error log - sRError(pReceiver, "snapshot receiver not follower"); - code = -1; + if (pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER && pSyncNode->state != TAOS_SYNC_STATE_LEARNER) { + sRError(pReceiver, "snapshot receiver not a follower or learner"); + return -1; } + if (pMsg->seq < SYNC_SNAPSHOT_SEQ_PREP || pMsg->seq > SYNC_SNAPSHOT_SEQ_END) { + sRError(pReceiver, "snap replication msg with invalid seq:%d", pMsg->seq); + return -1; + } + + // prepare + if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PREP) { + sInfo("vgId:%d, prepare snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", pSyncNode->vgId, pMsg->term, + pMsg->startTime); + code = syncNodeOnSnapshotPrep(pSyncNode, pMsg); + goto _out; + } + + // begin + if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { + sInfo("vgId:%d, begin snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", pSyncNode->vgId, pMsg->term, + pMsg->startTime); + code = syncNodeOnSnapshotBegin(pSyncNode, pMsg); + goto _out; + } + + // data + if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { + code = syncNodeOnSnapshotReceive(pSyncNode, ppMsg); + goto _out; + } + + // end + if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { + sInfo("vgId:%d, end snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", pSyncNode->vgId, pMsg->term, + pMsg->startTime); + code = syncNodeOnSnapshotEnd(pSyncNode, pMsg); + if (code != 0) { + sRError(pReceiver, "failed to end snapshot."); + goto _out; + } + + code = syncLogBufferReInit(pSyncNode->pLogBuf, pSyncNode); + if (code != 0) { + sRError(pReceiver, "failed to reinit log buffer since %s", terrstr()); + } + goto _out; + } + +_out:; syncNodeResetElectTimer(pSyncNode); return code; } @@ -1229,8 +1239,7 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { // check signature int32_t order = 0; if ((order = snapshotSenderSignatureCmp(pSender, pMsg)) > 0) { - sSError(pSender, "received a stale snapshot rsp, msg signature:(%" PRId64 ", %" PRId64 "), ignore it.", pMsg->term, - pMsg->startTime); + sSWarn(pSender, "ignore a stale snap rsp, msg signature:(%" PRId64 ", %" PRId64 ").", pMsg->term, pMsg->startTime); terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; return -1; } else if (order < 0) { @@ -1239,7 +1248,6 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { goto _ERROR; } - // state, term, seq/ack if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { sSError(pSender, "snapshot sender not leader"); terrno = TSDB_CODE_SYN_NOT_LEADER; @@ -1260,12 +1268,15 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { goto _ERROR; } - // prepare , send begin msg + // send begin if (pMsg->ack == SYNC_SNAPSHOT_SEQ_PREP) { - return syncNodeOnSnapshotPrepRsp(pSyncNode, pSender, pMsg); + sSInfo(pSender, "process prepare rsp"); + if (syncNodeOnSnapshotPrepRsp(pSyncNode, pSender, pMsg) != 0) { + goto _ERROR; + } } - // send next msg + // send msg of data or end if (pMsg->ack >= SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->ack < SYNC_SNAPSHOT_SEQ_END) { if (syncSnapBufferSend(pSender, ppMsg) != 0) { sSError(pSender, "failed to replicate snap since %s. seq:%d, pReader:%p, finish:%d", terrstr(), pSender->seq, @@ -1274,12 +1285,11 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { } } - // receive ack is finish, close sender + // end if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { - sSInfo(pSender, "process seq end"); + sSInfo(pSender, "process end rsp"); snapshotSenderStop(pSender, true); syncNodeReplicateReset(pSyncNode, &pMsg->srcId); - return 0; } return 0; From 55f6d2b7a96e50e364c286c0bdd813ba35ff5717 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Mon, 30 Oct 2023 09:30:35 +0800 Subject: [PATCH 24/37] use TSDB_MAX_KEEP --- source/libs/parser/src/parTranslater.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d65f97dce7..5c7a6180c0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3526,7 +3526,7 @@ static void convertVarDuration(SValueNode* pOffset, uint8_t precision) { pOffset->unit = units[precision]; } -static const int64_t maxKeepMS = (int64_t)3600 * 1000 * 24 * (365 * 1000 + 250); +static const int64_t tsdbMaxKeepMS = (int64_t)60 * 1000 * TSDB_MAX_KEEP; static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; @@ -3535,7 +3535,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime, getPrecisionStr(precision)); - } else if (pInter->datum.i / getPrecisionMultiple(precision) > maxKeepMS) { + } else if (pInter->datum.i / getPrecisionMultiple(precision) > tsdbMaxKeepMS) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_BIG, 1000, "years"); } From 4b6f927e6a719e763cc1b34f44fcd669cfa2f3be Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 30 Oct 2023 10:11:57 +0800 Subject: [PATCH 25/37] fix: add test case for bi tag scan --- source/libs/parser/src/parAstCreater.c | 7 +++--- tests/parallel_test/cases.task | 1 + tests/script/tsim/query/bi_tag_scan.sim | 31 +++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 tests/script/tsim/query/bi_tag_scan.sim diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 8196a9b329..de7ffce2d3 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1024,15 +1024,14 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr SNodeList* pHint) { CHECK_PARSER_STATUS(pCxt); SNode* select = createSelectStmtImpl(isDistinct, pProjectionList, pTable, pHint); - if (pCxt->pQueryCxt->biMode) { - setSelectStmtTagMode(pCxt, select, true); - } CHECK_OUT_OF_MEM(select); return select; } SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags) { - if (pStmt && QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + if (pCxt->pQueryCxt->biMode) { + ((SSelectStmt*)pStmt)->tagScan = true; + } else if (pStmt && QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { ((SSelectStmt*)pStmt)->tagScan = bSelectTags; } return pStmt; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 6915f802c3..1206e11c75 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1044,6 +1044,7 @@ e ,,y,script,./test.sh -f tsim/query/tableCount.sim ,,y,script,./test.sh -f tsim/query/show_db_table_kind.sim ,,y,script,./test.sh -f tsim/query/bi_star_table.sim +,,y,script,./test.sh -f tsim/query/bi_tag_scan.sim ,,y,script,./test.sh -f tsim/query/tag_scan.sim ,,y,script,./test.sh -f tsim/query/nullColSma.sim ,,y,script,./test.sh -f tsim/query/bug3398.sim diff --git a/tests/script/tsim/query/bi_tag_scan.sim b/tests/script/tsim/query/bi_tag_scan.sim new file mode 100644 index 0000000000..5b8af68b5a --- /dev/null +++ b/tests/script/tsim/query/bi_tag_scan.sim @@ -0,0 +1,31 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists db1; +sql create database db1 vgroups 3; +sql create database db1; +sql use db1; +sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create table tba1 using sta tags(1, 1, 1); +sql create table tba2 using sta tags(2, 2, 2); +sql insert into tba1 values(now, 1, "1")(now+3s, 3, "3")(now+5s, 5, "5"); +sql insert into tba2 values(now + 1s, 2, "2")(now+2s, 2, "2")(now+4s, 4, "4"); +sql create table tbn1 (ts timestamp, f1 int); + +set_bi_mode 1 +sql select t1,t2,t3 from db1.sta order by t1; +print $rows $data00 $data10 +if $rows != 2 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 2 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 70e261f66217e2d1ff8f1c6edd5374db7ed921b6 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 27 Oct 2023 19:21:54 +0800 Subject: [PATCH 26/37] enh: terminate snap replication on timeout --- include/libs/sync/sync.h | 2 +- source/libs/sync/src/syncSnapshot.c | 4 +--- source/libs/sync/src/syncTimeout.c | 16 +++++++++++++--- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index e3b5f3611c..a19b249bc7 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -46,7 +46,7 @@ extern "C" { #define SYNC_HEARTBEAT_SLOW_MS 1500 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 #define SYNC_SNAP_RESEND_MS 1000 * 60 -#define SYNC_SNAP_TIMEOUT_MS 1000 * 180 +#define SYNC_SNAP_TIMEOUT_MS 1000 * 600 #define SYNC_VND_COMMIT_MIN_MS 3000 diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 56735d479e..0b94d377f1 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -327,7 +327,6 @@ _OUT:; int32_t snapshotReSend(SSyncSnapshotSender *pSender) { SSyncSnapBuffer *pSndBuf = pSender->pSndBuf; int32_t code = -1; - taosThreadMutexLock(&pSndBuf->mutex); for (int32_t seq = pSndBuf->cursor + 1; seq < pSndBuf->end; ++seq) { @@ -366,12 +365,11 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { goto _out; } pBlk->sendTimeMs = nowMs; - pSender->lastSendTime = nowMs; } code = 0; _out:; taosThreadMutexUnlock(&pSndBuf->mutex); - return 0; + return code; } // return 0, start ok diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 5837308e59..a57dfbee53 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -77,9 +77,19 @@ static int32_t syncNodeTimerRoutine(SSyncNode* ths) { for (int i = 0; i < ths->peersNum; ++i) { SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(ths->peersId[i])); if (pSender != NULL) { - if (ths->isStart && ths->state == TAOS_SYNC_STATE_LEADER && pSender->start && - timeNow - pSender->lastSendTime > SYNC_SNAP_RESEND_MS) { - snapshotReSend(pSender); + if (ths->isStart && ths->state == TAOS_SYNC_STATE_LEADER && pSender->start) { + int64_t elapsedMs = timeNow - pSender->lastSendTime; + if (elapsedMs < SYNC_SNAP_RESEND_MS) { + continue; + } + + if (elapsedMs > SYNC_SNAP_TIMEOUT_MS) { + sSError(pSender, "snap replication timeout, terminate."); + snapshotSenderStop(pSender, false); + } else { + sSWarn(pSender, "snap replication resend."); + snapshotReSend(pSender); + } } } } From 78cce0003767bb44b4b876a1e8e4954a9bc2a8bc Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 30 Oct 2023 10:46:17 +0800 Subject: [PATCH 27/37] test: pause 1s after creating stream in testcase sliding.sim --- tests/script/tsim/stream/sliding.sim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index 18893245fa..a92da7f472 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -21,6 +21,7 @@ sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 in sql create stream streams2 trigger at_once watermark 1d IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s) sliding (5s); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s) sliding (5s); sql create stream stream_t2 trigger at_once watermark 1d IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s) sliding (5s); +sleep 1000 sql insert into t1 values(1648791210000,1,2,3,1.0); sql insert into t1 values(1648791216000,2,2,3,1.1); @@ -311,6 +312,7 @@ sql create table t2 using st tags(2,2,2); sql create stream streams11 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); sql create stream streams12 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); +sleep 1000 sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791223001,2,2,3,1.1); @@ -444,6 +446,7 @@ sql create table t2 using st tags(2,2,2); sql create stream streams21 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt21 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); sql create stream streams22 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt22 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); +sleep 1000 sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791223001,2,2,2,1.1); @@ -582,6 +585,7 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams23 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt23 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(20s) sliding(10s); +sleep 1000 sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791223001,2,2,2,1.1); @@ -706,6 +710,7 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart as ts, count(*),min(a) c1 from st interval(10s) sliding(5s); +sleep 1000 sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791243000,2,1,1,1.0); @@ -818,4 +823,4 @@ print ============loop_all=$loop_all #=goto looptest -system sh/stop_dnodes.sh \ No newline at end of file +system sh/stop_dnodes.sh From 58e61e21a5064ee9be6aeffe1e7bf77a9251d2d1 Mon Sep 17 00:00:00 2001 From: dapan1121 <72057773+dapan1121@users.noreply.github.com> Date: Mon, 30 Oct 2023 13:36:10 +0800 Subject: [PATCH 28/37] Update 10-function.md --- docs/zh/12-taos-sql/10-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 44ab3d5091..6840147112 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -539,7 +539,7 @@ TO_CHAR(ts, format_str_literal) - 使用`ms`, `us`, `ns`时, 以上三种格式的输出只在精度上不同, 比如ts为 `1697182085123`, `ms` 的输出为 `123`, `us` 的输出为 `123000`, `ns` 的输出为 `123000000`. - 时间格式中无法匹配规则的内容会直接输出. 如果想要在格式串中指定某些能够匹配规则的部分不做转换, 可以使用双引号, 如`to_char(ts, 'yyyy-mm-dd "is formated by yyyy-mm-dd"')`. 如果想要输出双引号, 那么在双引号之前加一个反斜杠, 如 `to_char(ts, '\"yyyy-mm-dd\"')` 将会输出 `"2023-10-10"`. - 那些输出是数字的格式, 如`YYYY`, `DD`, 大写与小写意义相同, 即`yyyy` 和 `YYYY` 可以互换. -- 默认输出的时间为本地时区的时间. +- 推荐在时间格式中带时区信息，如果不带则默认输出的时区为服务端或客户端所配置的时区. #### TO_TIMESTAMP From af1fed4f04f36a05975963c85dc44ff59b1a6997 Mon Sep 17 00:00:00 2001 From: dapan1121 <72057773+dapan1121@users.noreply.github.com> Date: Mon, 30 Oct 2023 13:41:28 +0800 Subject: [PATCH 29/37] Update 10-function.md --- docs/en/12-taos-sql/10-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index c986a98e46..5ff02b0f9d 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -539,7 +539,7 @@ TO_CHAR(ts, format_str_literal) - When `ms`,`us`,`ns` are used in `to_char`, like `to_char(ts, 'yyyy-mm-dd hh:mi:ss.ms.us.ns')`, The time of `ms`,`us`,`ns` corresponds to the same fraction seconds. When ts is `1697182085123`, the output of `ms` is `123`, `us` is `123000`, `ns` is `123000000`. - If we want to output some characters of format without converting, surround it with double quotes. `to_char(ts, 'yyyy-mm-dd "is formated by yyyy-mm-dd"')`. If want to output double quotes, add a back slash before double quote, like `to_char(ts, '\"yyyy-mm-dd\"')` will output `"2023-10-10"`. - For formats that output digits, the uppercase and lowercase formats are the same. -- The local time zone will be used to convert the timestamp. +- It's recommended to put time zone in the format, if not, the default time zone zone will be that in server or client. #### TO_TIMESTAMP From 46826ea600a1c7b8d5a8c0fd7f12f44b61b45b9e Mon Sep 17 00:00:00 2001 From: dapan1121 <72057773+dapan1121@users.noreply.github.com> Date: Mon, 30 Oct 2023 13:51:08 +0800 Subject: [PATCH 30/37] Update 10-function.md --- docs/zh/12-taos-sql/10-function.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 6840147112..4371124623 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -563,7 +563,8 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal) - 若`ms`, `us`, `ns`同时指定, 那么结果时间戳包含上述三个字段的和. 如 `to_timestamp('2023-10-10 10:10:10.123.000456.000000789', 'yyyy-mm-dd hh:mi:ss.ms.us.ns')` 输出是 `2023-10-10 10:10:10.123456789`. - `MONTH`, `MON`, `DAY`, `DY` 以及其他输出为数字的格式的大小写意义相同, 如 `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month`可以被替换为`MONTH` 或者`Month`. - 如果同一字段被指定了多次, 那么前面的指定将会被覆盖. 如 `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, 输出年份是`2022`. -- 如果某些部分没有指定 那么默认时间为本地时区的 `1970-01-01 00:00:00`, 未指定部分为对应默认值. +- 为避免转换时使用了非预期的时区，推荐在时间中携带时区信息，例如'2023-10-10 10:10:10+08'，如果未指定时区则默认时区为服务端或客户端指定的时区。 +- 如果没有指定完整的时间，那么默认时间值为指定或默认时区的 `1970-01-01 00:00:00`, 未指定部分使用该默认值中的对应部分. - 如果格式串中有`AM`, `PM`等, 那么小时必须是12小时制, 范围必须是01-12. - `to_timestamp`转换具有一定的容错机制, 在格式串和时间戳串不完全对应时, 有时也可转换, 如: `to_timestamp('200101/2', 'yyyyMM1/dd')`, 格式串中多出来的1会被丢弃. 格式串与时间戳串中多余的空格字符(空格, tab等)也会被 自动忽略. 如`to_timestamp(' 23 年 - 1 月 - 01 日 ', 'yy 年-MM月-dd日')` 可以被成功转换. 虽然`MM`等字段需要两个数字对应(只有一位时前面补0), 在`to_timestamp`时, 一个数字也可以成功转换. From d73968a6595c863f6991c8fc4bd254f86d1fd63f Mon Sep 17 00:00:00 2001 From: dapan1121 <72057773+dapan1121@users.noreply.github.com> Date: Mon, 30 Oct 2023 13:56:20 +0800 Subject: [PATCH 31/37] Update 10-function.md --- docs/en/12-taos-sql/10-function.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 5ff02b0f9d..18c7ffc345 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -563,7 +563,8 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal) - When `ms`, `us`, `ns` are used in `to_timestamp`, if multi of them are specified, the results are accumulated. For example, `to_timestamp('2023-10-10 10:10:10.123.000456.000000789', 'yyyy-mm-dd hh:mi:ss.ms.us.ns')` will output the timestamp of `2023-10-10 10:10:10.123456789`. - The uppercase or lowercase of `MONTH`, `MON`, `DAY`, `DY` and formtas that output digits have same effect when used in `to_timestamp`, like `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month` can be replaced by `MONTH`, or `month`. The cases are ignored. - If multi times are specified for one component, the previous will be overwritten. Like `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, the output year will be `2022`. -- The default timetsamp if some components are not specified will be: `1970-01-01 00:00:00` with your local timezone. +- To avoid unexpected time zone used during the convertion, it's recommended to put time zone in the ts string, e.g. '2023-10-10 10:10:10+08'. If time zone not specified, default will be that in server or client. +- The default timestamp if some components are not specified will be: `1970-01-01 00:00:00` with specified or default local timezone. - If `AM` or `PM` is specified in formats, the Hour must between `1-12`. - In some cases, `to_timestamp` can convert correctly even the format and the timestamp string are not totally matched. Like `to_timetamp('200101/2', 'yyyyMM1/dd')`, the digit `1` in format string are ignored, and the output timestsamp is `2001-01-02 00:00:00`. Spaces and tabs in formats and tiemstamp string are also ignored automatically. From be0a85a0ffd4fca365d150682af58ae1d562675e Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 30 Oct 2023 07:19:08 +0000 Subject: [PATCH 32/37] memory leak --- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 040ae96972..d4bea4332b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1008,7 +1008,7 @@ _exit: taosArrayDestroy(tbUids); tDecoderClear(&decoder); tEncoderClear(&encoder); - taosArrayDestroy(tbNames); + taosArrayDestroyP(tbNames, taosMemoryFree); return rcode; } @@ -1225,7 +1225,7 @@ _exit: tEncodeSVDropTbBatchRsp(&encoder, &rsp); tEncoderClear(&encoder); taosArrayDestroy(rsp.pArray); - taosArrayDestroy(tbNames); + taosArrayDestroyP(tbNames, taosMemoryFree); return 0; } From 0f40e1dbb90fb638328412d83354103eafccc889 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 30 Oct 2023 08:57:07 +0000 Subject: [PATCH 33/37] memory leak --- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d4bea4332b..81baa5a54e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1008,7 +1008,7 @@ _exit: taosArrayDestroy(tbUids); tDecoderClear(&decoder); tEncoderClear(&encoder); - taosArrayDestroyP(tbNames, taosMemoryFree); + taosArrayDestroyEx(tbNames, taosMemoryFree); return rcode; } @@ -1225,7 +1225,7 @@ _exit: tEncodeSVDropTbBatchRsp(&encoder, &rsp); tEncoderClear(&encoder); taosArrayDestroy(rsp.pArray); - taosArrayDestroyP(tbNames, taosMemoryFree); + taosArrayDestroyEx(tbNames, taosMemoryFree); return 0; } From f852f87f94d9fff0156852bfc9f7cc35119735e3 Mon Sep 17 00:00:00 2001 From: tjuzyp Date: Mon, 30 Oct 2023 17:31:42 +0800 Subject: [PATCH 34/37] feat: do not build taosx in ci --- tests/parallel_test/container_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 94704b1c25..f4db7edb8b 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -69,7 +69,7 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ if [[ -d ${WORKDIR}/debugNoSan ]] ;then @@ -99,7 +99,7 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan From fe044051c8ffd90d136bdf0192b9e53ae07ccf9f Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 30 Oct 2023 11:34:34 +0000 Subject: [PATCH 35/37] memory leak --- source/dnode/vnode/src/vnd/vnodeSvr.c | 34 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 81baa5a54e..b6a4aaf388 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -950,9 +950,11 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, taosArrayPush(rsp.pArray, &cRsp); - char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); - strcpy(str, pCreateReq->name); - taosArrayPush(tbNames, str); + if(tsEnableAuditCreateTable){ + char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); + strcpy(str, pCreateReq->name); + taosArrayPush(tbNames, &str); + } } vDebug("vgId:%d, add %d new created tables into query table list", TD_VID(pVnode), (int32_t)taosArrayGetSize(tbUids)); @@ -982,11 +984,12 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, SStringBuilder sb = {0}; for(int32_t iReq = 0; iReq < req.nReqs; iReq++){ - char* key = taosArrayGet(tbNames, iReq); - taosStringBuilderAppendStringLen(&sb, key, strlen(key)); + char** key = (char**)taosArrayGet(tbNames, iReq); + taosStringBuilderAppendStringLen(&sb, *key, strlen(*key)); if(iReq < req.nReqs - 1){ taosStringBuilderAppendChar(&sb, ','); } + taosMemoryFreeClear(*key); } size_t len = 0; @@ -1002,13 +1005,13 @@ _exit: pCreateReq = req.pReqs + iReq; taosMemoryFree(pCreateReq->sql); taosMemoryFree(pCreateReq->comment); - taosArrayDestroy(pCreateReq->ctb.tagName); + taosArrayDestroy(pCreateReq->ctb.tagName); } taosArrayDestroyEx(rsp.pArray, tFreeSVCreateTbRsp); taosArrayDestroy(tbUids); tDecoderClear(&decoder); tEncoderClear(&encoder); - taosArrayDestroyEx(tbNames, taosMemoryFree); + taosArrayDestroy(tbNames); return rcode; } @@ -1184,9 +1187,11 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in taosArrayPush(rsp.pArray, &dropTbRsp); - char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); - strcpy(str, pDropTbReq->name); - taosArrayPush(tbNames, str); + if(tsEnableAuditCreateTable){ + char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); + strcpy(str, pDropTbReq->name); + taosArrayPush(tbNames, &str); + } } tqUpdateTbUidList(pVnode->pTq, tbUids, false); @@ -1200,17 +1205,18 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in SStringBuilder sb = {0}; for(int32_t iReq = 0; iReq < req.nReqs; iReq++){ - char* key = taosArrayGet(tbNames, iReq); - taosStringBuilderAppendStringLen(&sb, key, strlen(key)); + char** key = (char**)taosArrayGet(tbNames, iReq); + taosStringBuilderAppendStringLen(&sb, *key, strlen(*key)); if(iReq < req.nReqs - 1){ taosStringBuilderAppendChar(&sb, ','); } + taosMemoryFreeClear(*key); } size_t len = 0; char* keyJoined = taosStringBuilderGetResult(&sb, &len); - auditRecord(NULL, clusterId, "createTable", name.dbname, "", keyJoined, len); + auditRecord(NULL, clusterId, "dropTable", name.dbname, "", keyJoined, len); taosStringBuilderDestroy(&sb); } @@ -1225,7 +1231,7 @@ _exit: tEncodeSVDropTbBatchRsp(&encoder, &rsp); tEncoderClear(&encoder); taosArrayDestroy(rsp.pArray); - taosArrayDestroyEx(tbNames, taosMemoryFree); + taosArrayDestroy(tbNames); return 0; } From e800c993dd81b16175cbbf7622402d2262673587 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 31 Oct 2023 07:37:56 +0800 Subject: [PATCH 36/37] fix: set scan tags when bi mode --- source/libs/parser/src/parAstCreater.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index de7ffce2d3..c6d70667bf 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1028,11 +1028,13 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr return select; } -SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags) { - if (pCxt->pQueryCxt->biMode) { - ((SSelectStmt*)pStmt)->tagScan = true; - } else if (pStmt && QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - ((SSelectStmt*)pStmt)->tagScan = bSelectTags; +SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags) { + if (pStmt && QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + if (pCxt->pQueryCxt->biMode) { + ((SSelectStmt*)pStmt)->tagScan = true; + } else { + ((SSelectStmt*)pStmt)->tagScan = bSelectTags; + } } return pStmt; } From 72a866988bb602a9d782b5c32780714004257eec Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 31 Oct 2023 11:42:00 +0800 Subject: [PATCH 37/37] enh: disable telemetry in enterprise version --- source/common/src/tglobal.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0a155f4ea1..c6cff27011 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -104,13 +104,21 @@ uint16_t tsAuditPort = 6043; bool tsEnableAuditCreateTable = true; // telem +#ifdef TD_ENTERPRISE +bool tsEnableTelem = false; +#else bool tsEnableTelem = true; +#endif int32_t tsTelemInterval = 43200; char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.tdengine.com"; uint16_t tsTelemPort = 80; char *tsTelemUri = "/report"; +#ifdef TD_ENTERPRISE +bool tsEnableCrashReport = false; +#else bool tsEnableCrashReport = true; +#endif char *tsClientCrashReportUri = "/ccrashreport"; char *tsSvrCrashReportUri = "/dcrashreport";