From 860a7bdcace0493c847fcde2ab23172e29c970ef Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 13 Sep 2022 14:51:04 +0800 Subject: [PATCH 01/13] enh: add queryRsmaTolerance conf for rsma query --- docs/en/14-reference/12-config/index.md | 302 ++++++++++++------------ docs/zh/14-reference/12-config/index.md | 301 +++++++++++------------ include/common/tglobal.h | 1 + source/common/src/tglobal.c | 5 + source/dnode/vnode/src/tsdb/tsdbRead.c | 7 +- 5 files changed, 314 insertions(+), 302 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 02921c3f6a..623e7d4e46 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -164,7 +164,7 @@ The parameters described in this document by the effect that they have on the sy | Attribute | Description | | -------- | -------------------- | | Applicable | Client only | -| 含义 | SMA index optimization policy | +| Meaning | SMA index optimization policy | | Unit | None | | Default Value | 0 | | Notes | @@ -697,152 +697,154 @@ To prevent system resource from being exhausted by multiple concurrent streams, | 15 | telemetryPort | No | Yes | | 16 | queryPolicy | No | Yes | | 17 | querySmaOptimize | No | Yes | -| 18 | queryBufferSize | Yes | Yes | -| 19 | maxNumOfDistinctRes | Yes | Yes | -| 20 | minSlidingTime | Yes | Yes | -| 21 | minIntervalTime | Yes | Yes | -| 22 | countAlwaysReturnValue | Yes | Yes | -| 23 | dataDir | Yes | Yes | -| 24 | minimalDataDirGB | Yes | Yes | -| 25 | supportVnodes | No | Yes | -| 26 | tempDir | Yes | Yes | -| 27 | minimalTmpDirGB | Yes | Yes | -| 28 | compressMsgSize | Yes | Yes | -| 29 | compressColData | Yes | Yes | -| 30 | smlChildTableName | Yes | Yes | -| 31 | smlTagName | Yes | Yes | -| 32 | smlDataFormat | No | Yes | -| 33 | statusInterval | Yes | Yes | -| 34 | shellActivityTimer | Yes | Yes | -| 35 | transPullupInterval | No | Yes | -| 36 | mqRebalanceInterval | No | Yes | -| 37 | ttlUnit | No | Yes | -| 38 | ttlPushInterval | No | Yes | -| 39 | numOfTaskQueueThreads | No | Yes | -| 40 | numOfRpcThreads | No | Yes | -| 41 | numOfCommitThreads | Yes | Yes | -| 42 | numOfMnodeReadThreads | No | Yes | -| 43 | numOfVnodeQueryThreads | No | Yes | -| 44 | numOfVnodeStreamThreads | No | Yes | -| 45 | numOfVnodeFetchThreads | No | Yes | -| 46 | numOfVnodeWriteThreads | No | Yes | -| 47 | numOfVnodeSyncThreads | No | Yes | -| 48 | numOfQnodeQueryThreads | No | Yes | -| 49 | numOfQnodeFetchThreads | No | Yes | -| 50 | numOfSnodeSharedThreads | No | Yes | -| 51 | numOfSnodeUniqueThreads | No | Yes | -| 52 | rpcQueueMemoryAllowed | No | Yes | -| 53 | logDir | Yes | Yes | -| 54 | minimalLogDirGB | Yes | Yes | -| 55 | numOfLogLines | Yes | Yes | -| 56 | asyncLog | Yes | Yes | -| 57 | logKeepDays | Yes | Yes | -| 58 | debugFlag | Yes | Yes | -| 59 | tmrDebugFlag | Yes | Yes | -| 60 | uDebugFlag | Yes | Yes | -| 61 | rpcDebugFlag | Yes | Yes | -| 62 | jniDebugFlag | Yes | Yes | -| 63 | qDebugFlag | Yes | Yes | -| 64 | cDebugFlag | Yes | Yes | -| 65 | dDebugFlag | Yes | Yes | -| 66 | vDebugFlag | Yes | Yes | -| 67 | mDebugFlag | Yes | Yes | -| 68 | wDebugFlag | Yes | Yes | -| 69 | sDebugFlag | Yes | Yes | -| 70 | tsdbDebugFlag | Yes | Yes | -| 71 | tqDebugFlag | No | Yes | -| 72 | fsDebugFlag | Yes | Yes | -| 73 | udfDebugFlag | No | Yes | -| 74 | smaDebugFlag | No | Yes | -| 75 | idxDebugFlag | No | Yes | -| 76 | tdbDebugFlag | No | Yes | -| 77 | metaDebugFlag | No | Yes | -| 78 | timezone | Yes | Yes | -| 79 | locale | Yes | Yes | -| 80 | charset | Yes | Yes | -| 81 | udf | Yes | Yes | -| 82 | enableCoreFile | Yes | Yes | -| 83 | arbitrator | Yes | No | -| 84 | numOfThreadsPerCore | Yes | No | -| 85 | numOfMnodes | Yes | No | -| 86 | vnodeBak | Yes | No | -| 87 | balance | Yes | No | -| 88 | balanceInterval | Yes | No | -| 89 | offlineThreshold | Yes | No | -| 90 | role | Yes | No | -| 91 | dnodeNopLoop | Yes | No | -| 92 | keepTimeOffset | Yes | No | -| 93 | rpcTimer | Yes | No | -| 94 | rpcMaxTime | Yes | No | -| 95 | rpcForceTcp | Yes | No | -| 96 | tcpConnTimeout | Yes | No | -| 97 | syncCheckInterval | Yes | No | -| 98 | maxTmrCtrl | Yes | No | -| 99 | monitorReplica | Yes | No | -| 100 | smlTagNullName | Yes | No | -| 101 | keepColumnName | Yes | No | -| 102 | ratioOfQueryCores | Yes | No | -| 103 | maxStreamCompDelay | Yes | No | -| 104 | maxFirstStreamCompDelay | Yes | No | -| 105 | retryStreamCompDelay | Yes | No | -| 106 | streamCompDelayRatio | Yes | No | -| 107 | maxVgroupsPerDb | Yes | No | -| 108 | maxTablesPerVnode | Yes | No | -| 109 | minTablesPerVnode | Yes | No | -| 110 | tableIncStepPerVnode | Yes | No | -| 111 | cache | Yes | No | -| 112 | blocks | Yes | No | -| 113 | days | Yes | No | -| 114 | keep | Yes | No | -| 115 | minRows | Yes | No | -| 116 | maxRows | Yes | No | -| 117 | quorum | Yes | No | -| 118 | comp | Yes | No | -| 119 | walLevel | Yes | No | -| 120 | fsync | Yes | No | -| 121 | replica | Yes | No | -| 122 | partitions | Yes | No | -| 123 | quorum | Yes | No | -| 124 | update | Yes | No | -| 125 | cachelast | Yes | No | -| 126 | maxSQLLength | Yes | No | -| 127 | maxWildCardsLength | Yes | No | -| 128 | maxRegexStringLen | Yes | No | -| 129 | maxNumOfOrderedRes | Yes | No | -| 130 | maxConnections | Yes | No | -| 131 | mnodeEqualVnodeNum | Yes | No | -| 132 | http | Yes | No | -| 133 | httpEnableRecordSql | Yes | No | -| 134 | httpMaxThreads | Yes | No | -| 135 | restfulRowLimit | Yes | No | -| 136 | httpDbNameMandatory | Yes | No | -| 137 | httpKeepAlive | Yes | No | -| 138 | enableRecordSql | Yes | No | -| 139 | maxBinaryDisplayWidth | Yes | No | -| 140 | stream | Yes | No | -| 141 | retrieveBlockingModel | Yes | No | -| 142 | tsdbMetaCompactRatio | Yes | No | -| 143 | defaultJSONStrType | Yes | No | -| 144 | walFlushSize | Yes | No | -| 145 | keepTimeOffset | Yes | No | -| 146 | flowctrl | Yes | No | -| 147 | slaveQuery | Yes | No | -| 148 | adjustMaster | Yes | No | -| 149 | topicBinaryLen | Yes | No | -| 150 | telegrafUseFieldNum | Yes | No | -| 151 | deadLockKillQuery | Yes | No | -| 152 | clientMerge | Yes | No | -| 153 | sdbDebugFlag | Yes | No | -| 154 | odbcDebugFlag | Yes | No | -| 155 | httpDebugFlag | Yes | No | -| 156 | monDebugFlag | Yes | No | -| 157 | cqDebugFlag | Yes | No | -| 158 | shortcutFlag | Yes | No | -| 159 | probeSeconds | Yes | No | -| 160 | probeKillSeconds | Yes | No | -| 161 | probeInterval | Yes | No | -| 162 | lossyColumns | Yes | No | -| 163 | fPrecision | Yes | No | -| 164 | dPrecision | Yes | No | -| 165 | maxRange | Yes | No | -| 166 | range | Yes | No | +| 18 | queryRsmaTolerance | No | Yes | +| 19 | queryBufferSize | Yes | Yes | +| 20 | maxNumOfDistinctRes | Yes | Yes | +| 21 | minSlidingTime | Yes | Yes | +| 22 | minIntervalTime | Yes | Yes | +| 23 | countAlwaysReturnValue | Yes | Yes | +| 24 | dataDir | Yes | Yes | +| 25 | minimalDataDirGB | Yes | Yes | +| 26 | supportVnodes | No | Yes | +| 27 | tempDir | Yes | Yes | +| 28 | minimalTmpDirGB | Yes | Yes | +| 29 | compressMsgSize | Yes | Yes | +| 30 | compressColData | Yes | Yes | +| 31 | smlChildTableName | Yes | Yes | +| 32 | smlTagName | Yes | Yes | +| 33 | smlDataFormat | No | Yes | +| 34 | statusInterval | Yes | Yes | +| 35 | shellActivityTimer | Yes | Yes | +| 36 | transPullupInterval | No | Yes | +| 37 | mqRebalanceInterval | No | Yes | +| 38 | ttlUnit | No | Yes | +| 39 | ttlPushInterval | No | Yes | +| 40 | numOfTaskQueueThreads | No | Yes | +| 41 | numOfRpcThreads | No | Yes | +| 42 | numOfCommitThreads | Yes | Yes | +| 43 | numOfMnodeReadThreads | No | Yes | +| 44 | numOfVnodeQueryThreads | No | Yes | +| 45 | numOfVnodeStreamThreads | No | Yes | +| 46 | numOfVnodeFetchThreads | No | Yes | +| 47 | numOfVnodeWriteThreads | No | Yes | +| 48 | numOfVnodeSyncThreads | No | Yes | +| 49 | numOfVnodeRsmaThreads | No | Yes | +| 50 | numOfQnodeQueryThreads | No | Yes | +| 51 | numOfQnodeFetchThreads | No | Yes | +| 52 | numOfSnodeSharedThreads | No | Yes | +| 53 | numOfSnodeUniqueThreads | No | Yes | +| 54 | rpcQueueMemoryAllowed | No | Yes | +| 55 | logDir | Yes | Yes | +| 56 | minimalLogDirGB | Yes | Yes | +| 57 | numOfLogLines | Yes | Yes | +| 58 | asyncLog | Yes | Yes | +| 59 | logKeepDays | Yes | Yes | +| 60 | debugFlag | Yes | Yes | +| 61 | tmrDebugFlag | Yes | Yes | +| 62 | uDebugFlag | Yes | Yes | +| 63 | rpcDebugFlag | Yes | Yes | +| 64 | jniDebugFlag | Yes | Yes | +| 65 | qDebugFlag | Yes | Yes | +| 66 | cDebugFlag | Yes | Yes | +| 67 | dDebugFlag | Yes | Yes | +| 68 | vDebugFlag | Yes | Yes | +| 69 | mDebugFlag | Yes | Yes | +| 70 | wDebugFlag | Yes | Yes | +| 71 | sDebugFlag | Yes | Yes | +| 72 | tsdbDebugFlag | Yes | Yes | +| 73 | tqDebugFlag | No | Yes | +| 74 | fsDebugFlag | Yes | Yes | +| 75 | udfDebugFlag | No | Yes | +| 76 | smaDebugFlag | No | Yes | +| 77 | idxDebugFlag | No | Yes | +| 78 | tdbDebugFlag | No | Yes | +| 79 | metaDebugFlag | No | Yes | +| 80 | timezone | Yes | Yes | +| 81 | locale | Yes | Yes | +| 82 | charset | Yes | Yes | +| 83 | udf | Yes | Yes | +| 84 | enableCoreFile | Yes | Yes | +| 85 | arbitrator | Yes | No | +| 86 | numOfThreadsPerCore | Yes | No | +| 87 | numOfMnodes | Yes | No | +| 88 | vnodeBak | Yes | No | +| 89 | balance | Yes | No | +| 90 | balanceInterval | Yes | No | +| 91 | offlineThreshold | Yes | No | +| 92 | role | Yes | No | +| 93 | dnodeNopLoop | Yes | No | +| 94 | keepTimeOffset | Yes | No | +| 95 | rpcTimer | Yes | No | +| 96 | rpcMaxTime | Yes | No | +| 97 | rpcForceTcp | Yes | No | +| 98 | tcpConnTimeout | Yes | No | +| 99 | syncCheckInterval | Yes | No | +| 100 | maxTmrCtrl | Yes | No | +| 101 | monitorReplica | Yes | No | +| 102 | smlTagNullName | Yes | No | +| 103 | keepColumnName | Yes | No | +| 104 | ratioOfQueryCores | Yes | No | +| 105 | maxStreamCompDelay | Yes | No | +| 106 | maxFirstStreamCompDelay | Yes | No | +| 107 | retryStreamCompDelay | Yes | No | +| 108 | streamCompDelayRatio | Yes | No | +| 109 | maxVgroupsPerDb | Yes | No | +| 110 | maxTablesPerVnode | Yes | No | +| 111 | minTablesPerVnode | Yes | No | +| 112 | tableIncStepPerVnode | Yes | No | +| 113 | cache | Yes | No | +| 114 | blocks | Yes | No | +| 115 | days | Yes | No | +| 116 | keep | Yes | No | +| 117 | minRows | Yes | No | +| 118 | maxRows | Yes | No | +| 119 | quorum | Yes | No | +| 120 | comp | Yes | No | +| 121 | walLevel | Yes | No | +| 122 | fsync | Yes | No | +| 123 | replica | Yes | No | +| 124 | partitions | Yes | No | +| 125 | quorum | Yes | No | +| 126 | update | Yes | No | +| 127 | cachelast | Yes | No | +| 128 | maxSQLLength | Yes | No | +| 129 | maxWildCardsLength | Yes | No | +| 130 | maxRegexStringLen | Yes | No | +| 131 | maxNumOfOrderedRes | Yes | No | +| 132 | maxConnections | Yes | No | +| 133 | mnodeEqualVnodeNum | Yes | No | +| 134 | http | Yes | No | +| 135 | httpEnableRecordSql | Yes | No | +| 136 | httpMaxThreads | Yes | No | +| 137 | restfulRowLimit | Yes | No | +| 138 | httpDbNameMandatory | Yes | No | +| 139 | httpKeepAlive | Yes | No | +| 140 | enableRecordSql | Yes | No | +| 141 | maxBinaryDisplayWidth | Yes | No | +| 142 | stream | Yes | No | +| 143 | retrieveBlockingModel | Yes | No | +| 144 | tsdbMetaCompactRatio | Yes | No | +| 145 | defaultJSONStrType | Yes | No | +| 146 | walFlushSize | Yes | No | +| 147 | keepTimeOffset | Yes | No | +| 148 | flowctrl | Yes | No | +| 149 | slaveQuery | Yes | No | +| 150 | adjustMaster | Yes | No | +| 151 | topicBinaryLen | Yes | No | +| 152 | telegrafUseFieldNum | Yes | No | +| 153 | deadLockKillQuery | Yes | No | +| 154 | clientMerge | Yes | No | +| 155 | sdbDebugFlag | Yes | No | +| 156 | odbcDebugFlag | Yes | No | +| 157 | httpDebugFlag | Yes | No | +| 158 | monDebugFlag | Yes | No | +| 159 | cqDebugFlag | Yes | No | +| 160 | shortcutFlag | Yes | No | +| 161 | probeSeconds | Yes | No | +| 162 | probeKillSeconds | Yes | No | +| 163 | probeInterval | Yes | No | +| 164 | lossyColumns | Yes | No | +| 165 | fPrecision | Yes | No | +| 166 | dPrecision | Yes | No | +| 167 | maxRange | Yes | No | +| 168 | range | Yes | No | diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 7b31e10572..f994d0d4cf 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -668,153 +668,154 @@ charset 的有效值是 UTF-8。 | 15 | telemetryPort | 否 | 是 | | 16 | queryPolicy | 否 | 是 | | 17 | querySmaOptimize | 否 | 是 | -| 18 | queryBufferSize | 是 | 是 | -| 19 | maxNumOfDistinctRes | 是 | 是 | -| 20 | minSlidingTime | 是 | 是 | -| 21 | minIntervalTime | 是 | 是 | -| 22 | countAlwaysReturnValue | 是 | 是 | -| 23 | dataDir | 是 | 是 | -| 24 | minimalDataDirGB | 是 | 是 | -| 25 | supportVnodes | 否 | 是 | -| 26 | tempDir | 是 | 是 | -| 27 | minimalTmpDirGB | 是 | 是 | -| 28 | compressMsgSize | 是 | 是 | -| 29 | compressColData | 是 | 是 | -| 30 | smlChildTableName | 是 | 是 | -| 31 | smlTagName | 是 | 是 | -| 32 | smlDataFormat | 否 | 是 | -| 33 | statusInterval | 是 | 是 | -| 34 | shellActivityTimer | 是 | 是 | -| 35 | transPullupInterval | 否 | 是 | -| 36 | mqRebalanceInterval | 否 | 是 | -| 37 | ttlUnit | 否 | 是 | -| 38 | ttlPushInterval | 否 | 是 | -| 39 | numOfTaskQueueThreads | 否 | 是 | -| 40 | numOfRpcThreads | 否 | 是 | -| 41 | numOfCommitThreads | 是 | 是 | -| 42 | numOfMnodeReadThreads | 否 | 是 | -| 43 | numOfVnodeQueryThreads | 否 | 是 | -| 44 | numOfVnodeStreamThreads | 否 | 是 | -| 45 | numOfVnodeFetchThreads | 否 | 是 | -| 46 | numOfVnodeWriteThreads | 否 | 是 | -| 47 | numOfVnodeSyncThreads | 否 | 是 | -| 48 | numOfVnodeRsmaThreads | 否 | 是 | -| 49 | numOfQnodeQueryThreads | 否 | 是 | -| 50 | numOfQnodeFetchThreads | 否 | 是 | -| 51 | numOfSnodeSharedThreads | 否 | 是 | -| 52 | numOfSnodeUniqueThreads | 否 | 是 | -| 53 | rpcQueueMemoryAllowed | 否 | 是 | -| 54 | logDir | 是 | 是 | -| 55 | minimalLogDirGB | 是 | 是 | -| 56 | numOfLogLines | 是 | 是 | -| 57 | asyncLog | 是 | 是 | -| 58 | logKeepDays | 是 | 是 | -| 59 | debugFlag | 是 | 是 | -| 60 | tmrDebugFlag | 是 | 是 | -| 61 | uDebugFlag | 是 | 是 | -| 62 | rpcDebugFlag | 是 | 是 | -| 63 | jniDebugFlag | 是 | 是 | -| 64 | qDebugFlag | 是 | 是 | -| 65 | cDebugFlag | 是 | 是 | -| 66 | dDebugFlag | 是 | 是 | -| 67 | vDebugFlag | 是 | 是 | -| 68 | mDebugFlag | 是 | 是 | -| 69 | wDebugFlag | 是 | 是 | -| 70 | sDebugFlag | 是 | 是 | -| 71 | tsdbDebugFlag | 是 | 是 | -| 72 | tqDebugFlag | 否 | 是 | -| 73 | fsDebugFlag | 是 | 是 | -| 74 | udfDebugFlag | 否 | 是 | -| 75 | smaDebugFlag | 否 | 是 | -| 76 | idxDebugFlag | 否 | 是 | -| 77 | tdbDebugFlag | 否 | 是 | -| 78 | metaDebugFlag | 否 | 是 | -| 79 | timezone | 是 | 是 | -| 80 | locale | 是 | 是 | -| 81 | charset | 是 | 是 | -| 82 | udf | 是 | 是 | -| 83 | enableCoreFile | 是 | 是 | -| 84 | arbitrator | 是 | 否 | -| 85 | numOfThreadsPerCore | 是 | 否 | -| 86 | numOfMnodes | 是 | 否 | -| 87 | vnodeBak | 是 | 否 | -| 88 | balance | 是 | 否 | -| 89 | balanceInterval | 是 | 否 | -| 90 | offlineThreshold | 是 | 否 | -| 91 | role | 是 | 否 | -| 92 | dnodeNopLoop | 是 | 否 | -| 93 | keepTimeOffset | 是 | 否 | -| 94 | rpcTimer | 是 | 否 | -| 95 | rpcMaxTime | 是 | 否 | -| 96 | rpcForceTcp | 是 | 否 | -| 97 | tcpConnTimeout | 是 | 否 | -| 98 | syncCheckInterval | 是 | 否 | -| 99 | maxTmrCtrl | 是 | 否 | -| 100 | monitorReplica | 是 | 否 | -| 101 | smlTagNullName | 是 | 否 | -| 102 | keepColumnName | 是 | 否 | -| 103 | ratioOfQueryCores | 是 | 否 | -| 104 | maxStreamCompDelay | 是 | 否 | -| 105 | maxFirstStreamCompDelay | 是 | 否 | -| 106 | retryStreamCompDelay | 是 | 否 | -| 107 | streamCompDelayRatio | 是 | 否 | -| 108 | maxVgroupsPerDb | 是 | 否 | -| 109 | maxTablesPerVnode | 是 | 否 | -| 110 | minTablesPerVnode | 是 | 否 | -| 111 | tableIncStepPerVnode | 是 | 否 | -| 112 | cache | 是 | 否 | -| 113 | blocks | 是 | 否 | -| 114 | days | 是 | 否 | -| 115 | keep | 是 | 否 | -| 116 | minRows | 是 | 否 | -| 117 | maxRows | 是 | 否 | -| 118 | quorum | 是 | 否 | -| 119 | comp | 是 | 否 | -| 120 | walLevel | 是 | 否 | -| 121 | fsync | 是 | 否 | -| 122 | replica | 是 | 否 | -| 123 | partitions | 是 | 否 | -| 124 | quorum | 是 | 否 | -| 125 | update | 是 | 否 | -| 126 | cachelast | 是 | 否 | -| 127 | maxSQLLength | 是 | 否 | -| 128 | maxWildCardsLength | 是 | 否 | -| 129 | maxRegexStringLen | 是 | 否 | -| 130 | maxNumOfOrderedRes | 是 | 否 | -| 131 | maxConnections | 是 | 否 | -| 132 | mnodeEqualVnodeNum | 是 | 否 | -| 133 | http | 是 | 否 | -| 134 | httpEnableRecordSql | 是 | 否 | -| 135 | httpMaxThreads | 是 | 否 | -| 136 | restfulRowLimit | 是 | 否 | -| 137 | httpDbNameMandatory | 是 | 否 | -| 138 | httpKeepAlive | 是 | 否 | -| 139 | enableRecordSql | 是 | 否 | -| 140 | maxBinaryDisplayWidth | 是 | 否 | -| 141 | stream | 是 | 否 | -| 142 | retrieveBlockingModel | 是 | 否 | -| 143 | tsdbMetaCompactRatio | 是 | 否 | -| 144 | defaultJSONStrType | 是 | 否 | -| 145 | walFlushSize | 是 | 否 | -| 146 | keepTimeOffset | 是 | 否 | -| 147 | flowctrl | 是 | 否 | -| 148 | slaveQuery | 是 | 否 | -| 149 | adjustMaster | 是 | 否 | -| 150 | topicBinaryLen | 是 | 否 | -| 151 | telegrafUseFieldNum | 是 | 否 | -| 152 | deadLockKillQuery | 是 | 否 | -| 153 | clientMerge | 是 | 否 | -| 154 | sdbDebugFlag | 是 | 否 | -| 155 | odbcDebugFlag | 是 | 否 | -| 156 | httpDebugFlag | 是 | 否 | -| 157 | monDebugFlag | 是 | 否 | -| 158 | cqDebugFlag | 是 | 否 | -| 159 | shortcutFlag | 是 | 否 | -| 160 | probeSeconds | 是 | 否 | -| 161 | probeKillSeconds | 是 | 否 | -| 162 | probeInterval | 是 | 否 | -| 163 | lossyColumns | 是 | 否 | -| 164 | fPrecision | 是 | 否 | -| 165 | dPrecision | 是 | 否 | -| 166 | maxRange | 是 | 否 | -| 167 | range | 是 | 否 | +| 18 | queryRsmaTolerance | 否 | 是 | +| 19 | queryBufferSize | 是 | 是 | +| 20 | maxNumOfDistinctRes | 是 | 是 | +| 21 | minSlidingTime | 是 | 是 | +| 22 | minIntervalTime | 是 | 是 | +| 23 | countAlwaysReturnValue | 是 | 是 | +| 24 | dataDir | 是 | 是 | +| 25 | minimalDataDirGB | 是 | 是 | +| 26 | supportVnodes | 否 | 是 | +| 27 | tempDir | 是 | 是 | +| 28 | minimalTmpDirGB | 是 | 是 | +| 29 | compressMsgSize | 是 | 是 | +| 30 | compressColData | 是 | 是 | +| 31 | smlChildTableName | 是 | 是 | +| 32 | smlTagName | 是 | 是 | +| 33 | smlDataFormat | 否 | 是 | +| 34 | statusInterval | 是 | 是 | +| 35 | shellActivityTimer | 是 | 是 | +| 36 | transPullupInterval | 否 | 是 | +| 37 | mqRebalanceInterval | 否 | 是 | +| 38 | ttlUnit | 否 | 是 | +| 39 | ttlPushInterval | 否 | 是 | +| 40 | numOfTaskQueueThreads | 否 | 是 | +| 41 | numOfRpcThreads | 否 | 是 | +| 42 | numOfCommitThreads | 是 | 是 | +| 43 | numOfMnodeReadThreads | 否 | 是 | +| 44 | numOfVnodeQueryThreads | 否 | 是 | +| 45 | numOfVnodeStreamThreads | 否 | 是 | +| 46 | numOfVnodeFetchThreads | 否 | 是 | +| 47 | numOfVnodeWriteThreads | 否 | 是 | +| 48 | numOfVnodeSyncThreads | 否 | 是 | +| 49 | numOfVnodeRsmaThreads | 否 | 是 | +| 50 | numOfQnodeQueryThreads | 否 | 是 | +| 51 | numOfQnodeFetchThreads | 否 | 是 | +| 52 | numOfSnodeSharedThreads | 否 | 是 | +| 53 | numOfSnodeUniqueThreads | 否 | 是 | +| 54 | rpcQueueMemoryAllowed | 否 | 是 | +| 55 | logDir | 是 | 是 | +| 56 | minimalLogDirGB | 是 | 是 | +| 57 | numOfLogLines | 是 | 是 | +| 58 | asyncLog | 是 | 是 | +| 59 | logKeepDays | 是 | 是 | +| 60 | debugFlag | 是 | 是 | +| 61 | tmrDebugFlag | 是 | 是 | +| 62 | uDebugFlag | 是 | 是 | +| 63 | rpcDebugFlag | 是 | 是 | +| 64 | jniDebugFlag | 是 | 是 | +| 65 | qDebugFlag | 是 | 是 | +| 66 | cDebugFlag | 是 | 是 | +| 67 | dDebugFlag | 是 | 是 | +| 68 | vDebugFlag | 是 | 是 | +| 69 | mDebugFlag | 是 | 是 | +| 70 | wDebugFlag | 是 | 是 | +| 71 | sDebugFlag | 是 | 是 | +| 72 | tsdbDebugFlag | 是 | 是 | +| 73 | tqDebugFlag | 否 | 是 | +| 74 | fsDebugFlag | 是 | 是 | +| 75 | udfDebugFlag | 否 | 是 | +| 76 | smaDebugFlag | 否 | 是 | +| 77 | idxDebugFlag | 否 | 是 | +| 78 | tdbDebugFlag | 否 | 是 | +| 79 | metaDebugFlag | 否 | 是 | +| 80 | timezone | 是 | 是 | +| 81 | locale | 是 | 是 | +| 82 | charset | 是 | 是 | +| 83 | udf | 是 | 是 | +| 84 | enableCoreFile | 是 | 是 | +| 85 | arbitrator | 是 | 否 | +| 86 | numOfThreadsPerCore | 是 | 否 | +| 87 | numOfMnodes | 是 | 否 | +| 88 | vnodeBak | 是 | 否 | +| 89 | balance | 是 | 否 | +| 90 | balanceInterval | 是 | 否 | +| 91 | offlineThreshold | 是 | 否 | +| 92 | role | 是 | 否 | +| 93 | dnodeNopLoop | 是 | 否 | +| 94 | keepTimeOffset | 是 | 否 | +| 95 | rpcTimer | 是 | 否 | +| 96 | rpcMaxTime | 是 | 否 | +| 97 | rpcForceTcp | 是 | 否 | +| 98 | tcpConnTimeout | 是 | 否 | +| 99 | syncCheckInterval | 是 | 否 | +| 100 | maxTmrCtrl | 是 | 否 | +| 101 | monitorReplica | 是 | 否 | +| 102 | smlTagNullName | 是 | 否 | +| 103 | keepColumnName | 是 | 否 | +| 104 | ratioOfQueryCores | 是 | 否 | +| 105 | maxStreamCompDelay | 是 | 否 | +| 106 | maxFirstStreamCompDelay | 是 | 否 | +| 107 | retryStreamCompDelay | 是 | 否 | +| 108 | streamCompDelayRatio | 是 | 否 | +| 109 | maxVgroupsPerDb | 是 | 否 | +| 110 | maxTablesPerVnode | 是 | 否 | +| 111 | minTablesPerVnode | 是 | 否 | +| 112 | tableIncStepPerVnode | 是 | 否 | +| 113 | cache | 是 | 否 | +| 114 | blocks | 是 | 否 | +| 115 | days | 是 | 否 | +| 116 | keep | 是 | 否 | +| 117 | minRows | 是 | 否 | +| 118 | maxRows | 是 | 否 | +| 119 | quorum | 是 | 否 | +| 120 | comp | 是 | 否 | +| 121 | walLevel | 是 | 否 | +| 122 | fsync | 是 | 否 | +| 123 | replica | 是 | 否 | +| 124 | partitions | 是 | 否 | +| 125 | quorum | 是 | 否 | +| 126 | update | 是 | 否 | +| 127 | cachelast | 是 | 否 | +| 128 | maxSQLLength | 是 | 否 | +| 129 | maxWildCardsLength | 是 | 否 | +| 130 | maxRegexStringLen | 是 | 否 | +| 131 | maxNumOfOrderedRes | 是 | 否 | +| 132 | maxConnections | 是 | 否 | +| 133 | mnodeEqualVnodeNum | 是 | 否 | +| 134 | http | 是 | 否 | +| 135 | httpEnableRecordSql | 是 | 否 | +| 136 | httpMaxThreads | 是 | 否 | +| 137 | restfulRowLimit | 是 | 否 | +| 138 | httpDbNameMandatory | 是 | 否 | +| 139 | httpKeepAlive | 是 | 否 | +| 140 | enableRecordSql | 是 | 否 | +| 141 | maxBinaryDisplayWidth | 是 | 否 | +| 142 | stream | 是 | 否 | +| 143 | retrieveBlockingModel | 是 | 否 | +| 144 | tsdbMetaCompactRatio | 是 | 否 | +| 145 | defaultJSONStrType | 是 | 否 | +| 146 | walFlushSize | 是 | 否 | +| 147 | keepTimeOffset | 是 | 否 | +| 148 | flowctrl | 是 | 否 | +| 149 | slaveQuery | 是 | 否 | +| 150 | adjustMaster | 是 | 否 | +| 151 | topicBinaryLen | 是 | 否 | +| 152 | telegrafUseFieldNum | 是 | 否 | +| 153 | deadLockKillQuery | 是 | 否 | +| 154 | clientMerge | 是 | 否 | +| 155 | sdbDebugFlag | 是 | 否 | +| 156 | odbcDebugFlag | 是 | 否 | +| 157 | httpDebugFlag | 是 | 否 | +| 158 | monDebugFlag | 是 | 否 | +| 159 | cqDebugFlag | 是 | 否 | +| 160 | shortcutFlag | 是 | 否 | +| 161 | probeSeconds | 是 | 否 | +| 162 | probeKillSeconds | 是 | 否 | +| 163 | probeInterval | 是 | 否 | +| 164 | lossyColumns | 是 | 否 | +| 165 | fPrecision | 是 | 否 | +| 166 | dPrecision | 是 | 否 | +| 167 | maxRange | 是 | 否 | +| 168 | range | 是 | 否 | diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 2de4ffdc17..c5b3f8f9f0 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -94,6 +94,7 @@ extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in // query client extern int32_t tsQueryPolicy; extern int32_t tsQuerySmaOptimize; +extern int32_t tsQueryRsmaTolerance; extern bool tsQueryPlannerTrace; // client diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c436e4ffd2..3d9608ad05 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -91,6 +91,7 @@ bool tsSmlDataFormat = // query int32_t tsQueryPolicy = 1; int32_t tsQuerySmaOptimize = 0; +int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which rsma level to query. bool tsQueryPlannerTrace = false; /* @@ -419,6 +420,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, 1) != 0) return -1; if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 100000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, 0) != 0) return -1; if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1; GRANT_CFG_ADD; @@ -713,6 +715,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; tsTtlPushInterval = cfgGetItem(pCfg, "ttlPushInterval")->i32; tsUptimeInterval = cfgGetItem(pCfg, "uptimeInterval")->i32; + tsQueryRsmaTolerance = cfgGetItem(pCfg, "queryRsmaTolerance")->i32; tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; @@ -972,6 +975,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; } else if (strcasecmp("queryPlannerTrace", name) == 0) { tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; + } else if (strcasecmp("queryRsmaTolerance", name) == 0) { + tsQueryRsmaTolerance = cfgGetItem(pCfg, "queryRsmaTolerance")->i32; } break; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 60d967681b..e3a5ef414c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2814,8 +2814,11 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret int8_t* pLevel) { if (VND_IS_RSMA(pVnode)) { int8_t level = 0; - int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); - int64_t offset = TSDB_TICK_PER_SECOND(pVnode->config.tsdbCfg.precision); + int8_t precision = pVnode->config.tsdbCfg.precision; + int64_t now = taosGetTimestamp(precision); + int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1 + : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000 + : 1000000); for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) { SRetention* pRetention = retentions + level; From c2c86159447eba6e87503a9c6e3b716e78049e8f Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 13 Sep 2022 14:56:49 +0800 Subject: [PATCH 02/13] chore: adjust the description of tsQueryRsmaTolerance --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3d9608ad05..fc4e28f9cd 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -91,7 +91,7 @@ bool tsSmlDataFormat = // query int32_t tsQueryPolicy = 1; int32_t tsQuerySmaOptimize = 0; -int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which rsma level to query. +int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. bool tsQueryPlannerTrace = false; /* From 11ffbfa70f66c357d74540f7aa36d3cabbe57ed9 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 13 Sep 2022 16:18:37 +0800 Subject: [PATCH 03/13] chore: code optimization for rsma fs build --- source/dnode/vnode/src/sma/smaFS.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c index 8e8611f0e8..a5f4e8d2e8 100644 --- a/source/dnode/vnode/src/sma/smaFS.c +++ b/source/dnode/vnode/src/sma/smaFS.c @@ -150,6 +150,8 @@ static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **outpu regex_t regex; int code = 0; + terrno = TSDB_CODE_SUCCESS; + tdGetVndDirName(TD_VID(pVnode), tfsGetPrimaryPath(pVnode->pTfs), VNODE_RSMA_DIR, true, dir); if (!taosCheckExistFile(dir)) { From ce60bd0c202a909699a36981a5d2fe8740ed0fad Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 14 Sep 2022 18:01:59 +0800 Subject: [PATCH 04/13] enh: tsdb retention and fs optimization --- include/libs/tfs/tfs.h | 8 ++++++++ source/dnode/vnode/src/tsdb/tsdbFS.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbRetention.c | 15 +++++++++++++-- source/libs/tfs/src/tfs.c | 2 ++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 1dc154ce48..2edb046492 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -69,6 +69,14 @@ void tfsUpdateSize(STfs *pTfs); */ SDiskSize tfsGetSize(STfs *pTfs); +/** + * @brief Get level of multi-tier storage + * + * @param pTfs + * @return int32_t + */ +int32_t tfsGetLevel(STfs *pTfs); + /** * @brief Allocate an existing available tier level from fs. * diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 31bdb784dd..ab16147337 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -58,7 +58,7 @@ static int32_t tsdbGnrtCurrent(STsdb *pTsdb, STsdbFS *pFS, char *fname) { taosCalcChecksumAppend(0, pData, size); // create and write - pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE); + pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1033,7 +1033,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - iOld++; + iNew++; continue; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index d99bf2aa5c..eb4151079b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -16,9 +16,19 @@ #include "tsdb.h" static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { + STsdbKeepCfg *keepCfg = &pTsdb->keepCfg; + + if ((keepCfg->keep0 == keepCfg->keep1) && (keepCfg->keep1 == keepCfg->keep2)) { + return false; + } + + if (tfsGetLevel(pTsdb->pVnode->pTfs) <= 1) { + return false; + } + for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + int32_t expLevel = tsdbFidLevel(pSet->fid, keepCfg, now); SDiskID did; if (expLevel == pSet->diskId.level) continue; @@ -53,7 +63,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { if (code) goto _err; for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); + SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); SDiskID did; @@ -65,6 +75,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { taosArrayRemove(fs.aDFileSet, iSet); iSet--; } else { + if (expLevel == 0) continue; if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { code = terrno; goto _exit; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 62aec219df..4600e5e568 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -113,6 +113,8 @@ SDiskSize tfsGetSize(STfs *pTfs) { return size; } +int32_t tfsGetLevel(STfs *pTfs) { return pTfs->nlevel; } + int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) { pDiskId->level = expLevel; pDiskId->id = -1; From 999d32cdbda788b254d549d3149340997e11f6eb Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 15 Sep 2022 17:59:09 +0800 Subject: [PATCH 05/13] chore: revert the code change --- include/libs/tfs/tfs.h | 2 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 2edb046492..6f71fd4cd0 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -70,7 +70,7 @@ void tfsUpdateSize(STfs *pTfs); SDiskSize tfsGetSize(STfs *pTfs); /** - * @brief Get level of multi-tier storage + * @brief Get level of multi-tier storage. * * @param pTfs * @return int32_t diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index ab16147337..10926ae6ad 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -1033,7 +1033,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - + iOld++; iNew++; continue; } From ab49ce6eafc3abc7d56820e8d57d302337d594f6 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 19 Sep 2022 11:16:20 +0800 Subject: [PATCH 06/13] enh: rsma exec logic optimization --- source/dnode/vnode/src/sma/smaCommit.c | 11 ++++++-- source/dnode/vnode/src/sma/smaRollup.c | 39 ++------------------------ 2 files changed, 10 insertions(+), 40 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 07ec7d0694..2b44cdcef1 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -295,12 +295,17 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } /** - * @brief step 3: consume the SubmitReq in buffer + * @brief step 3: commit should wait for all SubmitReq in buffer be consumed * 1) This is high cost task and should not put in asyncPreCommit originally. * 2) But, if put in asyncCommit, would trigger taskInfo cloning frequently. */ - if (tdRSmaProcessExecImpl(pSma, RSMA_EXEC_COMMIT) < 0) { - return TSDB_CODE_FAILED; + nLoops = 0; + while (atomic_load_64(&pRSmaStat->nBufItems) > 0) { + ++nLoops; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } } smaInfo("vgId:%d, rsma commit, wait for all items to be consumed, TID:%p", SMA_VID(pSma), diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ec8fcb2932..a9c8f69166 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -1822,7 +1822,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { while (true) { // step 1: rsma exec - consume data in buffer queue for all suids - if (type == RSMA_EXEC_OVERFLOW || type == RSMA_EXEC_COMMIT) { + if (type == RSMA_EXEC_OVERFLOW) { void *pIter = NULL; while ((pIter = taosHashIterate(infoHash, pIter))) { SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; @@ -1878,42 +1878,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0); } } - if (type == RSMA_EXEC_COMMIT) { - if (atomic_load_64(&pRSmaStat->nBufItems) <= 0) { - break; - } else { - // commit should wait for all items be consumed - continue; - } - } - } -#if 0 - else if (type == RSMA_EXEC_COMMIT) { - while (pIter) { - SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; - if (taosQueueItemSize(pInfo->iQueue)) { - if (atomic_val_compare_exchange_8(&pInfo->assigned, 0, 1) == 0) { - taosReadAllQitems(pInfo->iQueue, pInfo->iQall); // queue has mutex lock - int32_t qallItemSize = taosQallItemSize(pInfo->iQall); - if (qallItemSize > 0) { - atomic_fetch_sub_64(&pRSmaStat->nBufItems, qallItemSize); - nIdle = 0; - - // batch exec - tdRSmaBatchExec(pSma, pInfo, pInfo->qall, pSubmitArr, type); - } - - // tdRSmaFetchAllResult(pSma, pInfo, pSubmitArr); - atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0); - } - } - ASSERT(taosQueueItemSize(pInfo->iQueue) == 0); - pIter = taosHashIterate(infoHash, pIter); - } - break; - } -#endif - else { + } else { ASSERT(0); } From ef68cea88ed8601b65581ee1a40f3ccad2a4655d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 22 Sep 2022 08:47:08 +0800 Subject: [PATCH 07/13] fix: fix client memory leak --- source/client/src/clientImpl.c | 2 ++ source/libs/scheduler/src/schTask.c | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0e1d82b273..83714040df 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -681,6 +681,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList }; int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob); + + destroyQueryExecRes(&pRequest->body.resInfo.execRes); memcpy(&pRequest->body.resInfo.execRes, &res, sizeof(res)); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 8b3a2016b8..2cb227d84b 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -20,6 +20,7 @@ #include "tmsg.h" #include "tref.h" #include "trpc.h" +#include "tglobal.h" void schFreeTask(SSchJob *pJob, SSchTask *pTask) { schDeregisterTaskHb(pJob, pTask); @@ -870,8 +871,12 @@ int32_t schLaunchTaskImpl(void *param) { SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, pTask->msgLen); SCH_ERR_JRET(code); - } else { - SCH_TASK_DLOGL("physical plan len:%d, %s", pTask->msgLen, pTask->msg); + } else if (tsQueryPlannerTrace) { + char *msg = NULL; + int32_t msgLen = 0; + qSubPlanToString(plan, &msg, &msgLen); + SCH_TASK_DLOGL("physical plan len:%d, %s", msgLen, msg); + taosMemoryFree(msg); } } From 21629cc7074f254c98afe7dff78ac03402a461a9 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 22 Sep 2022 09:14:36 +0800 Subject: [PATCH 08/13] fix: adjust the destroy order of rsma stat --- source/dnode/vnode/src/sma/smaEnv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index b870ea1b62..ccf4ebb39f 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -278,7 +278,6 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { smaDebug("vgId:%d, destroy rsma stat %p", SMA_VID(pSma), pRSmaStat); // step 1: set rsma trigger stat cancelled atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED); - tsem_destroy(&(pStat->notEmpty)); // step 2: destroy the rsma info and associated fetch tasks taosHashCleanup(RSMA_INFO_HASH(pStat)); @@ -306,6 +305,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { tdRSmaFSClose(RSMA_FS(pStat)); // step 6: free pStat + tsem_destroy(&(pStat->notEmpty)); taosMemoryFreeClear(pStat); } } From 9f45f81aa0ec22074e26bbcdf83f785217b81dc0 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 22 Sep 2022 16:54:39 +0800 Subject: [PATCH 09/13] fix: join validity check --- include/util/taoserror.h | 2 +- source/libs/parser/src/parTranslater.c | 14 ++++++++++++++ source/libs/parser/test/parSelectTest.cpp | 7 +++++++ source/libs/planner/test/planJoinTest.cpp | 8 ++------ 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 840e7309fe..837d0c6303 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -552,7 +552,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_VALUE_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x2653) #define TSDB_CODE_PAR_INVALID_DELETE_WHERE TAOS_DEF_ERROR_CODE(0, 0x2655) #define TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG TAOS_DEF_ERROR_CODE(0, 0x2656) - #define TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2657) #define TSDB_CODE_PAR_INVALID_WINDOW_PC TAOS_DEF_ERROR_CODE(0, 0x2658) #define TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2659) @@ -565,6 +564,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_SELECTED_EXPR TAOS_DEF_ERROR_CODE(0, 0x2661) #define TSDB_CODE_PAR_GET_META_ERROR TAOS_DEF_ERROR_CODE(0, 0x2662) #define TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS TAOS_DEF_ERROR_CODE(0, 0x2663) +#define TSDB_CODE_PAR_NOT_SUPPORT_JOIN TAOS_DEF_ERROR_CODE(0, 0x2664) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f0d5e5d67e..7d16758e56 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2213,6 +2213,17 @@ static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSele return code; } +static int32_t checkJoinTable(STranslateContext* pCxt, SJoinTableNode* pJoinTable) { + if ((QUERY_NODE_TEMP_TABLE == nodeType(pJoinTable->pLeft) && + !isTimeLineQuery(((STempTableNode*)pJoinTable->pLeft)->pSubquery)) || + (QUERY_NODE_TEMP_TABLE == nodeType(pJoinTable->pRight) && + !isTimeLineQuery(((STempTableNode*)pJoinTable->pRight)->pSubquery))) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SUPPORT_JOIN, + "Join requires valid time series input"); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pTable)) { @@ -2259,6 +2270,9 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { if (TSDB_CODE_SUCCESS == code) { code = translateTable(pCxt, pJoinTable->pRight); } + if (TSDB_CODE_SUCCESS == code) { + code = checkJoinTable(pCxt, pJoinTable); + } if (TSDB_CODE_SUCCESS == code) { pJoinTable->table.precision = calcJoinTablePrecision(pJoinTable); pJoinTable->table.singleTable = joinTableIsSingleTable(pJoinTable); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 716dd7ffc0..be0ed72e23 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -445,4 +445,11 @@ TEST_F(ParserSelectTest, withoutFromSemanticCheck) { run("SELECT TBNAME", TSDB_CODE_PAR_INVALID_TBNAME); } +TEST_F(ParserSelectTest, joinSemanticCheck) { + useDb("root", "test"); + + run("SELECT * FROM (SELECT tag1, SUM(c1) s FROM st1 GROUP BY tag1) t1, st1 t2 where t1.tag1 = t2.tag1", + TSDB_CODE_PAR_NOT_SUPPORT_JOIN); +} + } // namespace ParserTest diff --git a/source/libs/planner/test/planJoinTest.cpp b/source/libs/planner/test/planJoinTest.cpp index 66ef4d3f19..535bb0b416 100644 --- a/source/libs/planner/test/planJoinTest.cpp +++ b/source/libs/planner/test/planJoinTest.cpp @@ -28,6 +28,8 @@ TEST_F(PlanJoinTest, basic) { run("SELECT t1.*, t2.* FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts"); run("SELECT t1.c1, t2.c1 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts"); + + run("SELECT t1.c1, t2.c1 FROM st1 t1 JOIN st2 t2 ON t1.ts = t2.ts"); } TEST_F(PlanJoinTest, complex) { @@ -56,9 +58,3 @@ TEST_F(PlanJoinTest, multiJoin) { run("SELECT t1.c1, t2.c1 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts JOIN st1s3 t3 ON t1.ts = t3.ts"); } - -TEST_F(PlanJoinTest, stable) { - useDb("root", "test"); - - run("SELECT t1.c1, t2.c1 FROM st1 t1 JOIN st2 t2 ON t1.ts = t2.ts "); -} From 62b9419351376d7e7f33bbff32ed3c17a28c0453 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 22 Sep 2022 17:27:30 +0800 Subject: [PATCH 10/13] fix: window pseudo column check --- source/libs/parser/src/parTranslater.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7d16758e56..23bc36e7ee 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1476,6 +1476,10 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SFunctio if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); } + if (beforeHaving(pCxt->currClause)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC, "There mustn't be %s", + pFunc->functionName); + } return TSDB_CODE_SUCCESS; } From cf39f2e20c0d281e958bf63c33f8fb37904a9273 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 22 Sep 2022 18:15:11 +0800 Subject: [PATCH 11/13] fix: window pseudo column check --- source/libs/parser/src/parTranslater.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 23bc36e7ee..88dd5afe57 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -264,6 +264,8 @@ static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; static bool afterHaving(ESqlClause clause) { return clause > SQL_CLAUSE_HAVING; } +static bool beforeWindow(ESqlClause clause) { return clause < SQL_CLAUSE_WINDOW; } + static bool hasSameTableAlias(SArray* pTables) { if (taosArrayGetSize(pTables) < 2) { return false; @@ -1476,7 +1478,7 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SFunctio if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); } - if (beforeHaving(pCxt->currClause)) { + if (beforeWindow(pCxt->currClause)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC, "There mustn't be %s", pFunc->functionName); } From ce79f915e463ca537a96c8f62696d25af34bb203 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 22 Sep 2022 20:28:38 +0800 Subject: [PATCH 12/13] fix: restore vnodes in multi-threads --- source/dnode/mgmt/mgmt_vnode/inc/vmInt.h | 1 + source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 98 ++++++++++++++++++++---- 2 files changed, 85 insertions(+), 14 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index ebbb9fa5d4..30f5483198 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -74,6 +74,7 @@ typedef struct { TdThread thread; SVnodeMgmt *pMgmt; SWrapperCfg *pCfgs; + SVnodeObj **ppVnodes; } SVnodeThread; // vmInt.c diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 1f981cc9e0..3940543e74 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -218,14 +218,14 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) { dInfo("start to close all vnodes"); int32_t numOfVnodes = 0; - SVnodeObj **pVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); + SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); for (int32_t i = 0; i < numOfVnodes; ++i) { - vmCloseVnode(pMgmt, pVnodes[i]); + vmCloseVnode(pMgmt, ppVnodes[i]); } - if (pVnodes != NULL) { - taosMemoryFree(pVnodes); + if (ppVnodes != NULL) { + taosMemoryFree(ppVnodes); } if (pMgmt->hash != NULL) { @@ -331,22 +331,92 @@ static int32_t vmRequire(const SMgmtInputOpt *pInput, bool *required) { return 0; } -static int32_t vmStart(SVnodeMgmt *pMgmt) { - int32_t numOfVnodes = 0; - SVnodeObj **pVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); +static void *vmRestoreVnodeInThread(void *param) { + SVnodeThread *pThread = param; + SVnodeMgmt *pMgmt = pThread->pMgmt; - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = pVnodes[i]; - vnodeStart(pVnode->pImpl); + dInfo("thread:%d, start to restore %d vnodes", pThread->threadIndex, pThread->vnodeNum); + setThreadName("restore-vnodes"); + + for (int32_t v = 0; v < pThread->vnodeNum; ++v) { + SVnodeObj *pVnode = pThread->ppVnodes[v]; + + char stepDesc[TSDB_STEP_DESC_LEN] = {0}; + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been restored", pVnode->vgId, + pMgmt->state.openVnodes, pMgmt->state.totalVnodes); + tmsgReportStartup("vnode-restore", stepDesc); + + int32_t code = vnodeStart(pVnode->pImpl); + if (code != 0) { + dError("vgId:%d, failed to restore vnode by thread:%d", pVnode->vgId, pThread->threadIndex); + pThread->failed++; + } else { + dDebug("vgId:%d, is restored by thread:%d", pVnode->vgId, pThread->threadIndex); + pThread->opened++; + atomic_add_fetch_32(&pMgmt->state.openVnodes, 1); + } } + dInfo("thread:%d, numOfVnodes:%d, restored:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + pThread->failed); + return NULL; +} + +static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) { + int32_t numOfVnodes = 0; + SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); + + int32_t threadNum = tsNumOfCores / 2; + if (threadNum < 1) threadNum = 1; + int32_t vnodesPerThread = numOfVnodes / threadNum + 1; + + SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread)); + for (int32_t t = 0; t < threadNum; ++t) { + threads[t].threadIndex = t; + threads[t].pMgmt = pMgmt; + threads[t].ppVnodes = taosMemoryCalloc(vnodesPerThread, sizeof(SVnode *)); + } + + for (int32_t v = 0; v < numOfVnodes; ++v) { + int32_t t = v % threadNum; + SVnodeThread *pThread = &threads[t]; + pThread->ppVnodes[pThread->vnodeNum++] = ppVnodes[v]; + } + + pMgmt->state.openVnodes = 0; + dInfo("restore %d vnodes with %d threads", numOfVnodes, threadNum); + + for (int32_t t = 0; t < threadNum; ++t) { + SVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum == 0) continue; + + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pThread->thread, &thAttr, vmRestoreVnodeInThread, pThread) != 0) { + dError("thread:%d, failed to create thread to restore vnode since %s", pThread->threadIndex, strerror(errno)); + } + + taosThreadAttrDestroy(&thAttr); + } + + for (int32_t t = 0; t < threadNum; ++t) { + SVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { + taosThreadJoin(pThread->thread, NULL); + taosThreadClear(&pThread->thread); + } + taosMemoryFree(pThread->pCfgs); + } + taosMemoryFree(threads); + for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = pVnodes[i]; + SVnodeObj *pVnode = ppVnodes[i]; vmReleaseVnode(pMgmt, pVnode); } - if (pVnodes != NULL) { - taosMemoryFree(pVnodes); + if (ppVnodes != NULL) { + taosMemoryFree(ppVnodes); } return 0; @@ -360,7 +430,7 @@ SMgmtFunc vmGetMgmtFunc() { SMgmtFunc mgmtFunc = {0}; mgmtFunc.openFp = vmInit; mgmtFunc.closeFp = (NodeCloseFp)vmCleanup; - mgmtFunc.startFp = (NodeStartFp)vmStart; + mgmtFunc.startFp = (NodeStartFp)vmStartVnodes; mgmtFunc.stopFp = (NodeStopFp)vmStop; mgmtFunc.requiredFp = vmRequire; mgmtFunc.getHandlesFp = vmGetMsgHandles; From ef859f7fe1ae959ed550cab33b5713100b3dfeba Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 22 Sep 2022 20:49:33 +0800 Subject: [PATCH 13/13] fix: definite lost --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 2 +- tests/script/tsim/insert/basic.sim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 3940543e74..f0c43d8b36 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -406,7 +406,7 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) { taosThreadJoin(pThread->thread, NULL); taosThreadClear(&pThread->thread); } - taosMemoryFree(pThread->pCfgs); + taosMemoryFree(pThread->ppVnodes); } taosMemoryFree(threads); diff --git a/tests/script/tsim/insert/basic.sim b/tests/script/tsim/insert/basic.sim index c4ef3e39da..c926cbc8b0 100644 --- a/tests/script/tsim/insert/basic.sim +++ b/tests/script/tsim/insert/basic.sim @@ -12,7 +12,7 @@ $tb = $tbPrefix . $i print =============== step1 sql drop database -x step1 step1: -sql create database $db +sql create database $db vgroups 2 sql use $db sql create table $tb (ts timestamp, speed int)