xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

merge from 3.0

This commit is contained in:
Liu Jicong 2022-07-08 17:53:27 +08:00
parent 3cd5a35054 8cbd88b6f4
commit 938868e39f
172 changed files with 5677 additions and 6622 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmake/cmake.define
View File

@ -97,10 +97,10 @@ ELSE ()
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
ENDIF () ENDIF ()
IF (${SANITIZER} MATCHES "true") IF (${BUILD_SANITIZER})
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
MESSAGE(STATUS "Will compile with Address Sanitizer!") MESSAGE(STATUS "Will compile with Address Sanitizer!")
ELSE () ELSE ()
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0")

6
cmake/cmake.options
View File

@ -77,6 +77,12 @@ ELSE ()
ENDIF () ENDIF ()
ENDIF () ENDIF ()
option(
BUILD_SANITIZER
"If build addr2line"
OFF
)
option( option(
BUILD_ADDR2LINE BUILD_ADDR2LINE
"If build addr2line" "If build addr2line"

4
docs/en/25-application/03-immigrate.md
View File

@ -379,11 +379,11 @@ We still use the hypothetical environment from Chapter 4. There are three measur
### Storage resource estimation ### Storage resource estimation
Assuming that the number of sensor devices that generate data and need to be stored is `n`, the frequency of data generation is `t` per second, and the length of each record is `L` bytes, the scale of data generated per day is `n * t * L` bytes. Assuming the compression ratio is `C`, the daily data size is `(n * t * L)/C` bytes. The storage resources are estimated to accommodate the data scale for 1.5 years. In the production environment, the compression ratio C of TDengine is generally between 5 and 7. Assuming that the number of sensor devices that generate data and need to be stored is `n`, the frequency of data generation is `t` per second, and the length of each record is `L` bytes, the scale of data generated per day is `86400 * n * t * L` bytes. Assuming the compression ratio is `C`, the daily data size is `(86400 * n * t * L)/C` bytes. The storage resources are estimated to accommodate the data scale for 1.5 years. In the production environment, the compression ratio C of TDengine is generally between 5 and 7.
With additional 20% redundancy, you can calculate the required storage resources: With additional 20% redundancy, you can calculate the required storage resources:
```matlab ```matlab
(n * t * L) * (365 * 1.5) * (1+20%)/C (86400 * n * t * L) * (365 * 1.5) * (1+20%)/C
```` ````
Substituting in the above formula, the raw data generated every year is 11.8TB without considering the label information. Note that tag information is associated with each timeline in TDengine, not every record. The amount of data to be recorded is somewhat reduced relative to the generated data, and label data can be ignored as a whole. Assuming a compression ratio of 5, the size of the retained data ends up being 2.56 TB. Substituting in the above formula, the raw data generated every year is 11.8TB without considering the label information. Note that tag information is associated with each timeline in TDengine, not every record. The amount of data to be recorded is somewhat reduced relative to the generated data, and label data can be ignored as a whole. Assuming a compression ratio of 5, the size of the retained data ends up being 2.56 TB.

4
docs/zh/25-application/03-immigrate.md
View File

@ -367,10 +367,10 @@ WHERE ts>=1510560000 AND ts<=1515000009
### 存储资源估算 ### 存储资源估算
假设产生数据并需要存储的传感器设备数量为 `n`，数据生成的频率为`t`条/秒，每条记录的长度为 `L` bytes，则每天产生的数据规模为 `n×t×L` bytes。假设压缩比为 C，则每日产生数据规模为 `(n×t×L)/C` bytes。存储资源预估为能够容纳 1.5 年的数据规模，生产环境下 TDengine 的压缩比 C 一般在 5 ~ 7 之间，同时为最后结果增加 20% 的冗余，可计算得到需要存储资源： 假设产生数据并需要存储的传感器设备数量为 `n`，数据生成的频率为`t`条/秒，每条记录的长度为 `L` bytes，则每天产生的数据规模为 `86400×n×t×L` bytes。假设压缩比为 C，则每日产生数据规模为 `(86400×n×t×L)/C` bytes。存储资源预估为能够容纳 1.5 年的数据规模，生产环境下 TDengine 的压缩比 C 一般在 5 ~ 7 之间，同时为最后结果增加 20% 的冗余，可计算得到需要存储资源：
```matlab ```matlab
(n×t×L)×(365×1.5)×(1+20%)/C (86400×n×t×L)×(365×1.5)×(1+20%)/C
``` ```
结合以上的计算公式，将参数带入计算公式，在不考虑标签信息的情况下，每年产生的原始数据规模是 11.8TB。需要注意的是，由于标签信息在 TDengine 中关联到每个时间线，并不是每条记录。所以需要记录的数据量规模相对于产生的数据有一定的降低，而这部分标签数据整体上可以忽略不记。假设压缩比为 5，则保留的数据规模最终为 2.56 TB。 结合以上的计算公式，将参数带入计算公式，在不考虑标签信息的情况下，每年产生的原始数据规模是 11.8TB。需要注意的是，由于标签信息在 TDengine 中关联到每个时间线，并不是每条记录。所以需要记录的数据量规模相对于产生的数据有一定的降低，而这部分标签数据整体上可以忽略不记。假设压缩比为 5，则保留的数据规模最终为 2.56 TB。

2
include/common/tcommon.h
View File

@ -89,8 +89,6 @@ typedef struct {
#pragma pack(push, 1) #pragma pack(push, 1)
typedef struct SColumnDataAgg { typedef struct SColumnDataAgg {
int16_t colId; int16_t colId;
int16_t minIndex;
int16_t maxIndex;
int16_t numOfNull; int16_t numOfNull;
int64_t sum; int64_t sum;
int64_t max; int64_t max;

6
include/common/tmsg.h
View File

@ -57,8 +57,8 @@ extern int32_t tMsgDict[];
#define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) #define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff)
#define TMSG_INFO(TYPE) \ #define TMSG_INFO(TYPE) \
((TYPE) >= 0 && \ ((TYPE) >= 0 && \
((TYPE) < TDMT_DND_MAX_MSG | (TYPE) < TDMT_MND_MAX_MSG | (TYPE) < TDMT_VND_MAX_MSG | (TYPE) < TDMT_SCH_MAX_MSG | \ ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || (TYPE) < TDMT_SCH_MAX_MSG || \
(TYPE) < TDMT_STREAM_MAX_MSG | (TYPE) < TDMT_MON_MAX_MSG | (TYPE) < TDMT_SYNC_MAX_MSG)) \ (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG)) \
? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \ ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \
: 0 : 0
#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)) #define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))
@ -665,6 +665,7 @@ typedef struct {
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
int32_t sversion; int32_t sversion;
int32_t tversion; int32_t tversion;
int64_t affectedRows;
} SQueryTableRsp; } SQueryTableRsp;
int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp);
@ -1510,6 +1511,7 @@ typedef struct SSubQueryMsg {
int32_t execId; int32_t execId;
int8_t taskType; int8_t taskType;
int8_t explain; int8_t explain;
int8_t needFetch;
uint32_t sqlLen; // the query sql, uint32_t sqlLen; // the query sql,
uint32_t phyLen; uint32_t phyLen;
char msg[]; char msg[];

379
include/common/ttokendef.h
View File

@ -79,195 +79,196 @@
#define TK_EXISTS 61 #define TK_EXISTS 61
#define TK_BUFFER 62 #define TK_BUFFER 62
#define TK_CACHELAST 63 #define TK_CACHELAST 63
#define TK_COMP 64 #define TK_CACHELASTSIZE 64
#define TK_DURATION 65 #define TK_COMP 65
#define TK_NK_VARIABLE 66 #define TK_DURATION 66
#define TK_FSYNC 67 #define TK_NK_VARIABLE 67
#define TK_MAXROWS 68 #define TK_FSYNC 68
#define TK_MINROWS 69 #define TK_MAXROWS 69
#define TK_KEEP 70 #define TK_MINROWS 70
#define TK_PAGES 71 #define TK_KEEP 71
#define TK_PAGESIZE 72 #define TK_PAGES 72
#define TK_PRECISION 73 #define TK_PAGESIZE 73
#define TK_REPLICA 74 #define TK_PRECISION 74
#define TK_STRICT 75 #define TK_REPLICA 75
#define TK_WAL 76 #define TK_STRICT 76
#define TK_VGROUPS 77 #define TK_WAL 77
#define TK_SINGLE_STABLE 78 #define TK_VGROUPS 78
#define TK_RETENTIONS 79 #define TK_SINGLE_STABLE 79
#define TK_SCHEMALESS 80 #define TK_RETENTIONS 80
#define TK_NK_COLON 81 #define TK_SCHEMALESS 81
#define TK_TABLE 82 #define TK_NK_COLON 82
#define TK_NK_LP 83 #define TK_TABLE 83
#define TK_NK_RP 84 #define TK_NK_LP 84
#define TK_STABLE 85 #define TK_NK_RP 85
#define TK_ADD 86 #define TK_STABLE 86
#define TK_COLUMN 87 #define TK_ADD 87
#define TK_MODIFY 88 #define TK_COLUMN 88
#define TK_RENAME 89 #define TK_MODIFY 89
#define TK_TAG 90 #define TK_RENAME 90
#define TK_SET 91 #define TK_TAG 91
#define TK_NK_EQ 92 #define TK_SET 92
#define TK_USING 93 #define TK_NK_EQ 93
#define TK_TAGS 94 #define TK_USING 94
#define TK_COMMENT 95 #define TK_TAGS 95
#define TK_BOOL 96 #define TK_COMMENT 96
#define TK_TINYINT 97 #define TK_BOOL 97
#define TK_SMALLINT 98 #define TK_TINYINT 98
#define TK_INT 99 #define TK_SMALLINT 99
#define TK_INTEGER 100 #define TK_INT 100
#define TK_BIGINT 101 #define TK_INTEGER 101
#define TK_FLOAT 102 #define TK_BIGINT 102
#define TK_DOUBLE 103 #define TK_FLOAT 103
#define TK_BINARY 104 #define TK_DOUBLE 104
#define TK_TIMESTAMP 105 #define TK_BINARY 105
#define TK_NCHAR 106 #define TK_TIMESTAMP 106
#define TK_UNSIGNED 107 #define TK_NCHAR 107
#define TK_JSON 108 #define TK_UNSIGNED 108
#define TK_VARCHAR 109 #define TK_JSON 109
#define TK_MEDIUMBLOB 110 #define TK_VARCHAR 110
#define TK_BLOB 111 #define TK_MEDIUMBLOB 111
#define TK_VARBINARY 112 #define TK_BLOB 112
#define TK_DECIMAL 113 #define TK_VARBINARY 113
#define TK_MAX_DELAY 114 #define TK_DECIMAL 114
#define TK_WATERMARK 115 #define TK_MAX_DELAY 115
#define TK_ROLLUP 116 #define TK_WATERMARK 116
#define TK_TTL 117 #define TK_ROLLUP 117
#define TK_SMA 118 #define TK_TTL 118
#define TK_FIRST 119 #define TK_SMA 119
#define TK_LAST 120 #define TK_FIRST 120
#define TK_SHOW 121 #define TK_LAST 121
#define TK_DATABASES 122 #define TK_SHOW 122
#define TK_TABLES 123 #define TK_DATABASES 123
#define TK_STABLES 124 #define TK_TABLES 124
#define TK_MNODES 125 #define TK_STABLES 125
#define TK_MODULES 126 #define TK_MNODES 126
#define TK_QNODES 127 #define TK_MODULES 127
#define TK_FUNCTIONS 128 #define TK_QNODES 128
#define TK_INDEXES 129 #define TK_FUNCTIONS 129
#define TK_ACCOUNTS 130 #define TK_INDEXES 130
#define TK_APPS 131 #define TK_ACCOUNTS 131
#define TK_CONNECTIONS 132 #define TK_APPS 132
#define TK_LICENCE 133 #define TK_CONNECTIONS 133
#define TK_GRANTS 134 #define TK_LICENCE 134
#define TK_QUERIES 135 #define TK_GRANTS 135
#define TK_SCORES 136 #define TK_QUERIES 136
#define TK_TOPICS 137 #define TK_SCORES 137
#define TK_VARIABLES 138 #define TK_TOPICS 138
#define TK_BNODES 139 #define TK_VARIABLES 139
#define TK_SNODES 140 #define TK_BNODES 140
#define TK_CLUSTER 141 #define TK_SNODES 141
#define TK_TRANSACTIONS 142 #define TK_CLUSTER 142
#define TK_DISTRIBUTED 143 #define TK_TRANSACTIONS 143
#define TK_CONSUMERS 144 #define TK_DISTRIBUTED 144
#define TK_SUBSCRIPTIONS 145 #define TK_CONSUMERS 145
#define TK_LIKE 146 #define TK_SUBSCRIPTIONS 146
#define TK_INDEX 147 #define TK_LIKE 147
#define TK_FUNCTION 148 #define TK_INDEX 148
#define TK_INTERVAL 149 #define TK_FUNCTION 149
#define TK_TOPIC 150 #define TK_INTERVAL 150
#define TK_AS 151 #define TK_TOPIC 151
#define TK_WITH 152 #define TK_AS 152
#define TK_META 153 #define TK_WITH 153
#define TK_CONSUMER 154 #define TK_META 154
#define TK_GROUP 155 #define TK_CONSUMER 155
#define TK_DESC 156 #define TK_GROUP 156
#define TK_DESCRIBE 157 #define TK_DESC 157
#define TK_RESET 158 #define TK_DESCRIBE 158
#define TK_QUERY 159 #define TK_RESET 159
#define TK_CACHE 160 #define TK_QUERY 160
#define TK_EXPLAIN 161 #define TK_CACHE 161
#define TK_ANALYZE 162 #define TK_EXPLAIN 162
#define TK_VERBOSE 163 #define TK_ANALYZE 163
#define TK_NK_BOOL 164 #define TK_VERBOSE 164
#define TK_RATIO 165 #define TK_NK_BOOL 165
#define TK_NK_FLOAT 166 #define TK_RATIO 166
#define TK_COMPACT 167 #define TK_NK_FLOAT 167
#define TK_VNODES 168 #define TK_COMPACT 168
#define TK_IN 169 #define TK_VNODES 169
#define TK_OUTPUTTYPE 170 #define TK_IN 170
#define TK_AGGREGATE 171 #define TK_OUTPUTTYPE 171
#define TK_BUFSIZE 172 #define TK_AGGREGATE 172
#define TK_STREAM 173 #define TK_BUFSIZE 173
#define TK_INTO 174 #define TK_STREAM 174
#define TK_TRIGGER 175 #define TK_INTO 175
#define TK_AT_ONCE 176 #define TK_TRIGGER 176
#define TK_WINDOW_CLOSE 177 #define TK_AT_ONCE 177
#define TK_IGNORE 178 #define TK_WINDOW_CLOSE 178
#define TK_EXPIRED 179 #define TK_IGNORE 179
#define TK_KILL 180 #define TK_EXPIRED 180
#define TK_CONNECTION 181 #define TK_KILL 181
#define TK_TRANSACTION 182 #define TK_CONNECTION 182
#define TK_BALANCE 183 #define TK_TRANSACTION 183
#define TK_VGROUP 184 #define TK_BALANCE 184
#define TK_MERGE 185 #define TK_VGROUP 185
#define TK_REDISTRIBUTE 186 #define TK_MERGE 186
#define TK_SPLIT 187 #define TK_REDISTRIBUTE 187
#define TK_SYNCDB 188 #define TK_SPLIT 188
#define TK_DELETE 189 #define TK_SYNCDB 189
#define TK_INSERT 190 #define TK_DELETE 190
#define TK_NULL 191 #define TK_INSERT 191
#define TK_NK_QUESTION 192 #define TK_NULL 192
#define TK_NK_ARROW 193 #define TK_NK_QUESTION 193
#define TK_ROWTS 194 #define TK_NK_ARROW 194
#define TK_TBNAME 195 #define TK_ROWTS 195
#define TK_QSTARTTS 196 #define TK_TBNAME 196
#define TK_QENDTS 197 #define TK_QSTARTTS 197
#define TK_WSTARTTS 198 #define TK_QENDTS 198
#define TK_WENDTS 199 #define TK_WSTARTTS 199
#define TK_WDURATION 200 #define TK_WENDTS 200
#define TK_CAST 201 #define TK_WDURATION 201
#define TK_NOW 202 #define TK_CAST 202
#define TK_TODAY 203 #define TK_NOW 203
#define TK_TIMEZONE 204 #define TK_TODAY 204
#define TK_CLIENT_VERSION 205 #define TK_TIMEZONE 205
#define TK_SERVER_VERSION 206 #define TK_CLIENT_VERSION 206
#define TK_SERVER_STATUS 207 #define TK_SERVER_VERSION 207
#define TK_CURRENT_USER 208 #define TK_SERVER_STATUS 208
#define TK_COUNT 209 #define TK_CURRENT_USER 209
#define TK_LAST_ROW 210 #define TK_COUNT 210
#define TK_BETWEEN 211 #define TK_LAST_ROW 211
#define TK_IS 212 #define TK_BETWEEN 212
#define TK_NK_LT 213 #define TK_IS 213
#define TK_NK_GT 214 #define TK_NK_LT 214
#define TK_NK_LE 215 #define TK_NK_GT 215
#define TK_NK_GE 216 #define TK_NK_LE 216
#define TK_NK_NE 217 #define TK_NK_GE 217
#define TK_MATCH 218 #define TK_NK_NE 218
#define TK_NMATCH 219 #define TK_MATCH 219
#define TK_CONTAINS 220 #define TK_NMATCH 220
#define TK_JOIN 221 #define TK_CONTAINS 221
#define TK_INNER 222 #define TK_JOIN 222
#define TK_SELECT 223 #define TK_INNER 223
#define TK_DISTINCT 224 #define TK_SELECT 224
#define TK_WHERE 225 #define TK_DISTINCT 225
#define TK_PARTITION 226 #define TK_WHERE 226
#define TK_BY 227 #define TK_PARTITION 227
#define TK_SESSION 228 #define TK_BY 228
#define TK_STATE_WINDOW 229 #define TK_SESSION 229
#define TK_SLIDING 230 #define TK_STATE_WINDOW 230
#define TK_FILL 231 #define TK_SLIDING 231
#define TK_VALUE 232 #define TK_FILL 232
#define TK_NONE 233 #define TK_VALUE 233
#define TK_PREV 234 #define TK_NONE 234
#define TK_LINEAR 235 #define TK_PREV 235
#define TK_NEXT 236 #define TK_LINEAR 236
#define TK_HAVING 237 #define TK_NEXT 237
#define TK_RANGE 238 #define TK_HAVING 238
#define TK_EVERY 239 #define TK_RANGE 239
#define TK_ORDER 240 #define TK_EVERY 240
#define TK_SLIMIT 241 #define TK_ORDER 241
#define TK_SOFFSET 242 #define TK_SLIMIT 242
#define TK_LIMIT 243 #define TK_SOFFSET 243
#define TK_OFFSET 244 #define TK_LIMIT 244
#define TK_ASC 245 #define TK_OFFSET 245
#define TK_NULLS 246 #define TK_ASC 246
#define TK_ID 247 #define TK_NULLS 247
#define TK_NK_BITNOT 248 #define TK_ID 248
#define TK_VALUES 249 #define TK_NK_BITNOT 249
#define TK_IMPORT 250 #define TK_VALUES 250
#define TK_NK_SEMI 251 #define TK_IMPORT 251
#define TK_FILE 252 #define TK_NK_SEMI 252
#define TK_FILE 253
#define TK_NK_SPACE 300 #define TK_NK_SPACE 300
#define TK_NK_COMMENT 301 #define TK_NK_COMMENT 301

6
include/libs/executor/dataSinkMgt.h
View File

@ -45,6 +45,10 @@ typedef struct SDeleterParam {
SArray* pUidList; SArray* pUidList;
} SDeleterParam; } SDeleterParam;
typedef struct SInserterParam {
SReadHandle* readHandle;
} SInserterParam;
typedef struct SDataSinkStat { typedef struct SDataSinkStat {
uint64_t cachedSize; uint64_t cachedSize;
} SDataSinkStat; } SDataSinkStat;
@ -96,7 +100,7 @@ void dsEndPut(DataSinkHandle handle, uint64_t useconds);
* @param handle * @param handle
* @param pLen data length * @param pLen data length
*/ */
void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd); void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, bool* pQueryEnd);
/** /**
* Get data, the caller needs to allocate data memory. * Get data, the caller needs to allocate data memory.

2
include/libs/executor/executor.h
View File

@ -157,7 +157,7 @@ int64_t qGetQueriedTableUid(qTaskInfo_t tinfo);
*/ */
int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t tagCondLen, SArray* pTableIdList); int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t tagCondLen, SArray* pTableIdList);
void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); void qProcessRspMsg(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);
int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes); int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes);

3
include/libs/nodes/cmdnodes.h
View File

@ -51,7 +51,8 @@ extern "C" {
typedef struct SDatabaseOptions { typedef struct SDatabaseOptions {
ENodeType type; ENodeType type;
int32_t buffer; int32_t buffer;
int8_t cachelast; int8_t cacheLast;
int32_t cacheLastSize;
int8_t compressionLevel; int8_t compressionLevel;
int32_t daysPerFile; int32_t daysPerFile;
SValueNode* pDaysPerFile; SValueNode* pDaysPerFile;

2
include/libs/qworker/qworker.h
View File

@ -67,7 +67,7 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts); int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts); int32_t qWorkerProcessRspMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts); int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);

3
include/libs/sync/sync.h
View File

@ -163,9 +163,6 @@ typedef struct SSyncLogStore {
// return commit index of log // return commit index of log
SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore);
// refactor, log[0 .. n] ==> log[m .. n]
// int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex);
SyncIndex (*syncLogBeginIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogBeginIndex)(struct SSyncLogStore* pLogStore);
SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore);
bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore); bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore);

2
include/util/tlog.h
View File

@ -60,7 +60,7 @@ extern int32_t tsdbDebugFlag;
extern int32_t tqDebugFlag; extern int32_t tqDebugFlag;
extern int32_t fsDebugFlag; extern int32_t fsDebugFlag;
extern int32_t metaDebugFlag; extern int32_t metaDebugFlag;
extern int32_t fnDebugFlag; extern int32_t udfDebugFlag;
extern int32_t smaDebugFlag; extern int32_t smaDebugFlag;
extern int32_t idxDebugFlag; extern int32_t idxDebugFlag;

6
source/client/src/TMQConnector.c
View File

@ -324,9 +324,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(
(*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfRowsFp, (jint)numOfRows); (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfRowsFp, (jint)numOfRows);
(*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfColsFp, (jint)numOfFields); (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfColsFp, (jint)numOfFields);
char *chars = (char *)data; int32_t len = *(int32_t *)data;
int32_t len = chars[0] + (chars[1] << 8) + (chars[2] << 16) + (chars[3] << 24); (*env)->CallVoidMethod(env, rowobj, g_blockdataSetByteArrayFp, jniFromNCharToByteArray(env, (char *)data, len));
(*env)->CallVoidMethod(env, rowobj, g_blockdataSetByteArrayFp, len, jniFromNCharToByteArray(env, (char *)data, len));
return JNI_SUCCESS; return JNI_SUCCESS;
} }

3
source/client/src/TSDBJNIConnector.c
View File

@ -592,8 +592,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI
(*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfRowsFp, (jint)numOfRows); (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfRowsFp, (jint)numOfRows);
(*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfColsFp, (jint)numOfFields); (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfColsFp, (jint)numOfFields);
char *chars = (char *)data; int32_t len = *(int32_t *)data;
int32_t len = chars[0] + (chars[1] << 8) + (chars[2] << 16) + (chars[3] << 24);
(*env)->CallVoidMethod(env, rowobj, g_blockdataSetByteArrayFp, jniFromNCharToByteArray(env, (char *)data, len)); (*env)->CallVoidMethod(env, rowobj, g_blockdataSetByteArrayFp, jniFromNCharToByteArray(env, (char *)data, len));
return JNI_SUCCESS; return JNI_SUCCESS;

28
source/client/test/clientTests.cpp
View File

@ -750,7 +750,6 @@ TEST(testCase, projection_query_stables) {
taos_close(pConn); taos_close(pConn);
} }
TEST(testCase, agg_query_tables) { TEST(testCase, agg_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr); ASSERT_NE(pConn, nullptr);
@ -763,7 +762,7 @@ TEST(testCase, agg_query_tables) {
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "show table distributed st1"); pRes = taos_query(pConn, "show table distributed tup");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes); taos_free_result(pRes);
@ -822,13 +821,29 @@ TEST(testCase, async_api_test) {
} }
#endif #endif
TEST(testCase, update_test) { TEST(testCase, update_test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr); ASSERT_NE(pConn, nullptr);
taos_query(pConn, "use abc1"); TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
printf("failed to create database, code:%s", taos_errstr(pRes));
taos_free_result(pRes);
return;
}
TAOS_RES* pRes = taos_query(pConn, "create table tup (ts timestamp, k int);"); taos_free_result(pRes);
pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
printf("failed to use db, code:%s", taos_errstr(pRes));
taos_free_result(pRes);
return;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create table tup (ts timestamp, k int);");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create table, reason:%s", taos_errstr(pRes)); printf("failed to create table, reason:%s", taos_errstr(pRes));
} }
@ -836,11 +851,10 @@ TEST(testCase, update_test) {
taos_free_result(pRes); taos_free_result(pRes);
char s[256] = {0}; char s[256] = {0};
for(int32_t i = 0; i < 7000; ++i) { for(int32_t i = 0; i < 17000; ++i) {
sprintf(s, "insert into tup values('2020-1-1 1:1:1', %d)", i); sprintf(s, "insert into tup values(now+%da, %d)", i, i);
pRes = taos_query(pConn, s); pRes = taos_query(pConn, s);
taos_free_result(pRes); taos_free_result(pRes);
} }
} }
#pragma GCC diagnostic pop #pragma GCC diagnostic pop

14
source/common/src/tdata.c
View File

@ -1,14 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

18
source/common/src/tdatablock.c
View File

@ -228,7 +228,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui
uint32_t finalNumOfRows = numOfRow1 + numOfRow2; uint32_t finalNumOfRows = numOfRow1 + numOfRow2;
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
// Handle the bitmap // Handle the bitmap
if (finalNumOfRows > *capacity) { if (finalNumOfRows > *capacity || numOfRow1 == 0) {
char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
if (p == NULL) { if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -262,7 +262,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui
memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len);
pColumnInfoData->varmeta.length = len + oldLen; pColumnInfoData->varmeta.length = len + oldLen;
} else { } else {
if (finalNumOfRows > *capacity) { if (finalNumOfRows > *capacity || numOfRow1 == 0) {
ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); ASSERT(finalNumOfRows * pColumnInfoData->info.bytes);
char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes);
if (tmp == NULL) { if (tmp == NULL) {
@ -1737,41 +1737,54 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
int32_t len = 0; int32_t len = 0;
len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id %lu|\n", flag, len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id %lu|\n", flag,
(int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId); (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
if (len >= size -1) return dumpBuf;
for (int32_t j = 0; j < rows; j++) { for (int32_t j = 0; j < rows; j++) {
len += snprintf(dumpBuf + len, size - len, "%s |", flag); len += snprintf(dumpBuf + len, size - len, "%s |", flag);
if (len >= size -1) return dumpBuf;
for (int32_t k = 0; k < colNum; k++) { for (int32_t k = 0; k < colNum; k++) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
if (colDataIsNull(pColInfoData, rows, j, NULL)) { if (colDataIsNull(pColInfoData, rows, j, NULL)) {
len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL"); len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL");
if (len >= size -1) return dumpBuf;
continue; continue;
} }
switch (pColInfoData->info.type) { switch (pColInfoData->info.type) {
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI);
len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf); len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf);
if (len >= size -1) return dumpBuf;
break; break;
case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_INT:
len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var); len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var);
if (len >= size -1) return dumpBuf;
break; break;
case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UINT:
len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var); len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var);
if (len >= size -1) return dumpBuf;
break; break;
case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_BIGINT:
len += snprintf(dumpBuf + len, size - len, " %15ld |", *(int64_t*)var); len += snprintf(dumpBuf + len, size - len, " %15ld |", *(int64_t*)var);
if (len >= size -1) return dumpBuf;
break; break;
case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_UBIGINT:
len += snprintf(dumpBuf + len, size - len, " %15lu |", *(uint64_t*)var); len += snprintf(dumpBuf + len, size - len, " %15lu |", *(uint64_t*)var);
if (len >= size -1) return dumpBuf;
break; break;
case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_FLOAT:
len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var); len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var);
if (len >= size -1) return dumpBuf;
break; break;
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var); len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var);
if (len >= size -1) return dumpBuf;
break; break;
} }
} }
len += snprintf(dumpBuf + len, size - len, "\n"); len += snprintf(dumpBuf + len, size - len, "\n");
if (len >= size -1) return dumpBuf;
} }
len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag); len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag);
return dumpBuf; return dumpBuf;
@ -2106,3 +2119,4 @@ const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRow
ASSERT(pStart - pData == dataLen); ASSERT(pStart - pData == dataLen);
return pStart; return pStart;
} }

8
source/common/src/tglobal.c
View File

@ -314,7 +314,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "fnDebugFlag", fnDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, 0) != 0) return -1;
return 0; return 0;
@ -504,7 +504,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32;
tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32;
fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32; udfDebugFlag = cfgGetItem(pCfg, "udfDebugFlag")->i32;
smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32; smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32;
idxDebugFlag = cfgGetItem(pCfg, "idxDebugFlag")->i32; idxDebugFlag = cfgGetItem(pCfg, "idxDebugFlag")->i32;
} }
@ -715,8 +715,6 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype);
} else if (strcasecmp("fsDebugFlag", name) == 0) { } else if (strcasecmp("fsDebugFlag", name) == 0) {
fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
} else if (strcasecmp("fnDebugFlag", name) == 0) {
fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32;
} }
break; break;
} }
@ -817,6 +815,8 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
case 'u': { case 'u': {
if (strcasecmp("multiProcess", name) == 0) { if (strcasecmp("multiProcess", name) == 0) {
tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->bval; tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->bval;
} else if (strcasecmp("udfDebugFlag", name) == 0) {
udfDebugFlag = cfgGetItem(pCfg, "udfDebugFlag")->i32;
} }
break; break;
} }

18
source/common/src/tmsgtype.c
View File

@ -1,18 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#define TSDB_SQL_C
#include "tmsgtype.h"

2
source/dnode/mgmt/exe/dmMain.c
View File

@ -216,7 +216,7 @@ int main(int argc, char const *argv[]) {
return -1; return -1;
} }
dInfo("start to open dnode"); dInfo("start to init service");
dmSetSignalHandle(); dmSetSignalHandle();
int32_t code = dmRun(); int32_t code = dmRun();
dInfo("shutting down the service"); dInfo("shutting down the service");

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -238,10 +238,12 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -377,10 +377,12 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_VNODE_STANDBY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_VNODE_STANDBY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;

4
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -173,7 +173,7 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v]; pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v];
} }
dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); dInfo("open %d vnodes with %d threads", numOfVnodes, threadNum);
for (int32_t t = 0; t < threadNum; ++t) { for (int32_t t = 0; t < threadNum; ++t) {
SVnodeThread *pThread = &threads[t]; SVnodeThread *pThread = &threads[t];
@ -204,7 +204,7 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes); dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes);
return -1; return -1;
} else { } else {
dInfo("total vnodes:%d open successfully", pMgmt->state.totalVnodes); dInfo("successfully opened %d vnodes", pMgmt->state.totalVnodes);
return 0; return 0;
} }
} }

2
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -128,7 +128,7 @@ static void dmClearVars(SDnode *pDnode) {
} }
int32_t dmInitDnode(SDnode *pDnode, EDndNodeType rtype) { int32_t dmInitDnode(SDnode *pDnode, EDndNodeType rtype) {
dInfo("start to create dnode"); dDebug("start to create dnode");
int32_t code = -1; int32_t code = -1;
char path[PATH_MAX + 100] = {0}; char path[PATH_MAX + 100] = {0};

3
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -89,7 +89,8 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
case TDMT_DND_SYSTABLE_RETRIEVE_RSP: case TDMT_DND_SYSTABLE_RETRIEVE_RSP:
case TDMT_SCH_FETCH_RSP: case TDMT_SCH_FETCH_RSP:
case TDMT_SCH_MERGE_FETCH_RSP: case TDMT_SCH_MERGE_FETCH_RSP:
qWorkerProcessFetchRsp(NULL, NULL, pRpc, 0); case TDMT_VND_SUBMIT_RSP:
qWorkerProcessRspMsg(NULL, NULL, pRpc, 0);
return; return;
case TDMT_MND_STATUS_RSP: case TDMT_MND_STATUS_RSP:
if (pEpSet != NULL) { if (pEpSet != NULL) {

2
source/dnode/mnode/impl/src/mndSync.c
View File

@ -188,7 +188,7 @@ int32_t mndInitSync(SMnode *pMnode) {
SNodeInfo *pNode = &pCfg->nodeInfo[0]; SNodeInfo *pNode = &pCfg->nodeInfo[0];
tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn)); tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn));
pNode->nodePort = pMgmt->replica.port; pNode->nodePort = pMgmt->replica.port;
mInfo("fqdn:%s port:%u", pNode->nodeFqdn, pNode->nodePort); mInfo("mnode ep:%s:%u", pNode->nodeFqdn, pNode->nodePort);
} }
tsem_init(&pMgmt->syncSem, 0, 0); tsem_init(&pMgmt->syncSem, 0, 0);

3
source/dnode/qnode/src/qnode.c
View File

@ -89,9 +89,6 @@ int32_t qndProcessQueryMsg(SQnode *pQnode, int64_t ts, SRpcMsg *pMsg) {
case TDMT_SCH_MERGE_FETCH: case TDMT_SCH_MERGE_FETCH:
code = qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg, ts); code = qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg, ts);
break; break;
case TDMT_SCH_FETCH_RSP:
code = qWorkerProcessFetchRsp(pQnode, pQnode->pQuery, pMsg, ts);
break;
case TDMT_SCH_CANCEL_TASK: case TDMT_SCH_CANCEL_TASK:
code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg, ts); code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg, ts);
break; break;

3
source/dnode/vnode/CMakeLists.txt
View File

@ -9,7 +9,6 @@ target_sources(
"src/vnd/vnodeCfg.c" "src/vnd/vnodeCfg.c"
"src/vnd/vnodeCommit.c" "src/vnd/vnodeCommit.c"
"src/vnd/vnodeQuery.c" "src/vnd/vnodeQuery.c"
"src/vnd/vnodeStateMgr.c"
"src/vnd/vnodeModule.c" "src/vnd/vnodeModule.c"
"src/vnd/vnodeSvr.c" "src/vnd/vnodeSvr.c"
"src/vnd/vnodeSync.c" "src/vnd/vnodeSync.c"
@ -32,7 +31,6 @@ target_sources(
"src/sma/smaUtil.c" "src/sma/smaUtil.c"
"src/sma/smaOpen.c" "src/sma/smaOpen.c"
"src/sma/smaCommit.c" "src/sma/smaCommit.c"
"src/sma/smaSnapshot.c"
"src/sma/smaRollup.c" "src/sma/smaRollup.c"
"src/sma/smaTimeRange.c" "src/sma/smaTimeRange.c"
@ -43,7 +41,6 @@ target_sources(
"src/tsdb/tsdbOpen.c" "src/tsdb/tsdbOpen.c"
"src/tsdb/tsdbMemTable.c" "src/tsdb/tsdbMemTable.c"
"src/tsdb/tsdbRead.c" "src/tsdb/tsdbRead.c"
"src/tsdb/tsdbReadImpl.c"
"src/tsdb/tsdbCache.c" "src/tsdb/tsdbCache.c"
"src/tsdb/tsdbWrite.c" "src/tsdb/tsdbWrite.c"
"src/tsdb/tsdbReaderWriter.c" "src/tsdb/tsdbReaderWriter.c"

3
source/dnode/vnode/inc/vnode.h
View File

@ -131,7 +131,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *pTabl
void tsdbReaderClose(STsdbReader *pReader); void tsdbReaderClose(STsdbReader *pReader);
bool tsdbNextDataBlock(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader);
void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo); void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo);
int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond, int32_t tWinIdx); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond, int32_t tWinIdx);
int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
@ -143,6 +143,7 @@ int32_t tsdbLastRowReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList,
void **pReader); void **pReader);
int32_t tsdbRetrieveLastRow(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds); int32_t tsdbRetrieveLastRow(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds);
int32_t tsdbLastrowReaderClose(void *pReader); int32_t tsdbLastrowReaderClose(void *pReader);
int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid);
// tq // tq

2
source/dnode/vnode/src/inc/tsdb.h
View File

@ -245,6 +245,8 @@ int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h);
int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);
// structs ======================= // structs =======================
typedef struct { typedef struct {
int minFid; int minFid;

1
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -439,6 +439,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv
tDecoderInit(&dc, pData, nData); tDecoderInit(&dc, pData, nData);
tDecodeSSchemaWrapper(&dc, pSchemaWrapper); tDecodeSSchemaWrapper(&dc, pSchemaWrapper);
tDecoderClear(&dc); tDecoderClear(&dc);
tdbFree(pData);
// convert // convert
STSchemaBuilder sb = {0}; STSchemaBuilder sb = {0};

85
source/dnode/vnode/src/meta/metaTable.c
View File

@ -16,6 +16,7 @@
#include "meta.h" #include "meta.h"
static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME); static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
@ -92,7 +93,65 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
} else if (type == TSDB_DATA_TYPE_BOOL) { } else if (type == TSDB_DATA_TYPE_BOOL) {
int val = *(int *)(&pTagVal->i64); int val = *(int *)(&pTagVal->i64);
int len = sizeof(val); int len = sizeof(val);
term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_INT, key, nKey, (const char *)&val, len); term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len);
}
if (term != NULL) {
indexMultiTermAdd(terms, term);
}
}
indexJsonPut(pMeta->pTagIvtIdx, terms, tuid);
indexMultiTermDestroy(terms);
#endif
return 0;
}
int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) {
#ifdef USE_INVERTED_INDEX
if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
return -1;
}
void * data = pCtbEntry->ctbEntry.pTags;
const char *tagName = pSchema->name;
tb_uid_t suid = pCtbEntry->ctbEntry.suid;
tb_uid_t tuid = pCtbEntry->uid;
const void *pTagData = pCtbEntry->ctbEntry.pTags;
int32_t nTagData = 0;
SArray *pTagVals = NULL;
if (tTagToValArray((const STag *)data, &pTagVals) != 0) {
return -1;
}
SIndexMultiTerm *terms = indexMultiTermCreate();
int16_t nCols = taosArrayGetSize(pTagVals);
for (int i = 0; i < nCols; i++) {
STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
char type = pTagVal->type;
char * key = pTagVal->pKey;
int32_t nKey = strlen(key);
SIndexTerm *term = NULL;
if (type == TSDB_DATA_TYPE_NULL) {
term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
} else if (type == TSDB_DATA_TYPE_NCHAR) {
if (pTagVal->nData > 0) {
char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
type = TSDB_DATA_TYPE_VARCHAR;
term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, val, len);
} else if (pTagVal->nData == 0) {
term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, pTagVal->pData, 0);
}
} else if (type == TSDB_DATA_TYPE_DOUBLE) {
double val = *(double *)(&pTagVal->i64);
int len = sizeof(val);
term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, (const char *)&val, len);
} else if (type == TSDB_DATA_TYPE_BOOL) {
int val = *(int *)(&pTagVal->i64);
int len = sizeof(val);
term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len);
} }
if (term != NULL) { if (term != NULL) {
indexMultiTermAdd(terms, term); indexMultiTermAdd(terms, term);
@ -434,9 +493,33 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
if (type) *type = e.type; if (type) *type = e.type;
if (e.type == TSDB_CHILD_TABLE) {
void *tData = NULL;
int tLen = 0;
if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) {
version = *(int64_t *)tData;
STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = version};
if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) {
SDecoder tdc = {0};
SMetaEntry stbEntry = {0};
tDecoderInit(&tdc, tData, tLen);
metaDecodeEntry(&tdc, &stbEntry);
const SSchema *pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0];
if (pTagColumn->type == TSDB_DATA_TYPE_JSON) {
metaDelJsonVarFromIdx(pMeta, &e, pTagColumn);
}
tDecoderClear(&tdc);
}
tdbFree(tData);
}
}
tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pMeta->txn); tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pMeta->txn);
tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, &pMeta->txn); tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, &pMeta->txn);
tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn); tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn);
if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e); if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e);
if (e.type == TSDB_CHILD_TABLE) { if (e.type == TSDB_CHILD_TABLE) {

7
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -277,6 +277,9 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo
pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY;
} }
pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2); pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2);
smaInfo("vgId:%d table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64
", finally maxdelay:%" PRIi32,
SMA_VID(pSma), pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_err: _err:
@ -570,11 +573,15 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
SSubmitReq *pReq = NULL; SSubmitReq *pReq = NULL;
// TODO: the schema update should be handled // TODO: the schema update should be handled
if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) { if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) {
smaError("vgId:%d, build submit req for rsma table %" PRIi64 "l evel %" PRIi8 " failed since %s", SMA_VID(pSma),
pRSmaInfo->suid, pItem->level, terrstr());
goto _err; goto _err;
} }
if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pRSmaInfo->pStat->submitVer, 1), pReq) < 0) { if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pRSmaInfo->pStat->submitVer, 1), pReq) < 0) {
taosMemoryFreeClear(pReq); taosMemoryFreeClear(pReq);
smaError("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma),
pRSmaInfo->suid, pItem->level, terrstr());
goto _err; goto _err;
} }

16
source/dnode/vnode/src/sma/smaSnapshot.c
View File

@ -1,16 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "sma.h"

125
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -59,6 +59,8 @@ static void getTableCacheKey(tb_uid_t uid, int cacheType, char *key, int *len) {
static void deleteTableCacheLastrow(const void *key, size_t keyLen, void *value) { taosMemoryFree(value); } static void deleteTableCacheLastrow(const void *key, size_t keyLen, void *value) { taosMemoryFree(value); }
static void deleteTableCacheLast(const void *key, size_t keyLen, void *value) { taosArrayDestroy(value); }
static int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { static int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) {
int32_t code = 0; int32_t code = 0;
char key[32] = {0}; char key[32] = {0};
@ -309,9 +311,11 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t
if (code) goto _err; if (code) goto _err;
// code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx); // code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx);
pDelIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ); SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ);
if (code) goto _err; if (code) goto _err;
*pDelIdx = *pIdx;
if (pDelIdxArray) { if (pDelIdxArray) {
taosArrayDestroy(pDelIdxArray); taosArrayDestroy(pDelIdxArray);
} }
@ -759,7 +763,6 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo
for (int i = 0; i < nMax; ++i) { for (int i = 0; i < nMax; ++i) {
TSDBKEY maxKey = TSDBROW_KEY(max[i]); TSDBKEY maxKey = TSDBROW_KEY(max[i]);
// bool deleted = false;
bool deleted = tsdbKeyDeleted(&maxKey, pSkyline, &iSkyline); bool deleted = tsdbKeyDeleted(&maxKey, pSkyline, &iSkyline);
if (!deleted) { if (!deleted) {
// iMerge[nMerge] = i; // iMerge[nMerge] = i;
@ -816,12 +819,22 @@ _err:
return code; return code;
} }
static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) { typedef struct {
int32_t code = 0; TSKEY ts;
SColVal colVal;
} SLastCol;
// static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) {
static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
int32_t code = 0;
SArray *pSkyline = NULL;
STSRow *pRow = NULL;
STSRow **ppRow = &pRow;
STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1); STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1);
int16_t nCol = pTSchema->numOfCols; int16_t nCol = pTSchema->numOfCols;
SArray *pColArray = taosArrayInit(nCol, sizeof(SColVal)); // SArray *pColArray = taosArrayInit(nCol, sizeof(SColVal));
SArray *pColArray = taosArrayInit(nCol, sizeof(SLastCol));
tb_uid_t suid = getTableSuidByUid(uid, pTsdb); tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
@ -835,9 +848,9 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) {
tsdbGetTbDataFromMemTable(pTsdb->imem, suid, uid, &pIMem); tsdbGetTbDataFromMemTable(pTsdb->imem, suid, uid, &pIMem);
} }
*ppRow = NULL; *ppLastArray = NULL;
SArray *pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); pSkyline = taosArrayInit(32, sizeof(TSDBKEY));
SDelIdx delIdx; SDelIdx delIdx;
@ -941,7 +954,6 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) {
for (int i = 0; i < nMax; ++i) { for (int i = 0; i < nMax; ++i) {
TSDBKEY maxKey = TSDBROW_KEY(max[i]); TSDBKEY maxKey = TSDBROW_KEY(max[i]);
// bool deleted = false;
bool deleted = tsdbKeyDeleted(&maxKey, pSkyline, &iSkyline); bool deleted = tsdbKeyDeleted(&maxKey, pSkyline, &iSkyline);
if (!deleted) { if (!deleted) {
iMerge[nMerge] = iMax[i]; iMerge[nMerge] = iMax[i];
@ -968,8 +980,9 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) {
tRowMergerClear(&merger); tRowMergerClear(&merger);
} }
} else { } else {
*ppRow = NULL; /* *ppRow = NULL; */
return code; /* return code; */
continue;
} }
if (iCol == 0) { if (iCol == 0) {
@ -978,7 +991,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) {
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = maxKey}); *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = maxKey});
if (taosArrayPush(pColArray, pColVal) == NULL) { // if (taosArrayPush(pColArray, pColVal) == NULL) {
if (taosArrayPush(pColArray, &(SLastCol){.ts = TSKEY_MAX, .colVal = *pColVal}) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
@ -989,7 +1003,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) {
for (int16_t i = iCol; i < nCol; ++i) { for (int16_t i = iCol; i < nCol; ++i) {
// tsdbRowGetColVal(*ppRow, pTSchema, i, pColVal); // tsdbRowGetColVal(*ppRow, pTSchema, i, pColVal);
tTSRowGetVal(*ppRow, pTSchema, i, pColVal); tTSRowGetVal(*ppRow, pTSchema, i, pColVal);
if (taosArrayPush(pColArray, pColVal) == NULL) { // if (taosArrayPush(pColArray, pColVal) == NULL) {
if (taosArrayPush(pColArray, &(SLastCol){.ts = maxKey, .colVal = *pColVal}) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
@ -1010,11 +1025,11 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) {
--nilColCount; --nilColCount;
} }
} }
/*
if (*ppRow) { if (*ppRow) {
taosMemoryFreeClear(*ppRow); taosMemoryFreeClear(*ppRow);
} }
*/
continue; continue;
} }
@ -1022,12 +1037,16 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) {
for (int16_t i = iCol; i < nCol; ++i) { for (int16_t i = iCol; i < nCol; ++i) {
SColVal colVal = {0}; SColVal colVal = {0};
tTSRowGetVal(*ppRow, pTSchema, i, &colVal); tTSRowGetVal(*ppRow, pTSchema, i, &colVal);
TSKEY rowTs = (*ppRow)->ts;
SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, i); // SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, i);
SLastCol *tTsVal = (SLastCol *)taosArrayGet(pColArray, i);
SColVal *tColVal = &tTsVal->colVal;
if (!colVal.isNone && !colVal.isNull) { if (!colVal.isNone && !colVal.isNull) {
if (tColVal->isNull || tColVal->isNone) { if (tColVal->isNull || tColVal->isNone) {
taosArraySet(pColArray, i, &colVal); // taosArraySet(pColArray, i, &colVal);
taosArraySet(pColArray, i, &(SLastCol){.ts = rowTs, .colVal = colVal});
--nilColCount; --nilColCount;
} }
} else { } else {
@ -1052,16 +1071,45 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) {
} while (nilColCount > 0); } while (nilColCount > 0);
// if () new ts row from pColArray if non empty // if () new ts row from pColArray if non empty
if (taosArrayGetSize(pColArray) == nCol) { /* if (taosArrayGetSize(pColArray) == nCol) { */
code = tdSTSRowNew(pColArray, pTSchema, ppRow); /* code = tdSTSRowNew(pColArray, pTSchema, ppRow); */
if (code) goto _err; /* if (code) goto _err; */
/* } */
/* taosArrayDestroy(pColArray); */
if (taosArrayGetSize(pColArray) <= 0) {
*ppLastArray = NULL;
taosArrayDestroy(pColArray);
} else {
*ppLastArray = pColArray;
}
if (*ppRow) {
taosMemoryFreeClear(*ppRow);
}
for (int i = 0; i < 3; ++i) {
if (input[i].nextRowClearFn) {
input[i].nextRowClearFn(input[i].iter);
}
}
if (pSkyline) {
taosArrayDestroy(pSkyline);
} }
taosArrayDestroy(pColArray);
taosMemoryFreeClear(pTSchema); taosMemoryFreeClear(pTSchema);
return code; return code;
_err: _err:
taosArrayDestroy(pColArray); taosArrayDestroy(pColArray);
if (*ppRow) {
taosMemoryFreeClear(*ppRow);
}
for (int i = 0; i < 3; ++i) {
if (input[i].nextRowClearFn) {
input[i].nextRowClearFn(input[i].iter);
}
}
if (pSkyline) {
taosArrayDestroy(pSkyline);
}
taosMemoryFreeClear(pTSchema); taosMemoryFreeClear(pTSchema);
tsdbError("vgId:%d merge last_row failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d merge last_row failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
@ -1101,6 +1149,30 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH
return code; return code;
} }
int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pTSchema) {
int32_t code = 0;
int16_t nCol = taosArrayGetSize(pLastArray);
SArray *pColArray = taosArrayInit(nCol, sizeof(SColVal));
for (int16_t iCol = 0; iCol < nCol; ++iCol) {
SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLastArray, iCol);
SColVal *tColVal = &tTsVal->colVal;
taosArrayPush(pColArray, tColVal);
}
code = tdSTSRowNew(pColArray, pTSchema, ppRow);
if (code) goto _err;
taosArrayDestroy(pColArray);
return code;
_err:
taosArrayDestroy(pColArray);
return code;
}
int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **handle) { int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **handle) {
int32_t code = 0; int32_t code = 0;
char key[32] = {0}; char key[32] = {0};
@ -1113,17 +1185,20 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand
//*ppRow = (STSRow *)taosLRUCacheValue(pCache, h); //*ppRow = (STSRow *)taosLRUCacheValue(pCache, h);
} else { } else {
STSRow *pRow = NULL; // STSRow *pRow = NULL;
code = mergeLast(uid, pTsdb, &pRow); // code = mergeLast(uid, pTsdb, &pRow);
SArray *pLastArray = NULL;
code = mergeLast(uid, pTsdb, &pLastArray);
// if table's empty or error, return code of -1 // if table's empty or error, return code of -1
if (code < 0 || pRow == NULL) { // if (code < 0 || pRow == NULL) {
if (code < 0 || pLastArray == NULL) {
*handle = NULL; *handle = NULL;
return 0; return 0;
} }
_taos_lru_deleter_t deleter = deleteTableCacheLastrow; _taos_lru_deleter_t deleter = deleteTableCacheLast;
LRUStatus status = LRUStatus status =
taosLRUCacheInsert(pCache, key, keyLen, pRow, TD_ROW_LEN(pRow), deleter, NULL, TAOS_LRU_PRIORITY_LOW); taosLRUCacheInsert(pCache, key, keyLen, pLastArray, pLastArray->capacity, deleter, NULL, TAOS_LRU_PRIORITY_LOW);
if (status != TAOS_LRU_STATUS_OK) { if (status != TAOS_LRU_STATUS_OK) {
code = -1; code = -1;
} }

7
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -128,6 +128,8 @@ int32_t tsdbRetrieveLastRow(void* pReader, SSDataBlock* pResBlock, const int32_t
} }
pRow = (STSRow*)taosLRUCacheValue(lruCache, h); pRow = (STSRow*)taosLRUCacheValue(lruCache, h);
// SArray* pLast = (SArray*)taosLRUCacheValue(lruCache, h);
// tsdbCacheLastArray2Row(pLast, &pRow, pr->pSchema);
if (pRow->ts > lastKey) { if (pRow->ts > lastKey) {
// Set result row into the same rowIndex repeatly, so we need to check if the internal result row has already // Set result row into the same rowIndex repeatly, so we need to check if the internal result row has already
// appended or not. // appended or not.
@ -140,6 +142,7 @@ int32_t tsdbRetrieveLastRow(void* pReader, SSDataBlock* pResBlock, const int32_t
lastKey = pRow->ts; lastKey = pRow->ts;
} }
// taosMemoryFree(pRow);
tsdbCacheRelease(lruCache, h); tsdbCacheRelease(lruCache, h);
} }
} else if (pr->type == LASTROW_RETRIEVE_TYPE_ALL) { } else if (pr->type == LASTROW_RETRIEVE_TYPE_ALL) {
@ -158,8 +161,12 @@ int32_t tsdbRetrieveLastRow(void* pReader, SSDataBlock* pResBlock, const int32_t
} }
pRow = (STSRow*)taosLRUCacheValue(lruCache, h); pRow = (STSRow*)taosLRUCacheValue(lruCache, h);
// SArray* pLast = (SArray*)taosLRUCacheValue(lruCache, h);
// tsdbCacheLastArray2Row(pLast, &pRow, pr->pSchema);
saveOneRow(pRow, pResBlock, pr, slotIds); saveOneRow(pRow, pResBlock, pr, slotIds);
// taosMemoryFree(pRow);
tsdbCacheRelease(lruCache, h); tsdbCacheRelease(lruCache, h);
pr->tableIndex += 1; pr->tableIndex += 1;

14
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -168,8 +168,6 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
tb_uid_t suid; tb_uid_t suid;
tb_uid_t uid; tb_uid_t uid;
taosArrayClear(pCommitter->aDelData);
if (pTbData) { if (pTbData) {
suid = pTbData->suid; suid = pTbData->suid;
uid = pTbData->uid; uid = pTbData->uid;
@ -185,6 +183,8 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, NULL); code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, NULL);
if (code) goto _err; if (code) goto _err;
} else {
taosArrayClear(pCommitter->aDelData);
} }
if (pTbData == NULL && pDelIdx == NULL) goto _exit; if (pTbData == NULL && pDelIdx == NULL) goto _exit;
@ -205,7 +205,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
if (code) goto _err; if (code) goto _err;
// put delIdx // put delIdx
if (taosArrayPush(pCommitter->aDelIdx, &delIdx) == NULL) { if (taosArrayPush(pCommitter->aDelIdxN, &delIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
@ -602,7 +602,7 @@ static int32_t tsdbGetOvlpNRow(STbDataIter *pIter, SBlock *pBlock) {
iter.pRow = NULL; iter.pRow = NULL;
while (true) { while (true) {
pRow = tsdbTbDataIterGet(pIter); pRow = tsdbTbDataIterGet(&iter);
if (pRow == NULL) break; if (pRow == NULL) break;
key = TSDBROW_KEY(pRow); key = TSDBROW_KEY(pRow);
@ -610,7 +610,7 @@ static int32_t tsdbGetOvlpNRow(STbDataIter *pIter, SBlock *pBlock) {
c = tBlockCmprFn(&(SBlock){.maxKey = key, .minKey = key}, pBlock); c = tBlockCmprFn(&(SBlock){.maxKey = key, .minKey = key}, pBlock);
if (c == 0) { if (c == 0) {
nRow++; nRow++;
tsdbTbDataIterNext(pIter); tsdbTbDataIterNext(&iter);
} else if (c > 0) { } else if (c > 0) {
break; break;
} else { } else {
@ -635,7 +635,7 @@ static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, S
code = tsdbCommitterUpdateRowSchema(pCommitter, pBlockIdx->suid, pBlockIdx->uid, TSDBROW_SVERSION(pRow)); code = tsdbCommitterUpdateRowSchema(pCommitter, pBlockIdx->suid, pBlockIdx->uid, TSDBROW_SVERSION(pRow));
if (code) goto _err; if (code) goto _err;
while (true) { while (true) {
if (pRow) break; if (pRow == NULL) break;
code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema);
if (code) goto _err; if (code) goto _err;
@ -854,7 +854,7 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) {
if (pCommitter->pReader) { if (pCommitter->pReader) {
code = tsdbDataFReaderClose(&pCommitter->pReader); code = tsdbDataFReaderClose(&pCommitter->pReader);
goto _err; if (code) goto _err;
} }
_exit: _exit:

2
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -520,7 +520,7 @@ static int32_t tsdbScanAndTryFixFS(STsdbFS *pFS, int8_t deepScan) {
return code; return code;
_err: _err:
tsdbError("vgId:%d tsdb can and try fix fs failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d tsdb scan and try fix fs failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
} }

4
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -105,7 +105,7 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
// check if table exists (todo: refact) // check if table exists (todo: refact)
SMetaReader mr = {0}; SMetaReader mr = {0};
SMetaEntry me = {0}; // SMetaEntry me = {0};
metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0); metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0);
if (metaGetTableEntryByUid(&mr, pMsgIter->uid) < 0) { if (metaGetTableEntryByUid(&mr, pMsgIter->uid) < 0) {
metaReaderClear(&mr); metaReaderClear(&mr);
@ -117,6 +117,8 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
if (mr.me.type == TSDB_NORMAL_TABLE) { if (mr.me.type == TSDB_NORMAL_TABLE) {
sverNew = mr.me.ntbEntry.schemaRow.version; sverNew = mr.me.ntbEntry.schemaRow.version;
} else { } else {
tDecoderClear(&mr.coder);
metaGetTableEntryByUid(&mr, mr.me.ctbEntry.suid); metaGetTableEntryByUid(&mr, mr.me.ctbEntry.suid);
sverNew = mr.me.stbEntry.schemaRow.version; sverNew = mr.me.stbEntry.schemaRow.version;
} }

931
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

File diff suppressed because it is too large Load Diff

16
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
View File

@ -1,16 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdb.h"

13
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -49,7 +49,7 @@ int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb
} }
pDelFWriter->fDel.size = TSDB_FHDR_SIZE; pDelFWriter->fDel.size = TSDB_FHDR_SIZE;
pDelFWriter->fDel.size = 0; pDelFWriter->fDel.offset = 0;
*ppWriter = pDelFWriter; *ppWriter = pDelFWriter;
return code; return code;
@ -969,6 +969,8 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl
SBlockData *pBlockData1 = &(SBlockData){0}; SBlockData *pBlockData1 = &(SBlockData){0};
SBlockData *pBlockData2 = &(SBlockData){0}; SBlockData *pBlockData2 = &(SBlockData){0};
tBlockDataInit(pBlockData1);
tBlockDataInit(pBlockData2);
for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) {
code = tsdbReadSubColData(pReader, pBlockIdx, pBlock, iSubBlock, aColId, nCol, pBlockData1, ppBuf1, ppBuf2); code = tsdbReadSubColData(pReader, pBlockIdx, pBlock, iSubBlock, aColId, nCol, pBlockData1, ppBuf1, ppBuf2);
if (code) goto _err; if (code) goto _err;
@ -1106,6 +1108,8 @@ int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *p
SBlockData *pBlockData1 = &(SBlockData){0}; SBlockData *pBlockData1 = &(SBlockData){0};
SBlockData *pBlockData2 = &(SBlockData){0}; SBlockData *pBlockData2 = &(SBlockData){0};
tBlockDataInit(pBlockData1);
tBlockDataInit(pBlockData2);
for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) {
code = tsdbReadSubBlockData(pReader, pBlockIdx, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); code = tsdbReadSubBlockData(pReader, pBlockIdx, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2);
if (code) { if (code) {
@ -1152,7 +1156,7 @@ _err:
int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf) { int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf) {
int32_t code = 0; int32_t code = 0;
TdFilePtr pFD = pReader->pSmaFD; TdFilePtr pFD = pReader->pSmaFD;
int64_t offset = pBlock->aSubBlock[0].offset; int64_t offset = pBlock->aSubBlock[0].sOffset;
int64_t size = pBlock->aSubBlock[0].nSma * sizeof(SColumnDataAgg) + sizeof(TSCKSUM); int64_t size = pBlock->aSubBlock[0].nSma * sizeof(SColumnDataAgg) + sizeof(TSCKSUM);
uint8_t *pBuf = NULL; uint8_t *pBuf = NULL;
int64_t n; int64_t n;
@ -1175,10 +1179,13 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} else if (n < size) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _err;
} }
// check // check
if (!taosCheckChecksumWhole(NULL, size)) { if (!taosCheckChecksumWhole(*ppBuf, size)) {
code = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
goto _err; goto _err;
} }

32
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -821,16 +821,20 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
int32_t code = 0; int32_t code = 0;
int32_t size; int32_t size;
ASSERT(pColDataSrc->nVal > 0);
pColDataDest->cid = pColDataSrc->cid; pColDataDest->cid = pColDataSrc->cid;
pColDataDest->type = pColDataSrc->type; pColDataDest->type = pColDataSrc->type;
pColDataDest->smaOn = pColDataSrc->smaOn; pColDataDest->smaOn = pColDataSrc->smaOn;
pColDataDest->nVal = pColDataSrc->nVal; pColDataDest->nVal = pColDataSrc->nVal;
pColDataDest->flag = pColDataSrc->flag; pColDataDest->flag = pColDataSrc->flag;
size = BIT2_SIZE(pColDataSrc->nVal); if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) {
code = tRealloc(&pColDataDest->pBitMap, size); size = BIT2_SIZE(pColDataSrc->nVal);
if (code) goto _exit; code = tRealloc(&pColDataDest->pBitMap, size);
memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); if (code) goto _exit;
memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size);
}
if (IS_VAR_DATA_TYPE(pColDataDest->type)) { if (IS_VAR_DATA_TYPE(pColDataDest->type)) {
size = sizeof(int32_t) * pColDataSrc->nVal; size = sizeof(int32_t) * pColDataSrc->nVal;
@ -1230,10 +1234,26 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) {
break; break;
case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_SMALLINT:
break; break;
case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_INT: {
pColAgg->sum += colVal.value.i32;
if (pColAgg->min > colVal.value.i32) {
pColAgg->min = colVal.value.i32;
}
if (pColAgg->max < colVal.value.i32) {
pColAgg->max = colVal.value.i32;
}
break; break;
case TSDB_DATA_TYPE_BIGINT: }
case TSDB_DATA_TYPE_BIGINT: {
pColAgg->sum += colVal.value.i64;
if (pColAgg->min > colVal.value.i64) {
pColAgg->min = colVal.value.i64;
}
if (pColAgg->max < colVal.value.i64) {
pColAgg->max = colVal.value.i64;
}
break; break;
}
case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_FLOAT:
break; break;
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:

14
source/dnode/vnode/src/vnd/vnodeStateMgr.c
View File

@ -1,14 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

16
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -281,8 +281,9 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
vTrace("message in fetch queue is processing"); vTrace("message in fetch queue is processing");
if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG) if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META ||
&& !vnodeIsLeader(pVnode)) { pMsg->msgType == TDMT_VND_TABLE_CFG) &&
!vnodeIsLeader(pVnode)) {
vnodeRedirectRpcMsg(pVnode, pMsg); vnodeRedirectRpcMsg(pVnode, pMsg);
return 0; return 0;
} }
@ -295,7 +296,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
case TDMT_SCH_MERGE_FETCH: case TDMT_SCH_MERGE_FETCH:
return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg, 0); return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_FETCH_RSP: case TDMT_SCH_FETCH_RSP:
return qWorkerProcessFetchRsp(pVnode, pVnode->pQuery, pMsg, 0); return qWorkerProcessRspMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_CANCEL_TASK: case TDMT_SCH_CANCEL_TASK:
return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg, 0); return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_DROP_TASK: case TDMT_SCH_DROP_TASK:
@ -348,7 +349,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p
if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY;
int32_t t = ntohl(*(int32_t *)pReq); int32_t t = ntohl(*(int32_t *)pReq);
vError("rec ttl time:%d", t); vDebug("rec ttl time:%d", t);
int32_t ret = metaTtlDropTable(pVnode->pMeta, t, tbUids); int32_t ret = metaTtlDropTable(pVnode->pMeta, t, tbUids);
if (ret != 0) { if (ret != 0) {
goto end; goto end;
@ -389,10 +390,14 @@ static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *p
goto _err; goto _err;
} }
taosMemoryFree(req.schemaRow.pSchema);
taosMemoryFree(req.schemaTag.pSchema);
tDecoderClear(&coder); tDecoderClear(&coder);
return 0; return 0;
_err: _err:
taosMemoryFree(req.schemaRow.pSchema);
taosMemoryFree(req.schemaTag.pSchema);
tDecoderClear(&coder); tDecoderClear(&coder);
return -1; return -1;
} }
@ -811,7 +816,8 @@ _exit:
taosArrayDestroy(submitRsp.pArray); taosArrayDestroy(submitRsp.pArray);
// TODO: the partial success scenario and the error case // TODO: the partial success scenario and the error case
// => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level 1/level 2. // => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level
// 1/level 2.
// TODO: refactor // TODO: refactor
if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) { if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT); tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);

3
source/libs/executor/inc/dataSinkInt.h
View File

@ -34,7 +34,7 @@ typedef struct SDataSinkManager {
typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue);
typedef void (*FEndPut)(struct SDataSinkHandle* pHandle, uint64_t useconds); typedef void (*FEndPut)(struct SDataSinkHandle* pHandle, uint64_t useconds);
typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd); typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd);
typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutputData* pOutput); typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutputData* pOutput);
typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle); typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle);
typedef int32_t (*FGetCacheSize)(struct SDataSinkHandle* pHandle, uint64_t* size); typedef int32_t (*FGetCacheSize)(struct SDataSinkHandle* pHandle, uint64_t* size);
@ -50,6 +50,7 @@ typedef struct SDataSinkHandle {
int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle); int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle);
int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void *pParam); int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void *pParam);
int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void *pParam);
#ifdef __cplusplus #ifdef __cplusplus
} }

568
source/libs/executor/inc/executorimpl.h
View File

@ -40,8 +40,8 @@ extern "C" {
#include "tpagedbuf.h" #include "tpagedbuf.h"
#include "tstreamUpdate.h" #include "tstreamUpdate.h"
#include "executorInt.h"
#include "vnode.h" #include "vnode.h"
#include "executorInt.h"
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
@ -171,10 +171,10 @@ typedef struct SExecTaskInfo {
} SExecTaskInfo; } SExecTaskInfo;
enum { enum {
OP_NOT_OPENED = 0x0, OP_NOT_OPENED = 0x0,
OP_OPENED = 0x1, OP_OPENED = 0x1,
OP_RES_TO_RETURN = 0x5, OP_RES_TO_RETURN = 0x5,
OP_EXEC_DONE = 0x9, OP_EXEC_DONE = 0x9,
}; };
typedef struct SOperatorFpSet { typedef struct SOperatorFpSet {
@ -212,7 +212,7 @@ typedef struct SOperatorInfo {
typedef enum { typedef enum {
EX_SOURCE_DATA_NOT_READY = 0x1, EX_SOURCE_DATA_NOT_READY = 0x1,
EX_SOURCE_DATA_READY = 0x2, EX_SOURCE_DATA_READY = 0x2,
EX_SOURCE_DATA_EXHAUSTED = 0x3, EX_SOURCE_DATA_EXHAUSTED = 0x3,
} EX_SOURCE_STATUS; } EX_SOURCE_STATUS;
@ -247,22 +247,22 @@ typedef struct SExchangeInfo {
} SExchangeInfo; } SExchangeInfo;
typedef struct SColMatchInfo { typedef struct SColMatchInfo {
int32_t srcSlotId; // source slot id int32_t srcSlotId; // source slot id
int32_t colId; int32_t colId;
int32_t targetSlotId; int32_t targetSlotId;
bool output; // todo remove this? bool output; // todo remove this?
bool reserved; bool reserved;
int32_t matchType; // determinate the source according to col id or slot id int32_t matchType; // determinate the source according to col id or slot id
} SColMatchInfo; } SColMatchInfo;
typedef struct SScanInfo { typedef struct SScanInfo {
int32_t numOfAsc; int32_t numOfAsc;
int32_t numOfDesc; int32_t numOfDesc;
} SScanInfo; } SScanInfo;
typedef struct SSampleExecInfo { typedef struct SSampleExecInfo {
double sampleRatio; // data block sample ratio, 1 by default double sampleRatio; // data block sample ratio, 1 by default
uint32_t seed; // random seed value uint32_t seed; // random seed value
} SSampleExecInfo; } SSampleExecInfo;
enum { enum {
@ -271,38 +271,36 @@ enum {
}; };
typedef struct STableScanInfo { typedef struct STableScanInfo {
STsdbReader* dataReader; STsdbReader* dataReader;
SReadHandle readHandle; SReadHandle readHandle;
SFileBlockLoadRecorder readRecorder; SFileBlockLoadRecorder readRecorder;
SScanInfo scanInfo; SScanInfo scanInfo;
int32_t scanTimes; int32_t scanTimes;
SNode* pFilterNode; // filter info, which is push down by optimizer SNode* pFilterNode; // filter info, which is push down by optimizer
SqlFunctionCtx* SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context,todo: remove this by using SExprSup
pCtx; // which belongs to the direct upstream operator operator query context,todo: remove this by using SExprSup int32_t* rowEntryInfoOffset; // todo: remove this by using SExprSup
int32_t* rowEntryInfoOffset; // todo: remove this by using SExprSup SExprInfo* pExpr;// todo: remove this by using SExprSup
SExprInfo* pExpr; // todo: remove this by using SExprSup
SSDataBlock* pResBlock; SSDataBlock* pResBlock;
SArray* pColMatchInfo; SArray* pColMatchInfo;
SExprSupp pseudoSup; SExprSupp pseudoSup;
SQueryTableDataCond cond; SQueryTableDataCond cond;
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
int32_t dataBlockLoadFlag; int32_t dataBlockLoadFlag;
SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded.
// window to check if current data block needs to be loaded.
SSampleExecInfo sample; // sample execution info SSampleExecInfo sample; // sample execution info
int32_t curTWinIdx; int32_t curTWinIdx;
int32_t currentGroupId; int32_t currentGroupId;
int32_t currentTable; int32_t currentTable;
uint64_t queryId; // todo remove it uint64_t queryId; // todo remove it
uint64_t taskId; // todo remove it uint64_t taskId; // todo remove it
struct { struct {
uint64_t uid; uint64_t uid;
int64_t ts; int64_t ts;
} lastStatus; } lastStatus;
int8_t scanMode; int8_t scanMode;
@ -310,28 +308,28 @@ typedef struct STableScanInfo {
} STableScanInfo; } STableScanInfo;
typedef struct STagScanInfo { typedef struct STagScanInfo {
SColumnInfo* pCols; SColumnInfo *pCols;
SSDataBlock* pRes; SSDataBlock *pRes;
SArray* pColMatchInfo; SArray *pColMatchInfo;
int32_t curPos; int32_t curPos;
SReadHandle readHandle; SReadHandle readHandle;
STableListInfo* pTableList; STableListInfo *pTableList;
} STagScanInfo; } STagScanInfo;
typedef struct SLastrowScanInfo { typedef struct SLastrowScanInfo {
SSDataBlock* pRes; SSDataBlock *pRes;
SArray* pTableList; SArray *pTableList;
SReadHandle readHandle; SReadHandle readHandle;
void* pLastrowReader; void *pLastrowReader;
SArray* pColMatchInfo; SArray *pColMatchInfo;
int32_t* pSlotIds; int32_t *pSlotIds;
} SLastrowScanInfo; } SLastrowScanInfo;
typedef enum EStreamScanMode { typedef enum EStreamScanMode {
STREAM_SCAN_FROM_READERHANDLE = 1, STREAM_SCAN_FROM_READERHANDLE = 1,
STREAM_SCAN_FROM_RES, STREAM_SCAN_FROM_RES,
STREAM_SCAN_FROM_UPDATERES, STREAM_SCAN_FROM_UPDATERES,
STREAM_SCAN_FROM_DATAREADER, // todo(liuyao) delete it STREAM_SCAN_FROM_DATAREADER, // todo(liuyao) delete it
STREAM_SCAN_FROM_DATAREADER_RETRIEVE, STREAM_SCAN_FROM_DATAREADER_RETRIEVE,
STREAM_SCAN_FROM_DATAREADER_RANGE, STREAM_SCAN_FROM_DATAREADER_RANGE,
} EStreamScanMode; } EStreamScanMode;
@ -348,9 +346,9 @@ typedef struct SStreamAggSupporter {
SArray* pCurWins; SArray* pCurWins;
int32_t valueSize; int32_t valueSize;
int32_t keySize; int32_t keySize;
char* pKeyBuf; // window key buffer char* pKeyBuf; // window key buffer
SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file
int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row
SArray* pScanWindow; SArray* pScanWindow;
} SStreamAggSupporter; } SStreamAggSupporter;
@ -424,27 +422,27 @@ typedef struct SBlockDistInfo {
SSDataBlock* pResBlock; SSDataBlock* pResBlock;
void* pHandle; void* pHandle;
SReadHandle readHandle; SReadHandle readHandle;
uint64_t uid; // table uid uint64_t uid; // table uid
} SBlockDistInfo; } SBlockDistInfo;
// todo remove this // todo remove this
typedef struct SOptrBasicInfo { typedef struct SOptrBasicInfo {
SResultRowInfo resultRowInfo; SResultRowInfo resultRowInfo;
SSDataBlock* pRes; SSDataBlock* pRes;
} SOptrBasicInfo; } SOptrBasicInfo;
typedef struct SAggSupporter { typedef struct SAggSupporter {
SHashObj* pResultRowHashTable; // quick locate the window object for each result SHashObj* pResultRowHashTable; // quick locate the window object for each result
char* keyBuf; // window key buffer char* keyBuf; // window key buffer
SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file
int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row
} SAggSupporter; } SAggSupporter;
typedef struct STimeWindowSupp { typedef struct STimeWindowSupp {
int8_t calTrigger; int8_t calTrigger;
int64_t waterMark; int64_t waterMark;
TSKEY maxTs; TSKEY maxTs;
SColumnInfoData timeWindowData; // query time window info for scalar function execution. SColumnInfoData timeWindowData; // query time window info for scalar function execution.
} STimeWindowAggSupp; } STimeWindowAggSupp;
typedef struct SIntervalAggOperatorInfo { typedef struct SIntervalAggOperatorInfo {
@ -463,78 +461,79 @@ typedef struct SIntervalAggOperatorInfo {
EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model]
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
bool invertible; bool invertible;
SArray* pPrevValues; // SArray<SGroupKeys> used to keep the previous not null value for interpolation. SArray* pPrevValues; // SArray<SGroupKeys> used to keep the previous not null value for interpolation.
bool ignoreExpiredData; bool ignoreExpiredData;
SArray* pRecycledPages; SArray* pRecycledPages;
SArray* pDelWins; // SWinRes SArray* pDelWins; // SWinRes
int32_t delIndex; int32_t delIndex;
SSDataBlock* pDelRes; SSDataBlock* pDelRes;
SNode* pCondition; SNode *pCondition;
} SIntervalAggOperatorInfo; } SIntervalAggOperatorInfo;
typedef struct SStreamFinalIntervalOperatorInfo { typedef struct SStreamFinalIntervalOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
SOptrBasicInfo binfo; // basic info SOptrBasicInfo binfo; // basic info
SAggSupporter aggSup; // aggregate supporter SAggSupporter aggSup; // aggregate supporter
SGroupResInfo groupResInfo; // multiple results build supporter SGroupResInfo groupResInfo; // multiple results build supporter
SInterval interval; // interval info SInterval interval; // interval info
int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator.
int32_t order; // current SSDataBlock scan order int32_t order; // current SSDataBlock scan order
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
SArray* pChildren; SArray* pChildren;
SSDataBlock* pUpdateRes; SSDataBlock* pUpdateRes;
bool returnUpdate; bool returnUpdate;
SPhysiNode* pPhyNode; // create new child SPhysiNode* pPhyNode; // create new child
bool isFinal; bool isFinal;
SHashObj* pPullDataMap; SHashObj* pPullDataMap;
SArray* pPullWins; // SPullWindowInfo SArray* pPullWins; // SPullWindowInfo
int32_t pullIndex; int32_t pullIndex;
SSDataBlock* pPullDataRes; SSDataBlock* pPullDataRes;
bool ignoreExpiredData; bool ignoreExpiredData;
SArray* pRecycledPages; SArray* pRecycledPages;
SArray* pDelWins; // SWinRes SArray* pDelWins; // SWinRes
int32_t delIndex; int32_t delIndex;
SSDataBlock* pDelRes; SSDataBlock* pDelRes;
} SStreamFinalIntervalOperatorInfo; } SStreamFinalIntervalOperatorInfo;
typedef struct SAggOperatorInfo { typedef struct SAggOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
SAggSupporter aggSup; SAggSupporter aggSup;
STableQueryInfo* current; STableQueryInfo *current;
uint64_t groupId; uint64_t groupId;
SGroupResInfo groupResInfo; SGroupResInfo groupResInfo;
SExprSupp scalarExprSup; SExprSupp scalarExprSup;
SNode* pCondition; SNode *pCondition;
} SAggOperatorInfo; } SAggOperatorInfo;
typedef struct SProjectOperatorInfo { typedef struct SProjectOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
SAggSupporter aggSup; SAggSupporter aggSup;
SNode* pFilterNode; // filter info, which is push down by optimizer SNode* pFilterNode; // filter info, which is push down by optimizer
SSDataBlock* existDataBlock; SSDataBlock* existDataBlock;
SArray* pPseudoColInfo; SArray* pPseudoColInfo;
SLimit limit; SLimit limit;
SLimit slimit; SLimit slimit;
uint64_t groupId; uint64_t groupId;
int64_t curSOffset; int64_t curSOffset;
int64_t curGroupOutput; int64_t curGroupOutput;
int64_t curOffset; int64_t curOffset;
int64_t curOutput; int64_t curOutput;
} SProjectOperatorInfo; } SProjectOperatorInfo;
typedef struct SIndefOperatorInfo { typedef struct SIndefOperatorInfo {
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
SAggSupporter aggSup; SAggSupporter aggSup;
SArray* pPseudoColInfo; SArray* pPseudoColInfo;
SExprSupp scalarSup; SExprSupp scalarSup;
SNode* pCondition;
} SIndefOperatorInfo; } SIndefOperatorInfo;
typedef struct SFillOperatorInfo { typedef struct SFillOperatorInfo {
@ -544,27 +543,29 @@ typedef struct SFillOperatorInfo {
void** p; void** p;
SSDataBlock* existNewGroupBlock; SSDataBlock* existNewGroupBlock;
bool multigroupResult; bool multigroupResult;
STimeWindow win;
SNode* pCondition;
} SFillOperatorInfo; } SFillOperatorInfo;
typedef struct SGroupbyOperatorInfo { typedef struct SGroupbyOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
SAggSupporter aggSup; SAggSupporter aggSup;
SArray* pGroupCols; // group by columns, SArray<SColumn> SArray* pGroupCols; // group by columns, SArray<SColumn>
SArray* pGroupColVals; // current group column values, SArray<SGroupKeys> SArray* pGroupColVals; // current group column values, SArray<SGroupKeys>
SNode* pCondition; SNode* pCondition;
bool isInit; // denote if current val is initialized or not bool isInit; // denote if current val is initialized or not
char* keyBuf; // group by keys for hash char* keyBuf; // group by keys for hash
int32_t groupKeyLen; // total group by column width int32_t groupKeyLen; // total group by column width
SGroupResInfo groupResInfo; SGroupResInfo groupResInfo;
SExprSupp scalarSup; SExprSupp scalarSup;
} SGroupbyOperatorInfo; } SGroupbyOperatorInfo;
typedef struct SDataGroupInfo { typedef struct SDataGroupInfo {
uint64_t groupId; uint64_t groupId;
int64_t numOfRows; int64_t numOfRows;
SArray* pPageList; SArray* pPageList;
} SDataGroupInfo; } SDataGroupInfo;
// The sort in partition may be needed later. // The sort in partition may be needed later.
@ -576,12 +577,12 @@ typedef struct SPartitionOperatorInfo {
int32_t groupKeyLen; // total group by column width int32_t groupKeyLen; // total group by column width
SHashObj* pGroupSet; // quick locate the window object for each result SHashObj* pGroupSet; // quick locate the window object for each result
SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file
int32_t rowCapacity; // maximum number of rows for each buffer page int32_t rowCapacity; // maximum number of rows for each buffer page
int32_t* columnOffset; // start position for each column data int32_t* columnOffset; // start position for each column data
SArray* sortedGroupArray; // SDataGroupInfo sorted by group id SArray* sortedGroupArray; // SDataGroupInfo sorted by group id
int32_t groupIndex; // group index int32_t groupIndex; // group index
int32_t pageIndex; // page index of current group int32_t pageIndex; // page index of current group
SSDataBlock* pUpdateRes; SSDataBlock* pUpdateRes;
SExprSupp scalarSup; SExprSupp scalarSup;
} SPartitionOperatorInfo; } SPartitionOperatorInfo;
@ -595,67 +596,68 @@ typedef struct SWindowRowsSup {
typedef struct SSessionAggOperatorInfo { typedef struct SSessionAggOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
SAggSupporter aggSup; SAggSupporter aggSup;
SGroupResInfo groupResInfo; SGroupResInfo groupResInfo;
SWindowRowsSup winSup; SWindowRowsSup winSup;
bool reptScan; // next round scan bool reptScan; // next round scan
int64_t gap; // session window gap int64_t gap; // session window gap
int32_t tsSlotId; // primary timestamp slot id int32_t tsSlotId; // primary timestamp slot id
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
const SNode* pCondition;
} SSessionAggOperatorInfo; } SSessionAggOperatorInfo;
typedef struct SResultWindowInfo { typedef struct SResultWindowInfo {
SResultRowPosition pos; SResultRowPosition pos;
STimeWindow win; STimeWindow win;
bool isOutput; bool isOutput;
bool isClosed; bool isClosed;
} SResultWindowInfo; } SResultWindowInfo;
typedef struct SStateWindowInfo { typedef struct SStateWindowInfo {
SResultWindowInfo winInfo; SResultWindowInfo winInfo;
SStateKeys stateKey; SStateKeys stateKey;
} SStateWindowInfo; } SStateWindowInfo;
typedef struct SStreamSessionAggOperatorInfo { typedef struct SStreamSessionAggOperatorInfo {
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
SStreamAggSupporter streamAggSup; SStreamAggSupporter streamAggSup;
SGroupResInfo groupResInfo; SGroupResInfo groupResInfo;
int64_t gap; // session window gap int64_t gap; // session window gap
int32_t primaryTsIndex; // primary timestamp slot id int32_t primaryTsIndex; // primary timestamp slot id
int32_t endTsIndex; // window end timestamp slot id int32_t endTsIndex; // window end timestamp slot id
int32_t order; // current SSDataBlock scan order int32_t order; // current SSDataBlock scan order
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
SSDataBlock* pWinBlock; // window result SSDataBlock* pWinBlock; // window result
SqlFunctionCtx* pDummyCtx; // for combine SqlFunctionCtx* pDummyCtx; // for combine
SSDataBlock* pDelRes; // delete result SSDataBlock* pDelRes; // delete result
bool returnDelete; bool returnDelete;
SSDataBlock* pUpdateRes; // update window SSDataBlock* pUpdateRes; // update window
SHashObj* pStDeleted; SHashObj* pStDeleted;
void* pDelIterator; void* pDelIterator;
SArray* pChildren; // cache for children's result; final stream operator SArray* pChildren; // cache for children's result; final stream operator
SPhysiNode* pPhyNode; // create new child SPhysiNode* pPhyNode; // create new child
bool isFinal; bool isFinal;
bool ignoreExpiredData; bool ignoreExpiredData;
} SStreamSessionAggOperatorInfo; } SStreamSessionAggOperatorInfo;
typedef struct STimeSliceOperatorInfo { typedef struct STimeSliceOperatorInfo {
SSDataBlock* pRes; SSDataBlock* pRes;
STimeWindow win; STimeWindow win;
SInterval interval; SInterval interval;
int64_t current; int64_t current;
SArray* pPrevRow; // SArray<SGroupValue> SArray* pPrevRow; // SArray<SGroupValue>
int32_t fillType; // fill type int32_t fillType; // fill type
SColumn tsCol; // primary timestamp column SColumn tsCol; // primary timestamp column
SExprSupp scalarSup; // scalar calculation SExprSupp scalarSup; // scalar calculation
struct SFillColInfo* pFillColInfo; // fill column info struct SFillColInfo* pFillColInfo; // fill column info
} STimeSliceOperatorInfo; } STimeSliceOperatorInfo;
typedef struct SStateWindowOperatorInfo { typedef struct SStateWindowOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
SAggSupporter aggSup; SAggSupporter aggSup;
SGroupResInfo groupResInfo; SGroupResInfo groupResInfo;
SWindowRowsSup winSup; SWindowRowsSup winSup;
@ -665,55 +667,56 @@ typedef struct SStateWindowOperatorInfo {
int32_t tsSlotId; // primary timestamp column slot id int32_t tsSlotId; // primary timestamp column slot id
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
// bool reptScan; // bool reptScan;
const SNode* pCondition;
} SStateWindowOperatorInfo; } SStateWindowOperatorInfo;
typedef struct SStreamStateAggOperatorInfo { typedef struct SStreamStateAggOperatorInfo {
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
SStreamAggSupporter streamAggSup; SStreamAggSupporter streamAggSup;
SGroupResInfo groupResInfo; SGroupResInfo groupResInfo;
int32_t primaryTsIndex; // primary timestamp slot id int32_t primaryTsIndex; // primary timestamp slot id
int32_t order; // current SSDataBlock scan order int32_t order; // current SSDataBlock scan order
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
SColumn stateCol; // start row index SColumn stateCol; // start row index
SqlFunctionCtx* pDummyCtx; // for combine SqlFunctionCtx* pDummyCtx; // for combine
SSDataBlock* pDelRes; SSDataBlock* pDelRes;
SHashObj* pSeDeleted; SHashObj* pSeDeleted;
void* pDelIterator; void* pDelIterator;
SArray* pScanWindow; SArray* pScanWindow;
SArray* pChildren; // cache for children's result; SArray* pChildren; // cache for children's result;
bool ignoreExpiredData; bool ignoreExpiredData;
} SStreamStateAggOperatorInfo; } SStreamStateAggOperatorInfo;
typedef struct SSortedMergeOperatorInfo { typedef struct SSortedMergeOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
SAggSupporter aggSup; SAggSupporter aggSup;
SArray* pSortInfo; SArray* pSortInfo;
int32_t numOfSources; int32_t numOfSources;
SSortHandle* pSortHandle; SSortHandle *pSortHandle;
int32_t bufPageSize; int32_t bufPageSize;
uint32_t sortBufSize; // max buffer size for in-memory sort uint32_t sortBufSize; // max buffer size for in-memory sort
int32_t resultRowFactor; int32_t resultRowFactor;
bool hasGroupVal; bool hasGroupVal;
SDiskbasedBuf* pTupleStore; // keep the final results SDiskbasedBuf *pTupleStore; // keep the final results
int32_t numOfResPerPage; int32_t numOfResPerPage;
char** groupVal; char** groupVal;
SArray* groupInfo; SArray *groupInfo;
} SSortedMergeOperatorInfo; } SSortedMergeOperatorInfo;
typedef struct SSortOperatorInfo { typedef struct SSortOperatorInfo {
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
uint32_t sortBufSize; // max buffer size for in-memory sort uint32_t sortBufSize; // max buffer size for in-memory sort
SArray* pSortInfo; SArray* pSortInfo;
SSortHandle* pSortHandle; SSortHandle* pSortHandle;
SArray* pColMatchInfo; // for index map from table scan output SArray* pColMatchInfo; // for index map from table scan output
int32_t bufPageSize; int32_t bufPageSize;
int64_t startTs; // sort start time int64_t startTs; // sort start time
uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included.
SNode* pCondition; SNode* pCondition;
} SSortOperatorInfo; } SSortOperatorInfo;
typedef struct STagFilterOperatorInfo { typedef struct STagFilterOperatorInfo {
@ -721,17 +724,17 @@ typedef struct STagFilterOperatorInfo {
} STagFilterOperatorInfo; } STagFilterOperatorInfo;
typedef struct SJoinOperatorInfo { typedef struct SJoinOperatorInfo {
SSDataBlock* pRes; SSDataBlock *pRes;
int32_t joinType; int32_t joinType;
SSDataBlock* pLeft; SSDataBlock *pLeft;
int32_t leftPos; int32_t leftPos;
SColumnInfo leftCol; SColumnInfo leftCol;
SSDataBlock* pRight; SSDataBlock *pRight;
int32_t rightPos; int32_t rightPos;
SColumnInfo rightCol; SColumnInfo rightCol;
SNode* pCondAfterMerge; SNode *pCondAfterMerge;
} SJoinOperatorInfo; } SJoinOperatorInfo;
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
@ -740,8 +743,8 @@ typedef struct SJoinOperatorInfo {
void doDestroyExchangeOperatorInfo(void* param); void doDestroyExchangeOperatorInfo(void* param);
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn,
__optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode,
__optr_decode_fn_t decode, __optr_explain_fn_t explain); __optr_decode_fn_t decode, __optr_explain_fn_t explain);
int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); int32_t operatorDummyOpenFn(SOperatorInfo* pOperator);
void operatorDummyCloseFn(void* param, int32_t numOfCols); void operatorDummyCloseFn(void* param, int32_t numOfCols);
@ -751,30 +754,28 @@ void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock);
void cleanupBasicInfo(SOptrBasicInfo* pInfo); void cleanupBasicInfo(SOptrBasicInfo* pInfo);
int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr);
void cleanupExprSupp(SExprSupp* pSup); void cleanupExprSupp(SExprSupp* pSup);
int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, int32_t initAggInfo(SExprSupp *pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey); const char* pkey);
void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows);
void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf);
SDiskbasedBuf* pBuf);
void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset,
SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order);
int32_t numOfTotal, int32_t numOfOutput, int32_t order);
int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData,
int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total,
SArray* pColList); SArray* pColList);
void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win); void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win);
int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag);
int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
void doSetOperatorCompleted(SOperatorInfo* pOperator); void doSetOperatorCompleted(SOperatorInfo* pOperator);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock);
void cleanupAggSup(SAggSupporter* pAggSup); void cleanupAggSup(SAggSupporter* pAggSup);
void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); void destroyBasicOperatorInfo(void* param, int32_t numOfOutput);
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId); void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId);
int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts); int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts);
int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts); int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts);
@ -783,108 +784,93 @@ SSDataBlock* loadNextDataBlock(void* param);
void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset);
SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo,
int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, char* pData, int16_t bytes, bool masterscan, uint64_t groupId,
bool isIntervalQuery, SAggSupporter* pSup); SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup);
SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo);
const char* pUser, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo,
SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo,
int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo);
SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo);
SExecTaskInfo* pTaskInfo); SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams,
SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo,
int32_t num, SArray* pSortInfo, SArray* pGroupInfo,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
SArray* pTableList, SExecTaskInfo* pTaskInfo); SArray* pTableList, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream);
SExecTaskInfo* pTaskInfo, bool isStream);
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
int32_t primaryTsSlotId, SExecTaskInfo* pTaskInfo); SNode* pCondition, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
SExecTaskInfo* pTaskInfo, int32_t numOfChild); SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild);
SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo); STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp,
STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo); SNode* pCondition, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition, SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition,
SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SBlockDistScanPhysiNode* pBlockScanNode,
SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle,
SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId);
uint64_t taskId);
SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, bool multigroupResult, SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, bool multigroupResult,
SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols,
SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSupp, int32_t tsSlotId, SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, int32_t tsSlotId,
SColumn* pStateKeyCol, SExecTaskInfo* pTaskInfo); SColumn* pStateKeyCol, SNode* pCondition, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, /*SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResultBlock, const SNodeListNode* pValNode, */SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SJoinPhysiNode* pJoinNode,
SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, /*SExprInfo* pExprInfo, int32_t SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream,
numOfCols, SSDataBlock* pResultBlock, const SNodeListNode* pValNode, */ SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream,
SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild);
SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
SJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo, int32_t numOfChild);
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo);
#if 0 #if 0
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
#endif #endif
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
int32_t numOfOutput, SArray* pPseudoList); int32_t numOfOutput, SArray* pPseudoList);
void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol);
int32_t scanFlag, bool createDummyCol);
bool isTaskKilled(SExecTaskInfo* pTaskInfo); bool isTaskKilled(SExecTaskInfo* pTaskInfo);
int32_t checkForQueryBuf(size_t numOfTables); int32_t checkForQueryBuf(size_t numOfTables);
void setTaskKilled(SExecTaskInfo* pTaskInfo); void setTaskKilled(SExecTaskInfo* pTaskInfo);
void queryCostStatis(SExecTaskInfo* pTaskInfo); void queryCostStatis(SExecTaskInfo* pTaskInfo);
void doDestroyTask(SExecTaskInfo* pTaskInfo); void doDestroyTask(SExecTaskInfo* pTaskInfo);
int32_t getMaximumIdleDurationSec(); int32_t getMaximumIdleDurationSec();
@ -896,7 +882,7 @@ int32_t getMaximumIdleDurationSec();
* nOptrWithVal: *nOptrWithVal save the number of optr with value * nOptrWithVal: *nOptrWithVal save the number of optr with value
* return: result code, 0 means success * return: result code, 0 means success
*/ */
int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t* length, int32_t* nOptrWithVal); int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length, int32_t *nOptrWithVal);
/* /*
* ops: root operator, created by caller * ops: root operator, created by caller
@ -909,7 +895,7 @@ int32_t decodeOperator(SOperatorInfo* ops, const char* data, int32_t length);
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
const char* sql, EOPTR_EXEC_MODEL model); const char* sql, EOPTR_EXEC_MODEL model);
int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo); int32_t createDataSinkParam(SDataSinkNode *pNode, void **pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle);
int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo** pRes, int32_t* capacity, int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo** pRes, int32_t* capacity,
int32_t* resNum); int32_t* resNum);
@ -919,37 +905,35 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len
STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
int32_t precision, STimeWindow* win); int32_t precision, STimeWindow* win);
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
__block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order);
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput,
int32_t size); int32_t size);
SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize); SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize);
SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs,
int64_t gap, int32_t* pIndex); TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex);
SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs,
int64_t gap, int32_t* pIndex); TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex);
bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap);
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows, int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs,
int32_t start, int64_t gap, SHashObj* pStDeleted); TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted);
bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool functionNeedToExecute(SqlFunctionCtx* pCtx);
int32_t compareTimeWindow(const void* p1, const void* p2, const void* param); int32_t compareTimeWindow(const void* p1, const void* p2, const void* param);
int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition,
SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset,
const int32_t* rowCellOffset, SSDataBlock* pBlock, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
SExecTaskInfo* pTaskInfo);
int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId);
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo,
SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId);
uint64_t taskId);
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey);
SSDataBlock* createPullDataBlock(); SSDataBlock* createPullDataBlock();
#ifdef __cplusplus #ifdef __cplusplus

4
source/libs/executor/src/dataDeleter.c
View File

@ -154,7 +154,7 @@ static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
taosThreadMutexUnlock(&pDeleter->mutex); taosThreadMutexUnlock(&pDeleter->mutex);
} }
static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd) { static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
if (taosQueueEmpty(pDeleter->pDataBlocks)) { if (taosQueueEmpty(pDeleter->pDataBlocks)) {
*pQueryEnd = pDeleter->queryEnd; *pQueryEnd = pDeleter->queryEnd;
@ -168,7 +168,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryE
taosFreeQitem(pBuf); taosFreeQitem(pBuf);
*pLen = ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->dataLen; *pLen = ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->dataLen;
*pQueryEnd = pDeleter->queryEnd; *pQueryEnd = pDeleter->queryEnd;
qDebug("got data len %d, row num %d in sink", *pLen, ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows); qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows);
} }
static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {

4
source/libs/executor/src/dataDispatcher.c
View File

@ -156,7 +156,7 @@ static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
taosThreadMutexUnlock(&pDispatcher->mutex); taosThreadMutexUnlock(&pDispatcher->mutex);
} }
static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd) { static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
if (taosQueueEmpty(pDispatcher->pDataBlocks)) { if (taosQueueEmpty(pDispatcher->pDataBlocks)) {
*pQueryEnd = pDispatcher->queryEnd; *pQueryEnd = pDispatcher->queryEnd;
@ -170,7 +170,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryE
taosFreeQitem(pBuf); taosFreeQitem(pBuf);
*pLen = ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen; *pLen = ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen;
*pQueryEnd = pDispatcher->queryEnd; *pQueryEnd = pDispatcher->queryEnd;
qDebug("got data len %d, row num %d in sink", *pLen, ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows); qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows);
} }
static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {

404
source/libs/executor/src/dataInserter.c
View File

@ -24,195 +24,266 @@
extern SDataSinkStat gDataSinkStat; extern SDataSinkStat gDataSinkStat;
typedef struct SDataInserterBuf { typedef struct SSubmitRes {
int32_t useSize; int64_t affectedRows;
int32_t allocSize; int32_t code;
char* pData; SSubmitRsp *pRsp;
} SDataInserterBuf; } SSubmitRes;
typedef struct SDataCacheEntry {
int32_t dataLen;
int32_t numOfRows;
int32_t numOfCols;
int8_t compressed;
char data[];
} SDataCacheEntry;
typedef struct SDataInserterHandle { typedef struct SDataInserterHandle {
SDataSinkHandle sink; SDataSinkHandle sink;
SDataSinkManager* pManager; SDataSinkManager* pManager;
SDataBlockDescNode* pSchema; STSchema* pSchema;
SDataDeleterNode* pDeleter; SQueryInserterNode* pNode;
SDeleterParam* pParam; SSubmitRes submitRes;
STaosQueue* pDataBlocks; SInserterParam* pParam;
SDataInserterBuf nextOutput; SArray* pDataBlocks;
SHashObj* pCols;
int32_t status; int32_t status;
bool queryEnd; bool queryEnd;
uint64_t useconds; uint64_t useconds;
uint64_t cachedSize; uint64_t cachedSize;
TdThreadMutex mutex; TdThreadMutex mutex;
tsem_t ready;
} SDataInserterHandle; } SDataInserterHandle;
static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { typedef struct SSubmitRspParam {
if (tsCompressColData < 0 || 0 == pData->info.rows) { SDataInserterHandle* pInserter;
return false; } SSubmitRspParam;
}
for (int32_t col = 0; col < numOfCols; ++col) { int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
SColumnInfoData* pColRes = taosArrayGet(pData->pDataBlock, col); SSubmitRspParam* pParam = (SSubmitRspParam*)param;
int32_t colSize = pColRes->info.bytes * pData->info.rows; SDataInserterHandle* pInserter = pParam->pInserter;
if (NEEDTO_COMPRESS_QUERY(colSize)) {
return true; pInserter->submitRes.code = code;
if (code == TSDB_CODE_SUCCESS) {
pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp));
SDecoder coder = {0};
tDecoderInit(&coder, pMsg->pData, pMsg->len);
code = tDecodeSSubmitRsp(&coder, pInserter->submitRes.pRsp);
if (code) {
tFreeSSubmitRsp(pInserter->submitRes.pRsp);
pInserter->submitRes.code = code;
goto _return;
} }
if (pInserter->submitRes.pRsp->nBlocks > 0) {
for (int32_t i = 0; i < pInserter->submitRes.pRsp->nBlocks; ++i) {
SSubmitBlkRsp *blk = pInserter->submitRes.pRsp->pBlocks + i;
if (TSDB_CODE_SUCCESS != blk->code) {
code = blk->code;
tFreeSSubmitRsp(pInserter->submitRes.pRsp);
pInserter->submitRes.code = code;
goto _return;
}
}
}
pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows;
qDebug("submit rsp received, affectedRows:%d, total:%d", pInserter->submitRes.pRsp->affectedRows, pInserter->submitRes.affectedRows);
tFreeSSubmitRsp(pInserter->submitRes.pRsp);
} }
return false; _return:
tsem_post(&pInserter->ready);
taosMemoryFree(param);
return TSDB_CODE_SUCCESS;
} }
static void toDataCacheEntry(SDataInserterHandle* pHandle, const SInputData* pInput, SDataInserterBuf* pBuf) {
int32_t numOfCols = LIST_LENGTH(pHandle->pSchema->pSlots);
SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMsg, void* pTransporter, SEpSet* pEpset) {
pEntry->compressed = 0; // send the fetch remote task result reques
pEntry->numOfRows = pInput->pData->info.rows; SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
pEntry->numOfCols = taosArrayGetSize(pInput->pData->pDataBlock); if (NULL == pMsgSendInfo) {
pEntry->dataLen = sizeof(SDeleterRes); taosMemoryFreeClear(pMsg);
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
ASSERT(1 == pEntry->numOfRows); return terrno;
ASSERT(1 == pEntry->numOfCols);
pBuf->useSize = sizeof(SDataCacheEntry);
SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0);
SDeleterRes* pRes = (SDeleterRes*)pEntry->data;
pRes->suid = pHandle->pParam->suid;
pRes->uidList = pHandle->pParam->pUidList;
pRes->skey = pHandle->pDeleter->deleteTimeRange.skey;
pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey;
pRes->affectedRows = *(int64_t*)pColRes->pData;
pBuf->useSize += pEntry->dataLen;
atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
}
static bool allocBuf(SDataInserterHandle* pDeleter, const SInputData* pInput, SDataInserterBuf* pBuf) {
uint32_t capacity = pDeleter->pManager->cfg.maxDataBlockNumPerQuery;
if (taosQueueItemSize(pDeleter->pDataBlocks) > capacity) {
qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity,
taosQueueItemSize(pDeleter->pDataBlocks));
return false;
} }
pBuf->allocSize = sizeof(SDataCacheEntry) + sizeof(SDeleterRes); SSubmitRspParam* pParam = taosMemoryCalloc(1, sizeof(SSubmitRspParam));
pParam->pInserter = pInserter;
pBuf->pData = taosMemoryMalloc(pBuf->allocSize); pMsgSendInfo->param = pParam;
if (pBuf->pData == NULL) { pMsgSendInfo->msgInfo.pData = pMsg;
qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno)); pMsgSendInfo->msgInfo.len = ntohl(pMsg->length);
pMsgSendInfo->msgType = TDMT_VND_SUBMIT;
pMsgSendInfo->fp = inserterCallback;
int64_t transporterId = 0;
return asyncSendMsgToServer(pTransporter, pEpset, &transporterId, pMsgSendInfo);
}
int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) {
const SArray* pBlocks = pInserter->pDataBlocks;
const STSchema* pTSchema = pInserter->pSchema;
int64_t uid = pInserter->pNode->tableId;
int64_t suid = pInserter->pNode->stableId;
int32_t vgId = pInserter->pNode->vgId;
bool fullCol = (pInserter->pNode->pCols->length == pTSchema->numOfCols);
SSubmitReq* ret = NULL;
int32_t sz = taosArrayGetSize(pBlocks);
// cal size
int32_t cap = sizeof(SSubmitReq);
for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i);
int32_t rows = pDataBlock->info.rows;
// TODO min
int32_t rowSize = pDataBlock->info.rowSize;
int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
cap += sizeof(SSubmitBlk) + rows * maxLen;
} }
return NULL != pBuf->pData; // assign data
// TODO
ret = taosMemoryCalloc(1, cap);
ret->header.vgId = htonl(vgId);
ret->version = htonl(pTSchema->version);
ret->length = sizeof(SSubmitReq);
ret->numOfBlocks = htonl(sz);
SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq));
for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i);
blkHead->sversion = htonl(pTSchema->version);
// TODO
blkHead->suid = htobe64(suid);
blkHead->uid = htobe64(uid);
blkHead->schemaLen = htonl(0);
int32_t rows = 0;
int32_t dataLen = 0;
STSRow* rowData = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk));
int64_t lastTs = TSKEY_MIN;
bool ignoreRow = false;
for (int32_t j = 0; j < pDataBlock->info.rows; j++) {
SRowBuilder rb = {0};
tdSRowInit(&rb, pTSchema->version);
tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen);
tdSRowResetBuf(&rb, rowData);
ignoreRow = false;
for (int32_t k = 0; k < pTSchema->numOfCols; k++) {
const STColumn* pColumn = &pTSchema->columns[k];
SColumnInfoData* pColData = NULL;
int16_t colIdx = k;
if (!fullCol) {
int16_t *slotId = taosHashGet(pInserter->pCols, &pColumn->colId, sizeof(pColumn->colId));
if (NULL == slotId) {
continue;
}
colIdx = *slotId;
}
pColData = taosArrayGet(pDataBlock->pDataBlock, colIdx);
if (pColData->info.type != pColumn->type) {
qError("col type mis-match, schema type:%d, type in block:%d", pColumn->type, pColData->info.type);
terrno = TSDB_CODE_APP_ERROR;
return TSDB_CODE_APP_ERROR;
}
if (colDataIsNull_s(pColData, j)) {
if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) {
ignoreRow = true;
break;
}
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k);
} else {
void* data = colDataGetData(pColData, j);
if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) {
if (*(int64_t*)data == lastTs) {
ignoreRow = true;
break;
} else {
lastTs = *(int64_t*)data;
}
}
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k);
}
}
if (ignoreRow) {
continue;
}
rows++;
int32_t rowLen = TD_ROW_LEN(rowData);
rowData = POINTER_SHIFT(rowData, rowLen);
dataLen += rowLen;
}
blkHead->dataLen = htonl(dataLen);
blkHead->numOfRows = htons(rows);
ret->length += sizeof(SSubmitBlk) + dataLen;
blkHead = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk) + dataLen);
}
ret->length = htonl(ret->length);
*pReq = ret;
return TSDB_CODE_SUCCESS;
} }
static int32_t updateStatus(SDataInserterHandle* pDeleter) {
taosThreadMutexLock(&pDeleter->mutex);
int32_t blockNums = taosQueueItemSize(pDeleter->pDataBlocks);
int32_t status =
(0 == blockNums ? DS_BUF_EMPTY
: (blockNums < pDeleter->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL));
pDeleter->status = status;
taosThreadMutexUnlock(&pDeleter->mutex);
return status;
}
static int32_t getStatus(SDataInserterHandle* pDeleter) {
taosThreadMutexLock(&pDeleter->mutex);
int32_t status = pDeleter->status;
taosThreadMutexUnlock(&pDeleter->mutex);
return status;
}
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
SDataInserterHandle* pDeleter = (SDataInserterHandle*)pHandle; SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
SDataInserterBuf* pBuf = taosAllocateQitem(sizeof(SDataInserterBuf), DEF_QITEM); taosArrayPush(pInserter->pDataBlocks, &pInput->pData);
if (NULL == pBuf || !allocBuf(pDeleter, pInput, pBuf)) { SSubmitReq* pMsg = NULL;
return TSDB_CODE_QRY_OUT_OF_MEMORY; int32_t code = dataBlockToSubmit(pInserter, &pMsg);
if (code) {
return code;
} }
toDataCacheEntry(pDeleter, pInput, pBuf);
taosWriteQitem(pDeleter->pDataBlocks, pBuf); code = sendSubmitRequest(pInserter, pMsg, pInserter->pParam->readHandle->pMsgCb->clientRpc, &pInserter->pNode->epSet);
*pContinue = (DS_BUF_LOW == updateStatus(pDeleter) ? true : false); if (code) {
return code;
}
tsem_wait(&pInserter->ready);
if (pInserter->submitRes.code) {
return pInserter->submitRes.code;
}
*pContinue = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
SDataInserterHandle* pDeleter = (SDataInserterHandle*)pHandle; SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
taosThreadMutexLock(&pDeleter->mutex); taosThreadMutexLock(&pInserter->mutex);
pDeleter->queryEnd = true; pInserter->queryEnd = true;
pDeleter->useconds = useconds; pInserter->useconds = useconds;
taosThreadMutexUnlock(&pDeleter->mutex); taosThreadMutexUnlock(&pInserter->mutex);
} }
static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd) { static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd) {
SDataInserterHandle* pDeleter = (SDataInserterHandle*)pHandle; SDataInserterHandle* pDispatcher = (SDataInserterHandle*)pHandle;
if (taosQueueEmpty(pDeleter->pDataBlocks)) { *pLen = pDispatcher->submitRes.affectedRows;
*pQueryEnd = pDeleter->queryEnd; qDebug("got total affectedRows %" PRId64 , *pLen);
*pLen = 0;
return;
}
SDataInserterBuf* pBuf = NULL;
taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataInserterBuf));
taosFreeQitem(pBuf);
*pLen = ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->dataLen;
*pQueryEnd = pDeleter->queryEnd;
qDebug("got data len %d, row num %d in sink", *pLen, ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows);
} }
static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
SDataInserterHandle* pDeleter = (SDataInserterHandle*)pHandle;
if (NULL == pDeleter->nextOutput.pData) {
assert(pDeleter->queryEnd);
pOutput->useconds = pDeleter->useconds;
pOutput->precision = pDeleter->pSchema->precision;
pOutput->bufStatus = DS_BUF_EMPTY;
pOutput->queryEnd = pDeleter->queryEnd;
return TSDB_CODE_SUCCESS;
}
SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDeleter->nextOutput.pData);
memcpy(pOutput->pData, pEntry->data, pEntry->dataLen);
pOutput->numOfRows = pEntry->numOfRows;
pOutput->numOfCols = pEntry->numOfCols;
pOutput->compressed = pEntry->compressed;
atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen);
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
taosMemoryFreeClear(pDeleter->nextOutput.pData); // todo persistent
pOutput->bufStatus = updateStatus(pDeleter);
taosThreadMutexLock(&pDeleter->mutex);
pOutput->queryEnd = pDeleter->queryEnd;
pOutput->useconds = pDeleter->useconds;
pOutput->precision = pDeleter->pSchema->precision;
taosThreadMutexUnlock(&pDeleter->mutex);
return TSDB_CODE_SUCCESS;
}
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
SDataInserterHandle* pDeleter = (SDataInserterHandle*)pHandle; SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDeleter->cachedSize); atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pInserter->cachedSize);
taosMemoryFreeClear(pDeleter->nextOutput.pData); taosArrayDestroy(pInserter->pDataBlocks);
while (!taosQueueEmpty(pDeleter->pDataBlocks)) { taosMemoryFree(pInserter->pSchema);
SDataInserterBuf* pBuf = NULL; taosThreadMutexDestroy(&pInserter->mutex);
taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
taosMemoryFreeClear(pBuf->pData);
taosFreeQitem(pBuf);
}
taosCloseQueue(pDeleter->pDataBlocks);
taosThreadMutexDestroy(&pDeleter->mutex);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -230,25 +301,46 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
return TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY;
} }
SDataDeleterNode* pDeleterNode = (SDataDeleterNode *)pDataSink; SQueryInserterNode* pInserterNode = (SQueryInserterNode *)pDataSink;
inserter->sink.fPut = putDataBlock; inserter->sink.fPut = putDataBlock;
inserter->sink.fEndPut = endPut; inserter->sink.fEndPut = endPut;
inserter->sink.fGetLen = getDataLength; inserter->sink.fGetLen = getDataLength;
inserter->sink.fGetData = getDataBlock; inserter->sink.fGetData = NULL;
inserter->sink.fDestroy = destroyDataSinker; inserter->sink.fDestroy = destroyDataSinker;
inserter->sink.fGetCacheSize = getCacheSize; inserter->sink.fGetCacheSize = getCacheSize;
inserter->pManager = pManager; inserter->pManager = pManager;
inserter->pDeleter = pDeleterNode; inserter->pNode = pInserterNode;
inserter->pSchema = pDataSink->pInputDataBlockDesc;
inserter->pParam = pParam; inserter->pParam = pParam;
inserter->status = DS_BUF_EMPTY; inserter->status = DS_BUF_EMPTY;
inserter->queryEnd = false; inserter->queryEnd = false;
inserter->pDataBlocks = taosOpenQueue();
int64_t suid = 0;
int32_t code = tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid);
if (code) {
return code;
}
if (pInserterNode->stableId != suid) {
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
return terrno;
}
inserter->pDataBlocks = taosArrayInit(1, POINTER_BYTES);
taosThreadMutexInit(&inserter->mutex, NULL); taosThreadMutexInit(&inserter->mutex, NULL);
if (NULL == inserter->pDataBlocks) { if (NULL == inserter->pDataBlocks) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY;
} }
inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK);
SNode* pNode = NULL;
FOREACH(pNode, pInserterNode->pCols) {
SColumnNode* pCol = (SColumnNode*)pNode;
taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId));
}
tsem_init(&inserter->ready, 0, 0);
*pHandle = inserter; *pHandle = inserter;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

4
source/libs/executor/src/dataSinkMgt.c
View File

@ -40,6 +40,8 @@ int32_t dsCreateDataSinker(const SDataSinkNode *pDataSink, DataSinkHandle* pHand
return createDataDispatcher(&gDataSinkManager, pDataSink, pHandle); return createDataDispatcher(&gDataSinkManager, pDataSink, pHandle);
case QUERY_NODE_PHYSICAL_PLAN_DELETE: case QUERY_NODE_PHYSICAL_PLAN_DELETE:
return createDataDeleter(&gDataSinkManager, pDataSink, pHandle, pParam); return createDataDeleter(&gDataSinkManager, pDataSink, pHandle, pParam);
case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT:
return createDataInserter(&gDataSinkManager, pDataSink, pHandle, pParam);
} }
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -54,7 +56,7 @@ void dsEndPut(DataSinkHandle handle, uint64_t useconds) {
return pHandleImpl->fEndPut(pHandleImpl, useconds); return pHandleImpl->fEndPut(pHandleImpl, useconds);
} }
void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, bool* pQueryEnd) {
SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle;
pHandleImpl->fGetLen(pHandleImpl, pLen, pQueryEnd); pHandleImpl->fGetLen(pHandleImpl, pLen, pQueryEnd);
} }

2
source/libs/executor/src/executorMain.c
View File

@ -52,7 +52,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
if (handle) { if (handle) {
void* pSinkParam = NULL; void* pSinkParam = NULL;
code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo); code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo, readHandle);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }

98
source/libs/executor/src/executorimpl.c
View File

@ -741,10 +741,10 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF
if (type == TSDB_DATA_TYPE_BIGINT) { if (type == TSDB_DATA_TYPE_BIGINT) {
int64_t v = pFuncParam->param.i; int64_t v = pFuncParam->param.i;
*da = (SColumnDataAgg){.numOfNull = 0, .min = v, .max = v, .maxIndex = 0, .minIndex = 0, .sum = v * numOfRows}; *da = (SColumnDataAgg){.numOfNull = 0, .min = v, .max = v, .sum = v * numOfRows};
} else if (type == TSDB_DATA_TYPE_DOUBLE) { } else if (type == TSDB_DATA_TYPE_DOUBLE) {
double v = pFuncParam->param.d; double v = pFuncParam->param.d;
*da = (SColumnDataAgg){.numOfNull = 0, .maxIndex = 0, .minIndex = 0}; *da = (SColumnDataAgg){.numOfNull = 0};
*(double*)&da->min = v; *(double*)&da->min = v;
*(double*)&da->max = v; *(double*)&da->max = v;
@ -752,7 +752,7 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF
} else if (type == TSDB_DATA_TYPE_BOOL) { // todo validate this data type } else if (type == TSDB_DATA_TYPE_BOOL) { // todo validate this data type
bool v = pFuncParam->param.i; bool v = pFuncParam->param.i;
*da = (SColumnDataAgg){.numOfNull = 0, .maxIndex = 0, .minIndex = 0}; *da = (SColumnDataAgg){.numOfNull = 0};
*(bool*)&da->min = 0; *(bool*)&da->min = 0;
*(bool*)&da->max = v; *(bool*)&da->max = v;
*(bool*)&da->sum = v * numOfRows; *(bool*)&da->sum = v * numOfRows;
@ -1130,7 +1130,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc
} else if ((*status) == BLK_DATA_SMA_LOAD) { } else if ((*status) == BLK_DATA_SMA_LOAD) {
// this function never returns error? // this function never returns error?
pCost->loadBlockStatis += 1; pCost->loadBlockStatis += 1;
// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); // tsdbRetrieveDatablockSMA(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg);
if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block
// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); // pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
@ -1141,7 +1141,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc
// load the data block statistics to perform further filter // load the data block statistics to perform further filter
pCost->loadBlockStatis += 1; pCost->loadBlockStatis += 1;
// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); // tsdbRetrieveDatablockSMA(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg);
if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) { if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) {
{ // set previous window { // set previous window
@ -1340,6 +1340,8 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); extractQualifiedTupleByFilterResult(pBlock, rowRes, keep);
blockDataUpdateTsWindow(pBlock, 0); blockDataUpdateTsWindow(pBlock, 0);
taosMemoryFree(rowRes);
} }
void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) { void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) {
@ -1992,7 +1994,7 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
taosMemoryFreeClear(pMsgBody); taosMemoryFreeClear(pMsgBody);
} }
void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle;
assert(pMsg->info.ahandle != NULL); assert(pMsg->info.ahandle != NULL);
@ -2461,6 +2463,8 @@ static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) {
blockDataDestroy(pInfo->binfo.pRes); blockDataDestroy(pInfo->binfo.pRes);
cleanupAggSup(&pInfo->aggSup); cleanupAggSup(&pInfo->aggSup);
taosMemoryFreeClear(param);
} }
static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int32_t rowIndex) { static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int32_t rowIndex) {
@ -3337,7 +3341,7 @@ static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResult
SExecTaskInfo* pTaskInfo) { SExecTaskInfo* pTaskInfo) {
pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows;
int64_t ekey = Q_STATUS_EQUAL(pTaskInfo->status, TASK_COMPLETED) ? pTaskInfo->window.ekey int64_t ekey = Q_STATUS_EQUAL(pTaskInfo->status, TASK_COMPLETED) ? pInfo->win.ekey
: pInfo->existNewGroupBlock->info.window.ekey; : pInfo->existNewGroupBlock->info.window.ekey;
taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo));
@ -3365,7 +3369,7 @@ static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo* pInfo, SResultInf
} }
} }
static SSDataBlock* doFill(SOperatorInfo* pOperator) { static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
SFillOperatorInfo* pInfo = pOperator->info; SFillOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -3373,9 +3377,6 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
SSDataBlock* pResBlock = pInfo->pRes; SSDataBlock* pResBlock = pInfo->pRes;
blockDataCleanup(pResBlock); blockDataCleanup(pResBlock);
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
// todo handle different group data interpolation // todo handle different group data interpolation
bool n = false; bool n = false;
@ -3398,7 +3399,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
// Fill the previous group data block, before handle the data block of new group. // Fill the previous group data block, before handle the data block of new group.
// Close the fill operation for previous group data block // Close the fill operation for previous group data block
taosFillSetStartInfo(pInfo->pFillInfo, 0, pTaskInfo->window.ekey); taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey);
} else { } else {
if (pBlock == NULL) { if (pBlock == NULL) {
if (pInfo->totalInputRows == 0) { if (pInfo->totalInputRows == 0) {
@ -3406,7 +3407,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
return NULL; return NULL;
} }
taosFillSetStartInfo(pInfo->pFillInfo, 0, pTaskInfo->window.ekey); taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey);
} else { } else {
pInfo->totalInputRows += pBlock->info.rows; pInfo->totalInputRows += pBlock->info.rows;
taosFillSetStartInfo(pInfo->pFillInfo, pBlock->info.rows, pBlock->info.window.ekey); taosFillSetStartInfo(pInfo->pFillInfo, pBlock->info.rows, pBlock->info.window.ekey);
@ -3441,6 +3442,39 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
} }
} }
static SSDataBlock* doFill(SOperatorInfo* pOperator) {
SFillOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
SSDataBlock* fillResult = NULL;
while (true) {
fillResult = doFillImpl(pOperator);
if (fillResult != NULL) {
doFilter(pInfo->pCondition, fillResult);
}
if (fillResult == NULL) {
doSetOperatorCompleted(pOperator);
break;
}
if (fillResult->info.rows > 0) {
break;
}
}
if (fillResult != NULL) {
size_t rows = fillResult->info.rows;
pOperator->resultInfo.totalRows += rows;
}
return fillResult;
}
static void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { static void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) {
for (int32_t i = 0; i < numOfExprs; ++i) { for (int32_t i = 0; i < numOfExprs; ++i) {
SExprInfo* pExprInfo = &pExpr[i]; SExprInfo* pExprInfo = &pExpr[i];
@ -3475,7 +3509,6 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) {
} }
taosMemoryFreeClear(pOperator->exprSupp.pExprInfo); taosMemoryFreeClear(pOperator->exprSupp.pExprInfo);
taosMemoryFreeClear(pOperator->info);
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
} }
@ -3645,11 +3678,15 @@ void cleanupBasicInfo(SOptrBasicInfo* pInfo) {
void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) {
SOptrBasicInfo* pInfo = (SOptrBasicInfo*)param; SOptrBasicInfo* pInfo = (SOptrBasicInfo*)param;
cleanupBasicInfo(pInfo); cleanupBasicInfo(pInfo);
taosMemoryFreeClear(param);
} }
void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { void destroyAggOperatorInfo(void* param, int32_t numOfOutput) {
SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param; SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param;
cleanupBasicInfo(&pInfo->binfo); cleanupBasicInfo(&pInfo->binfo);
taosMemoryFreeClear(param);
} }
void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { void destroySFillOperatorInfo(void* param, int32_t numOfOutput) {
@ -3657,6 +3694,8 @@ void destroySFillOperatorInfo(void* param, int32_t numOfOutput) {
pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo);
pInfo->pRes = blockDataDestroy(pInfo->pRes); pInfo->pRes = blockDataDestroy(pInfo->pRes);
taosMemoryFreeClear(pInfo->p); taosMemoryFreeClear(pInfo->p);
taosMemoryFreeClear(param);
} }
static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
@ -3667,6 +3706,8 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
cleanupBasicInfo(&pInfo->binfo); cleanupBasicInfo(&pInfo->binfo);
cleanupAggSup(&pInfo->aggSup); cleanupAggSup(&pInfo->aggSup);
taosArrayDestroy(pInfo->pPseudoColInfo); taosArrayDestroy(pInfo->pPseudoColInfo);
taosMemoryFreeClear(param);
} }
void cleanupExprSupp(SExprSupp* pSupp) { void cleanupExprSupp(SExprSupp* pSupp) {
@ -3683,6 +3724,8 @@ static void destroyIndefinitOperatorInfo(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->pPseudoColInfo); taosArrayDestroy(pInfo->pPseudoColInfo);
cleanupAggSup(&pInfo->aggSup); cleanupAggSup(&pInfo->aggSup);
cleanupExprSupp(&pInfo->scalarSup); cleanupExprSupp(&pInfo->scalarSup);
taosMemoryFreeClear(param);
} }
void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) {
@ -3700,6 +3743,8 @@ void doDestroyExchangeOperatorInfo(void* param) {
} }
tsem_destroy(&pExInfo->ready); tsem_destroy(&pExInfo->ready);
taosMemoryFreeClear(param);
} }
static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) { static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) {
@ -3833,6 +3878,8 @@ static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
} }
} }
doFilter(pIndefInfo->pCondition, pInfo->pRes);
size_t rows = pInfo->pRes->info.rows; size_t rows = pInfo->pRes->info.rows;
pOperator->resultInfo.totalRows += rows; pOperator->resultInfo.totalRows += rows;
@ -3886,6 +3933,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
pInfo->binfo.pRes = pResBlock; pInfo->binfo.pRes = pResBlock;
pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr);
pInfo->pCondition = pPhyNode->node.pConditions;
pOperator->name = "IndefinitOperator"; pOperator->name = "IndefinitOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT;
@ -3923,7 +3971,8 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t
int32_t order = TSDB_ORDER_ASC; int32_t order = TSDB_ORDER_ASC;
pInfo->pFillInfo = taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval, fillType, pColInfo, id); pInfo->pFillInfo = taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval, fillType, pColInfo, id);
pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); pInfo->win = win;
pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES);
if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { if (pInfo->pFillInfo == NULL || pInfo->p == NULL) {
taosMemoryFree(pInfo->pFillInfo); taosMemoryFree(pInfo->pFillInfo);
taosMemoryFree(pInfo->p); taosMemoryFree(pInfo->p);
@ -3958,6 +4007,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
pInfo->pRes = pResBlock; pInfo->pRes = pResBlock;
pInfo->multigroupResult = multigroupResult; pInfo->multigroupResult = multigroupResult;
pInfo->pCondition = pPhyFillNode->node.pConditions;
pOperator->name = "FillOperator"; pOperator->name = "FillOperator";
pOperator->blocking = false; pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED; pOperator->status = OP_NOT_OPENED;
@ -4415,7 +4465,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
.precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pTaskInfo); pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pPhyNode->pConditions, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) {
SMergeIntervalPhysiNode* pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode; SMergeIntervalPhysiNode* pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode;
@ -4455,7 +4505,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
pOptr = pOptr =
createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pTaskInfo); createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pPhyNode->pConditions, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION == type) {
pOptr = createStreamSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); pOptr = createStreamSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION == type) {
@ -4477,7 +4527,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
SColumn col = extractColumnFromColumnNode(pColNode); SColumn col = extractColumnFromColumnNode(pColNode);
pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pTaskInfo); pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pPhyNode->pConditions, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE == type) {
pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN == type) {
@ -4738,10 +4788,20 @@ int32_t decodeOperator(SOperatorInfo* ops, const char* result, int32_t length) {
return TDB_CODE_SUCCESS; return TDB_CODE_SUCCESS;
} }
int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo) { int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle) {
SExecTaskInfo* pTask = *(SExecTaskInfo**)pTaskInfo; SExecTaskInfo* pTask = *(SExecTaskInfo**)pTaskInfo;
switch (pNode->type) { switch (pNode->type) {
case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: {
SInserterParam* pInserterParam = taosMemoryCalloc(1, sizeof(SInserterParam));
if (NULL == pInserterParam) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pInserterParam->readHandle = readHandle;
*pParam = pInserterParam;
break;
}
case QUERY_NODE_PHYSICAL_PLAN_DELETE: { case QUERY_NODE_PHYSICAL_PLAN_DELETE: {
SDeleterParam* pDeleterParam = taosMemoryCalloc(1, sizeof(SDeleterParam)); SDeleterParam* pDeleterParam = taosMemoryCalloc(1, sizeof(SDeleterParam));
if (NULL == pDeleterParam) { if (NULL == pDeleterParam) {

23
source/libs/executor/src/groupoperator.c
View File

@ -38,6 +38,8 @@ static void destroyGroupOperatorInfo(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->pGroupCols); taosArrayDestroy(pInfo->pGroupCols);
taosArrayDestroy(pInfo->pGroupColVals); taosArrayDestroy(pInfo->pGroupColVals);
cleanupExprSupp(&pInfo->scalarSup); cleanupExprSupp(&pInfo->scalarSup);
taosMemoryFreeClear(param);
} }
static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) {
@ -302,14 +304,21 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
SSDataBlock* pRes = pInfo->binfo.pRes; SSDataBlock* pRes = pInfo->binfo.pRes;
if (pOperator->status == OP_RES_TO_RETURN) { if (pOperator->status == OP_RES_TO_RETURN) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); while(1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pRes);
size_t rows = pRes->info.rows; bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
if (rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { if (!hasRemain) {
doSetOperatorCompleted(pOperator); doSetOperatorCompleted(pOperator);
break;
}
if (pRes->info.rows > 0) {
break;
}
} }
pOperator->resultInfo.totalRows += pRes->info.rows;
pOperator->resultInfo.totalRows += rows;
return (pRes->info.rows == 0)? NULL:pRes; return (pRes->info.rows == 0)? NULL:pRes;
} }
@ -717,6 +726,8 @@ static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) {
taosMemoryFree(pInfo->columnOffset); taosMemoryFree(pInfo->columnOffset);
cleanupExprSupp(&pInfo->scalarSup); cleanupExprSupp(&pInfo->scalarSup);
taosMemoryFreeClear(param);
} }
SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) {

2
source/libs/executor/src/joinoperator.c
View File

@ -104,6 +104,8 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) {
void destroyMergeJoinOperator(void* param, int32_t numOfOutput) { void destroyMergeJoinOperator(void* param, int32_t numOfOutput) {
SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param;
nodesDestroyNode(pJoinOperator->pCondAfterMerge); nodesDestroyNode(pJoinOperator->pCondAfterMerge);
taosMemoryFreeClear(param);
} }
static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) { static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) {

31
source/libs/executor/src/scanoperator.c
View File

@ -210,7 +210,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
bool allColumnsHaveAgg = true; bool allColumnsHaveAgg = true;
SColumnDataAgg** pColAgg = NULL; SColumnDataAgg** pColAgg = NULL;
tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg);
if (code != TSDB_CODE_SUCCESS) {
longjmp(pTaskInfo->env, code);
}
if (allColumnsHaveAgg == true) { if (allColumnsHaveAgg == true) {
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
@ -592,6 +595,8 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) {
if (pTableScanInfo->pColMatchInfo != NULL) { if (pTableScanInfo->pColMatchInfo != NULL) {
taosArrayDestroy(pTableScanInfo->pColMatchInfo); taosArrayDestroy(pTableScanInfo->pColMatchInfo);
} }
taosMemoryFreeClear(param);
} }
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
@ -740,6 +745,8 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
static void destroyBlockDistScanOperatorInfo(void* param, int32_t numOfOutput) { static void destroyBlockDistScanOperatorInfo(void* param, int32_t numOfOutput) {
SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param;
blockDataDestroy(pDistInfo->pResBlock); blockDataDestroy(pDistInfo->pResBlock);
taosMemoryFreeClear(param);
} }
SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid,
@ -1349,11 +1356,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
pInfo->pRes->info.type = STREAM_NORMAL; pInfo->pRes->info.type = STREAM_NORMAL;
pInfo->pRes->info.capacity = block.info.rows; pInfo->pRes->info.capacity = block.info.rows;
// for generating rollup SMA result, each time is an independent time serie.
// TODO temporarily used, when the statement of "partition by tbname" is ready, remove this
if (pInfo->assignBlockUid) {
pInfo->pRes->info.groupId = block.info.uid;
}
uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &block.info.uid, sizeof(int64_t)); uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &block.info.uid, sizeof(int64_t));
if (groupIdPre) { if (groupIdPre) {
@ -1362,6 +1365,12 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
pInfo->pRes->info.groupId = 0; pInfo->pRes->info.groupId = 0;
} }
// for generating rollup SMA result, each time is an independent time serie.
// TODO temporarily used, when the statement of "partition by tbname" is ready, remove this
if (pInfo->assignBlockUid) {
pInfo->pRes->info.groupId = block.info.uid;
}
// todo extract method // todo extract method
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) {
SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i);
@ -1593,6 +1602,8 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->scanCols); taosArrayDestroy(pInfo->scanCols);
taosMemoryFreeClear(pInfo->pUser); taosMemoryFreeClear(pInfo->pUser);
taosMemoryFreeClear(param);
} }
static int32_t getSysTableDbNameColId(const char* pTable) { static int32_t getSysTableDbNameColId(const char* pTable) {
@ -2271,6 +2282,8 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
STagScanInfo* pInfo = (STagScanInfo*)param; STagScanInfo* pInfo = (STagScanInfo*)param;
pInfo->pRes = blockDataDestroy(pInfo->pRes); pInfo->pRes = blockDataDestroy(pInfo->pRes);
taosMemoryFreeClear(param);
} }
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
@ -2452,7 +2465,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
bool allColumnsHaveAgg = true; bool allColumnsHaveAgg = true;
SColumnDataAgg** pColAgg = NULL; SColumnDataAgg** pColAgg = NULL;
STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx); STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
tsdbRetrieveDataBlockStatisInfo(reader, &pColAgg, &allColumnsHaveAgg); tsdbRetrieveDatablockSMA(reader, &pColAgg, &allColumnsHaveAgg);
if (allColumnsHaveAgg == true) { if (allColumnsHaveAgg == true) {
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
@ -2761,6 +2774,8 @@ void destroyTableMergeScanOperatorInfo(void* param, int32_t numOfOutput) {
pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock);
taosArrayDestroy(pTableScanInfo->pSortInfo); taosArrayDestroy(pTableScanInfo->pSortInfo);
taosMemoryFreeClear(param);
} }
typedef struct STableMergeScanExecInfo { typedef struct STableMergeScanExecInfo {
@ -2892,6 +2907,8 @@ static void destroyLastrowScanOperator(void* param, int32_t numOfOutput) {
SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param; SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param;
blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pRes);
tsdbLastrowReaderClose(pInfo->pLastrowReader); tsdbLastrowReaderClose(pInfo->pLastrowReader);
taosMemoryFreeClear(param);
} }
SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, SArray* pTableList, SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, SArray* pTableList,

6
source/libs/executor/src/sortoperator.c
View File

@ -235,6 +235,8 @@ void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pSortInfo);
taosArrayDestroy(pInfo->pColMatchInfo); taosArrayDestroy(pInfo->pColMatchInfo);
taosMemoryFreeClear(param);
} }
int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
@ -451,6 +453,8 @@ void destroyGroupSortOperatorInfo(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pSortInfo);
taosArrayDestroy(pInfo->pColMatchInfo); taosArrayDestroy(pInfo->pColMatchInfo);
taosMemoryFreeClear(param);
} }
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
@ -670,6 +674,8 @@ void destroyMultiwayMergeOperatorInfo(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pSortInfo);
taosArrayDestroy(pInfo->pColMatchInfo); taosArrayDestroy(pInfo->pColMatchInfo);
taosMemoryFreeClear(param);
} }
int32_t getMultiwayMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { int32_t getMultiwayMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {

159
source/libs/executor/src/timewindowoperator.c
View File

@ -1146,13 +1146,22 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
SOptrBasicInfo* pBInfo = &pInfo->binfo; SOptrBasicInfo* pBInfo = &pInfo->binfo;
if (pOperator->status == OP_RES_TO_RETURN) { if (pOperator->status == OP_RES_TO_RETURN) {
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); while(1) {
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doSetOperatorCompleted(pOperator); doFilter(pInfo->pCondition, pBInfo->pRes);
return NULL;
}
return pBInfo->pRes; bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
if (!hasRemain) {
doSetOperatorCompleted(pOperator);
break;
}
if (pBInfo->pRes->info.rows > 0) {
break;
}
}
pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes;
} }
int32_t order = TSDB_ORDER_ASC; int32_t order = TSDB_ORDER_ASC;
@ -1178,15 +1187,22 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC); initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); while(1) {
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doSetOperatorCompleted(pOperator); doFilter(pInfo->pCondition, pBInfo->pRes);
bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
if (!hasRemain) {
doSetOperatorCompleted(pOperator);
break;
}
if (pBInfo->pRes->info.rows > 0) {
break;
}
} }
pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
size_t rows = pBInfo->pRes->info.rows; return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes;
pOperator->resultInfo.totalRows += rows;
return (rows == 0) ? NULL : pBInfo->pRes;
} }
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
@ -1300,7 +1316,7 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId)
// window has been closed // window has been closed
return false; return false;
} }
SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId); // SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId);
// dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage); // dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage);
taosHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); taosHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
return true; return true;
@ -1403,7 +1419,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
ASSERT(pRecyPages != NULL); ASSERT(pRecyPages != NULL);
taosArrayPush(pRecyPages, &pPos->pageId); taosArrayPush(pRecyPages, &pPos->pageId);
} else { } else {
SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId); // SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId);
// dBufSetBufPageRecycled(pDiscBuf, bufPage); // dBufSetBufPageRecycled(pDiscBuf, bufPage);
} }
char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
@ -1430,7 +1446,7 @@ static void freeAllPages(SArray* pageIds, SDiskbasedBuf* pDiskBuf) {
int32_t size = taosArrayGetSize(pageIds); int32_t size = taosArrayGetSize(pageIds);
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i); int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i);
SFilePage* bufPage = getBufPage(pDiskBuf, pageId); // SFilePage* bufPage = getBufPage(pDiskBuf, pageId);
// dBufSetBufPageRecycled(pDiskBuf, bufPage); // dBufSetBufPageRecycled(pDiskBuf, bufPage);
} }
taosArrayClear(pageIds); taosArrayClear(pageIds);
@ -1541,6 +1557,8 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) {
SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param; SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
cleanupBasicInfo(&pInfo->binfo); cleanupBasicInfo(&pInfo->binfo);
taosMemoryFreeClear(pInfo->stateKey.pData); taosMemoryFreeClear(pInfo->stateKey.pData);
taosMemoryFreeClear(param);
} }
void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) {
@ -1548,6 +1566,8 @@ void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) {
cleanupBasicInfo(&pInfo->binfo); cleanupBasicInfo(&pInfo->binfo);
cleanupAggSup(&pInfo->aggSup); cleanupAggSup(&pInfo->aggSup);
taosArrayDestroy(pInfo->pRecycledPages); taosArrayDestroy(pInfo->pRecycledPages);
taosMemoryFreeClear(param);
} }
void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
@ -1570,6 +1590,8 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
} }
} }
nodesDestroyNode((SNode*)pInfo->pPhyNode); nodesDestroyNode((SNode*)pInfo->pPhyNode);
taosMemoryFreeClear(param);
} }
static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
@ -1880,12 +1902,22 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
if (pOperator->status == OP_RES_TO_RETURN) { if (pOperator->status == OP_RES_TO_RETURN) {
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); while(1) {
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doSetOperatorCompleted(pOperator); doFilter(pInfo->pCondition, pBInfo->pRes);
}
return pBInfo->pRes->info.rows > 0 ? pBInfo->pRes : NULL; bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
if (!hasRemain) {
doSetOperatorCompleted(pOperator);
break;
}
if (pBInfo->pRes->info.rows > 0) {
break;
}
}
pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes;
} }
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
@ -1914,15 +1946,22 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC); initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); while(1) {
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doSetOperatorCompleted(pOperator); doFilter(pInfo->pCondition, pBInfo->pRes);
bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
if (!hasRemain) {
doSetOperatorCompleted(pOperator);
break;
}
if (pBInfo->pRes->info.rows > 0) {
break;
}
} }
pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
size_t rows = pBInfo->pRes->info.rows; return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes;
pOperator->resultInfo.totalRows += rows;
return (rows == 0) ? NULL : pBInfo->pRes;
} }
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) { static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
@ -2235,7 +2274,7 @@ _error:
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols,
SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, int32_t tsSlotId, SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, int32_t tsSlotId,
SColumn* pStateKeyCol, SExecTaskInfo* pTaskInfo) { SColumn* pStateKeyCol, SNode* pCondition, SExecTaskInfo* pTaskInfo) {
SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -2246,6 +2285,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf
pInfo->stateKey.type = pInfo->stateCol.type; pInfo->stateKey.type = pInfo->stateCol.type;
pInfo->stateKey.bytes = pInfo->stateCol.bytes; pInfo->stateKey.bytes = pInfo->stateCol.bytes;
pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes); pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
pInfo->pCondition = pCondition;
if (pInfo->stateKey.pData == NULL) { if (pInfo->stateKey.pData == NULL) {
goto _error; goto _error;
} }
@ -2285,11 +2325,13 @@ _error:
void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) {
SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param; SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param;
cleanupBasicInfo(&pInfo->binfo); cleanupBasicInfo(&pInfo->binfo);
taosMemoryFreeClear(param);
} }
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId,
STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { STimeWindowAggSupp* pTwAggSupp, SNode* pCondition, SExecTaskInfo* pTaskInfo) {
SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -2315,6 +2357,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo
pInfo->binfo.pRes = pResBlock; pInfo->binfo.pRes = pResBlock;
pInfo->winSup.prevTs = INT64_MIN; pInfo->winSup.prevTs = INT64_MIN;
pInfo->reptScan = false; pInfo->reptScan = false;
pInfo->pCondition = pCondition;
pOperator->name = "SessionWindowAggOperator"; pOperator->name = "SessionWindowAggOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION;
pOperator->blocking = true; pOperator->blocking = true;
@ -2960,6 +3003,8 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) {
taosMemoryFreeClear(pChInfo); taosMemoryFreeClear(pChInfo);
} }
} }
taosMemoryFreeClear(param);
} }
int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols, int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,
@ -3154,7 +3199,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start
*pIndex = index + 1; *pIndex = index + 1;
return pWin; return pWin;
} else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) { } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
*pIndex = index; *pIndex = index + 1;
return pWin; return pWin;
} }
} }
@ -3407,7 +3452,7 @@ void deleteWindow(SArray* pWinInfos, int32_t index) {
taosArrayRemove(pWinInfos, index); taosArrayRemove(pWinInfos, index);
} }
static void doDeleteSessionWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) { static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) {
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; TSKEY* startDatas = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
@ -3665,13 +3710,13 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
// gap must be 0 // gap must be 0
doDeleteSessionWindows(&pInfo->streamAggSup, pBlock, 0, pWins); doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins);
if (IS_FINAL_OP(pInfo)) { if (IS_FINAL_OP(pInfo)) {
int32_t childIndex = getChildIndex(pBlock); int32_t childIndex = getChildIndex(pBlock);
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
// gap must be 0 // gap must be 0
doDeleteSessionWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL); doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL);
rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator); rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator);
} }
copyDeleteWindowInfo(pWins, pInfo->pStDeleted); copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
@ -3805,7 +3850,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
break; break;
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
// gap must be 0 // gap must be 0
doDeleteSessionWindows(&pInfo->streamAggSup, pBlock, 0, NULL); doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, NULL);
copyDataBlock(pInfo->pDelRes, pBlock); copyDataBlock(pInfo->pDelRes, pBlock);
pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
break; break;
@ -3919,6 +3964,8 @@ void destroyStreamStateOperatorInfo(void* param, int32_t numOfOutput) {
taosMemoryFreeClear(pChInfo); taosMemoryFreeClear(pChInfo);
} }
} }
taosMemoryFreeClear(param);
} }
int64_t getStateWinTsKey(void* data, int32_t index) { int64_t getStateWinTsKey(void* data, int32_t index) {
@ -4197,6 +4244,12 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId, doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
pSeUpdated, pInfo->pSeDeleted); pSeUpdated, pInfo->pSeDeleted);
continue; continue;
} else if (pBlock->info.type == STREAM_DELETE_DATA) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins);
copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
taosArrayDestroy(pWins);
continue;
} else if (pBlock->info.type == STREAM_GET_ALL) { } else if (pBlock->info.type == STREAM_GET_ALL) {
getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState); getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState);
continue; continue;
@ -4316,23 +4369,27 @@ _error:
} }
typedef struct SMergeAlignedIntervalAggOperatorInfo { typedef struct SMergeAlignedIntervalAggOperatorInfo {
SIntervalAggOperatorInfo intervalAggOperatorInfo; SIntervalAggOperatorInfo *intervalAggOperatorInfo;
bool hasGroupId; bool hasGroupId;
uint64_t groupId; uint64_t groupId;
SSDataBlock* prefetchedBlock; SSDataBlock* prefetchedBlock;
bool inputBlocksFinished; bool inputBlocksFinished;
SNode* pCondition;
} SMergeAlignedIntervalAggOperatorInfo; } SMergeAlignedIntervalAggOperatorInfo;
void destroyMergeAlignedIntervalOperatorInfo(void* param, int32_t numOfOutput) { void destroyMergeAlignedIntervalOperatorInfo(void* param, int32_t numOfOutput) {
SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param; SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param;
destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); destroyIntervalOperatorInfo(miaInfo->intervalAggOperatorInfo, numOfOutput);
taosMemoryFreeClear(param);
} }
static int32_t outputMergeAlignedIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, static int32_t outputMergeAlignedIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId,
SSDataBlock* pResultBlock, TSKEY wstartTs) { SSDataBlock* pResultBlock, TSKEY wstartTs) {
SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
SExprSupp* pSup = &pOperatorInfo->exprSupp; SExprSupp* pSup = &pOperatorInfo->exprSupp;
@ -4353,7 +4410,7 @@ static int32_t outputMergeAlignedIntervalResult(SOperatorInfo* pOperatorInfo, ui
static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo,
SSDataBlock* pBlock, int32_t scanFlag, SSDataBlock* pResultBlock) { SSDataBlock* pBlock, int32_t scanFlag, SSDataBlock* pResultBlock) {
SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
SExprSupp* pSup = &pOperatorInfo->exprSupp; SExprSupp* pSup = &pOperatorInfo->exprSupp;
@ -4418,7 +4475,7 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperator->info; SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperator->info;
SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
return NULL; return NULL;
} }
@ -4457,8 +4514,8 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
getTableScanInfo(pOperator, &iaInfo->order, &scanFlag); getTableScanInfo(pOperator, &iaInfo->order, &scanFlag);
setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->order, scanFlag, true); setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->order, scanFlag, true);
doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);
doFilter(miaInfo->pCondition, pRes);
if (pRes->info.rows >= pOperator->resultInfo.threshold) { if (pRes->info.rows > 0) {
break; break;
} }
} }
@ -4466,7 +4523,7 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
pRes->info.groupId = miaInfo->groupId; pRes->info.groupId = miaInfo->groupId;
} }
if (pRes->info.rows == 0) { if (miaInfo->inputBlocksFinished) {
doSetOperatorCompleted(pOperator); doSetOperatorCompleted(pOperator);
} }
@ -4477,16 +4534,22 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo,
int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval,
int32_t primaryTsSlotId, SExecTaskInfo* pTaskInfo) { int32_t primaryTsSlotId, SNode* pCondition, SExecTaskInfo* pTaskInfo) {
SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo)); SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (miaInfo == NULL || pOperator == NULL) { if (miaInfo == NULL || pOperator == NULL) {
goto _error; goto _error;
} }
SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; miaInfo->intervalAggOperatorInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
if (miaInfo->intervalAggOperatorInfo == NULL) {
goto _error;
}
SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
miaInfo->pCondition = pCondition;
iaInfo->win = pTaskInfo->window; iaInfo->win = pTaskInfo->window;
iaInfo->order = TSDB_ORDER_ASC; iaInfo->order = TSDB_ORDER_ASC;
iaInfo->interval = *pInterval; iaInfo->interval = *pInterval;
@ -4561,6 +4624,8 @@ void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) {
SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param; SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
tdListFree(miaInfo->groupIntervals); tdListFree(miaInfo->groupIntervals);
destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput);
taosMemoryFreeClear(param);
} }
static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win, static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win,

14
source/libs/function/inc/fnLog.h
View File

@ -10,12 +10,14 @@
extern "C" { extern "C" {
#endif #endif
#define fnFatal(...) { if (fnDebugFlag & DEBUG_FATAL) { taosPrintLog("FN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} // clang-format off
#define fnError(...) { if (fnDebugFlag & DEBUG_ERROR) { taosPrintLog("FN ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} #define fnFatal(...) { if (udfDebugFlag & DEBUG_FATAL) { taosPrintLog("UDF FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define fnWarn(...) { if (fnDebugFlag & DEBUG_WARN) { taosPrintLog("FN WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} #define fnError(...) { if (udfDebugFlag & DEBUG_ERROR) { taosPrintLog("UDF ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define fnInfo(...) { if (fnDebugFlag & DEBUG_INFO) { taosPrintLog("FN ", DEBUG_INFO, 255, __VA_ARGS__); }} #define fnWarn(...) { if (udfDebugFlag & DEBUG_WARN) { taosPrintLog("UDF WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define fnDebug(...) { if (fnDebugFlag & DEBUG_DEBUG) { taosPrintLog("FN ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} #define fnInfo(...) { if (udfDebugFlag & DEBUG_INFO) { taosPrintLog("UDF ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define fnTrace(...) { if (fnDebugFlag & DEBUG_TRACE) { taosPrintLog("FN ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} #define fnDebug(...) { if (udfDebugFlag & DEBUG_DEBUG) { taosPrintLog("UDF ", DEBUG_DEBUG, udfDebugFlag, __VA_ARGS__); }}
#define fnTrace(...) { if (udfDebugFlag & DEBUG_TRACE) { taosPrintLog("UDF ", DEBUG_TRACE, udfDebugFlag, __VA_ARGS__); }}
// clang-format on
#ifdef __cplusplus #ifdef __cplusplus
} }

51
source/libs/function/src/builtinsimpl.c
View File

@ -1080,6 +1080,19 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
static void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); static void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
static void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); static void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, const char* tval) {
// the data is loaded, not only the block SMA value
for(int32_t i = start; i < num + start; ++i) {
char* p = colDataGetData(pCol, i);
if (memcpy((void*)tval, p, pCol->info.bytes) == 0) {
return i;
}
}
ASSERT(0);
}
int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
int32_t numOfElems = 0; int32_t numOfElems = 0;
@ -1111,15 +1124,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if (isMinFunc) { if (isMinFunc) {
tval = &pInput->pColumnDataAgg[0]->min; tval = &pInput->pColumnDataAgg[0]->min;
index = pInput->pColumnDataAgg[0]->minIndex;
} else { } else {
tval = &pInput->pColumnDataAgg[0]->max; tval = &pInput->pColumnDataAgg[0]->max;
index = pInput->pColumnDataAgg[0]->maxIndex;
} }
if (!pBuf->assign) { if (!pBuf->assign) {
pBuf->v = *(int64_t*)tval; pBuf->v = *(int64_t*)tval;
if (pCtx->subsidiaries.num > 0) { if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos);
} }
} else { } else {
@ -1131,6 +1143,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if ((prev < val) ^ isMinFunc) { if ((prev < val) ^ isMinFunc) {
pBuf->v = val; pBuf->v = val;
if (pCtx->subsidiaries.num > 0) { if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos);
} }
} }
@ -1143,6 +1156,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if ((prev < val) ^ isMinFunc) { if ((prev < val) ^ isMinFunc) {
pBuf->v = val; pBuf->v = val;
if (pCtx->subsidiaries.num > 0) { if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos);
} }
} }
@ -1154,6 +1168,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if ((prev < val) ^ isMinFunc) { if ((prev < val) ^ isMinFunc) {
pBuf->v = val; pBuf->v = val;
if (pCtx->subsidiaries.num > 0) { if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos);
} }
} }
@ -1167,6 +1182,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
} }
if (pCtx->subsidiaries.num > 0) { if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos);
} }
} }
@ -5547,30 +5563,18 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
} }
} }
int32_t delta = maxVal - minVal; // maximum number of step is 80
int32_t step = delta / 50; double factor = pData->numOfBlocks / 80.0;
if (step == 0) {
step = 1;
}
int32_t numOfBuckets = sizeof(pData->blockRowsHisto) / sizeof(pData->blockRowsHisto[0]); int32_t numOfBuckets = sizeof(pData->blockRowsHisto) / sizeof(pData->blockRowsHisto[0]);
int32_t bucketRange = (pData->maxRows - pData->minRows) / numOfBuckets; int32_t bucketRange = (pData->defMaxRows - pData->defMinRows) / numOfBuckets;
bool singleModel = false;
if (bucketRange == 0) {
singleModel = true;
step = 20;
bucketRange = (pData->defMaxRows - pData->defMinRows) / numOfBuckets;
}
for (int32_t i = 0; i < tListLen(pData->blockRowsHisto); ++i) { for (int32_t i = 0; i < tListLen(pData->blockRowsHisto); ++i) {
len = sprintf(st + VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * (i + 1)); len = sprintf(st + VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * i);
int32_t num = 0; int32_t num = 0;
if (singleModel && pData->blockRowsHisto[i] > 0) { if (pData->blockRowsHisto[i] > 0) {
num = 20; num = (pData->blockRowsHisto[i]) / factor;
} else {
num = (pData->blockRowsHisto[i] + step - 1) / step;
} }
for (int32_t j = 0; j < num; ++j) { for (int32_t j = 0; j < num; ++j) {
@ -5578,9 +5582,10 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
len += x; len += x;
} }
double v = pData->blockRowsHisto[i] * 100.0 / pData->numOfBlocks; if (num > 0) {
len += sprintf(st + VARSTR_HEADER_SIZE + len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%'); double v = pData->blockRowsHisto[i] * 100.0 / pData->numOfBlocks;
printf("%s\n", st); len += sprintf(st + VARSTR_HEADER_SIZE + len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%');
}
varDataSetLen(st, len); varDataSetLen(st, len);
colDataAppend(pColInfo, row++, st, false); colDataAppend(pColInfo, row++, st, false);

4
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -3605,7 +3605,7 @@ static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) {
int32_t code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsBuffer, pNode->buffer); int32_t code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsBuffer, pNode->buffer);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCachelast, pNode->cachelast); code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCachelast, pNode->cacheLast);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCompressionLevel, pNode->compressionLevel); code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCompressionLevel, pNode->compressionLevel);
@ -3667,7 +3667,7 @@ static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) {
int32_t code = tjsonGetIntValue(pJson, jkDatabaseOptionsBuffer, &pNode->buffer); int32_t code = tjsonGetIntValue(pJson, jkDatabaseOptionsBuffer, &pNode->buffer);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCachelast, &pNode->cachelast); code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCachelast, &pNode->cacheLast);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCompressionLevel, &pNode->compressionLevel); code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCompressionLevel, &pNode->compressionLevel);

1
source/libs/parser/inc/parAst.h
View File

@ -39,6 +39,7 @@ typedef struct SAstCreateContext {
typedef enum EDatabaseOptionType { typedef enum EDatabaseOptionType {
DB_OPTION_BUFFER = 1, DB_OPTION_BUFFER = 1,
DB_OPTION_CACHELAST, DB_OPTION_CACHELAST,
DB_OPTION_CACHELASTSIZE,
DB_OPTION_COMP, DB_OPTION_COMP,
DB_OPTION_DAYS, DB_OPTION_DAYS,
DB_OPTION_FSYNC, DB_OPTION_FSYNC,

2
source/libs/parser/inc/sql.y
View File

@ -172,6 +172,7 @@ exists_opt(A) ::= .
db_options(A) ::= . { A = createDefaultDatabaseOptions(pCxt); } db_options(A) ::= . { A = createDefaultDatabaseOptions(pCxt); }
db_options(A) ::= db_options(B) BUFFER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BUFFER, &C); } db_options(A) ::= db_options(B) BUFFER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BUFFER, &C); }
db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELAST, &C); } db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELAST, &C); }
db_options(A) ::= db_options(B) CACHELASTSIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELASTSIZE, &C); }
db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMP, &C); } db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMP, &C); }
db_options(A) ::= db_options(B) DURATION NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } db_options(A) ::= db_options(B) DURATION NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); }
db_options(A) ::= db_options(B) DURATION NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } db_options(A) ::= db_options(B) DURATION NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); }
@ -198,6 +199,7 @@ alter_db_options(A) ::= alter_db_options(B) alter_db_option(C).
%destructor alter_db_option { } %destructor alter_db_option { }
alter_db_option(A) ::= BUFFER NK_INTEGER(B). { A.type = DB_OPTION_BUFFER; A.val = B; } alter_db_option(A) ::= BUFFER NK_INTEGER(B). { A.type = DB_OPTION_BUFFER; A.val = B; }
alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; } alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; }
alter_db_option(A) ::= CACHELASTSIZE NK_INTEGER(B). { A.type = DB_OPTION_CACHELASTSIZE; A.val = B; }
alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.val = B; } alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.val = B; }
alter_db_option(A) ::= KEEP integer_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; } alter_db_option(A) ::= KEEP integer_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; }
alter_db_option(A) ::= KEEP variable_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; } alter_db_option(A) ::= KEEP variable_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; }

11
source/libs/parser/src/parAstCreater.c
View File

@ -746,7 +746,8 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions); CHECK_OUT_OF_MEM(pOptions);
pOptions->buffer = TSDB_DEFAULT_BUFFER_PER_VNODE; pOptions->buffer = TSDB_DEFAULT_BUFFER_PER_VNODE;
pOptions->cachelast = TSDB_DEFAULT_CACHE_LAST_ROW; pOptions->cacheLast = TSDB_DEFAULT_CACHE_LAST_ROW;
pOptions->cacheLastSize = TSDB_DEFAULT_LAST_ROW_MEM;
pOptions->compressionLevel = TSDB_DEFAULT_COMP_LEVEL; pOptions->compressionLevel = TSDB_DEFAULT_COMP_LEVEL;
pOptions->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; pOptions->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
pOptions->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; pOptions->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
@ -772,7 +773,8 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) {
SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions); CHECK_OUT_OF_MEM(pOptions);
pOptions->buffer = -1; pOptions->buffer = -1;
pOptions->cachelast = -1; pOptions->cacheLast = -1;
pOptions->cacheLastSize = -1;
pOptions->compressionLevel = -1; pOptions->compressionLevel = -1;
pOptions->daysPerFile = -1; pOptions->daysPerFile = -1;
pOptions->fsyncPeriod = -1; pOptions->fsyncPeriod = -1;
@ -800,7 +802,10 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti
((SDatabaseOptions*)pOptions)->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); ((SDatabaseOptions*)pOptions)->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_CACHELAST: case DB_OPTION_CACHELAST:
((SDatabaseOptions*)pOptions)->cachelast = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); ((SDatabaseOptions*)pOptions)->cacheLast = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_CACHELASTSIZE:
((SDatabaseOptions*)pOptions)->cacheLastSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_COMP: case DB_OPTION_COMP:
((SDatabaseOptions*)pOptions)->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); ((SDatabaseOptions*)pOptions)->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);

1
source/libs/parser/src/parTokenizer.c
View File

@ -53,6 +53,7 @@ static SKeyword keywordTable[] = {
{"BY", TK_BY}, {"BY", TK_BY},
{"CACHE", TK_CACHE}, {"CACHE", TK_CACHE},
{"CACHELAST", TK_CACHELAST}, {"CACHELAST", TK_CACHELAST},
{"CACHELASTSIZE", TK_CACHELASTSIZE},
{"CAST", TK_CAST}, {"CAST", TK_CAST},
{"CLIENT_VERSION", TK_CLIENT_VERSION}, {"CLIENT_VERSION", TK_CLIENT_VERSION},
{"CLUSTER", TK_CLUSTER}, {"CLUSTER", TK_CLUSTER},

12
source/libs/parser/src/parTranslater.c
View File

@ -2988,7 +2988,8 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
pReq->compression = pStmt->pOptions->compressionLevel; pReq->compression = pStmt->pOptions->compressionLevel;
pReq->replications = pStmt->pOptions->replica; pReq->replications = pStmt->pOptions->replica;
pReq->strict = pStmt->pOptions->strict; pReq->strict = pStmt->pOptions->strict;
pReq->cacheLastRow = pStmt->pOptions->cachelast; pReq->cacheLastRow = pStmt->pOptions->cacheLast;
pReq->lastRowMem = pStmt->pOptions->cacheLastSize;
pReq->schemaless = pStmt->pOptions->schemaless; pReq->schemaless = pStmt->pOptions->schemaless;
pReq->ignoreExist = pStmt->ignoreExists; pReq->ignoreExist = pStmt->ignoreExists;
return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq); return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq);
@ -3149,9 +3150,13 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
int32_t code = int32_t code =
checkRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE); checkRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "cacheLast", pOptions->cachelast, TSDB_MIN_DB_CACHE_LAST_ROW, code = checkRangeOption(pCxt, "cacheLast", pOptions->cacheLast, TSDB_MIN_DB_CACHE_LAST_ROW,
TSDB_MAX_DB_CACHE_LAST_ROW); TSDB_MAX_DB_CACHE_LAST_ROW);
} }
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "cacheLastSize", pOptions->cacheLastSize, TSDB_MIN_DB_LAST_ROW_MEM,
TSDB_MAX_DB_LAST_ROW_MEM);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "compression", pOptions->compressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); code = checkRangeOption(pCxt, "compression", pOptions->compressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL);
} }
@ -3271,7 +3276,8 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt,
pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod;
pReq->walLevel = pStmt->pOptions->walLevel; pReq->walLevel = pStmt->pOptions->walLevel;
pReq->strict = pStmt->pOptions->strict; pReq->strict = pStmt->pOptions->strict;
pReq->cacheLastRow = pStmt->pOptions->cachelast; pReq->cacheLastRow = pStmt->pOptions->cacheLast;
pReq->lastRowMem = pStmt->pOptions->cacheLastSize;
pReq->replications = pStmt->pOptions->replica; pReq->replications = pStmt->pOptions->replica;
return; return;
} }

3
source/libs/parser/src/parser.c
View File

@ -39,6 +39,9 @@ bool qIsInsertValuesSql(const char* pStr, size_t length) {
if (TK_USING == t.type || TK_VALUES == t.type) { if (TK_USING == t.type || TK_VALUES == t.type) {
return true; return true;
} }
if (0 == t.type) {
break;
}
} while (pStr - pSql < length); } while (pStr - pSql < length);
return false; return false;
} }

5865
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

7
source/libs/parser/test/parInitialCTest.cpp
View File

@ -77,6 +77,7 @@ TEST_F(ParserInitialCTest, createDatabase) {
expect.ignoreExist = igExists; expect.ignoreExist = igExists;
expect.buffer = TSDB_DEFAULT_BUFFER_PER_VNODE; expect.buffer = TSDB_DEFAULT_BUFFER_PER_VNODE;
expect.cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; expect.cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
expect.lastRowMem = TSDB_DEFAULT_LAST_ROW_MEM;
expect.compression = TSDB_DEFAULT_COMP_LEVEL; expect.compression = TSDB_DEFAULT_COMP_LEVEL;
expect.daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; expect.daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
expect.fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; expect.fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
@ -97,7 +98,8 @@ TEST_F(ParserInitialCTest, createDatabase) {
}; };
auto setDbBufferFunc = [&](int32_t buffer) { expect.buffer = buffer; }; auto setDbBufferFunc = [&](int32_t buffer) { expect.buffer = buffer; };
auto setDbCachelastFunc = [&](int8_t CACHELAST) { expect.cacheLastRow = CACHELAST; }; auto setDbCachelastFunc = [&](int8_t cachelast) { expect.cacheLastRow = cachelast; };
auto setDbCachelastSize = [&](int8_t cachelastSize) { expect.lastRowMem = cachelastSize; };
auto setDbCompressionFunc = [&](int8_t compressionLevel) { expect.compression = compressionLevel; }; auto setDbCompressionFunc = [&](int8_t compressionLevel) { expect.compression = compressionLevel; };
auto setDbDaysFunc = [&](int32_t daysPerFile) { expect.daysPerFile = daysPerFile; }; auto setDbDaysFunc = [&](int32_t daysPerFile) { expect.daysPerFile = daysPerFile; };
auto setDbFsyncFunc = [&](int32_t fsyncPeriod) { expect.fsyncPeriod = fsyncPeriod; }; auto setDbFsyncFunc = [&](int32_t fsyncPeriod) { expect.fsyncPeriod = fsyncPeriod; };
@ -154,6 +156,7 @@ TEST_F(ParserInitialCTest, createDatabase) {
ASSERT_EQ(req.replications, expect.replications); ASSERT_EQ(req.replications, expect.replications);
ASSERT_EQ(req.strict, expect.strict); ASSERT_EQ(req.strict, expect.strict);
ASSERT_EQ(req.cacheLastRow, expect.cacheLastRow); ASSERT_EQ(req.cacheLastRow, expect.cacheLastRow);
ASSERT_EQ(req.lastRowMem, expect.lastRowMem);
// ASSERT_EQ(req.schemaless, expect.schemaless); // ASSERT_EQ(req.schemaless, expect.schemaless);
ASSERT_EQ(req.ignoreExist, expect.ignoreExist); ASSERT_EQ(req.ignoreExist, expect.ignoreExist);
ASSERT_EQ(req.numOfRetensions, expect.numOfRetensions); ASSERT_EQ(req.numOfRetensions, expect.numOfRetensions);
@ -179,6 +182,7 @@ TEST_F(ParserInitialCTest, createDatabase) {
setCreateDbReqFunc("wxy_db", 1); setCreateDbReqFunc("wxy_db", 1);
setDbBufferFunc(64); setDbBufferFunc(64);
setDbCachelastFunc(2); setDbCachelastFunc(2);
setDbCachelastSize(20);
setDbCompressionFunc(1); setDbCompressionFunc(1);
setDbDaysFunc(100 * 1440); setDbDaysFunc(100 * 1440);
setDbFsyncFunc(100); setDbFsyncFunc(100);
@ -200,6 +204,7 @@ TEST_F(ParserInitialCTest, createDatabase) {
run("CREATE DATABASE IF NOT EXISTS wxy_db " run("CREATE DATABASE IF NOT EXISTS wxy_db "
"BUFFER 64 " "BUFFER 64 "
"CACHELAST 2 " "CACHELAST 2 "
"CACHELASTSIZE 20 "
"COMP 1 " "COMP 1 "
"DURATION 100 " "DURATION 100 "
"FSYNC 100 " "FSYNC 100 "

17
source/libs/planner/src/planLogicCreater.c
View File

@ -1340,6 +1340,17 @@ static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) {
static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); } static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); }
static void setLogicSubplanType(SLogicSubplan* pSubplan) {
if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY != nodeType(pSubplan->pNode)) {
pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
} else {
SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)pSubplan->pNode;
pSubplan->subplanType = (MODIFY_TABLE_TYPE_INSERT == pModify->modifyType && NULL != pModify->node.pChildren)
? SUBPLAN_TYPE_SCAN
: SUBPLAN_TYPE_MODIFY;
}
}
int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) {
SLogicPlanContext cxt = {.pPlanCxt = pCxt}; SLogicPlanContext cxt = {.pPlanCxt = pCxt};
@ -1354,11 +1365,7 @@ int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) {
int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, &pSubplan->pNode); int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, &pSubplan->pNode);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
setLogicNodeParent(pSubplan->pNode); setLogicNodeParent(pSubplan->pNode);
if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY == nodeType(pSubplan->pNode)) { setLogicSubplanType(pSubplan);
pSubplan->subplanType = SUBPLAN_TYPE_MODIFY;
} else {
pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
}
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {

4
source/libs/planner/src/planOptimizer.c
View File

@ -890,7 +890,7 @@ static int32_t pushDownCondOptDealProject(SOptimizeContext* pCxt, SProjectLogicN
return code; return code;
} }
static int32_t pushDownCondOptDealLogicNode(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { static int32_t pushDownCondOptTrivialPushDown(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
if (NULL == pLogicNode->pConditions || if (NULL == pLogicNode->pConditions ||
OPTIMIZE_FLAG_TEST_MASK(pLogicNode->optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) { OPTIMIZE_FLAG_TEST_MASK(pLogicNode->optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -921,7 +921,7 @@ static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLog
break; break;
case QUERY_NODE_LOGIC_PLAN_SORT: case QUERY_NODE_LOGIC_PLAN_SORT:
case QUERY_NODE_LOGIC_PLAN_PARTITION: case QUERY_NODE_LOGIC_PLAN_PARTITION:
code = pushDownCondOptDealLogicNode(pCxt, pLogicNode); code = pushDownCondOptTrivialPushDown(pCxt, pLogicNode);
break; break;
default: default:
break; break;

2
source/libs/planner/src/planPhysiCreater.c
View File

@ -1566,7 +1566,7 @@ static int32_t buildInsertSelectSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLog
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = createQueryInserter(pCxt, pModify, pSubplan, &pSubplan->pDataSink); code = createQueryInserter(pCxt, pModify, pSubplan, &pSubplan->pDataSink);
} }
pSubplan->msgType = TDMT_VND_SUBMIT; pSubplan->msgType = TDMT_SCH_MERGE_QUERY;
return code; return code;
} }

123
source/libs/planner/src/planSpliter.c
View File

@ -39,7 +39,6 @@ typedef struct SSplitRule {
FSplit splitFunc; FSplit splitFunc;
} SSplitRule; } SSplitRule;
// typedef bool (*FSplFindSplitNode)(SSplitContext* pCxt, SLogicSubplan* pSubplan, void* pInfo);
typedef bool (*FSplFindSplitNode)(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode, void* pInfo); typedef bool (*FSplFindSplitNode)(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode, void* pInfo);
static void splSetSubplanVgroups(SLogicSubplan* pSubplan, SLogicNode* pNode) { static void splSetSubplanVgroups(SLogicSubplan* pSubplan, SLogicNode* pNode) {
@ -67,6 +66,19 @@ static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SLogicNode* pNod
return pSubplan; return pSubplan;
} }
static SLogicSubplan* splCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode, ESubplanType subplanType) {
SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN);
if (NULL == pSubplan) {
return NULL;
}
pSubplan->id.queryId = pCxt->queryId;
pSubplan->id.groupId = pCxt->groupId;
pSubplan->subplanType = subplanType;
pSubplan->pNode = pNode;
pNode->pParent = NULL;
return pSubplan;
}
static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pChild, SExchangeLogicNode** pOutput) { static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pChild, SExchangeLogicNode** pOutput) {
SExchangeLogicNode* pExchange = (SExchangeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); SExchangeLogicNode* pExchange = (SExchangeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE);
if (NULL == pExchange) { if (NULL == pExchange) {
@ -98,6 +110,43 @@ static int32_t splCreateExchangeNodeForSubplan(SSplitContext* pCxt, SLogicSubpla
return code; return code;
} }
static bool splIsChildSubplan(SLogicNode* pLogicNode, int32_t groupId) {
if (QUERY_NODE_LOGIC_PLAN_EXCHANGE == nodeType(pLogicNode)) {
return ((SExchangeLogicNode*)pLogicNode)->srcGroupId == groupId;
}
if (QUERY_NODE_LOGIC_PLAN_MERGE == nodeType(pLogicNode)) {
return ((SMergeLogicNode*)pLogicNode)->srcGroupId == groupId;
}
SNode* pChild;
FOREACH(pChild, pLogicNode->pChildren) {
bool isChild = splIsChildSubplan((SLogicNode*)pChild, groupId);
if (isChild) {
return isChild;
}
}
return false;
}
static int32_t splMountSubplan(SLogicSubplan* pParent, SNodeList* pChildren) {
SNode* pChild = NULL;
WHERE_EACH(pChild, pChildren) {
if (splIsChildSubplan(pParent->pNode, ((SLogicSubplan*)pChild)->id.groupId)) {
int32_t code = nodesListMakeAppend(&pParent->pChildren, pChild);
if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(NULL);
ERASE_NODE(pChildren);
continue;
} else {
return code;
}
}
WHERE_NEXT;
}
return TSDB_CODE_SUCCESS;
}
static bool splMatchByNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode, FSplFindSplitNode func, static bool splMatchByNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode, FSplFindSplitNode func,
void* pInfo) { void* pInfo) {
if (func(pCxt, pSubplan, pNode, pInfo)) { if (func(pCxt, pSubplan, pNode, pInfo)) {
@ -982,56 +1031,6 @@ static int32_t singleTableJoinSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan
return code; return code;
} }
static bool unionIsChildSubplan(SLogicNode* pLogicNode, int32_t groupId) {
if (QUERY_NODE_LOGIC_PLAN_EXCHANGE == nodeType(pLogicNode)) {
return ((SExchangeLogicNode*)pLogicNode)->srcGroupId == groupId;
}
if (QUERY_NODE_LOGIC_PLAN_MERGE == nodeType(pLogicNode)) {
return ((SMergeLogicNode*)pLogicNode)->srcGroupId == groupId;
}
SNode* pChild;
FOREACH(pChild, pLogicNode->pChildren) {
bool isChild = unionIsChildSubplan((SLogicNode*)pChild, groupId);
if (isChild) {
return isChild;
}
}
return false;
}
static int32_t unionMountSubplan(SLogicSubplan* pParent, SNodeList* pChildren) {
SNode* pChild = NULL;
WHERE_EACH(pChild, pChildren) {
if (unionIsChildSubplan(pParent->pNode, ((SLogicSubplan*)pChild)->id.groupId)) {
int32_t code = nodesListMakeAppend(&pParent->pChildren, pChild);
if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(NULL);
ERASE_NODE(pChildren);
continue;
} else {
return code;
}
}
WHERE_NEXT;
}
return TSDB_CODE_SUCCESS;
}
static SLogicSubplan* unionCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode, ESubplanType subplanType) {
SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN);
if (NULL == pSubplan) {
return NULL;
}
pSubplan->id.queryId = pCxt->queryId;
pSubplan->id.groupId = pCxt->groupId;
pSubplan->subplanType = subplanType;
pSubplan->pNode = pNode;
pNode->pParent = NULL;
return pSubplan;
}
static int32_t unionSplitSubplan(SSplitContext* pCxt, SLogicSubplan* pUnionSubplan, SLogicNode* pSplitNode) { static int32_t unionSplitSubplan(SSplitContext* pCxt, SLogicSubplan* pUnionSubplan, SLogicNode* pSplitNode) {
SNodeList* pSubplanChildren = pUnionSubplan->pChildren; SNodeList* pSubplanChildren = pUnionSubplan->pChildren;
pUnionSubplan->pChildren = NULL; pUnionSubplan->pChildren = NULL;
@ -1040,11 +1039,11 @@ static int32_t unionSplitSubplan(SSplitContext* pCxt, SLogicSubplan* pUnionSubpl
SNode* pChild = NULL; SNode* pChild = NULL;
FOREACH(pChild, pSplitNode->pChildren) { FOREACH(pChild, pSplitNode->pChildren) {
SLogicSubplan* pNewSubplan = unionCreateSubplan(pCxt, (SLogicNode*)pChild, pUnionSubplan->subplanType); SLogicSubplan* pNewSubplan = splCreateSubplan(pCxt, (SLogicNode*)pChild, pUnionSubplan->subplanType);
code = nodesListMakeStrictAppend(&pUnionSubplan->pChildren, (SNode*)pNewSubplan); code = nodesListMakeStrictAppend(&pUnionSubplan->pChildren, (SNode*)pNewSubplan);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(NULL); REPLACE_NODE(NULL);
code = unionMountSubplan(pNewSubplan, pSubplanChildren); code = splMountSubplan(pNewSubplan, pSubplanChildren);
} }
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
break; break;
@ -1219,14 +1218,24 @@ static int32_t insertSelectSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t code = splCreateExchangeNodeForSubplan(pCxt, info.pSubplan, info.pQueryRoot, info.pSubplan->subplanType); SLogicSubplan* pNewSubplan = NULL;
SNodeList* pSubplanChildren = info.pSubplan->pChildren;
ESubplanType subplanType = info.pSubplan->subplanType;
int32_t code = splCreateExchangeNodeForSubplan(pCxt, info.pSubplan, info.pQueryRoot, SUBPLAN_TYPE_MODIFY);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, (SNode*)splCreateScanSubplan(pCxt, info.pQueryRoot, 0)); pNewSubplan = splCreateSubplan(pCxt, info.pQueryRoot, subplanType);
if (NULL == pNewSubplan) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
info.pSubplan->subplanType = SUBPLAN_TYPE_MODIFY; code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, (SNode*)pNewSubplan);
SPLIT_FLAG_SET_MASK(info.pSubplan->splitFlag, SPLIT_FLAG_INSERT_SPLIT);
} }
if (TSDB_CODE_SUCCESS == code) {
code = splMountSubplan(pNewSubplan, pSubplanChildren);
}
SPLIT_FLAG_SET_MASK(info.pSubplan->splitFlag, SPLIT_FLAG_INSERT_SPLIT);
++(pCxt->groupId); ++(pCxt->groupId);
pCxt->split = true; pCxt->split = true;
return code; return code;

23
source/libs/qworker/inc/qwInt.h
View File

@ -80,12 +80,19 @@ typedef struct SQWDebug {
extern SQWDebug gQWDebug; extern SQWDebug gQWDebug;
typedef struct SQWMsgInfo {
int8_t taskType;
int8_t explain;
int8_t needFetch;
} SQWMsgInfo;
typedef struct SQWMsg { typedef struct SQWMsg {
void *node; void *node;
int32_t code; int32_t code;
int32_t msgType; int32_t msgType;
char *msg; char *msg;
int32_t msgLen; int32_t msgLen;
SQWMsgInfo msgInfo;
SRpcHandleInfo connInfo; SRpcHandleInfo connInfo;
} SQWMsg; } SQWMsg;
@ -122,15 +129,18 @@ typedef struct SQWTaskCtx {
int8_t phase; int8_t phase;
int8_t taskType; int8_t taskType;
int8_t explain; int8_t explain;
int8_t needFetch;
int32_t queryType; int32_t queryType;
int32_t fetchType; int32_t fetchType;
int32_t execId; int32_t execId;
bool queryRsped;
bool queryFetched; bool queryFetched;
bool queryEnd; bool queryEnd;
bool queryContinue; bool queryContinue;
bool queryInQueue; bool queryInQueue;
int32_t rspCode; int32_t rspCode;
int64_t affectedRows; // for insert ...select stmt
SRpcHandleInfo ctrlConnInfo; SRpcHandleInfo ctrlConnInfo;
SRpcHandleInfo dataConnInfo; SRpcHandleInfo dataConnInfo;
@ -162,7 +172,7 @@ typedef struct SQWMsgStat {
uint64_t queryProcessed; uint64_t queryProcessed;
uint64_t cqueryProcessed; uint64_t cqueryProcessed;
uint64_t fetchProcessed; uint64_t fetchProcessed;
uint64_t fetchRspProcessed; uint64_t rspProcessed;
uint64_t cancelProcessed; uint64_t cancelProcessed;
uint64_t dropProcessed; uint64_t dropProcessed;
uint64_t hbProcessed; uint64_t hbProcessed;
@ -212,8 +222,8 @@ typedef struct SQWorkerMgmt {
#define QW_STAT_GET(_item) atomic_load_64(&(_item)) #define QW_STAT_GET(_item) atomic_load_64(&(_item))
#define QW_GET_EVENT(ctx, event) atomic_load_8(&(ctx)->events[event]) #define QW_GET_EVENT(ctx, event) atomic_load_8(&(ctx)->events[event])
#define QW_IS_EVENT_RECEIVED(ctx, event) (QW_GET_EVENT(ctx, event) == QW_EVENT_RECEIVED) #define QW_EVENT_RECEIVED(ctx, event) (QW_GET_EVENT(ctx, event) == QW_EVENT_RECEIVED)
#define QW_IS_EVENT_PROCESSED(ctx, event) (QW_GET_EVENT(ctx, event) == QW_EVENT_PROCESSED) #define QW_EVENT_PROCESSED(ctx, event) (QW_GET_EVENT(ctx, event) == QW_EVENT_PROCESSED)
#define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED) #define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED)
#define QW_SET_EVENT_PROCESSED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_PROCESSED) #define QW_SET_EVENT_PROCESSED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_PROCESSED)
@ -222,13 +232,8 @@ typedef struct SQWorkerMgmt {
#define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code) #define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code)
#define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) #define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code)
#define QW_IS_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY) #define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY)
#define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code))
#define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code))
#define QW_TASK_READY(status) \
(status == JOB_TASK_STATUS_SUCC || status == JOB_TASK_STATUS_FAIL || status == JOB_TASK_STATUS_CANCELLED || \
status == JOB_TASK_STATUS_PART_SUCC)
#define QW_SET_QTID(id, qId, tId, eId) \ #define QW_SET_QTID(id, qId, tId, eId) \
do { \ do { \
*(uint64_t *)(id) = (qId); \ *(uint64_t *)(id) = (qId); \

4
source/libs/qworker/inc/qwMsg.h
View File

@ -25,7 +25,7 @@ extern "C" {
int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF); int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF);
int32_t qwPrerocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwPrerocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain, const char* sql); int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql);
int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg);
@ -38,7 +38,7 @@ int32_t qwBuildAndSendCancelRsp(SRpcHandleInfo *pConn, int32_t code);
int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code); int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code);
void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete); void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete);
int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn);
int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, STbVerInfo* tbInfo); int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SQWTaskCtx *ctx);
int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SExplainExecInfo *execInfo, int32_t num); int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SExplainExecInfo *execInfo, int32_t num);
void qwFreeFetchRsp(void *msg); void qwFreeFetchRsp(void *msg);
int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp); int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp);

23
source/libs/qworker/src/qwMsg.c
View File

@ -43,13 +43,16 @@ void qwFreeFetchRsp(void *msg) {
} }
} }
int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, STbVerInfo* tbInfo) { int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SQWTaskCtx *ctx) {
STbVerInfo* tbInfo = ctx ? &ctx->tbInfo : NULL;
int64_t affectedRows = ctx ? ctx->affectedRows : 0;
SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp)); SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp));
pRsp->code = code; pRsp->code = htonl(code);
pRsp->affectedRows = htobe64(affectedRows);
if (tbInfo) { if (tbInfo) {
strcpy(pRsp->tbFName, tbInfo->tbFName); strcpy(pRsp->tbFName, tbInfo->tbFName);
pRsp->sversion = tbInfo->sversion; pRsp->sversion = htonl(tbInfo->sversion);
pRsp->tversion = tbInfo->tversion; pRsp->tversion = htonl(tbInfo->tversion);
} }
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
@ -366,10 +369,14 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
int32_t eId = msg->execId; int32_t eId = msg->execId;
SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connInfo = pMsg->info, .msgType = pMsg->msgType}; SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connInfo = pMsg->info, .msgType = pMsg->msgType};
qwMsg.msgInfo.explain = msg->explain;
qwMsg.msgInfo.taskType = msg->taskType;
qwMsg.msgInfo.needFetch = msg->needFetch;
char * sql = strndup(msg->msg, msg->sqlLen); char * sql = strndup(msg->msg, msg->sqlLen);
QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, sql:%s", node, TMSG_INFO(pMsg->msgType), pMsg->info.handle, sql); QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, sql:%s", node, TMSG_INFO(pMsg->msgType), pMsg->info.handle, sql);
QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType, msg->explain, sql)); QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, sql));
QW_SCH_TASK_DLOG("processQuery end, node:%p", node); QW_SCH_TASK_DLOG("processQuery end, node:%p", node);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -447,14 +454,14 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) { int32_t qWorkerProcessRspMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) {
SQWorker * mgmt = (SQWorker *)qWorkerMgmt; SQWorker * mgmt = (SQWorker *)qWorkerMgmt;
if (mgmt) { if (mgmt) {
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE);
QW_STAT_INC(mgmt->stat.msgStat.fetchRspProcessed, 1); QW_STAT_INC(mgmt->stat.msgStat.rspProcessed, 1);
} }
qProcessFetchRsp(NULL, pMsg, NULL); qProcessRspMsg(NULL, pMsg, NULL);
pMsg->pCont = NULL; pMsg->pCont = NULL;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

61
source/libs/qworker/src/qworker.c
View File

@ -57,6 +57,10 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
connInfo.ahandle = NULL; connInfo.ahandle = NULL;
QW_ERR_RET(qwBuildAndSendExplainRsp(&connInfo, execInfo, resNum)); QW_ERR_RET(qwBuildAndSendExplainRsp(&connInfo, execInfo, resNum));
} }
if (!ctx->needFetch) {
dsGetDataLength(ctx->sinkHandle, &ctx->affectedRows, NULL);
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -123,11 +127,11 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) {
break; break;
} }
if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY) && execNum >= QW_DEFAULT_SHORT_RUN_TIMES) { if (ctx->needFetch && (!ctx->queryRsped) && execNum >= QW_DEFAULT_SHORT_RUN_TIMES) {
break; break;
} }
if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
break; break;
} }
@ -184,7 +188,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo)
} }
int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void **rspMsg, SOutputData *pOutput) { int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void **rspMsg, SOutputData *pOutput) {
int32_t len = 0; int64_t len = 0;
SRetrieveTableRsp *rsp = NULL; SRetrieveTableRsp *rsp = NULL;
bool queryEnd = false; bool queryEnd = false;
int32_t code = 0; int32_t code = 0;
@ -243,7 +247,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
} }
int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes) { int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes) {
int32_t len = 0; int64_t len = 0;
bool queryEnd = false; bool queryEnd = false;
int32_t code = 0; int32_t code = 0;
SOutputData output = {0}; SOutputData output = {0};
@ -251,7 +255,7 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes
dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); dsGetDataLength(ctx->sinkHandle, &len, &queryEnd);
if (len <= 0 || len != sizeof(SDeleterRes)) { if (len <= 0 || len != sizeof(SDeleterRes)) {
QW_TASK_ELOG("invalid length from dsGetDataLength, length:%d", len); QW_TASK_ELOG("invalid length from dsGetDataLength, length:%" PRId64, len);
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
@ -282,7 +286,6 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes
int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) {
int32_t code = 0; int32_t code = 0;
SQWTaskCtx *ctx = NULL; SQWTaskCtx *ctx = NULL;
SRpcHandleInfo *cancelConnection = NULL;
QW_TASK_DLOG("start to handle event at phase %s", qwPhaseStr(phase)); QW_TASK_DLOG("start to handle event at phase %s", qwPhaseStr(phase));
@ -303,13 +306,13 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
switch (phase) { switch (phase) {
case QW_PHASE_PRE_QUERY: { case QW_PHASE_PRE_QUERY: {
if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { if (QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) {
QW_TASK_ELOG("task already dropped at wrong phase %s", qwPhaseStr(phase)); QW_TASK_ELOG("task already dropped at wrong phase %s", qwPhaseStr(phase));
QW_ERR_JRET(TSDB_CODE_QRY_TASK_STATUS_ERROR); QW_ERR_JRET(TSDB_CODE_QRY_TASK_STATUS_ERROR);
break; break;
} }
if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) {
QW_ERR_JRET(qwDropTask(QW_FPARAMS())); QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
//qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code); //qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code);
@ -323,29 +326,29 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
break; break;
} }
case QW_PHASE_PRE_FETCH: { case QW_PHASE_PRE_FETCH: {
if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP) || QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { if (QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP) || QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) {
QW_TASK_WLOG("task dropping or already dropped, phase:%s", qwPhaseStr(phase)); QW_TASK_WLOG("task dropping or already dropped, phase:%s", qwPhaseStr(phase));
QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
} }
if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
QW_TASK_WLOG("last fetch still not processed, phase:%s", qwPhaseStr(phase)); QW_TASK_WLOG("last fetch still not processed, phase:%s", qwPhaseStr(phase));
QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION);
} }
if (!QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_READY)) { if (!ctx->queryRsped) {
QW_TASK_ELOG("ready msg has not been processed, phase:%s", qwPhaseStr(phase)); QW_TASK_ELOG("ready msg has not been processed, phase:%s", qwPhaseStr(phase));
QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR); QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR);
} }
break; break;
} }
case QW_PHASE_PRE_CQUERY: { case QW_PHASE_PRE_CQUERY: {
if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { if (QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) {
QW_TASK_WLOG("task already dropped, phase:%s", qwPhaseStr(phase)); QW_TASK_WLOG("task already dropped, phase:%s", qwPhaseStr(phase));
QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
} }
if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) {
QW_ERR_JRET(qwDropTask(QW_FPARAMS())); QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
//qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code); //qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code);
@ -374,11 +377,6 @@ _return:
qwReleaseTaskCtx(mgmt, ctx); qwReleaseTaskCtx(mgmt, ctx);
} }
if (cancelConnection) {
qwBuildAndSendCancelRsp(cancelConnection, code);
QW_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", cancelConnection->handle, code, tstrerror(code));
}
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
QW_TASK_ELOG("end to handle event at phase %s, code:%s", qwPhaseStr(phase), tstrerror(code)); QW_TASK_ELOG("end to handle event at phase %s, code:%s", qwPhaseStr(phase), tstrerror(code));
} else { } else {
@ -400,7 +398,7 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp
QW_LOCK(QW_WRITE, &ctx->lock); QW_LOCK(QW_WRITE, &ctx->lock);
if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { if (QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) {
QW_TASK_WLOG("task already dropped, phase:%s", qwPhaseStr(phase)); QW_TASK_WLOG("task already dropped, phase:%s", qwPhaseStr(phase));
QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
} }
@ -409,10 +407,10 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp
connInfo = ctx->ctrlConnInfo; connInfo = ctx->ctrlConnInfo;
rspConnection = &connInfo; rspConnection = &connInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); ctx->queryRsped = true;
} }
if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) {
if (QW_PHASE_POST_FETCH == phase) { if (QW_PHASE_POST_FETCH == phase) {
QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase)); QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase));
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
@ -440,7 +438,7 @@ _return:
} }
if (rspConnection) { if (rspConnection) {
qwBuildAndSendQueryRsp(input->msgType + 1, rspConnection, code, ctx ? &ctx->tbInfo : NULL); qwBuildAndSendQueryRsp(input->msgType + 1, rspConnection, code, ctx);
QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", rspConnection->handle, code, tstrerror(code)); QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", rspConnection->handle, code, tstrerror(code));
} }
@ -501,7 +499,7 @@ _return:
} }
int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain, const char* sql) { int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql) {
int32_t code = 0; int32_t code = 0;
bool queryRsped = false; bool queryRsped = false;
SSubplan *plan = NULL; SSubplan *plan = NULL;
@ -514,8 +512,9 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex
QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx));
ctx->taskType = taskType; ctx->taskType = qwMsg->msgInfo.taskType;
ctx->explain = explain; ctx->explain = qwMsg->msgInfo.explain;
ctx->needFetch = qwMsg->msgInfo.needFetch;
ctx->queryType = qwMsg->msgType; ctx->queryType = qwMsg->msgType;
QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg);
@ -585,7 +584,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryEnd)); QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryEnd));
if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
SOutputData sOutput = {0}; SOutputData sOutput = {0};
QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput));
@ -622,7 +621,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
break; break;
} }
if (code && QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { if (code && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwFreeFetchRsp(rsp); qwFreeFetchRsp(rsp);
rsp = NULL; rsp = NULL;
@ -686,7 +685,7 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
locked = true; locked = true;
// RC WARNING // RC WARNING
if (QW_IS_QUERY_RUNNING(ctx)) { if (QW_QUERY_RUNNING(ctx)) {
atomic_store_8((int8_t *)&ctx->queryContinue, 1); atomic_store_8((int8_t *)&ctx->queryContinue, 1);
} else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) { } else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC);
@ -714,7 +713,7 @@ _return:
if (code || rsp) { if (code || rsp) {
qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code);
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), QW_TASK_DLOG("%s send, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1), qwMsg->connInfo.handle, code, tstrerror(code),
dataLen); dataLen);
} }
@ -733,12 +732,12 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
locked = true; locked = true;
if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) {
QW_TASK_WLOG_E("task already dropping"); QW_TASK_WLOG_E("task already dropping");
QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION);
} }
if (QW_IS_QUERY_RUNNING(ctx)) { if (QW_QUERY_RUNNING(ctx)) {
QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx));
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROP); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROP);
} else if (ctx->phase > 0) { } else if (ctx->phase > 0) {

2
source/libs/qworker/test/qworkerTests.cpp
View File

@ -332,7 +332,7 @@ void qwtEndPut(DataSinkHandle handle, uint64_t useconds) {
qwtTestSinkQueryEnd = true; qwtTestSinkQueryEnd = true;
} }
void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { void qwtGetDataLength(DataSinkHandle handle, int64_t* pLen, bool* pQueryEnd) {
static int32_t in = 0; static int32_t in = 0;
if (in > 0) { if (in > 0) {

9
source/libs/scheduler/inc/schInt.h
View File

@ -296,8 +296,8 @@ extern SSchedulerMgmt schMgmt;
#define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1) #define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1)
#define SCH_TASK_EID(_task) ((_task) ? (_task)->execId : -1) #define SCH_TASK_EID(_task) ((_task) ? (_task)->execId : -1)
#define SCH_IS_DATA_SRC_QRY_TASK(task) ((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) #define SCH_IS_DATA_BIND_QRY_TASK(task) ((task)->plan->subplanType == SUBPLAN_TYPE_SCAN)
#define SCH_IS_DATA_SRC_TASK(task) (((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) || ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY)) #define SCH_IS_DATA_BIND_TASK(task) (((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) || ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY))
#define SCH_IS_LEAF_TASK(_job, _task) (((_task)->level->level + 1) == (_job)->levelNum) #define SCH_IS_LEAF_TASK(_job, _task) (((_task)->level->level + 1) == (_job)->levelNum)
#define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) #define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st)
@ -317,8 +317,9 @@ extern SSchedulerMgmt schMgmt;
#define SCH_SET_JOB_NEED_FLOW_CTRL(_job) (_job)->attr.needFlowCtrl = true #define SCH_SET_JOB_NEED_FLOW_CTRL(_job) (_job)->attr.needFlowCtrl = true
#define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl) #define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl)
#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_SRC_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) #define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_BIND_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level))
#define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_SRC_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH) #define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_BIND_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH)
#define SCH_TASK_NEED_FETCH(_task) ((_task)->plan->subplanType != SUBPLAN_TYPE_MODIFY)
#define SCH_SET_JOB_TYPE(_job, type) do { if ((type) != SUBPLAN_TYPE_MODIFY) { (_job)->attr.queryJob = true; } } while (0) #define SCH_SET_JOB_TYPE(_job, type) do { if ((type) != SUBPLAN_TYPE_MODIFY) { (_job)->attr.queryJob = true; } } while (0)
#define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) #define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob)

4
source/libs/scheduler/src/schJob.c
View File

@ -247,7 +247,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
int32_t schAppendJobDataSrc(SSchJob *pJob, SSchTask *pTask) { int32_t schAppendJobDataSrc(SSchJob *pJob, SSchTask *pTask) {
if (!SCH_IS_DATA_SRC_QRY_TASK(pTask)) { if (!SCH_IS_DATA_BIND_QRY_TASK(pTask)) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -879,7 +879,7 @@ int32_t schProcessOnCbBegin(SSchJob** job, SSchTask** task, uint64_t qId, int64_
} }
if (schJobNeedToStop(pJob, &status)) { if (schJobNeedToStop(pJob, &status)) {
SCH_TASK_ELOG("will not do further processing cause of job status %s", jobTaskStatusStr(status)); SCH_TASK_DLOG("will not do further processing cause of job status %s", jobTaskStatusStr(status));
SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR);
} }

44
source/libs/scheduler/src/schRemote.c
View File

@ -21,44 +21,30 @@
#include "tref.h" #include "tref.h"
#include "trpc.h" #include "trpc.h"
int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { int32_t schValidateRspMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) {
int32_t lastMsgType = pTask->lastMsgType; int32_t lastMsgType = pTask->lastMsgType;
int32_t taskStatus = SCH_GET_TASK_STATUS(pTask); int32_t taskStatus = SCH_GET_TASK_STATUS(pTask);
int32_t reqMsgType = msgType - 1; int32_t reqMsgType = (msgType & 1U) ? msgType : (msgType - 1);
switch (msgType) { switch (msgType) {
case TDMT_SCH_LINK_BROKEN: case TDMT_SCH_LINK_BROKEN:
case TDMT_SCH_EXPLAIN_RSP: case TDMT_SCH_EXPLAIN_RSP:
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
case TDMT_SCH_MERGE_QUERY_RSP:
case TDMT_SCH_QUERY_RSP: // query_rsp may be processed later than ready_rsp
if (lastMsgType != reqMsgType && -1 != lastMsgType) {
SCH_TASK_DLOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType),
TMSG_INFO(msgType));
}
if (taskStatus != JOB_TASK_STATUS_EXEC && taskStatus != JOB_TASK_STATUS_PART_SUCC) {
SCH_TASK_DLOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus),
TMSG_INFO(msgType));
}
// SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
return TSDB_CODE_SUCCESS;
case TDMT_SCH_FETCH_RSP: case TDMT_SCH_FETCH_RSP:
case TDMT_SCH_MERGE_FETCH_RSP: case TDMT_SCH_MERGE_FETCH_RSP:
if (lastMsgType != reqMsgType && -1 != lastMsgType) { if (lastMsgType != reqMsgType) {
SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType),
TMSG_INFO(msgType)); TMSG_INFO(msgType));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
if (taskStatus != JOB_TASK_STATUS_PART_SUCC) {
if (taskStatus != JOB_TASK_STATUS_EXEC && taskStatus != JOB_TASK_STATUS_PART_SUCC) {
SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus),
TMSG_INFO(msgType)); TMSG_INFO(msgType));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
// SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
case TDMT_SCH_MERGE_QUERY_RSP:
case TDMT_SCH_QUERY_RSP:
case TDMT_VND_CREATE_TABLE_RSP: case TDMT_VND_CREATE_TABLE_RSP:
case TDMT_VND_DROP_TABLE_RSP: case TDMT_VND_DROP_TABLE_RSP:
case TDMT_VND_ALTER_TABLE_RSP: case TDMT_VND_ALTER_TABLE_RSP:
@ -77,14 +63,12 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
if (taskStatus != JOB_TASK_STATUS_EXEC && taskStatus != JOB_TASK_STATUS_PART_SUCC) { if (taskStatus != JOB_TASK_STATUS_EXEC) {
SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus),
TMSG_INFO(msgType)); TMSG_INFO(msgType));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
// SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -98,7 +82,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode)); bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode));
SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, execId)); SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, execId));
SCH_ERR_JRET(schValidateReceivedMsgType(pJob, pTask, msgType)); SCH_ERR_JRET(schValidateRspMsgType(pJob, pTask, msgType));
int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1); int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1);
if (SCH_NEED_REDIRECT(reqType, rspCode, pMsg->len)) { if (SCH_NEED_REDIRECT(reqType, rspCode, pMsg->len)) {
@ -259,16 +243,23 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
} }
case TDMT_SCH_QUERY_RSP: case TDMT_SCH_QUERY_RSP:
case TDMT_SCH_MERGE_QUERY_RSP: { case TDMT_SCH_MERGE_QUERY_RSP: {
SQueryTableRsp *rsp = (SQueryTableRsp *)msg;
SCH_ERR_JRET(rspCode); SCH_ERR_JRET(rspCode);
if (NULL == msg) { if (NULL == msg) {
SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
} }
SQueryTableRsp *rsp = (SQueryTableRsp *)msg;
rsp->code = ntohl(rsp->code);
rsp->sversion = ntohl(rsp->sversion);
rsp->tversion = ntohl(rsp->tversion);
rsp->affectedRows = be64toh(rsp->affectedRows);
SCH_ERR_JRET(rsp->code); SCH_ERR_JRET(rsp->code);
SCH_ERR_JRET(schSaveJobQueryRes(pJob, rsp)); SCH_ERR_JRET(schSaveJobQueryRes(pJob, rsp));
atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows);
taosMemoryFreeClear(msg); taosMemoryFreeClear(msg);
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
@ -1010,6 +1001,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
pMsg->execId = htonl(pTask->execId); pMsg->execId = htonl(pTask->execId);
pMsg->taskType = TASK_TYPE_TEMP; pMsg->taskType = TASK_TYPE_TEMP;
pMsg->explain = SCH_IS_EXPLAIN_JOB(pJob); pMsg->explain = SCH_IS_EXPLAIN_JOB(pJob);
pMsg->needFetch = SCH_TASK_NEED_FETCH(pTask);
pMsg->phyLen = htonl(pTask->msgLen); pMsg->phyLen = htonl(pTask->msgLen);
pMsg->sqlLen = htonl(len); pMsg->sqlLen = htonl(len);

12
source/libs/scheduler/src/schTask.c
View File

@ -276,7 +276,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
} }
int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) { int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) {
if (SCH_IS_DATA_SRC_QRY_TASK(pTask)) { if (SCH_IS_DATA_BIND_TASK(pTask)) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -313,7 +313,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32
pTask->lastMsgType = 0; pTask->lastMsgType = 0;
memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr));
if (SCH_IS_DATA_SRC_QRY_TASK(pTask)) { if (SCH_IS_DATA_BIND_TASK(pTask)) {
if (pData) { if (pData) {
SCH_ERR_JRET(schUpdateTaskCandidateAddr(pJob, pTask, pData->pEpSet)); SCH_ERR_JRET(schUpdateTaskCandidateAddr(pJob, pTask, pData->pEpSet));
} }
@ -358,7 +358,7 @@ _return:
int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32_t rspCode) { int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32_t rspCode) {
int32_t code = 0; int32_t code = 0;
if (SCH_IS_DATA_SRC_QRY_TASK(pTask)) { if (SCH_IS_DATA_BIND_TASK(pTask)) {
if (NULL == pData->pEpSet) { if (NULL == pData->pEpSet) {
SCH_TASK_ELOG("no epset updated while got error %s", tstrerror(rspCode)); SCH_TASK_ELOG("no epset updated while got error %s", tstrerror(rspCode));
SCH_ERR_JRET(rspCode); SCH_ERR_JRET(rspCode);
@ -492,7 +492,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (SCH_IS_DATA_SRC_TASK(pTask)) { if (SCH_IS_DATA_BIND_TASK(pTask)) {
if ((pTask->execId + 1) >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) { if ((pTask->execId + 1) >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) {
*needRetry = false; *needRetry = false;
SCH_TASK_DLOG("task no more retry since all ep tried, execId:%d, epNum:%d", pTask->execId, SCH_TASK_DLOG("task no more retry since all ep tried, execId:%d, epNum:%d", pTask->execId,
@ -528,7 +528,7 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) {
schDeregisterTaskHb(pJob, pTask); schDeregisterTaskHb(pJob, pTask);
if (SCH_IS_DATA_SRC_TASK(pTask)) { if (SCH_IS_DATA_BIND_TASK(pTask)) {
SCH_SWITCH_EPSET(&pTask->plan->execNode); SCH_SWITCH_EPSET(&pTask->plan->execNode);
} else { } else {
int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs); int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs);
@ -596,7 +596,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (SCH_IS_DATA_SRC_QRY_TASK(pTask)) { if (SCH_IS_DATA_BIND_TASK(pTask)) {
SCH_TASK_ELOG("no execNode specifed for data src task, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps); SCH_TASK_ELOG("no execNode specifed for data src task, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps);
SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
} }

1
source/libs/sync/inc/syncInt.h
View File

@ -67,7 +67,6 @@ typedef struct SSyncNode {
char path[TSDB_FILENAME_LEN]; char path[TSDB_FILENAME_LEN];
char raftStorePath[TSDB_FILENAME_LEN * 2]; char raftStorePath[TSDB_FILENAME_LEN * 2];
char configPath[TSDB_FILENAME_LEN * 2]; char configPath[TSDB_FILENAME_LEN * 2];
int32_t batchSize;
// sync io // sync io
SWal* pWal; SWal* pWal;

72
source/libs/sync/inc/syncOnMessage.h
View File

@ -1,72 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_ON_MESSAGE_H
#define _TD_LIBS_SYNC_ON_MESSAGE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "taosdef.h"
// TLA+ Spec
// Receive(m) ==
// LET i == m.mdest
// j == m.msource
// IN \* Any RPC with a newer term causes the recipient to advance
// \* its term first. Responses with stale terms are ignored.
// \/ UpdateTerm(i, j, m)
// \/ /\ m.mtype = RequestVoteRequest
// /\ HandleRequestVoteRequest(i, j, m)
// \/ /\ m.mtype = RequestVoteResponse
// /\ \/ DropStaleResponse(i, j, m)
// \/ HandleRequestVoteResponse(i, j, m)
// \/ /\ m.mtype = AppendEntriesRequest
// /\ HandleAppendEntriesRequest(i, j, m)
// \/ /\ m.mtype = AppendEntriesResponse
// /\ \/ DropStaleResponse(i, j, m)
// \/ HandleAppendEntriesResponse(i, j, m)
// DuplicateMessage(m) ==
// /\ Send(m)
// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
// DropMessage(m) ==
// /\ Discard(m)
// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
// Next == /\ \/ \E i \in Server : Restart(i)
// \/ \E i \in Server : Timeout(i)
// \/ \E i,j \in Server : RequestVote(i, j)
// \/ \E i \in Server : BecomeLeader(i)
// \/ \E i \in Server, v \in Value : ClientRequest(i, v)
// \/ \E i \in Server : AdvanceCommitIndex(i)
// \/ \E i,j \in Server : AppendEntries(i, j)
// \/ \E m \in DOMAIN messages : Receive(m)
// \/ \E m \in DOMAIN messages : DuplicateMessage(m)
// \/ \E m \in DOMAIN messages : DropMessage(m)
// \* History variable that tracks every log ever:
// /\ allLogs' = allLogs \cup {log[i] : i \in Server}
//
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_ON_MESSAGE_H*/

370
source/libs/sync/src/syncAppendEntries.c
View File

@ -162,6 +162,17 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
pReply->success = false; pReply->success = false;
pReply->matchIndex = SYNC_INDEX_INVALID; pReply->matchIndex = SYNC_INDEX_INVALID;
// msg event log
do {
char host[128];
uint16_t port;
syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port);
sDebug(
"vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, "
"match-index:%ld}",
ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex);
} while (0);
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
@ -334,270 +345,16 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
pReply->matchIndex = pMsg->prevLogIndex; pReply->matchIndex = pMsg->prevLogIndex;
} }
SRpcMsg rpcMsg; // msg event log
syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); do {
syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); char host[128];
syncAppendEntriesReplyDestroy(pReply); uint16_t port;
syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port);
// maybe update commit index from leader sDebug(
if (pMsg->commitIndex > ths->commitIndex) { "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, "
// has commit entry in local "match-index:%ld}",
if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex);
SyncIndex beginIndex = ths->commitIndex + 1; } while (0);
SyncIndex endIndex = pMsg->commitIndex;
// update commit index
ths->commitIndex = pMsg->commitIndex;
// call back Wal
ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex);
int32_t code = syncNodeCommit(ths, beginIndex, endIndex, ths->state);
ASSERT(code == 0);
}
}
}
return ret;
}
#if 0
int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
int32_t ret = 0;
char logBuf[128] = {0};
snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesCb== term:%lu", ths->pRaftStore->currentTerm);
syncAppendEntriesLog2(logBuf, pMsg);
if (pMsg->term > ths->pRaftStore->currentTerm) {
syncNodeUpdateTerm(ths, pMsg->term);
}
ASSERT(pMsg->term <= ths->pRaftStore->currentTerm);
// reset elect timer
if (pMsg->term == ths->pRaftStore->currentTerm) {
ths->leaderCache = pMsg->srcId;
syncNodeResetElectTimer(ths);
}
ASSERT(pMsg->dataLen >= 0);
SyncTerm localPreLogTerm = 0;
if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) {
SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, pMsg->prevLogIndex);
if (pEntry == NULL) {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "getEntry error, index:%ld, since %s", pMsg->prevLogIndex, terrstr());
syncNodeErrorLog(ths, logBuf);
return -1;
}
localPreLogTerm = pEntry->term;
syncEntryDestory(pEntry);
}
bool logOK =
(pMsg->prevLogIndex == SYNC_INDEX_INVALID) ||
((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) &&
(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogTerm == localPreLogTerm));
// reject request
if ((pMsg->term < ths->pRaftStore->currentTerm) ||
((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) {
sTrace(
"syncNodeOnAppendEntriesCb --> reject, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, ths->state:%d, "
"logOK:%d",
pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK);
SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
pReply->srcId = ths->myRaftId;
pReply->destId = pMsg->srcId;
pReply->term = ths->pRaftStore->currentTerm;
pReply->success = false;
pReply->matchIndex = SYNC_INDEX_INVALID;
SRpcMsg rpcMsg;
syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
syncAppendEntriesReplyDestroy(pReply);
return ret;
}
// return to follower state
if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) {
sTrace(
"syncNodeOnAppendEntriesCb --> return to follower, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, "
"ths->state:%d, logOK:%d",
pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK);
syncNodeBecomeFollower(ths, "from candidate by append entries");
// ret or reply?
return ret;
}
// accept request
if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) {
// preIndex = -1, or has preIndex entry in local log
ASSERT(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore));
// has extra entries (> preIndex) in local log
bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore);
// has entries in SyncAppendEntries msg
bool hasAppendEntries = pMsg->dataLen > 0;
sTrace(
"syncNodeOnAppendEntriesCb --> accept, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, ths->state:%d, "
"logOK:%d, hasExtraEntries:%d, hasAppendEntries:%d",
pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK, hasExtraEntries, hasAppendEntries);
if (hasExtraEntries && hasAppendEntries) {
// not conflict by default
bool conflict = false;
SyncIndex extraIndex = pMsg->prevLogIndex + 1;
SSyncRaftEntry* pExtraEntry = ths->pLogStore->getEntry(ths->pLogStore, extraIndex);
if (pExtraEntry == NULL) {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "getEntry error2, index:%ld, since %s", extraIndex, terrstr());
syncNodeErrorLog(ths, logBuf);
return -1;
}
SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen);
if (pAppendEntry == NULL) {
syncNodeErrorLog(ths, "syncEntryDeserialize pAppendEntry error");
return -1;
}
// log not match, conflict
ASSERT(extraIndex == pAppendEntry->index);
if (pExtraEntry->term != pAppendEntry->term) {
conflict = true;
}
if (conflict) {
// roll back
SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore);
SyncIndex delEnd = extraIndex;
sTrace("syncNodeOnAppendEntriesCb --> conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, delEnd);
// notice! reverse roll back!
for (SyncIndex index = delEnd; index >= delBegin; --index) {
if (ths->pFsm->FpRollBackCb != NULL) {
SSyncRaftEntry* pRollBackEntry = ths->pLogStore->getEntry(ths->pLogStore, index);
if (pRollBackEntry == NULL) {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "getEntry error3, index:%ld, since %s", index, terrstr());
syncNodeErrorLog(ths, logBuf);
return -1;
}
// if (pRollBackEntry->msgType != TDMT_SYNC_NOOP) {
if (syncUtilUserRollback(pRollBackEntry->msgType)) {
SRpcMsg rpcMsg;
syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
SFsmCbMeta cbMeta = {0};
cbMeta.index = pRollBackEntry->index;
cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index);
cbMeta.isWeak = pRollBackEntry->isWeak;
cbMeta.code = 0;
cbMeta.state = ths->state;
cbMeta.seqNum = pRollBackEntry->seqNum;
ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta);
rpcFreeCont(rpcMsg.pCont);
}
syncEntryDestory(pRollBackEntry);
}
}
// delete confict entries
ths->pLogStore->truncate(ths->pLogStore, extraIndex);
// append new entries
ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry);
// pre commit
SRpcMsg rpcMsg;
syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
if (ths->pFsm != NULL) {
// if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) {
if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) {
SFsmCbMeta cbMeta = {0};
cbMeta.index = pAppendEntry->index;
cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index);
cbMeta.isWeak = pAppendEntry->isWeak;
cbMeta.code = 2;
cbMeta.state = ths->state;
cbMeta.seqNum = pAppendEntry->seqNum;
ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta);
}
}
rpcFreeCont(rpcMsg.pCont);
}
// free memory
syncEntryDestory(pExtraEntry);
syncEntryDestory(pAppendEntry);
} else if (hasExtraEntries && !hasAppendEntries) {
// do nothing
} else if (!hasExtraEntries && hasAppendEntries) {
SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen);
if (pAppendEntry == NULL) {
syncNodeErrorLog(ths, "syncEntryDeserialize pAppendEntry2 error");
return -1;
}
// append new entries
ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry);
// pre commit
SRpcMsg rpcMsg;
syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
if (ths->pFsm != NULL) {
// if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) {
if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) {
SFsmCbMeta cbMeta = {0};
cbMeta.index = pAppendEntry->index;
cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index);
cbMeta.isWeak = pAppendEntry->isWeak;
cbMeta.code = 3;
cbMeta.state = ths->state;
cbMeta.seqNum = pAppendEntry->seqNum;
ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta);
}
}
rpcFreeCont(rpcMsg.pCont);
// free memory
syncEntryDestory(pAppendEntry);
} else if (!hasExtraEntries && !hasAppendEntries) {
// do nothing
} else {
syncNodeLog3("", ths);
ASSERT(0);
}
SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
pReply->srcId = ths->myRaftId;
pReply->destId = pMsg->srcId;
pReply->term = ths->pRaftStore->currentTerm;
pReply->success = true;
if (hasAppendEntries) {
pReply->matchIndex = pMsg->prevLogIndex + 1;
} else {
pReply->matchIndex = pMsg->prevLogIndex;
}
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
@ -626,8 +383,6 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
return ret; return ret;
} }
#endif
static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) {
int32_t code; int32_t code;
@ -842,8 +597,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
do { do {
char logBuf[128]; char logBuf[128];
snprintf(logBuf, sizeof(logBuf), snprintf(logBuf, sizeof(logBuf),
"recv sync-append-entries-batch, fake match2, pre-index:%ld, pre-term:%lu, datalen:%d", "recv sync-append-entries-batch, fake match2, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}",
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount);
syncNodeEventLog(ths, logBuf); syncNodeEventLog(ths, logBuf);
} while (0); } while (0);
@ -876,7 +631,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
code = syncNodePreCommit(ths, pAppendEntry); code = syncNodePreCommit(ths, pAppendEntry);
ASSERT(code == 0); ASSERT(code == 0);
syncEntryDestory(pAppendEntry); // syncEntryDestory(pAppendEntry);
} }
// fsync once // fsync once
@ -897,6 +652,17 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
pReply->success = true; pReply->success = true;
pReply->matchIndex = matchIndex; pReply->matchIndex = matchIndex;
// msg event log
do {
char host[128];
uint16_t port;
syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port);
sDebug(
"vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, "
"match-index:%ld}",
ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex);
} while (0);
// send response // send response
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
@ -931,8 +697,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
do { do {
char logBuf[128]; char logBuf[128];
snprintf(logBuf, sizeof(logBuf), snprintf(logBuf, sizeof(logBuf),
"recv sync-append-entries-batch, not match, pre-index:%ld, pre-term:%lu, datalen:%d", "recv sync-append-entries-batch, not match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}",
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount);
syncNodeEventLog(ths, logBuf); syncNodeEventLog(ths, logBuf);
} while (0); } while (0);
@ -945,6 +711,17 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
pReply->success = false; pReply->success = false;
pReply->matchIndex = SYNC_INDEX_INVALID; pReply->matchIndex = SYNC_INDEX_INVALID;
// msg event log
do {
char host[128];
uint16_t port;
syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port);
sDebug(
"vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, "
"match-index:%ld}",
ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex);
} while (0);
// send response // send response
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
@ -976,8 +753,9 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
do { do {
char logBuf[128]; char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, match, pre-index:%ld, pre-term:%lu, datalen:%d", snprintf(logBuf, sizeof(logBuf),
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); "recv sync-append-entries-batch, match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}",
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount);
syncNodeEventLog(ths, logBuf); syncNodeEventLog(ths, logBuf);
} while (0); } while (0);
@ -999,7 +777,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
code = syncNodePreCommit(ths, pAppendEntry); code = syncNodePreCommit(ths, pAppendEntry);
ASSERT(code == 0); ASSERT(code == 0);
syncEntryDestory(pAppendEntry); // syncEntryDestory(pAppendEntry);
} }
// fsync once // fsync once
@ -1017,6 +795,17 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
pReply->success = true; pReply->success = true;
pReply->matchIndex = hasAppendEntries ? pMsg->prevLogIndex + pMsg->dataCount : pMsg->prevLogIndex; pReply->matchIndex = hasAppendEntries ? pMsg->prevLogIndex + pMsg->dataCount : pMsg->prevLogIndex;
// msg event log
do {
char host[128];
uint16_t port;
syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port);
sDebug(
"vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, "
"match-index:%ld}",
ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex);
} while (0);
// send response // send response
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
@ -1226,6 +1015,17 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
pReply->success = true; pReply->success = true;
pReply->matchIndex = matchIndex; pReply->matchIndex = matchIndex;
// msg event log
do {
char host[128];
uint16_t port;
syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port);
sDebug(
"vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, "
"match-index:%ld}",
ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex);
} while (0);
// send response // send response
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
@ -1271,6 +1071,17 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
pReply->success = false; pReply->success = false;
pReply->matchIndex = SYNC_INDEX_INVALID; pReply->matchIndex = SYNC_INDEX_INVALID;
// msg event log
do {
char host[128];
uint16_t port;
syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port);
sDebug(
"vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, "
"match-index:%ld}",
ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex);
} while (0);
// send response // send response
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
@ -1336,6 +1147,17 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
pReply->success = true; pReply->success = true;
pReply->matchIndex = hasAppendEntries ? pMsg->prevLogIndex + 1 : pMsg->prevLogIndex; pReply->matchIndex = hasAppendEntries ? pMsg->prevLogIndex + 1 : pMsg->prevLogIndex;
// msg event log
do {
char host[128];
uint16_t port;
syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port);
sDebug(
"vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, "
"match-index:%ld}",
ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex);
} while (0);
// send response // send response
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);

48
source/libs/sync/src/syncAppendEntriesReply.c
View File

@ -174,8 +174,12 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
SyncIndex newNextIndex = pMsg->matchIndex + 1; SyncIndex newNextIndex = pMsg->matchIndex + 1;
SyncIndex newMatchIndex = pMsg->matchIndex; SyncIndex newMatchIndex = pMsg->matchIndex;
if (ths->pLogStore->syncLogExist(ths->pLogStore, newNextIndex) && bool needStartSnapshot = false;
ths->pLogStore->syncLogExist(ths->pLogStore, newNextIndex - 1)) { if (newMatchIndex >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, newMatchIndex)) {
needStartSnapshot = true;
}
if (!needStartSnapshot) {
// update next-index, match-index // update next-index, match-index
syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), newNextIndex); syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), newNextIndex);
syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex); syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex);
@ -197,15 +201,36 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex); syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex);
} }
// event log, update next-index
do {
char host[64];
int16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
char logBuf[256];
snprintf(logBuf, sizeof(logBuf), "reset next-index:%ld, match-index:%ld for %s:%d", newNextIndex, newMatchIndex,
host, port);
syncNodeEventLog(ths, logBuf);
} while (0);
} else { } else {
SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId));
if (nextIndex > SYNC_INDEX_BEGIN) { if (nextIndex > SYNC_INDEX_BEGIN) {
--nextIndex; --nextIndex;
if (ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex) && bool needStartSnapshot = false;
ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex - 1)) { if (nextIndex >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex)) {
needStartSnapshot = true;
}
if (nextIndex - 1 >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex - 1)) {
needStartSnapshot = true;
}
if (!needStartSnapshot) {
// do nothing // do nothing
} else { } else {
SSyncRaftEntry* pEntry; SSyncRaftEntry* pEntry;
int32_t code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, nextIndex, &pEntry); int32_t code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, nextIndex, &pEntry);
@ -227,6 +252,21 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
nextIndex = SYNC_INDEX_BEGIN; nextIndex = SYNC_INDEX_BEGIN;
} }
syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex);
// event log, update next-index
do {
char host[64];
int16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
SyncIndex newNextIndex = nextIndex;
SyncIndex newMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId));
char logBuf[256];
snprintf(logBuf, sizeof(logBuf), "reset2 next-index:%ld, match-index:%ld for %s:%d", newNextIndex, newMatchIndex,
host, port);
syncNodeEventLog(ths, logBuf);
} while (0);
} }
return 0; return 0;

22
source/libs/sync/src/syncMain.c
View File

@ -1025,14 +1025,14 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
pSyncNode->FpOnSnapshotRsp = syncNodeOnSnapshotRspCb; pSyncNode->FpOnSnapshotRsp = syncNodeOnSnapshotRspCb;
if (pSyncNode->pRaftCfg->snapshotStrategy) { if (pSyncNode->pRaftCfg->snapshotStrategy) {
sInfo("sync node use snapshot"); sInfo("vgId:%d, sync node use snapshot", pSyncNode->vgId);
pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteSnapshotCb; pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteSnapshotCb;
pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplySnapshotCb; pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplySnapshotCb;
pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesSnapshotCb; pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesSnapshotCb;
pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplySnapshotCb; pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplySnapshotCb;
} else { } else {
sInfo("sync node do not use snapshot"); sInfo("vgId:%d, sync node do not use snapshot", pSyncNode->vgId);
pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb;
pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb; pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb;
pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb; pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb;
@ -1311,8 +1311,10 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRp
pMsg->info.noResp = 1; pMsg->info.noResp = 1;
pSyncNode->FpSendMsg(&epSet, pMsg); pSyncNode->FpSendMsg(&epSet, pMsg);
} else { } else {
sTrace("syncNodeSendMsgById pSyncNode->FpSendMsg is NULL"); sError("vgId:%d, sync send msg by id error, fp-send-msg is null", pSyncNode->vgId);
return -1;
} }
return 0; return 0;
} }
@ -1326,7 +1328,7 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S
pMsg->info.noResp = 1; pMsg->info.noResp = 1;
pSyncNode->FpSendMsg(&epSet, pMsg); pSyncNode->FpSendMsg(&epSet, pMsg);
} else { } else {
sTrace("syncNodeSendMsgByInfo pSyncNode->FpSendMsg is NULL"); sError("vgId:%d, sync send msg by info error, fp-send-msg is null", pSyncNode->vgId);
} }
return 0; return 0;
} }
@ -1526,12 +1528,14 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) {
snprintf(logBuf, sizeof(logBuf), snprintf(logBuf, sizeof(logBuf),
"vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, "
"strategy:%d, batch:%d, "
"replica-num:%d, " "replica-num:%d, "
"lconfig:%ld, changing:%d, restore:%d, %s", "lconfig:%ld, changing:%d, restore:%d, %s",
pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm,
pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex,
pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize,
pSyncNode->changing, pSyncNode->restoreFinish, printStr); pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing,
pSyncNode->restoreFinish, printStr);
} else { } else {
snprintf(logBuf, sizeof(logBuf), "%s", str); snprintf(logBuf, sizeof(logBuf), "%s", str);
} }
@ -1543,12 +1547,14 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) {
snprintf(s, len, snprintf(s, len,
"vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, "
"strategy:%d, batch:%d, "
"replica-num:%d, " "replica-num:%d, "
"lconfig:%ld, changing:%d, restore:%d, %s", "lconfig:%ld, changing:%d, restore:%d, %s",
pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm,
pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex,
pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize,
pSyncNode->changing, pSyncNode->restoreFinish, printStr); pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing,
pSyncNode->restoreFinish, printStr);
} else { } else {
snprintf(s, len, "%s", str); snprintf(s, len, "%s", str);
} }

2
source/libs/sync/src/syncMessage.c
View File

@ -1605,7 +1605,7 @@ void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) {
SyncAppendEntriesBatch* syncAppendEntriesBatchBuild(SSyncRaftEntry** entryPArr, int32_t arrSize, int32_t vgId) { SyncAppendEntriesBatch* syncAppendEntriesBatchBuild(SSyncRaftEntry** entryPArr, int32_t arrSize, int32_t vgId) {
ASSERT(entryPArr != NULL); ASSERT(entryPArr != NULL);
ASSERT(arrSize > 0); ASSERT(arrSize >= 0);
int32_t dataLen = 0; int32_t dataLen = 0;
int32_t metaArrayLen = sizeof(SOffsetAndContLen) * arrSize; // <offset, contLen> int32_t metaArrayLen = sizeof(SOffsetAndContLen) * arrSize; // <offset, contLen>

56
source/libs/sync/src/syncOnMessage.c
View File

@ -1,56 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncOnMessage.h"
// TLA+ Spec
// Receive(m) ==
// LET i == m.mdest
// j == m.msource
// IN \* Any RPC with a newer term causes the recipient to advance
// \* its term first. Responses with stale terms are ignored.
// \/ UpdateTerm(i, j, m)
// \/ /\ m.mtype = RequestVoteRequest
// /\ HandleRequestVoteRequest(i, j, m)
// \/ /\ m.mtype = RequestVoteResponse
// /\ \/ DropStaleResponse(i, j, m)
// \/ HandleRequestVoteResponse(i, j, m)
// \/ /\ m.mtype = AppendEntriesRequest
// /\ HandleAppendEntriesRequest(i, j, m)
// \/ /\ m.mtype = AppendEntriesResponse
// /\ \/ DropStaleResponse(i, j, m)
// \/ HandleAppendEntriesResponse(i, j, m)
// DuplicateMessage(m) ==
// /\ Send(m)
// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
// DropMessage(m) ==
// /\ Discard(m)
// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
// Next == /\ \/ \E i \in Server : Restart(i)
// \/ \E i \in Server : Timeout(i)
// \/ \E i,j \in Server : RequestVote(i, j)
// \/ \E i \in Server : BecomeLeader(i)
// \/ \E i \in Server, v \in Value : ClientRequest(i, v)
// \/ \E i \in Server : AdvanceCommitIndex(i)
// \/ \E i,j \in Server : AppendEntries(i, j)
// \/ \E m \in DOMAIN messages : Receive(m)
// \/ \E m \in DOMAIN messages : DuplicateMessage(m)
// \/ \E m \in DOMAIN messages : DropMessage(m)
// \* History variable that tracks every log ever:
// /\ allLogs' = allLogs \cup {log[i] : i \in Server}
//

8
source/libs/sync/src/syncRaftCfg.c
View File

@ -101,7 +101,7 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) {
char *syncCfg2Str(SSyncCfg *pSyncCfg) { char *syncCfg2Str(SSyncCfg *pSyncCfg) {
cJSON *pJson = syncCfg2Json(pSyncCfg); cJSON *pJson = syncCfg2Json(pSyncCfg);
char * serialized = cJSON_Print(pJson); char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson); cJSON_Delete(pJson);
return serialized; return serialized;
} }
@ -109,7 +109,7 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) {
char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) {
if (pSyncCfg != NULL) { if (pSyncCfg != NULL) {
int32_t len = 512; int32_t len = 512;
char * s = taosMemoryMalloc(len); char *s = taosMemoryMalloc(len);
memset(s, 0, len); memset(s, 0, len);
snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex);
@ -206,7 +206,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
char *raftCfg2Str(SRaftCfg *pRaftCfg) { char *raftCfg2Str(SRaftCfg *pRaftCfg) {
cJSON *pJson = raftCfg2Json(pRaftCfg); cJSON *pJson = raftCfg2Json(pRaftCfg);
char * serialized = cJSON_Print(pJson); char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson); cJSON_Delete(pJson);
return serialized; return serialized;
} }
@ -285,7 +285,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) {
(pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring); (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring);
} }
cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg));
ASSERT(code == 0); ASSERT(code == 0);

16
source/libs/sync/src/syncRaftLog.c
View File

@ -326,6 +326,14 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn
ASSERT(0); ASSERT(0);
} }
// event log
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "wal truncate, from-index:%ld", fromIndex);
syncNodeEventLog(pData->pSyncNode, logBuf);
} while (0);
return code; return code;
} }
@ -463,6 +471,14 @@ int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) {
ASSERT(0); ASSERT(0);
} }
// event log
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "wal truncate, from-index:%ld", fromIndex);
syncNodeEventLog(pData->pSyncNode, logBuf);
} while (0);
return 0; return 0;
} }

14
source/libs/sync/src/syncReplication.c
View File

@ -148,7 +148,7 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) {
// get entry batch // get entry batch
int32_t getCount = 0; int32_t getCount = 0;
SyncIndex getEntryIndex = nextIndex; SyncIndex getEntryIndex = nextIndex;
for (int32_t i = 0; i < pSyncNode->batchSize; ++i) { for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) {
SSyncRaftEntry* pEntry = NULL; SSyncRaftEntry* pEntry = NULL;
int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, getEntryIndex, &pEntry); int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, getEntryIndex, &pEntry);
if (code == 0) { if (code == 0) {
@ -162,12 +162,22 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) {
} }
} }
// event log
do {
char logBuf[128];
char host[64];
uint16_t port;
syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port);
snprintf(logBuf, sizeof(logBuf), "build batch:%d for %s:%d", getCount, host, port);
syncNodeEventLog(pSyncNode, logBuf);
} while (0);
// build msg // build msg
SyncAppendEntriesBatch* pMsg = syncAppendEntriesBatchBuild(entryPArr, getCount, pSyncNode->vgId); SyncAppendEntriesBatch* pMsg = syncAppendEntriesBatchBuild(entryPArr, getCount, pSyncNode->vgId);
ASSERT(pMsg != NULL); ASSERT(pMsg != NULL);
// free entries // free entries
for (int32_t i = 0; i < pSyncNode->batchSize; ++i) { for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) {
SSyncRaftEntry* pEntry = entryPArr[i]; SSyncRaftEntry* pEntry = entryPArr[i];
if (pEntry != NULL) { if (pEntry != NULL) {
syncEntryDestory(pEntry); syncEntryDestory(pEntry);

14
source/libs/sync/test/CMakeLists.txt
View File

@ -54,6 +54,7 @@ add_executable(syncRaftLogTest2 "")
add_executable(syncRaftLogTest3 "") add_executable(syncRaftLogTest3 "")
add_executable(syncLeaderTransferTest "") add_executable(syncLeaderTransferTest "")
add_executable(syncReconfigFinishTest "") add_executable(syncReconfigFinishTest "")
add_executable(syncRestoreFromSnapshot "")
target_sources(syncTest target_sources(syncTest
@ -280,6 +281,10 @@ target_sources(syncReconfigFinishTest
PRIVATE PRIVATE
"syncReconfigFinishTest.cpp" "syncReconfigFinishTest.cpp"
) )
target_sources(syncRestoreFromSnapshot
PRIVATE
"syncRestoreFromSnapshot.cpp"
)
target_include_directories(syncTest target_include_directories(syncTest
@ -562,6 +567,11 @@ target_include_directories(syncReconfigFinishTest
"${TD_SOURCE_DIR}/include/libs/sync" "${TD_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc" "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
) )
target_include_directories(syncRestoreFromSnapshot
PUBLIC
"${TD_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries(syncTest target_link_libraries(syncTest
@ -788,6 +798,10 @@ target_link_libraries(syncReconfigFinishTest
sync sync
gtest_main gtest_main
) )
target_link_libraries(syncRestoreFromSnapshot
sync
gtest_main
)
enable_testing() enable_testing()

1
source/libs/sync/test/syncLogStoreTest.cpp
View File

@ -85,6 +85,7 @@ void logStoreTest() {
} }
int main(int argc, char** argv) { int main(int argc, char** argv) {
gRaftDetailLog = true;
tsAsyncLog = 0; tsAsyncLog = 0;
sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;

78
source/libs/sync/test/syncRestoreFromSnapshot.cpp Normal file
View File

@ -0,0 +1,78 @@
#include <gtest/gtest.h>
#include <stdio.h>
#include "syncEnv.h"
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftLog.h"
#include "syncRaftStore.h"
#include "syncUtil.h"
#include "wal.h"
void logTest() {
sTrace("--- sync log test: trace");
sDebug("--- sync log test: debug");
sInfo("--- sync log test: info");
sWarn("--- sync log test: warn");
sError("--- sync log test: error");
sFatal("--- sync log test: fatal");
}
void init() {
int code = walInit();
assert(code == 0);
}
void cleanup() { walCleanUp(); }
SWal* createWal(char* path, int32_t vgId) {
SWalCfg walCfg;
memset(&walCfg, 0, sizeof(SWalCfg));
walCfg.vgId = vgId;
walCfg.fsyncPeriod = 1000;
walCfg.retentionPeriod = 1000;
walCfg.rollPeriod = 1000;
walCfg.retentionSize = 1000;
walCfg.segSize = 1000;
walCfg.level = TAOS_WAL_FSYNC;
SWal* pWal = walOpen(path, &walCfg);
assert(pWal != NULL);
return pWal;
}
SSyncNode* createSyncNode(SWal* pWal) {
SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode));
memset(pSyncNode, 0, sizeof(SSyncNode));
pSyncNode->pWal = pWal;
return pSyncNode;
}
void usage(char* exe) { printf("usage: %s path vgId snapshotIndex \n", exe); }
int main(int argc, char** argv) {
if (argc != 4) {
usage(argv[0]);
exit(-1);
}
char* path = argv[1];
int32_t vgId = atoi(argv[2]);
int64_t snapshotIndex = atoll(argv[3]);
init();
SWal* pWal = createWal(path, vgId);
assert(pWal != NULL);
SSyncNode* pSyncNode = createSyncNode(pWal);
assert(pSyncNode != NULL);
SSyncLogStore* pLog = logStoreCreate(pSyncNode);
assert(pLog != NULL);
int32_t code = pLog->syncLogRestoreFromSnapshot(pLog, snapshotIndex);
assert(code == 0);
walClose(pWal);
logStoreDestory(pLog);
taosMemoryFree(pSyncNode);
cleanup();
return 0;
}

17
source/common/src/tmsgcb.c → source/libs/transport/src/tmsgcb.c
View File

@ -16,20 +16,33 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "tmsgcb.h" #include "tmsgcb.h"
#include "taoserror.h" #include "taoserror.h"
#include "trpc.h"
static SMsgCb defaultMsgCb; static SMsgCb defaultMsgCb;
void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; } void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; }
int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) { int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) {
return (*msgcb->putToQueueFp)(msgcb->mgmt, qtype, pMsg); int32_t code = (*msgcb->putToQueueFp)(msgcb->mgmt, qtype, pMsg);
if (code != 0) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
}
return code;
} }
int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype) { int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype) {
return (*msgcb->qsizeFp)(msgcb->mgmt, vgId, qtype); return (*msgcb->qsizeFp)(msgcb->mgmt, vgId, qtype);
} }
int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { return (*defaultMsgCb.sendReqFp)(epSet, pMsg); } int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) {
int32_t code = (*defaultMsgCb.sendReqFp)(epSet, pMsg);
if (code != 0) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
}
return code;
}
void tmsgSendRsp(SRpcMsg* pMsg) { return (*defaultMsgCb.sendRspFp)(pMsg); } void tmsgSendRsp(SRpcMsg* pMsg) { return (*defaultMsgCb.sendRspFp)(pMsg); }

4
source/libs/transport/src/transCli.c
View File

@ -573,8 +573,8 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
return; return;
} }
if (nread < 0) { if (nread < 0) {
tError("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), tWarn("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread),
T_REF_VAL_GET(conn)); T_REF_VAL_GET(conn));
conn->broken = true; conn->broken = true;
cliHandleExcept(conn); cliHandleExcept(conn);
} }

10
source/libs/transport/src/transSvr.c
View File

@ -305,7 +305,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
return; return;
} }
tError("%s conn %p read error: %s", transLabel(pTransInst), conn, uv_err_name(nread)); tWarn("%s conn %p read error: %s", transLabel(pTransInst), conn, uv_err_name(nread));
if (nread < 0) { if (nread < 0) {
conn->broken = true; conn->broken = true;
if (conn->status == ConnAcquire) { if (conn->status == ConnAcquire) {
@ -390,8 +390,9 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
pHead->traceId = pMsg->info.traceId; pHead->traceId = pMsg->info.traceId;
pHead->hasEpSet = pMsg->info.hasEpSet; pHead->hasEpSet = pMsg->info.hasEpSet;
if (pConn->status == ConnNormal) { if (pConn->status == ConnNormal) {
pHead->msgType = pConn->inType + 1; pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType);
} else { } else {
if (smsg->type == Release) { if (smsg->type == Release) {
pHead->msgType = 0; pHead->msgType = 0;
@ -400,11 +401,8 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
destroyConnRegArg(pConn); destroyConnRegArg(pConn);
transUnrefSrvHandle(pConn); transUnrefSrvHandle(pConn);
} else { } else {
pHead->msgType = pMsg->msgType;
// set up resp msg type // set up resp msg type
if (pHead->msgType == 0 && transMsgLenFromCont(pMsg->contLen) == sizeof(STransMsgHead)) { pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType);
pHead->msgType = pConn->inType + 1;
}
} }
} }

58
source/libs/transport/test/CMakeLists.txt
View File

@ -1,30 +1,16 @@
add_executable(transportTest "") add_executable(transportTest "")
add_executable(client "")
add_executable(server "")
add_executable(transUT "") add_executable(transUT "")
add_executable(syncClient "")
add_executable(pushServer "") add_executable(pushServer "")
target_sources(transUT target_sources(transUT
PRIVATE PRIVATE
"transUT.cpp" "transUT.cpp"
) )
target_sources(transportTest target_sources(transportTest
PRIVATE PRIVATE
"transportTests.cpp" "transportTests.cpp"
) )
target_sources (client
PRIVATE
"rclient.c"
)
target_sources (server
PRIVATE
"rserver.c"
)
target_sources (syncClient
PRIVATE
"syncClient.c"
)
target_sources(pushServer target_sources(pushServer
PRIVATE PRIVATE
@ -44,6 +30,7 @@ target_link_libraries (transportTest
gtest_main gtest_main
transport transport
) )
target_link_libraries (transUT target_link_libraries (transUT
os os
util util
@ -52,56 +39,18 @@ target_link_libraries (transUT
transport transport
) )
target_include_directories(client
PUBLIC
"${TD_SOURCE_DIR}/include/libs/transport"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (client
os
util
common
gtest_main
transport
)
target_include_directories(server
PUBLIC
"${TD_SOURCE_DIR}/include/libs/transport"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories(transUT target_include_directories(transUT
PUBLIC PUBLIC
"${TD_SOURCE_DIR}/include/libs/transport" "${TD_SOURCE_DIR}/include/libs/transport"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc" "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
) )
target_link_libraries (server
os
util
common
gtest_main
transport
)
target_include_directories(syncClient
PUBLIC
"${TD_SOURCE_DIR}/include/libs/transport"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (syncClient
os
util
common
gtest_main
transport
)
target_include_directories(pushServer target_include_directories(pushServer
PUBLIC PUBLIC
"${TD_SOURCE_DIR}/include/libs/transport" "${TD_SOURCE_DIR}/include/libs/transport"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc" "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
) )
target_link_libraries (pushServer target_link_libraries (pushServer
os os
util util
@ -110,7 +59,6 @@ target_link_libraries (pushServer
transport transport
) )
add_test( add_test(
NAME transUT NAME transUT
COMMAND transUT COMMAND transUT

Some files were not shown because too many files have changed in this diff Show More