xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'n2' into new

This commit is contained in:
yihaoDeng 2022-10-16 21:33:12 +08:00
parent 4ff0b21bb4 9b70b429a5
commit 1132f72440
109 changed files with 1628 additions and 979 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG d58230c
GIT_TAG 2849aa4
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

4
docs/en/12-taos-sql/28-recovery.md
View File

@ -16,10 +16,10 @@ You can use the SHOW CONNECTIONS statement to find the conn_id.
## Terminate a Query
```sql
SHOW QUERY query_id;
KILL QUERY kill_id;
```
You can use the SHOW QUERIES statement to find the query_id.
You can use the SHOW QUERIES statement to find the kill_id.
## Terminate a Transaction

2
docs/en/14-reference/05-taosbenchmark.md
View File

@ -231,7 +231,7 @@ The parameters related to database creation are configured in `dbinfo` in the js
- **name**: specify the name of the database.
- **drop**: indicate whether to delete the database before inserting. The default is true.
- **drop**: indicate whether to delete the database before inserting. The value can be 'yes' or 'no'. No means do not drop. The default is to drop.
#### Stream processing related configuration parameters

2
docs/zh/12-taos-sql/10-function.md
View File

@ -340,7 +340,7 @@ LTRIM(expr)
#### RTRIM
```sql
LTRIM(expr)
RTRIM(expr)
```
**功能说明**：返回清除右边空格后的字符串。

4
docs/zh/12-taos-sql/28-recovery.md
View File

@ -17,10 +17,10 @@ conn_id 可以通过 `SHOW CONNECTIONS` 获取。
## 终止查询
```sql
SHOW QUERY query_id;
KILL QUERY kill_id;
```
query_id 可以通过 `SHOW QUERIES` 获取。
kill_id 可以通过 `SHOW QUERIES` 获取。
## 终止事务

7
docs/zh/12-taos-sql/29-changes.md
View File

@ -94,3 +94,10 @@ description: "TDengine 3.0 版本的语法变更说明"
| 9 | SAMPLE | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
| 10 | STATECOUNT | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
| 11 | STATEDURATION | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
## SCHEMALESS 变更
| # | **元素** | **<div style={{width: 60}}>差异性</div>** | **说明** |
| - | :------- | :-------- | :------- |
| 1 | 主键ts 变更为 _ts | 变更 | schemaless自动建的列名用 _ 开头，不同于2.x。

2
docs/zh/14-reference/05-taosbenchmark.md
View File

@ -231,7 +231,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
- **name** : 数据库名。
- **drop** : 插入前是否删除数据库，默认为 true。
- **drop** : 插入前是否删除数据库，可选项为 "yes" 或者 "no", 为 "no" 时不创建。默认删除。
#### 流式计算相关配置参数

2
docs/zh/14-reference/13-schemaless/13-schemaless.md
View File

@ -115,7 +115,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000
## 数据模式映射规则
本节将说明行协议的数据如何映射成为具有模式的数据。每个行协议中数据 measurement 映射为
本节将说明 InfluxDB 行协议(Line Protocol)的数据如何映射成为具有模式的数据。每个行协议中数据 measurement 映射为
超级表名称。tag_set 中的 标签名称为 数据模式中的标签名，field_set 中的名称为列名称。以如下数据为例，说明映射规则：
```json

68
examples/c/demoapi.c
View File

@ -1,5 +1,17 @@
// C api call sequence demo
// to compile: gcc -o apidemo apidemo.c -ltaos
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdio.h>
#include <stdlib.h>
@ -13,8 +25,8 @@
#include "taos.h"
#define debugPrint(fmt, ...) \
do { if (g_args.debug_print || g_args.verbose_print) \
fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); } while(0)
do { if (g_args.debug_print || g_args.verbose_print) {\
fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); }} while (0)
#define warnPrint(fmt, ...) \
do { fprintf(stderr, "\033[33m"); \
@ -74,10 +86,16 @@ static void prepare_data(TAOS* taos) {
res = taos_query(taos, "create database test;");
taos_free_result(res);
taosMsleep(100);
taos_select_db(taos, "test");
if (taos_select_db(taos, "test")) {
errorPrint("%s() LN%d: taos_select_db() failed\n",
__func__, __LINE__);
return;
}
char command[1024] = {0};
sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, city binary(20), dist nchar(20), street binary(20));");
sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, "
"bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, "
"city binary(20), dist nchar(20), street binary(20));");
res = taos_query(taos, command);
if ((res) && (0 == taos_errno(res))) {
okPrint("%s created\n", "meters");
@ -117,12 +135,11 @@ static void prepare_data(TAOS* taos) {
sprintf(command, "insert into t%"PRId64" "
"values(%" PRId64 ", %f, %"PRId64", "
"'%c%d', '%s%c%d', '%c%d')",
i, 1650000000000+j, (float)j, j,
i, 1650000000000+j, j * 1.0, j,
'a'+(int)j%25, rand(),
// "涛思", 'z' - (int)j%25, rand(),
"TAOS", 'z' - (int)j%25, rand(),
'b' - (int)j%25, rand()
);
'b' - (int)j%25, rand());
res = taos_query(taos, command);
if ((res) && (0 == taos_errno(res))) {
affected = taos_affected_rows(res);
@ -150,8 +167,7 @@ static void prepare_data(TAOS* taos) {
i, 1650000000000+j+1, (float)j, j,
'a'+(int)j%25, rand(),
"数据", 'z' - (int)j%25, rand(),
'b' - (int)j%25, rand()
);
'b' - (int)j%25, rand());
res = taos_query(taos, command);
if ((res) && (0 == taos_errno(res))) {
affected = taos_affected_rows(res);
@ -179,7 +195,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
}
if (block) {
warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n");
warnPrint("%s", "call taos_fetch_block(), "
"don't call taos_fetch_lengths()\n");
int rows = 0;
while ((rows = taos_fetch_block(res, &row))) {
int *lengths = taos_fetch_lengths(res);
@ -199,7 +216,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d, row: %"PRId64", "
"value: %"PRId64"\n",
f, c,
*(int64_t*)((char*)(row[f])+c*sizeof(int64_t)));
*(int64_t*)((char*)(row[f])
+c*sizeof(int64_t)));
}
break;
@ -211,7 +229,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d, row: %"PRId64", "
"value: %d\n",
f, c,
*(int32_t*)((char*)(row[f])+c*sizeof(int32_t)));
*(int32_t*)((char*)(row[f])
+c*sizeof(int32_t)));
}
break;
@ -223,7 +242,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d, row: %"PRId64", "
"value: %f\n",
f, c,
*(float*)((char*)(row[f])+c*sizeof(float)));
*(float*)((char*)(row[f])
+c*sizeof(float)));
}
break;
@ -238,14 +258,18 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
if (offsets) {
for (int c = 0; c < rows; c++) {
if (offsets[c] != -1) {
int length = *(int16_t*)((char*)(row[f]) + offsets[c]);
int length = *(int16_t*)((char*)(row[f])
+ offsets[c]);
char *buf = calloc(1, length + 1);
strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length);
printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n",
strncpy(buf, (char *)((char*)(row[f])
+ offsets[c] + 2), length);
printf("row: %d, col: %d, offset: %d, "
"length: %d, content: %s\n",
c, f, offsets[c], length, buf);
free(buf);
} else {
printf("row: %d, col: %d, offset: -1, means content is NULL\n",
printf("row: %d, col: %d, offset: -1, "
"means content is NULL\n",
c, f);
}
}
@ -267,7 +291,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
int* lengths = taos_fetch_lengths(res);
if (lengths) {
for (int c = 0; c < num_fields; c++) {
printf("row: %"PRId64", col: %d, is_null: %s, length of column %d is %d\n",
printf("row: %"PRId64", col: %d, is_null: %s, "
"length of column %d is %d\n",
num_rows, c,
taos_is_null(res, num_rows, c)?"True":"False",
c, lengths[c]);
@ -322,7 +347,8 @@ int main(int argc, char *argv[]) {
#endif
TAOS* taos = taos_connect(host, user, passwd, "", 0);
if (taos == NULL) {
printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos));
printf("\033[31mfailed to connect to db, reason:%s\033[0m\n",
taos_errstr(taos));
exit(1);
}

1
include/common/tglobal.h
View File

@ -93,6 +93,7 @@ extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in
// query client
extern int32_t tsQueryPolicy;
extern int32_t tsQueryRspPolicy;
extern int32_t tsQuerySmaOptimize;
extern int32_t tsQueryRsmaTolerance;
extern bool tsQueryPlannerTrace;

5
include/dnode/mnode/mnode.h
View File

@ -30,9 +30,10 @@ typedef struct SMnode SMnode;
typedef struct {
int32_t dnodeId;
bool standby;
bool deploy;
SReplica replica;
int8_t selfIndex;
int8_t numOfReplicas;
SReplica replicas[TSDB_MAX_REPLICA];
SMsgCb msgCb;
} SMnodeOpt;

3
include/libs/qcom/query.h
View File

@ -54,6 +54,9 @@ typedef enum {
#define QUERY_POLICY_QNODE 3
#define QUERY_POLICY_CLIENT 4
#define QUERY_RSP_POLICY_DELAY 0
#define QUERY_RSP_POLICY_QUICK 1
typedef struct STableComInfo {
uint8_t numOfTags; // the number of tags in schema
uint8_t precision; // the number of precision

4
include/libs/sync/sync.h
View File

@ -211,7 +211,7 @@ typedef struct SSyncInfo {
int32_t syncInit();
void syncCleanUp();
int64_t syncOpen(const SSyncInfo* pSyncInfo);
int64_t syncOpen(SSyncInfo* pSyncInfo);
void syncStart(int64_t rid);
void syncStop(int64_t rid);
int32_t syncSetStandby(int64_t rid);
@ -233,7 +233,7 @@ const char* syncStr(ESyncState state);
bool syncIsRestoreFinish(int64_t rid);
int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapshot);
int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg);
int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg);
// build SRpcMsg, need to call syncPropose with SRpcMsg
int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg);

6
include/util/trbtree.h
View File

@ -26,7 +26,7 @@ typedef struct SRBTree SRBTree;
typedef struct SRBTreeNode SRBTreeNode;
typedef struct SRBTreeIter SRBTreeIter;
typedef int32_t (*tRBTreeCmprFn)(const void *, const void *);
typedef int32_t (*tRBTreeCmprFn)(const SRBTreeNode *, const SRBTreeNode *);
// SRBTree =============================================
#define tRBTreeMin(T) ((T)->min == ((T)->NIL) ? NULL : (T)->min)
@ -36,7 +36,7 @@ void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn);
SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z);
void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z);
SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey);
SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey);
SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode);
// SRBTreeIter =============================================
#define tRBTreeIterCreate(tree, ascend) \
@ -53,8 +53,6 @@ struct SRBTreeNode {
SRBTreeNode *right;
};
#define RBTREE_NODE_PAYLOAD(N) ((const void *)&(N)[1])
struct SRBTree {
tRBTreeCmprFn cmprFn;
int64_t n;

10
source/client/src/clientImpl.c
View File

@ -1036,6 +1036,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
pRequest->type = pQuery->msgType;
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
SPlanContext cxt = {.queryId = pRequest->requestId,
.acctId = pRequest->pTscObj->acctId,
.mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
@ -1046,7 +1047,11 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
.pUser = pRequest->pTscObj->user,
.sysInfo = pRequest->pTscObj->sysInfo,
.allocatorId = pRequest->allocatorRefId};
SAppInstInfo* pAppInfo = getAppInfo(pRequest);
SQueryPlan* pDag = NULL;
int64_t st = taosGetTimestampUs();
int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
if (code) {
tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
@ -1056,7 +1061,10 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
}
pRequest->metric.planEnd = taosGetTimestampUs();
if (code == TSDB_CODE_SUCCESS) {
tscDebug("0x%" PRIx64 " create query plan success, elapsed time:%.2f ms, 0x%" PRIx64, pRequest->self,
(pRequest->metric.planEnd - st)/1000.0, pRequest->requestId);
}
if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
SArray* pNodeList = NULL;
buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);

2
source/client/src/clientMain.c
View File

@ -222,7 +222,9 @@ void taos_kill_query(TAOS *taos) {
int64_t rid = *(int64_t *)taos;
STscObj *pTscObj = acquireTscObj(rid);
if (pTscObj) {
stopAllRequests(pTscObj->pRequests);
}
releaseTscObj(rid);
}

5
source/client/src/clientTmq.c
View File

@ -972,6 +972,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
req.topicNames = taosArrayInit(sz, sizeof(void*));
if (req.topicNames == NULL) goto FAIL;
tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz);
for (int32_t i = 0; i < sz; i++) {
char* topic = taosArrayGetP(container, i);
@ -1297,7 +1299,8 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) {
pParam->code = code;
if (code != 0) {
tscError("consumer:%" PRId64 ", get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async);
tscError("consumer:%" PRId64 ", get topic endpoint error, not ready, wait:%d, code %x", tmq->consumerId,
pParam->async, code);
goto END;
}

84
source/client/test/clientTests.cpp
View File

@ -52,15 +52,15 @@ void printResult(TAOS_RES* pRes) {
int32_t n = 0;
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t* length = taos_fetch_lengths(pRes);
for (int32_t i = 0; i < numOfFields; ++i) {
printf("(%d):%d ", i, length[i]);
}
printf("\n");
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str);
memset(str, 0, sizeof(str));
// int32_t* length = taos_fetch_lengths(pRes);
// for(int32_t i = 0; i < numOfFields; ++i) {
// printf("(%d):%d " , i, length[i]);
// }
// printf("\n");
//
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// memset(str, 0, sizeof(str));
}
}
@ -102,17 +102,6 @@ void queryCallback(void* param, void* res, int32_t code) {
taos_fetch_raw_block_a(res, fetchCallback, param);
}
void queryCallback1(void* param, void* res, int32_t code) {
if (code != TSDB_CODE_SUCCESS) {
printf("failed to execute, reason:%s\n", taos_errstr(res));
}
taos_free_result(res);
printf("exec query:\n");
taos_query_a(param, "select * from tm1", queryCallback, param);
}
void createNewTable(TAOS* pConn, int32_t index) {
char str[1024] = {0};
sprintf(str, "create table tu%d using st2 tags(%d)", index, index);
@ -123,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) {
}
taos_free_result(pRes);
for (int32_t i = 0; i < 3280; i += 20) {
for(int32_t i = 0; i < 10000; i += 20) {
char sql[1024] = {0};
sprintf(sql,
"insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
@ -141,10 +130,49 @@ void createNewTable(TAOS* pConn, int32_t index) {
taos_free_result(p);
}
}
void *queryThread(void *arg) {
TAOS* pConn = taos_connect("192.168.0.209", "root", "taosdata", NULL, 0);
if (pConn == NULL) {
printf("failed to connect to db, reason:%s", taos_errstr(pConn));
return NULL;
}
int64_t el = 0;
for (int32_t i = 0; i < 5000000; ++i) {
int64_t st = taosGetTimestampUs();
TAOS_RES* pRes = taos_query(pConn,
"SELECT _wstart as ts,max(usage_user) FROM benchmarkcpu.host_49 WHERE ts >= 1451618560000 AND ts < 1451622160000 INTERVAL(1m) ;");
if (taos_errno(pRes) != 0) {
printf("failed, reason:%s\n", taos_errstr(pRes));
} else {
printResult(pRes);
}
taos_free_result(pRes);
el += (taosGetTimestampUs() - st);
if (i % 1000 == 0 && i != 0) {
printf("total:%d, avg time:%.2fms\n", i, el/(double)(i*1000));
}
}
taos_close(pConn);
return NULL;
}
static int32_t numOfThreads = 1;
} // namespace
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
if (argc > 1) {
numOfThreads = atoi(argv[1]);
}
numOfThreads = TMAX(numOfThreads, 1);
printf("the runing threads is:%d", numOfThreads);
return RUN_ALL_TESTS();
}
@ -664,7 +692,6 @@ TEST(testCase, insert_test) {
taos_free_result(pRes);
taos_close(pConn);
}
#endif
TEST(testCase, projection_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
@ -676,13 +703,14 @@ TEST(testCase, projection_query_tables) {
// }
// taos_free_result(pRes);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
TAOS_RES* pRes = taos_query(pConn, "use benchmarkcpu");
taos_free_result(pRes);
pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
if (taos_errno(pRes) != 0) {
printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)");
@ -722,6 +750,16 @@ TEST(testCase, projection_query_tables) {
taos_free_result(pRes);
taos_close(pConn);
}
#endif
TEST(testCase, tsbs_perf_test) {
TdThread qid[20] = {0};
for(int32_t i = 0; i < numOfThreads; ++i) {
taosThreadCreate(&qid[i], NULL, queryThread, NULL);
}
getchar();
}
#if 0
TEST(testCase, projection_query_stables) {

3
source/common/src/tglobal.c
View File

@ -90,6 +90,7 @@ bool tsSmlDataFormat = false;
// query
int32_t tsQueryPolicy = 1;
int32_t tsQueryRspPolicy = 0;
int32_t tsQuerySmaOptimize = 0;
int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data.
bool tsQueryPlannerTrace = false;
@ -350,6 +351,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, 0) != 0) return -1;
if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "multiProcess", tsMultiProcess, 0, 2, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "mnodeShmSize", tsMnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1;
@ -728,6 +730,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32;
tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32;
tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval;
tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32;
tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32;

4
source/dnode/mgmt/mgmt_mnode/inc/mmInt.h
View File

@ -41,8 +41,8 @@ typedef struct SMnodeMgmt {
} SMnodeMgmt;
// mmFile.c
int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed);
int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed);
int32_t mmReadFile(const char *path, SMnodeOpt *pOption);
int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption);
// mmHandle.c
SArray *mmGetMsgHandles();

70
source/dnode/mgmt/mgmt_mnode/src/mmFile.c
View File

@ -16,7 +16,7 @@
#define _DEFAULT_SOURCE
#include "mmInt.h"
int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
int32_t code = TSDB_CODE_INVALID_JSON_FORMAT;
int32_t len = 0;
int32_t maxLen = 4096;
@ -25,7 +25,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
char file[PATH_MAX] = {0};
TdFilePtr pFile = NULL;
snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
code = 0;
@ -50,9 +50,38 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
dError("failed to read %s since deployed not found", file);
goto _OVER;
}
*pDeployed = deployed->valueint;
pOption->deploy = deployed->valueint;
cJSON *id = cJSON_GetObjectItem(root, "id");
cJSON *selfIndex = cJSON_GetObjectItem(root, "selfIndex");
if (selfIndex) {
if (selfIndex->type != cJSON_Number) {
dError("failed to read %s since selfIndex not found", file);
goto _OVER;
}
pOption->selfIndex = selfIndex->valueint;
}
cJSON *replicas = cJSON_GetObjectItem(root, "replicas");
if (replicas) {
if (replicas->type != cJSON_Array) {
dError("failed to read %s since replicas not found", file);
goto _OVER;
}
int32_t numOfReplicas = cJSON_GetArraySize(replicas);
if (numOfReplicas <= 0) {
dError("failed to read %s since numOfReplicas:%d invalid", file, numOfReplicas);
goto _OVER;
}
pOption->numOfReplicas = numOfReplicas;
for (int32_t i = 0; i < numOfReplicas; ++i) {
SReplica *pReplica = pOption->replicas + i;
cJSON *replica = cJSON_GetArrayItem(replicas, i);
if (replica == NULL) break;
cJSON *id = cJSON_GetObjectItem(replica, "id");
if (id) {
if (id->type != cJSON_Number) {
dError("failed to read %s since id not found", file);
@ -63,7 +92,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
}
}
cJSON *fqdn = cJSON_GetObjectItem(root, "fqdn");
cJSON *fqdn = cJSON_GetObjectItem(replica, "fqdn");
if (fqdn) {
if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
dError("failed to read %s since fqdn not found", file);
@ -74,7 +103,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
}
}
cJSON *port = cJSON_GetObjectItem(root, "port");
cJSON *port = cJSON_GetObjectItem(replica, "port");
if (port) {
if (port->type != cJSON_Number) {
dError("failed to read %s since port not found", file);
@ -84,6 +113,8 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
pReplica->port = (uint16_t)port->valueint;
}
}
}
}
code = 0;
@ -92,18 +123,18 @@ _OVER:
if (root != NULL) cJSON_Delete(root);
if (pFile != NULL) taosCloseFile(&pFile);
if (code == 0) {
dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed);
dDebug("succcessed to read file %s, deployed:%d", file, pOption->deploy);
}
terrno = code;
return code;
}
int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) {
int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP);
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
@ -117,12 +148,25 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed)
char *content = taosMemoryCalloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
if (pOption->deploy && pOption->numOfReplicas > 0) {
len += snprintf(content + len, maxLen - len, " \"selfIndex\": %d,\n", pOption->selfIndex);
len += snprintf(content + len, maxLen - len, " \"replicas\": [{\n");
for (int32_t i = 0; i < pOption->numOfReplicas; ++i) {
const SReplica *pReplica = pOption->replicas + i;
if (pReplica != NULL && pReplica->id > 0) {
len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id);
len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn);
len += snprintf(content + len, maxLen - len, " \"port\": %u\n,", pReplica->port);
len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port);
}
len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed);
if (i < pOption->numOfReplicas - 1) {
len += snprintf(content + len, maxLen - len, " },{\n");
} else {
len += snprintf(content + len, maxLen - len, " }],\n");
}
}
}
len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", pOption->deploy);
len += snprintf(content + len, maxLen - len, "}\n");
taosWriteFile(pFile, content, len);
@ -136,6 +180,6 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed)
return -1;
}
dDebug("successed to write %s, deployed:%d", realfile, deployed);
dDebug("successed to write %s, deployed:%d", realfile, pOption->deploy);
return 0;
}

28
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -80,18 +80,21 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
return -1;
}
if (createReq.replica != 1) {
SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, .selfIndex = -1};
memcpy(option.replicas, createReq.replicas, sizeof(createReq.replicas));
for (int32_t i = 0; i < option.numOfReplicas; ++i) {
if (createReq.replicas[i].id == pInput->pData->dnodeId) {
option.selfIndex = i;
}
}
if (option.selfIndex == -1) {
terrno = TSDB_CODE_INVALID_OPTION;
dGError("failed to create mnode since %s", terrstr());
dGError("failed to create mnode since %s, selfIndex is -1", terrstr());
return -1;
}
bool deployed = true;
SMnodeMgmt mgmt = {0};
mgmt.path = pInput->path;
mgmt.name = pInput->name;
if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) {
if (mmWriteFile(pInput->path, &option) != 0) {
dGError("failed to write mnode file since %s", terrstr());
return -1;
}
@ -113,12 +116,8 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
return -1;
}
bool deployed = false;
SMnodeMgmt mgmt = {0};
mgmt.path = pInput->path;
mgmt.name = pInput->name;
if (mmWriteFile(&mgmt, NULL, deployed) != 0) {
SMnodeOpt option = {.deploy = false};
if (mmWriteFile(pInput->path, &option) != 0) {
dGError("failed to write mnode file since %s", terrstr());
return -1;
}
@ -207,7 +206,6 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
// if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;

42
source/dnode/mgmt/mgmt_mnode/src/mmInt.c
View File

@ -25,38 +25,35 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) {
}
static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
SMnodeMgmt mgmt = {0};
mgmt.path = pInput->path;
if (mmReadFile(&mgmt, NULL, required) != 0) {
SMnodeOpt option = {0};
if (mmReadFile(pInput->path, &option) != 0) {
return -1;
}
if (!(*required)) {
if (!option.deploy) {
*required = mmDeployRequired(pInput);
} else {
*required = true;
}
return 0;
}
static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) {
pOption->standby = false;
pOption->deploy = true;
pOption->msgCb = pMgmt->msgCb;
pOption->dnodeId = pMgmt->pData->dnodeId;
pOption->replica.id = 1;
pOption->replica.port = tsServerPort;
tstrncpy(pOption->replica.fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
pOption->selfIndex = 0;
pOption->numOfReplicas = 1;
pOption->replicas[0].id = 1;
pOption->replicas[0].port = tsServerPort;
tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
}
static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) {
pOption->standby = false;
static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
pOption->deploy = false;
pOption->msgCb = pMgmt->msgCb;
pOption->dnodeId = pMgmt->pData->dnodeId;
if (pReplica->id > 0) {
pOption->standby = true;
pOption->replica = *pReplica;
}
}
static void mmClose(SMnodeMgmt *pMgmt) {
@ -95,22 +92,20 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->msgCb.mgmt = pMgmt;
taosThreadRwlockInit(&pMgmt->lock, NULL);
bool deployed = false;
SReplica replica = {0};
if (mmReadFile(pMgmt, &replica, &deployed) != 0) {
SMnodeOpt option = {0};
if (mmReadFile(pMgmt->path, &option) != 0) {
dError("failed to read file since %s", terrstr());
mmClose(pMgmt);
return -1;
}
SMnodeOpt option = {0};
if (!deployed) {
if (!option.deploy) {
dInfo("mnode start to deploy");
pMgmt->pData->dnodeId = 1;
mmBuildOptionForDeploy(pMgmt, pInput, &option);
} else {
dInfo("mnode start to open");
mmBuildOptionForOpen(pMgmt, &replica, &option);
mmBuildOptionForOpen(pMgmt, &option);
}
pMgmt->pMnode = mndOpen(pMgmt->path, &option);
@ -128,9 +123,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
}
tmsgReportStartup("mnode-worker", "initialized");
if (!deployed || replica.id > 0) {
deployed = true;
if (mmWriteFile(pMgmt, NULL, deployed) != 0) {
if (option.numOfReplicas > 0) {
option.deploy = true;
option.numOfReplicas = 0;
if (mmWriteFile(pMgmt->path, &option) != 0) {
dError("failed to write mnode file since %s", terrstr());
return -1;
}

2
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -72,7 +72,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) {
mndPostProcessQueryMsg(pMsg);
}
dGTrace("msg:%p, is freed, code:0x%x", pMsg, code);
dGTrace("msg:%p is freed, code:%s", pMsg, tstrerror(code));
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}

10
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -87,12 +87,13 @@ typedef struct {
typedef struct {
tsem_t syncSem;
int64_t sync;
SReplica replica;
int32_t errCode;
int32_t transId;
SRWLatch lock;
int8_t standby;
int8_t leaderTransferFinish;
int8_t selfIndex;
int8_t numOfReplicas;
SReplica replicas[TSDB_MAX_REPLICA];
} SSyncMgmt;
typedef struct {
@ -130,9 +131,8 @@ typedef struct SMnode {
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
int64_t mndGenerateUid(const char *name, int32_t len);
int32_t mndAcquireRpcRef(SMnode *pMnode);
void mndReleaseRpcRef(SMnode *pMnode);
void mndSetRestore(SMnode *pMnode, bool restored);
void mndSetRestored(SMnode *pMnode, bool restored);
bool mndGetRestored(SMnode *pMnode);
void mndSetStop(SMnode *pMnode);
bool mndGetStop(SMnode *pMnode);

2
source/dnode/mnode/impl/inc/mndSync.h
View File

@ -24,7 +24,7 @@ extern "C" {
int32_t mndInitSync(SMnode *pMnode);
void mndCleanupSync(SMnode *pMnode);
bool mndIsMaster(SMnode *pMnode);
bool mndIsLeader(SMnode *pMnode);
int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId);
void mndSyncStart(SMnode *pMnode);
void mndSyncStop(SMnode *pMnode);

2
source/dnode/mnode/impl/src/mndDb.c
View File

@ -1809,7 +1809,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
}
while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus);
pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus, true);
if (pShow->pIter == NULL) break;
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) {

3
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -423,7 +423,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
goto _OVER;
} else {
pDnode->accessTimes++;
mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes);
mDebug("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes);
}
}
@ -471,6 +471,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
}
pDnode->lastAccessTime = curMs;
pDnode->accessTimes++;
code = 0;
_OVER:

87
source/dnode/mnode/impl/src/mndMain.c
View File

@ -43,6 +43,37 @@
#include "mndUser.h"
#include "mndVgroup.h"
static inline int32_t mndAcquireRpc(SMnode *pMnode) {
int32_t code = 0;
taosThreadRwlockRdlock(&pMnode->lock);
if (pMnode->stopped) {
terrno = TSDB_CODE_APP_NOT_READY;
code = -1;
} else if (!mndIsLeader(pMnode)) {
code = -1;
} else {
#if 1
atomic_add_fetch_32(&pMnode->rpcRef, 1);
#else
int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1);
mTrace("mnode rpc is acquired, ref:%d", ref);
#endif
}
taosThreadRwlockUnlock(&pMnode->lock);
return code;
}
static inline void mndReleaseRpc(SMnode *pMnode) {
taosThreadRwlockRdlock(&pMnode->lock);
#if 1
atomic_sub_fetch_32(&pMnode->rpcRef, 1);
#else
int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1);
mTrace("mnode rpc is released, ref:%d", ref);
#endif
taosThreadRwlockUnlock(&pMnode->lock);
}
static void *mndBuildTimerMsg(int32_t *pContLen) {
SMTimerReq timerReq = {0};
@ -338,8 +369,9 @@ static int32_t mndExecSteps(SMnode *pMnode) {
static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
pMnode->msgCb = pOption->msgCb;
pMnode->selfDnodeId = pOption->dnodeId;
pMnode->syncMgmt.replica = pOption->replica;
pMnode->syncMgmt.standby = pOption->standby;
pMnode->syncMgmt.selfIndex = pOption->selfIndex;
pMnode->syncMgmt.numOfReplicas = pOption->numOfReplicas;
memcpy(pMnode->syncMgmt.replicas, pOption->replicas, sizeof(pOption->replicas));
}
SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
@ -430,7 +462,7 @@ int32_t mndStart(SMnode *pMnode) {
mError("failed to deploy sdb while start mnode");
return -1;
}
mndSetRestore(pMnode, true);
mndSetRestored(pMnode, true);
}
grantReset(pMnode, TSDB_GRANT_ALL, 0);
@ -570,7 +602,7 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) {
return 0;
}
if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0;
if (mndAcquireRpc(pMsg->info.node) == 0) return 0;
if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER ||
pMsg->msgType == TDMT_MND_UPTIME_TIMER) {
@ -578,15 +610,19 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
}
SEpSet epSet = {0};
mndGetMnodeEpSet(pMsg->info.node, &epSet);
SMnode *pMnode = pMsg->info.node;
mndGetMnodeEpSet(pMnode, &epSet);
const STraceId *trace = &pMsg->info.traceId;
mError("msg:%p, failed to check mnode state since %s, type:%s, numOfMnodes:%d inUse:%d", pMsg, terrstr(),
TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse);
mDebug(
"msg:%p, failed to check mnode state since %s, mnode restored:%d stopped:%d, sync restored:%d role:%s type:%s "
"numOfEps:%d inUse:%d",
pMsg, terrstr(), pMnode->restored, pMnode->stopped, syncIsRestoreFinish(pMnode->syncMgmt.sync),
syncGetMyRoleStr(pMnode->syncMgmt.sync), TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse);
if (epSet.numOfEps > 0) {
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
}
int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
@ -633,7 +669,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
int32_t code = (*fp)(pMsg);
mndReleaseRpcRef(pMnode);
mndReleaseRpc(pMnode);
if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
mGTrace("msg:%p, won't response immediately since in progress", pMsg);
@ -669,7 +705,7 @@ int64_t mndGenerateUid(const char *name, int32_t len) {
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) {
if (mndAcquireRpcRef(pMnode) != 0) return -1;
if (mndAcquireRpc(pMnode) != 0) return -1;
SSdb *pSdb = pMnode->pSdb;
int64_t ms = taosGetTimestampMs();
@ -680,7 +716,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
pStbInfo->stbs = taosArrayInit(sdbGetSize(pSdb, SDB_STB), sizeof(SMonStbDesc));
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL ||
pStbInfo->stbs == NULL) {
mndReleaseRpcRef(pMnode);
mndReleaseRpc(pMnode);
return -1;
}
@ -800,7 +836,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
pGrantInfo->timeseries_total = INT32_MAX;
}
mndReleaseRpcRef(pMnode);
mndReleaseRpc(pMnode);
return 0;
}
@ -810,32 +846,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
return 0;
}
int32_t mndAcquireRpcRef(SMnode *pMnode) {
int32_t code = 0;
taosThreadRwlockRdlock(&pMnode->lock);
if (pMnode->stopped) {
mTrace("mnode not running");
terrno = TSDB_CODE_APP_NOT_READY;
code = -1;
} else if (!mndIsMaster(pMnode)) {
mTrace("mnode not ready, role:%s restored:%d", syncGetMyRoleStr(pMnode->syncMgmt.sync), pMnode->restored);
code = -1;
} else {
int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1);
// mTrace("mnode rpc is acquired, ref:%d", ref);
}
taosThreadRwlockUnlock(&pMnode->lock);
return code;
}
void mndReleaseRpcRef(SMnode *pMnode) {
taosThreadRwlockRdlock(&pMnode->lock);
int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1);
// mTrace("mnode rpc is released, ref:%d", ref);
taosThreadRwlockUnlock(&pMnode->lock);
}
void mndSetRestore(SMnode *pMnode, bool restored) {
void mndSetRestored(SMnode *pMnode, bool restored) {
if (restored) {
taosThreadRwlockWrlock(&pMnode->lock);
pMnode->restored = true;

334
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -36,6 +36,7 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq);
static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq);
static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter);
static void mndReloadSyncConfig(SMnode *pMnode);
int32_t mndInitMnode(SMnode *pMnode) {
SSdbTable table = {
@ -187,6 +188,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
}
pObj->state = TAOS_SYNC_STATE_ERROR;
mndReloadSyncConfig(pSdb->pMnode);
return 0;
}
@ -203,6 +205,8 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) {
static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew) {
mTrace("mnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew);
pOld->updateTime = pNew->updateTime;
mndReloadSyncConfig(pSdb->pMnode);
return 0;
}
@ -233,7 +237,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
if (pIter == NULL) break;
if (pObj->id == pMnode->selfDnodeId) {
if (mndIsMaster(pMnode)) {
if (mndIsLeader(pMnode)) {
pEpSet->inUse = pEpSet->numOfEps;
} else {
pEpSet->inUse = (pEpSet->numOfEps + 1) % totalMnodes;
@ -248,6 +252,10 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
if (pEpSet->numOfEps == 0) {
syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet);
}
if (pEpSet->inUse >= pEpSet->numOfEps) {
pEpSet->inUse = 0;
}
}
static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
@ -274,59 +282,13 @@ static int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnod
return 0;
}
static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
int32_t numOfReplicas = 0;
SDAlterMnodeReq alterReq = {0};
SDCreateMnodeReq createReq = {0};
SEpSet alterEpset = {0};
SEpSet createEpset = {0};
while (1) {
SMnodeObj *pMObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
if (pIter == NULL) break;
alterReq.replicas[numOfReplicas].id = pMObj->id;
alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port;
memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
if (pMObj->state == TAOS_SYNC_STATE_LEADER) {
alterEpset.inUse = numOfReplicas;
}
numOfReplicas++;
sdbRelease(pSdb, pMObj);
}
alterReq.replica = numOfReplicas + 1;
alterReq.replicas[numOfReplicas].id = pDnode->id;
alterReq.replicas[numOfReplicas].port = pDnode->port;
memcpy(alterReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
alterEpset.numOfEps = numOfReplicas + 1;
alterEpset.eps[numOfReplicas].port = pDnode->port;
memcpy(alterEpset.eps[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
createReq.replica = 1;
createReq.replicas[0].id = pDnode->id;
createReq.replicas[0].port = pDnode->port;
memcpy(createReq.replicas[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
createEpset.numOfEps = 1;
createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
{
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq);
static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) {
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, &createReq);
tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq);
STransAction action = {
.epSet = createEpset,
.epSet = *pCreateEpSet,
.pCont = pReq,
.contLen = contLen,
.msgType = TDMT_DND_CREATE_MNODE,
@ -337,15 +299,16 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
taosMemoryFree(pReq);
return -1;
}
return 0;
}
{
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq);
static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) {
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq);
tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq);
STransAction action = {
.epSet = alterEpset,
.epSet = *pAlterEpSet,
.pCont = pReq,
.contLen = contLen,
.msgType = TDMT_MND_ALTER_MNODE,
@ -356,8 +319,62 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
taosMemoryFree(pReq);
return -1;
}
return 0;
}
static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) {
int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, pDropReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSCreateDropMQSBNodeReq(pReq, contLen, pDropReq);
STransAction action = {
.epSet = *pDroprEpSet,
.pCont = pReq,
.contLen = contLen,
.msgType = TDMT_DND_DROP_MNODE,
.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
return -1;
}
return 0;
}
static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
int32_t numOfReplicas = 0;
SDCreateMnodeReq createReq = {0};
SEpSet createEpset = {0};
while (1) {
SMnodeObj *pMObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
if (pIter == NULL) break;
createReq.replicas[numOfReplicas].id = pMObj->id;
createReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
numOfReplicas++;
sdbRelease(pSdb, pMObj);
}
createReq.replica = numOfReplicas + 1;
createReq.replicas[numOfReplicas].id = pDnode->id;
createReq.replicas[numOfReplicas].port = pDnode->port;
memcpy(createReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
createEpset.inUse = 0;
createEpset.numOfEps = 1;
createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1;
return 0;
}
@ -374,9 +391,9 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId);
if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER;
if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER;
if (mndTransAppendNullLog(pTrans) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
@ -462,112 +479,32 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
int32_t numOfReplicas = 0;
SDAlterMnodeReq alterReq = {0};
SDDropMnodeReq dropReq = {0};
SSetStandbyReq standbyReq = {0};
SEpSet alterEpset = {0};
SEpSet dropEpSet = {0};
while (1) {
SMnodeObj *pMObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
if (pIter == NULL) break;
if (pMObj->id == pObj->id) {
sdbRelease(pSdb, pMObj);
continue;
}
alterReq.replicas[numOfReplicas].id = pMObj->id;
alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port;
memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
if (pMObj->state == TAOS_SYNC_STATE_LEADER) {
alterEpset.inUse = numOfReplicas;
}
numOfReplicas++;
sdbRelease(pSdb, pMObj);
}
alterReq.replica = numOfReplicas;
alterEpset.numOfEps = numOfReplicas;
dropReq.dnodeId = pDnode->id;
dropEpSet.numOfEps = 1;
dropEpSet.eps[0].port = pDnode->port;
memcpy(dropEpSet.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
standbyReq.dnodeId = pDnode->id;
standbyReq.standby = 1;
{
int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq) + sizeof(SMsgHead);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq);
SMsgHead *pHead = pReq;
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(MNODE_HANDLE);
STransAction action = {
.epSet = dropEpSet,
.pCont = pReq,
.contLen = contLen,
.msgType = TDMT_SYNC_SET_MNODE_STANDBY,
.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE);
if (totalMnodes == 2) {
mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes);
if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
} else if (totalMnodes == 3) {
mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes);
if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
} else {
return -1;
}
}
{
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq);
STransAction action = {
.epSet = alterEpset,
.pCont = pReq,
.contLen = contLen,
.msgType = TDMT_MND_ALTER_MNODE,
.acceptableCode = 0,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
return -1;
}
}
{
int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq);
STransAction action = {
.epSet = dropEpSet,
.pCont = pReq,
.contLen = contLen,
.msgType = TDMT_DND_DROP_MNODE,
.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
return -1;
}
}
return 0;
}
int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
if (pObj == NULL) return 0;
if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1;
if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1;
if (mndTransAppendNullLog(pTrans) != 0) return -1;
@ -657,7 +594,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
int64_t curMs = taosGetTimestampMs();
while (numOfRows < rows) {
pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus);
pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus, true);
if (pShow->pIter == NULL) break;
cols = 0;
@ -712,6 +649,9 @@ static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter) {
}
static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
#if 1
return 0;
#else
SMnode *pMnode = pReq->info.node;
SDAlterMnodeReq alterReq = {0};
@ -720,41 +660,107 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
return -1;
}
SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1};
memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas));
for (int32_t i = 0; i < option.numOfReplicas; ++i) {
if (alterReq.replicas[i].id == pMnode->selfDnodeId) {
option.selfIndex = i;
}
}
if (option.selfIndex == -1) {
mInfo("alter mnode not processed since selfIndex is -1", terrstr());
return 0;
}
if (mndWriteFile(pMnode->path, &option) != 0) {
mError("failed to write mnode file since %s", terrstr());
return -1;
}
SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1};
for (int32_t i = 0; i < alterReq.replica; ++i) {
SNodeInfo *pNode = &cfg.nodeInfo[i];
tstrncpy(pNode->nodeFqdn, alterReq.replicas[i].fqdn, sizeof(pNode->nodeFqdn));
pNode->nodePort = alterReq.replicas[i].port;
if (alterReq.replicas[i].id == pMnode->selfDnodeId) cfg.myIndex = i;
if (alterReq.replicas[i].id == pMnode->selfDnodeId) {
cfg.myIndex = i;
}
}
if (cfg.myIndex == -1) {
mError("failed to alter mnode since myindex is -1");
return -1;
} else {
mInfo("start to alter mnode sync, replica:%d myindex:%d", cfg.replicaNum, cfg.myIndex);
mInfo("start to alter mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex);
for (int32_t i = 0; i < alterReq.replica; ++i) {
SNodeInfo *pNode = &cfg.nodeInfo[i];
mInfo("index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort);
}
}
mInfo("trans:-1, sync reconfig will be proposed");
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
pMgmt->standby = 0;
int32_t code = syncReconfig(pMgmt->sync, &cfg);
int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
if (code != 0) {
mError("trans:-1, failed to propose sync reconfig since %s", terrstr());
return code;
mError("failed to sync reconfig since %s", terrstr());
} else {
pMgmt->errCode = 0;
taosWLockLatch(&pMgmt->lock);
pMgmt->transId = -1;
taosWUnLockLatch(&pMgmt->lock);
tsem_wait(&pMgmt->syncSem);
mInfo("alter mnode sync result:0x%x %s", pMgmt->errCode, tstrerror(pMgmt->errCode));
terrno = pMgmt->errCode;
return pMgmt->errCode;
mInfo("alter mnode sync success");
}
return code;
#endif
}
static void mndReloadSyncConfig(SMnode *pMnode) {
SSdb *pSdb = pMnode->pSdb;
SMnodeObj *pObj = NULL;
ESdbStatus objStatus = 0;
void *pIter = NULL;
bool hasUpdatingMnode = false;
SSyncCfg cfg = {.myIndex = -1};
while (1) {
pIter = sdbFetchAll(pSdb, SDB_MNODE, pIter, (void **)&pObj, &objStatus, false);
if (pIter == NULL) break;
if (objStatus == SDB_STATUS_CREATING || objStatus == SDB_STATUS_DROPPING) {
mInfo("vgId:1, has updating mnode:%d, status:%s", pObj->id, sdbStatusName(objStatus));
hasUpdatingMnode = true;
}
if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) {
SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum];
tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, sizeof(pNode->nodeFqdn));
pNode->nodePort = pObj->pDnode->port;
if (pObj->pDnode->id == pMnode->selfDnodeId) {
cfg.myIndex = cfg.replicaNum;
}
cfg.replicaNum++;
}
sdbReleaseLock(pSdb, pObj, false);
}
if (cfg.myIndex == -1) {
mInfo("vgId:1, mnode not reload since selfIndex is -1");
return;
}
if (!mndGetRestored(pMnode)) {
mInfo("vgId:1, mnode not reload since restore not finished");
return;
}
if (hasUpdatingMnode) {
mInfo("vgId:1, start to reload mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex);
for (int32_t i = 0; i < cfg.replicaNum; ++i) {
SNodeInfo *pNode = &cfg.nodeInfo[i];
mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort);
}
int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
if (code != 0) {
mError("vgId:1, failed to reconfig mnode sync since %s", terrstr());
} else {
mInfo("vgId:1, reconfig mnode sync success");
}
}
}

29
source/dnode/mnode/impl/src/mndSync.c
View File

@ -107,7 +107,7 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) {
if (!pMnode->deploy) {
mInfo("vgId:1, sync restore finished, and will handle outstanding transactions");
mndTransPullup(pMnode);
mndSetRestore(pMnode, true);
mndSetRestored(pMnode, true);
} else {
mInfo("vgId:1, sync restore finished");
}
@ -225,18 +225,17 @@ int32_t mndInitSync(SMnode *pMnode) {
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP);
syncInfo.pWal = pMnode->pWal;
syncInfo.pFsm = mndSyncMakeFsm(pMnode);
syncInfo.isStandBy = pMgmt->standby;
syncInfo.snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT;
mInfo("vgId:1, start to open sync, standby:%d", pMgmt->standby);
if (pMgmt->standby || pMgmt->replica.id > 0) {
mInfo("vgId:1, start to open sync, selfIndex:%d replica:%d", pMgmt->selfIndex, pMgmt->numOfReplicas);
SSyncCfg *pCfg = &syncInfo.syncCfg;
pCfg->replicaNum = 1;
pCfg->myIndex = 0;
SNodeInfo *pNode = &pCfg->nodeInfo[0];
tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn));
pNode->nodePort = pMgmt->replica.port;
mInfo("vgId:1, ep:%s:%u", pNode->nodeFqdn, pNode->nodePort);
pCfg->replicaNum = pMgmt->numOfReplicas;
pCfg->myIndex = pMgmt->selfIndex;
for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) {
SNodeInfo *pNode = &pCfg->nodeInfo[i];
tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
pNode->nodePort = pMgmt->replicas[i].port;
mInfo("vgId:1, index:%d ep:%s:%u", i, pNode->nodeFqdn, pNode->nodePort);
}
tsem_init(&pMgmt->syncSem, 0, 0);
@ -250,10 +249,6 @@ int32_t mndInitSync(SMnode *pMnode) {
setPingTimerMS(pMgmt->sync, 5000);
setElectTimerMS(pMgmt->sync, 3000);
setHeartbeatTimerMS(pMgmt->sync, 500);
/*
setElectTimerMS(pMgmt->sync, 600);
setHeartbeatTimerMS(pMgmt->sync, 300);
*/
mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync);
return 0;
@ -319,7 +314,7 @@ void mndSyncStart(SMnode *pMnode) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
syncSetMsgCb(pMgmt->sync, &pMnode->msgCb);
syncStart(pMgmt->sync);
mInfo("vgId:1, sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby);
mInfo("vgId:1, sync started, id:%" PRId64, pMgmt->sync);
}
void mndSyncStop(SMnode *pMnode) {
@ -331,7 +326,7 @@ void mndSyncStop(SMnode *pMnode) {
taosWUnLockLatch(&pMnode->syncMgmt.lock);
}
bool mndIsMaster(SMnode *pMnode) {
bool mndIsLeader(SMnode *pMnode) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
if (!syncIsReady(pMgmt->sync)) {
@ -340,7 +335,7 @@ bool mndIsMaster(SMnode *pMnode) {
return false;
}
if (!pMnode->restored) {
if (!mndGetRestored(pMnode)) {
terrno = TSDB_CODE_APP_NOT_READY;
return false;
}

2
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -53,7 +53,7 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans);
static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans);
static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans);
static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans);
static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); }
static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); }
static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
static int32_t mndProcessTransTimer(SRpcMsg *pReq);

3
source/dnode/mnode/sdb/inc/sdb.h
View File

@ -298,6 +298,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey);
* @param pObj The object of the row.
*/
void sdbRelease(SSdb *pSdb, void *pObj);
void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock);
/**
* @brief Traverse a sdb table
@ -309,7 +310,7 @@ void sdbRelease(SSdb *pSdb, void *pObj);
* @return void* The next iterator of the table.
*/
void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj);
void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status);
void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock);
/**
* @brief Cancel a traversal

14
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -327,14 +327,16 @@ static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) {
taosThreadRwlockUnlock(pLock);
}
void sdbRelease(SSdb *pSdb, void *pObj) {
void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) {
if (pObj == NULL) return;
SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow));
if (pRow->type >= SDB_MAX) return;
TdThreadRwlock *pLock = &pSdb->locks[pRow->type];
if (lock) {
taosThreadRwlockWrlock(pLock);
}
int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
sdbPrintOper(pSdb, pRow, "release");
@ -342,8 +344,12 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
sdbFreeRow(pSdb, pRow, true);
}
if (lock) {
taosThreadRwlockUnlock(pLock);
}
}
void sdbRelease(SSdb *pSdb, void *pObj) { sdbReleaseLock(pSdb, pObj, true); }
void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
*ppObj = NULL;
@ -372,14 +378,16 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
return ppRow;
}
void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) {
void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock) {
*ppObj = NULL;
SHashObj *hash = sdbGetHash(pSdb, type);
if (hash == NULL) return NULL;
TdThreadRwlock *pLock = &pSdb->locks[type];
if (lock) {
taosThreadRwlockRdlock(pLock);
}
SSdbRow **ppRow = taosHashIterate(hash, pIter);
while (ppRow != NULL) {
@ -395,7 +403,9 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat
*status = pRow->status;
break;
}
if (lock) {
taosThreadRwlockUnlock(pLock);
}
return ppRow;
}

1
source/dnode/vnode/src/meta/metaTable.c
View File

@ -754,6 +754,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
if (iCol >= pSchema->nCols) break;
pColumn = &pSchema->pSchema[iCol];
ASSERT(pAlterTbReq->colName);
if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break;
iCol++;
}

8
source/dnode/vnode/src/tq/tq.c
View File

@ -171,11 +171,11 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0);
if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
if (pRsp->blockNum > 0) {
/*if (pRsp->blockNum > 0) {*/
/*ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version);*/
/*} else {*/
ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version);
} else {
ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version);
}
/*}*/
}
int32_t len = 0;

2
source/dnode/vnode/src/tq/tqPush.c
View File

@ -245,7 +245,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
tqDebug("vgId:%d cannot find handle %s", pTq->pVnode->config.vgId, pPushEntry->subKey);
continue;
}
if (pPushEntry->dataRsp.reqOffset.version > ver) {
if (pPushEntry->dataRsp.reqOffset.version >= ver) {
tqDebug("vgId:%d push entry req version %ld, while push version %ld, skip", pTq->pVnode->config.vgId,
pPushEntry->dataRsp.reqOffset.version, ver);
continue;

19
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -378,10 +378,10 @@ static int32_t getTableDelSkyline(STbData *pMem, STbData *pIMem, SDelFReader *pD
if (code) goto _err;
}
_err:
if (aDelData) {
taosArrayDestroy(aDelData);
}
_err:
return code;
}
@ -399,14 +399,13 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t
// code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx);
SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ);
if (code) goto _err;
*pDelIdx = *pIdx;
_err:
if (pDelIdxArray) {
taosArrayDestroy(pDelIdxArray);
}
_err:
return code;
}
@ -599,8 +598,6 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
* &state->blockIdx);
*/
state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ);
if (code) goto _err;
if (!state->pBlockIdx) {
goto _next_fileset;
}
@ -898,10 +895,16 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
if (code) goto _err;
code = getTableDelIdx(pDelFReader, suid, uid, &delIdx);
if (code) goto _err;
if (code) {
tsdbDelFReaderClose(&pDelFReader);
goto _err;
}
code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline);
if (code) goto _err;
if (code) {
tsdbDelFReaderClose(&pDelFReader);
goto _err;
}
tsdbDelFReaderClose(&pDelFReader);
} else {
@ -1231,6 +1234,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
_err:
nextRowIterClose(&iter);
taosMemoryFreeClear(pTSchema);
*ppLastArray = NULL;
taosArrayDestroy(pColArray);
return code;
}

9
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -396,12 +396,19 @@ _exit:
return code;
}
static int32_t tDataIterCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) {
SDataIter *pIter1 = (SDataIter *)((uint8_t *)n1 - offsetof(SDataIter, n));
SDataIter *pIter2 = (SDataIter *)((uint8_t *)n2 - offsetof(SDataIter, n));
return tRowInfoCmprFn(&pIter1->r, &pIter2->r);
}
static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
int32_t code = 0;
int32_t lino = 0;
pCommitter->pIter = NULL;
tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn);
tRBTreeCreate(&pCommitter->rbt, tDataIterCmprFn);
// memory
TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN};

13
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -610,9 +610,6 @@ int32_t tsdbFSRollback(STsdbFS *pFS) {
ASSERT(0);
return code;
_err:
return code;
}
int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile) {
@ -866,7 +863,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
nRef = atomic_sub_fetch_32(&fSet.pSmaF->nRef, 1);
if (nRef == 0) {
tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pSmaF, fname);
taosRemoveFile(fname);
(void)taosRemoveFile(fname);
taosMemoryFree(fSet.pSmaF);
}
} else {
@ -877,7 +874,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
// stt
if (sameDisk) {
if (pSetNew->nSttF > pSetOld->nSttF) {
ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1);
ASSERT(pSetNew->nSttF == pSetOld->nSttF + 1);
pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
if (pSetOld->aSttF[pSetOld->nSttF] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@ -964,7 +961,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1);
if (nRef == 0) {
tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname);
taosRemoveFile(fname);
(void)taosRemoveFile(fname);
taosMemoryFree(pSetOld->pHeadF);
}
@ -1104,7 +1101,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) {
ASSERT(nRef >= 0);
if (nRef == 0) {
tsdbDelFileName(pTsdb, pFS->pDelFile, fname);
taosRemoveFile(fname);
(void)taosRemoveFile(fname);
taosMemoryFree(pFS->pDelFile);
}
}
@ -1117,7 +1114,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) {
ASSERT(nRef >= 0);
if (nRef == 0) {
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
taosRemoveFile(fname);
(void)taosRemoveFile(fname);
taosMemoryFree(pSet->pHeadF);
}

2
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -559,6 +559,8 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i
// backward put first data
row.pTSRow = tGetSubmitBlkNext(&blkIter);
if (row.pTSRow == NULL) return code;
key.ts = row.pTSRow->ts;
nRow++;
tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD);

18
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -133,8 +133,15 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) {
id.uid = pIter->uid;
}
tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols);
code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols);
if (code != TSDB_CODE_SUCCESS) {
goto _exit;
}
code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock);
if (code != TSDB_CODE_SUCCESS) {
goto _exit;
}
double el = (taosGetTimestampUs() - st) / 1000.0;
pInfo->elapsedTime += el;
@ -504,9 +511,9 @@ _exit:
SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; }
// SMergeTree =================================================
static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) {
SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - sizeof(SRBTreeNode));
SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - sizeof(SRBTreeNode));
static FORCE_INLINE int32_t tLDataIterCmprFn(const SRBTreeNode *p1, const SRBTreeNode *p2) {
SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - offsetof(SLDataIter, node));
SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - offsetof(SLDataIter, node));
TSDBKEY key1 = TSDBROW_KEY(&pIter1->rInfo.row);
TSDBKEY key2 = TSDBROW_KEY(&pIter2->rInfo.row);
@ -543,6 +550,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
pMTree->pLoadInfo = pBlockLoadInfo;
pMTree->destroyLoadInfo = destroyLoadInfo;
ASSERT(pMTree->pLoadInfo != NULL);
for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file
struct SLDataIter *pIter = NULL;
@ -583,7 +591,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) {
// compare with min in RB Tree
pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt);
if (pMTree->pIter && pIter) {
int32_t c = pMTree->rbt.cmprFn(RBTREE_NODE_PAYLOAD(&pMTree->pIter->node), RBTREE_NODE_PAYLOAD(&pIter->node));
int32_t c = pMTree->rbt.cmprFn(&pMTree->pIter->node, &pIter->node);
if (c > 0) {
tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter);
pMTree->pIter = NULL;

6
source/dnode/vnode/src/tsdb/tsdbOpen.c
View File

@ -87,9 +87,13 @@ _err:
int tsdbClose(STsdb **pTsdb) {
if (*pTsdb) {
taosThreadRwlockDestroy(&(*pTsdb)->rwLock);
taosThreadRwlockWrlock(&(*pTsdb)->rwLock);
tsdbMemTableDestroy((*pTsdb)->mem);
(*pTsdb)->mem = NULL;
taosThreadRwlockUnlock(&(*pTsdb)->rwLock);
taosThreadRwlockDestroy(&(*pTsdb)->rwLock);
tsdbFSClose(*pTsdb);
tsdbCloseCache(*pTsdb);
taosMemoryFreeClear(*pTsdb);

17
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -2676,9 +2676,9 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
int8_t level = 0;
int8_t precision = pVnode->config.tsdbCfg.precision;
int64_t now = taosGetTimestamp(precision);
int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1
: (precision == TSDB_TIME_PRECISION_MICRO) ? 1000
: 1000000);
int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1L
: (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L
: 1000000L);
for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
SRetention* pRetention = retentions + level;
@ -3372,14 +3372,13 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
if (pCond->suid != 0) {
pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1, 1);
if (pReader->pSchema == NULL) {
tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:%" PRId64 " , %s", pReader->suid, -1,
pReader->idStr);
tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->suid, pReader->idStr);
}
} else if (taosArrayGetSize(pTableList) > 0) {
STableKeyInfo* pKey = taosArrayGet(pTableList, 0);
pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1);
if (pReader->pSchema == NULL) {
tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:%" PRId64 " , %s", pKey->uid, -1, pReader->idStr);
tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr);
}
}
@ -3395,6 +3394,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
goto _err;
}
if (numOfTables > 0) {
code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
@ -3437,6 +3437,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
return code;
}
}
}
tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr);
return code;
@ -3513,6 +3514,10 @@ void tsdbReaderClose(STsdbReader* pReader) {
taosMemoryFree(pLReader);
}
if (pReader->innerReader[0] != 0) {
tsdbUntakeReadSnap(pReader->innerReader[0]->pTsdb, pReader->innerReader[0]->pReadSnap, pReader->idStr);
}
tsdbDebug(
"%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64
" SMA-time:%.2f ms, fileBlocks:%" PRId64

95
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -18,7 +18,7 @@
// =============== PAGE-WISE FILE ===============
static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) {
int32_t code = 0;
STsdbFD *pFD;
STsdbFD *pFD = NULL;
*ppFD = NULL;
@ -35,6 +35,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd
pFD->pFD = taosOpenFile(path, flag);
if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pFD);
goto _exit;
}
pFD->szPage = szPage;
@ -42,11 +43,15 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd
pFD->pBuf = taosMemoryCalloc(1, szPage);
if (pFD->pBuf == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosCloseFile(&pFD->pFD);
taosMemoryFree(pFD);
goto _exit;
}
if (taosStatFile(path, &pFD->szFile, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pFD->pBuf);
taosCloseFile(&pFD->pFD);
taosMemoryFree(pFD);
goto _exit;
}
ASSERT(pFD->szFile % szPage == 0);
@ -59,11 +64,13 @@ _exit:
static void tsdbCloseFile(STsdbFD **ppFD) {
STsdbFD *pFD = *ppFD;
if (pFD) {
taosMemoryFree(pFD->pBuf);
taosCloseFile(&pFD->pFD);
taosMemoryFree(pFD);
*ppFD = NULL;
}
}
static int32_t tsdbWriteFilePage(STsdbFD *pFD) {
int32_t code = 0;
@ -117,7 +124,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
}
// check
if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _exit;
}
@ -443,7 +450,7 @@ int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *p
pBlockIdx->size = size;
pHeadFile->size += size;
tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64
tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%" PRId64 " suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64
" size:%" PRId64 " nItem:%d",
TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid,
pBlockIdx->offset, pBlockIdx->size, mDataBlk->nItem);
@ -457,7 +464,7 @@ _err:
int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk) {
int32_t code = 0;
SSttFile *pSttFile = &pWriter->fStt[pWriter->wSet.nSttF - 1];
int64_t size;
int64_t size = 0;
int64_t n;
// check
@ -836,7 +843,8 @@ _err:
// SDataFReader ====================================================
int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) {
int32_t code = 0;
SDataFReader *pReader;
int32_t lino = 0;
SDataFReader *pReader = NULL;
int32_t szPage = pTsdb->pVnode->config.tsdbPageSize;
char fname[TSDB_FILENAME_LEN];
@ -844,7 +852,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
pReader->pTsdb = pTsdb;
pReader->pSet = pSet;
@ -852,31 +860,40 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
// head
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
// data
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pDataFD);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
// sma
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
// stt
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSttFD[iStt]);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
*ppReader = pReader;
return code;
_err:
tsdbError("vgId:%d, tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
_exit:
if (code) {
*ppReader = NULL;
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
if (pReader) {
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) tsdbCloseFile(&pReader->aSttFD[iStt]);
tsdbCloseFile(&pReader->pSmaFD);
tsdbCloseFile(&pReader->pDataFD);
tsdbCloseFile(&pReader->pHeadFD);
taosMemoryFree(pReader);
}
} else {
*ppReader = pReader;
}
return code;
}
@ -906,10 +923,6 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) {
taosMemoryFree(*ppReader);
*ppReader = NULL;
return code;
_err:
tsdbError("vgId:%d, data file reader close failed since %s", TD_VID((*ppReader)->pTsdb->pVnode), tstrerror(code));
return code;
}
int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
@ -1289,16 +1302,17 @@ _exit:
// SDelFWriter ====================================================
int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) {
int32_t code = 0;
int32_t lino = 0;
char fname[TSDB_FILENAME_LEN];
uint8_t hdr[TSDB_FHDR_SIZE] = {0};
SDelFWriter *pDelFWriter;
SDelFWriter *pDelFWriter = NULL;
int64_t n;
// alloc
pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter));
if (pDelFWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
pDelFWriter->pTsdb = pTsdb;
pDelFWriter->fDel = *pFile;
@ -1306,21 +1320,28 @@ int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb
tsdbDelFileName(pTsdb, pFile, fname);
code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE,
&pDelFWriter->pWriteH);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
// update header
code = tsdbWriteFile(pDelFWriter->pWriteH, 0, hdr, TSDB_FHDR_SIZE);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
pDelFWriter->fDel.size = TSDB_FHDR_SIZE;
pDelFWriter->fDel.offset = 0;
*ppWriter = pDelFWriter;
return code;
_err:
tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
_exit:
if (code) {
if (pDelFWriter) {
taosMemoryFree(pDelFWriter);
tsdbCloseFile(&pDelFWriter->pWriteH);
}
*ppWriter = NULL;
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(errno));
} else {
*ppWriter = pDelFWriter;
}
return code;
}
@ -1456,15 +1477,15 @@ struct SDelFReader {
int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) {
int32_t code = 0;
int32_t lino = 0;
char fname[TSDB_FILENAME_LEN];
SDelFReader *pDelFReader;
int64_t n;
SDelFReader *pDelFReader = NULL;
// alloc
pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader));
if (pDelFReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
goto _exit;
}
// open impl
@ -1473,14 +1494,18 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
tsdbDelFileName(pTsdb, pFile, fname);
code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ, &pDelFReader->pReadH);
if (code) goto _err;
if (code) {
taosMemoryFree(pDelFReader);
goto _exit;
}
*ppReader = pDelFReader;
return code;
_err:
tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
_exit:
if (code) {
*ppReader = NULL;
tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
} else {
*ppReader = pDelFReader;
}
return code;
}

112
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -67,6 +67,13 @@ extern int32_t tRowInfoCmprFn(const void* p1, const void* p2);
extern int32_t tsdbReadDataBlockEx(SDataFReader* pReader, SDataBlk* pDataBlk, SBlockData* pBlockData);
extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo);
static int32_t tFDataIterCmprFn(const SRBTreeNode* pNode1, const SRBTreeNode* pNode2) {
SFDataIter* pIter1 = (SFDataIter*)(((uint8_t*)pNode1) - offsetof(SFDataIter, n));
SFDataIter* pIter2 = (SFDataIter*)(((uint8_t*)pNode2) - offsetof(SFDataIter, n));
return tRowInfoCmprFn(&pIter1->rInfo, &pIter2->rInfo);
}
static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
int32_t code = 0;
@ -79,7 +86,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
if (code) goto _err;
pReader->pIter = NULL;
tRBTreeCreate(&pReader->rbt, tRowInfoCmprFn);
tRBTreeCreate(&pReader->rbt, tFDataIterCmprFn);
// .data file
SFDataIter* pIter = &pReader->aFDataIter[0];
@ -421,7 +428,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
n += tPutDelData((*ppData) + n, pDelData);
}
tsdbInfo("vgId:%d, vnode snapshot tsdb read del data for %s, suid:%" PRId64 " uid:%d" PRId64 " size:%d",
tsdbInfo("vgId:%d, vnode snapshot tsdb read del data for %s, suid:%" PRId64 " uid:%" PRId64 " size:%d",
TD_VID(pTsdb->pVnode), pTsdb->path, pDelIdx->suid, pDelIdx->uid, size);
break;
@ -431,20 +438,21 @@ _exit:
return code;
_err:
tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->pVnode,
tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
tstrerror(code));
return code;
}
int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) {
int32_t code = 0;
int32_t lino = 0;
STsdbSnapReader* pReader = NULL;
// alloc
pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
pReader->pTsdb = pTsdb;
pReader->sver = sver;
@ -454,19 +462,19 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
code = taosThreadRwlockRdlock(&pTsdb->rwLock);
if (code) {
code = TAOS_SYSTEM_ERROR(code);
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbFSRef(pTsdb, &pReader->fs);
if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = taosThreadRwlockUnlock(&pTsdb->rwLock);
if (code) {
code = TAOS_SYSTEM_ERROR(code);
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
// data
@ -478,43 +486,52 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pIter->aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
} else {
pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (pIter->aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
}
code = tBlockDataCreate(&pIter->bData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataCreate(&pReader->bData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
// del
pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
if (pReader->aDelIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
pReader->aDelData = taosArrayInit(0, sizeof(SDelData));
if (pReader->aDelData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path);
*ppReader = pReader;
return code;
_err:
tsdbError("vgId:%d, vnode snapshot tsdb reader open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
tstrerror(code));
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s, TSDB path: %s", TD_VID(pTsdb->pVnode), lino, tstrerror(code),
pTsdb->path);
*ppReader = NULL;
if (pReader) {
taosArrayDestroy(pReader->aDelData);
taosArrayDestroy(pReader->aDelIdx);
tBlockDataDestroy(&pReader->bData, 1);
tsdbFSDestroy(&pReader->fs);
taosMemoryFree(pReader);
}
} else {
*ppReader = pReader;
tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path);
}
return code;
}
@ -1247,20 +1264,21 @@ _err:
// APIs
int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) {
int32_t code = 0;
int32_t lino = 0;
STsdbSnapWriter* pWriter = NULL;
// alloc
pWriter = (STsdbSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
pWriter->pTsdb = pTsdb;
pWriter->sver = sver;
pWriter->ever = ever;
code = tsdbFSCopy(pTsdb, &pWriter->fs);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
// config
pWriter->minutes = pTsdb->keepCfg.days;
@ -1272,7 +1290,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
// SNAP_DATA_TSDB
code = tBlockDataCreate(&pWriter->bData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
pWriter->fid = INT32_MIN;
pWriter->id = (TABLEID){0};
@ -1280,53 +1298,67 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
pWriter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pWriter->dReader.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataCreate(&pWriter->dReader.bData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
// Writer
pWriter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pWriter->dWriter.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
pWriter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (pWriter->dWriter.aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataCreate(&pWriter->dWriter.bData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
code = tBlockDataCreate(&pWriter->dWriter.sData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
// SNAP_DATA_DEL
pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx));
if (pWriter->aDelIdxR == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
pWriter->aDelData = taosArrayInit(0, sizeof(SDelData));
if (pWriter->aDelData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
pWriter->aDelIdxW = taosArrayInit(0, sizeof(SDelIdx));
if (pWriter->aDelIdxW == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
*ppWriter = pWriter;
tsdbInfo("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path);
return code;
_err:
tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
tstrerror(code));
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
*ppWriter = NULL;
if (pWriter) {
if (pWriter->aDelIdxW) taosArrayDestroy(pWriter->aDelIdxW);
if (pWriter->aDelData) taosArrayDestroy(pWriter->aDelData);
if (pWriter->aDelIdxR) taosArrayDestroy(pWriter->aDelIdxR);
tBlockDataDestroy(&pWriter->dWriter.sData, 1);
tBlockDataDestroy(&pWriter->dWriter.bData, 1);
if (pWriter->dWriter.aSttBlk) taosArrayDestroy(pWriter->dWriter.aSttBlk);
if (pWriter->dWriter.aBlockIdx) taosArrayDestroy(pWriter->dWriter.aBlockIdx);
tBlockDataDestroy(&pWriter->dReader.bData, 1);
if (pWriter->dReader.aBlockIdx) taosArrayDestroy(pWriter->dReader.aBlockIdx);
tBlockDataDestroy(&pWriter->bData, 1);
tsdbFSDestroy(&pWriter->fs);
taosMemoryFree(pWriter);
}
} else {
tsdbDebug("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path);
*ppWriter = pWriter;
}
return code;
}

4
source/dnode/vnode/src/tsdb/tsdbWrite.c
View File

@ -34,7 +34,9 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *
}
// loop to insert
tInitSubmitMsgIter(pMsg, &msgIter);
if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) {
return -1;
}
while (true) {
SSubmitBlkRsp r = {0};
tGetSubmitMsgNext(&msgIter, &pBlock);

18
source/dnode/vnode/src/vnd/vnodeBufPool.c
View File

@ -141,17 +141,17 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) {
}
void vnodeBufPoolFree(SVBufPool *pPool, void *p) {
uint8_t *ptr = (uint8_t *)p;
SVBufPoolNode *pNode;
// uint8_t *ptr = (uint8_t *)p;
// SVBufPoolNode *pNode;
if (ptr < pPool->node.data || ptr >= pPool->node.data + pPool->node.size) {
pNode = &((SVBufPoolNode *)p)[-1];
*pNode->pnext = pNode->prev;
pNode->prev->pnext = pNode->pnext;
// if (ptr < pPool->node.data || ptr >= pPool->node.data + pPool->node.size) {
// pNode = &((SVBufPoolNode *)p)[-1];
// *pNode->pnext = pNode->prev;
// pNode->prev->pnext = pNode->pnext;
pPool->size = pPool->size - sizeof(*pNode) - pNode->size;
taosMemoryFree(pNode);
}
// pPool->size = pPool->size - sizeof(*pNode) - pNode->size;
// taosMemoryFree(pNode);
// }
}
void vnodeBufPoolRef(SVBufPool *pPool) {

11
source/dnode/vnode/src/vnd/vnodeModule.c
View File

@ -46,11 +46,17 @@ int vnodeInit(int nthreads) {
return 0;
}
vnodeGlobal.stop = 0;
taosThreadMutexInit(&vnodeGlobal.mutex, NULL);
taosThreadCondInit(&vnodeGlobal.hasTask, NULL);
taosThreadMutexLock(&vnodeGlobal.mutex);
vnodeGlobal.stop = 0;
vnodeGlobal.queue.next = &vnodeGlobal.queue;
vnodeGlobal.queue.prev = &vnodeGlobal.queue;
taosThreadMutexUnlock(&(vnodeGlobal.mutex));
vnodeGlobal.nthreads = nthreads;
vnodeGlobal.threads = taosMemoryCalloc(nthreads, sizeof(TdThread));
if (vnodeGlobal.threads == NULL) {
@ -59,9 +65,6 @@ int vnodeInit(int nthreads) {
return -1;
}
taosThreadMutexInit(&vnodeGlobal.mutex, NULL);
taosThreadCondInit(&vnodeGlobal.hasTask, NULL);
for (int i = 0; i < nthreads; i++) {
taosThreadCreate(&(vnodeGlobal.threads[i]), NULL, loop, NULL);
}

4
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -38,7 +38,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
if (taosMkDir(path)) {
return TAOS_SYSTEM_ERROR(errno);
}
strcpy(dir, path);
snprintf(dir, TSDB_FILENAME_LEN, "%s", path);
}
if (pCfg) {
@ -51,7 +51,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
info.state.commitID = 0;
if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) {
vError("vgId:%d, failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno));
vError("vgId:%d, failed to save vnode config since %s", pCfg ? pCfg->vgId : 0, tstrerror(terrno));
return -1;
}

9
source/dnode/vnode/src/vnd/vnodeSnapshot.c
View File

@ -166,7 +166,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (*ppData) {
goto _exit;
} else {
pReader->tqHandleDone = 1;
pReader->tqOffsetDone = 1;
code = tqOffsetReaderClose(&pReader->pTqOffsetReader);
if (code) goto _err;
}
@ -219,7 +219,7 @@ _exit:
return code;
_err:
vError("vgId:% vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code));
vError("vgId:%d vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code));
return code;
}
@ -260,7 +260,10 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr
// commit it
code = vnodeCommit(pVnode);
if (code) goto _err;
if (code) {
taosMemoryFree(pWriter);
goto _err;
}
// inc commit ID
pVnode->state.commitID++;

35
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -43,14 +43,23 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead));
tStartDecode(&dc);
tDecodeI32v(&dc, &nReqs);
if (tDecodeI32v(&dc, &nReqs) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto _err;
}
for (int32_t iReq = 0; iReq < nReqs; iReq++) {
tb_uid_t uid = tGenIdPI64();
char *name = NULL;
tStartDecode(&dc);
tDecodeI32v(&dc, NULL);
tDecodeCStr(&dc, &name);
if (tDecodeI32v(&dc, NULL) < 0) {
code = TSDB_CODE_INVALID_MSG;
return code;
}
if (tDecodeCStr(&dc, &name) < 0) {
code = TSDB_CODE_INVALID_MSG;
return code;
}
*(int64_t *)(dc.data + dc.pos) = uid;
*(int64_t *)(dc.data + dc.pos + 8) = ctime;
@ -68,7 +77,10 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
int64_t ctime = taosGetTimestampMs();
tb_uid_t uid;
tInitSubmitMsgIter(pSubmitReq, &msgIter);
if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) {
code = terrno;
goto _err;
}
for (;;) {
tGetSubmitMsgNext(&msgIter, &pBlock);
@ -78,10 +90,19 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
char *name = NULL;
tDecoderInit(&dc, pBlock->data, msgIter.schemaLen);
tStartDecode(&dc);
if (tStartDecode(&dc) < 0) {
code = TSDB_CODE_INVALID_MSG;
return code;
}
tDecodeI32v(&dc, NULL);
tDecodeCStr(&dc, &name);
if (tDecodeI32v(&dc, NULL) < 0) {
code = TSDB_CODE_INVALID_MSG;
return code;
}
if (tDecodeCStr(&dc, &name) < 0) {
code = TSDB_CODE_INVALID_MSG;
return code;
}
uid = metaGetTableEntryUidByName(pVnode->pMeta, name);
if (uid == 0) {

5
source/libs/executor/inc/executorimpl.h
View File

@ -212,6 +212,7 @@ typedef struct SExprSupp {
int32_t numOfExprs; // the number of scalar expression in group operator
SqlFunctionCtx* pCtx;
int32_t* rowEntryInfoOffset; // offset value for each row result cell info
SFilterInfo* pFilterInfo;
} SExprSupp;
typedef struct SOperatorInfo {
@ -479,6 +480,7 @@ typedef struct SStreamScanInfo {
SExprInfo* pPseudoExpr;
int32_t numOfPseudoExpr;
SExprSupp tbnameCalSup;
SExprSupp tagCalSup;
int32_t primaryTsIndex; // primary time stamp slot id
SReadHandle readHandle;
SInterval interval; // if the upstream is an interval operator, the interval info is also kept here.
@ -775,6 +777,7 @@ typedef struct SStreamPartitionOperatorInfo {
SPartitionBySupporter partitionSup;
SExprSupp scalarSup;
SExprSupp tbnameCalSup;
SExprSupp tagCalSup;
SHashObj* pPartitions;
void* parIte;
SSDataBlock* pInputDataBlock;
@ -924,7 +927,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
void doSetOperatorCompleted(SOperatorInfo* pOperator);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo);
int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr,
SSDataBlock* pBlock, const char* idStr);

2
source/libs/executor/src/dataInserter.c
View File

@ -81,7 +81,7 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
}
pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows;
qDebug("submit rsp received, affectedRows:%d, total:%d", pInserter->submitRes.pRsp->affectedRows,
qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows,
pInserter->submitRes.affectedRows);
tFreeSSubmitRsp(pInserter->submitRes.pRsp);

17
source/libs/executor/src/executil.c
View File

@ -1191,14 +1191,21 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) {
pExp->base.resSchema =
createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pFuncNode->node.aliasName);
pExp->pExpr->_function.functionId = pFuncNode->funcId;
pExp->pExpr->_function.pFunctNode = pFuncNode;
tExprNode* pExprNode = pExp->pExpr;
pExprNode->_function.functionId = pFuncNode->funcId;
pExprNode->_function.pFunctNode = pFuncNode;
tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName));
strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName,
tListLen(pExp->pExpr->_function.functionName));
#if 1
// todo refactor: add the parameter for tbname function
if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) {
const char* name = "tbname";
int32_t len = strlen(name);
if (!pFuncNode->pParameterList && (memcmp(pExprNode->_function.functionName, name, len) == 0) &&
pExprNode->_function.functionName[len] == 0) {
pFuncNode->pParameterList = nodesMakeList();
ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0);
SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);

6
source/libs/executor/src/executor.c
View File

@ -856,15 +856,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema);
pTaskInfo->streamInfo.schema = mtInfo.schema;
qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %ld", mtInfo.uid, pOffset->ts);
qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %"PRId64, mtInfo.uid, pOffset->ts);
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) {
SStreamRawScanInfo* pInfo = pOperator->info;
SSnapContext* sContext = pInfo->sContext;
if (setForSnapShot(sContext, pOffset->uid) != 0) {
qError("setForSnapShot error. uid:%" PRIi64 " ,version:%" PRIi64, pOffset->uid);
qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version);
return -1;
}
qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %ld", pOffset->uid);
qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %"PRId64, pOffset->uid, pOffset->ts);
} else if (pOffset->type == TMQ_OFFSET__LOG) {
SStreamRawScanInfo* pInfo = pOperator->info;
tsdbReaderClose(pInfo->dataReader);

29
source/libs/executor/src/executorimpl.c
View File

@ -1113,15 +1113,24 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO
static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep,
int32_t status);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo) {
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo) {
if (pFilterNode == NULL || pBlock->info.rows == 0) {
return;
}
SFilterInfo* filter = NULL;
SFilterInfo* filter = pFilterInfo;
int64_t st = taosGetTimestampUs();
// pError("start filter");
// todo move to the initialization function
int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0);
int32_t code = 0;
bool needFree = false;
if (filter == NULL) {
needFree = true;
code = filterInitFromNode((SNode*)pFilterNode, &filter, 0);
}
SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock};
code = filterSetDataFromSlotId(filter, &param1);
@ -1130,7 +1139,10 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColM
// todo the keep seems never to be True??
bool keep = filterExecute(filter, pBlock, &p, NULL, param1.numOfCols, &status);
if (needFree) {
filterFreeInfo(filter);
}
extractQualifiedTupleByFilterResult(pBlock, p, keep, status);
@ -1845,6 +1857,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq));
if (NULL == pMsg) {
pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
taosMemoryFree(pWrapper);
return pTaskInfo->code;
}
@ -2479,7 +2492,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
while (1) {
doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf);
doFilter(pAggInfo->pCondition, pInfo->pRes, NULL);
doFilter(pAggInfo->pCondition, pInfo->pRes, NULL, NULL);
if (!hasRemainResults(&pAggInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
@ -2873,7 +2886,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
break;
}
doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo);
doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo, NULL);
if (fillResult->info.rows > 0) {
break;
}
@ -3049,6 +3062,12 @@ void cleanupExprSupp(SExprSupp* pSupp) {
destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs);
taosMemoryFreeClear(pSupp->pExprInfo);
}
if (pSupp->pFilterInfo != NULL) {
filterFreeInfo(pSupp->pFilterInfo);
pSupp->pFilterInfo = NULL;
}
taosMemoryFree(pSupp->rowEntryInfoOffset);
}

17
source/libs/executor/src/groupoperator.c
View File

@ -311,7 +311,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) {
SSDataBlock* pRes = pInfo->binfo.pRes;
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pRes, NULL);
doFilter(pInfo->pCondition, pRes, NULL, NULL);
if (!hasRemainResults(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
@ -991,6 +991,8 @@ static void destroyStreamPartitionOperatorInfo(void* param) {
taosMemoryFree(pInfo->partitionSup.keyBuf);
cleanupExprSupp(&pInfo->scalarSup);
cleanupExprSupp(&pInfo->tbnameCalSup);
cleanupExprSupp(&pInfo->tagCalSup);
blockDataDestroy(pInfo->pDelRes);
taosMemoryFreeClear(param);
}
@ -1037,6 +1039,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
}
}
if (pPartNode->pTags != NULL) {
int32_t numOfTags;
SExprInfo* pTagExpr = createExprInfo(pPartNode->pTags, NULL, &numOfTags);
if (pTagExpr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
}
int32_t keyLen = 0;
code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf,
pInfo->partitionSup.pGroupCols);

2
source/libs/executor/src/joinoperator.c
View File

@ -387,7 +387,7 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
break;
}
if (pJoinInfo->pCondAfterMerge != NULL) {
doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL);
doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL, NULL);
}
if (pRes->info.rows >= pOperator->resultInfo.threshold) {
break;

8
source/libs/executor/src/projectoperator.c
View File

@ -315,7 +315,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
}
// do apply filter
doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL);
doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL, NULL);
// when apply the limit/offset for each group, pRes->info.rows may be 0, due to limit constraint.
if (pFinalRes->info.rows > 0 || (pOperator->status == OP_EXEC_DONE)) {
@ -325,7 +325,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
} else {
// do apply filter
if (pRes->info.rows > 0) {
doFilter(pProjectInfo->pFilterNode, pRes, NULL);
doFilter(pProjectInfo->pFilterNode, pRes, NULL, NULL);
if (pRes->info.rows == 0) {
continue;
}
@ -518,7 +518,7 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
}
}
doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL);
doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL, NULL);
size_t rows = pInfo->pRes->info.rows;
if (rows > 0 || pOperator->status == OP_EXEC_DONE) {
break;
@ -620,7 +620,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) {
}
pRes->info.rows = 1;
doFilter(pProjectInfo->pFilterNode, pRes, NULL);
doFilter(pProjectInfo->pFilterNode, pRes, NULL, NULL);
/*int32_t status = */ doIngroupLimitOffset(&pProjectInfo->limitInfo, 0, pRes, pOperator);

56
source/libs/executor/src/scanoperator.c
View File

@ -421,7 +421,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
if (pTableScanInfo->pFilterNode != NULL) {
int64_t st = taosGetTimestampUs();
doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo);
doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, pOperator->exprSupp.pFilterInfo);
double el = (taosGetTimestampUs() - st) / 1000.0;
pTableScanInfo->readRecorder.filterTime += el;
@ -790,6 +790,11 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired;
pInfo->pResBlock = createResDataBlock(pDescNode);
pInfo->pFilterNode = pTableScanNode->scan.node.pConditions;
if (pInfo->pFilterNode != NULL) {
code = filterInitFromNode((SNode*)pInfo->pFilterNode, &pOperator->exprSupp.pFilterInfo, 0);
}
pInfo->scanFlag = MAIN_SCAN;
pInfo->pColMatchInfo = pColList;
pInfo->currentGroupId = -1;
@ -1159,7 +1164,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
return NULL;
}
doFilter(pInfo->pCondition, pResult, NULL);
doFilter(pInfo->pCondition, pResult, NULL, NULL);
if (pResult->info.rows == 0) {
continue;
}
@ -1347,10 +1352,15 @@ static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock*
blockDataEnsureCapacity(pResBlock, 1);
projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, 1, NULL);
projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, pTagCalSup->numOfExprs, NULL);
ASSERT(pResBlock->info.rows == 1);
// build tagArray
/*SArray* tagArray = taosArrayInit(0, sizeof(void*));*/
/*STagVal tagVal = {*/
/*.cid = 0,*/
/*.type = 0,*/
/*};*/
// build STag
// set STag
@ -1505,7 +1515,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
}
if (filter) {
doFilter(pInfo->pCondition, pInfo->pRes, NULL);
doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL);
}
blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
blockDataFreeRes((SSDataBlock*)pBlock);
@ -1927,7 +1937,7 @@ FETCH_NEXT_BLOCK:
}
}
doFilter(pInfo->pCondition, pInfo->pRes, NULL);
doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL);
blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) {
@ -2146,6 +2156,9 @@ static void destroyStreamScanOperatorInfo(void* param) {
taosMemoryFree(pStreamScan->pPseudoExpr);
}
cleanupExprSupp(&pStreamScan->tbnameCalSup);
cleanupExprSupp(&pStreamScan->tagCalSup);
updateInfoDestroy(pStreamScan->pUpdateInfo);
blockDataDestroy(pStreamScan->pRes);
blockDataDestroy(pStreamScan->pUpdateRes);
@ -2200,6 +2213,19 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
}
}
if (pTableScanNode->pTags != NULL) {
int32_t numOfTags;
SExprInfo* pTagExpr = createExprInfo(pTableScanNode->pTags, NULL, &numOfTags);
if (pTagExpr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
}
pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES);
if (pInfo->pBlockLists == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -2397,7 +2423,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) {
return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes;
}
doFilter(pInfo->pCondition, pInfo->pRes, NULL);
doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL);
return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes;
}
@ -2578,16 +2604,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
SMetaReader smrChildTable = {0};
metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0);
metaGetTableEntryByName(&smrChildTable, condTableName);
int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName);
if (code != TSDB_CODE_SUCCESS) {
// terrno has been set by metaGetTableEntryByName, therefore, return directly
return NULL;
}
if (smrChildTable.me.type != TSDB_CHILD_TABLE) {
metaReaderClear(&smrChildTable);
blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
}
SMetaReader smrSuperTable = {0};
metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK);
metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
if (code != TSDB_CODE_SUCCESS) {
// terrno has been set by metaGetTableEntryByUid
return NULL;
}
sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock);
metaReaderClear(&smrSuperTable);
metaReaderClear(&smrChildTable);
@ -3577,7 +3614,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
if (pTableScanInfo->pFilterNode != NULL) {
int64_t st = taosGetTimestampMs();
doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo);
doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, NULL);
double el = (taosGetTimestampUs() - st) / 1000.0;
pTableScanInfo->readRecorder.filterTime += el;
@ -3912,6 +3949,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pColList);
goto _error;
}

4
source/libs/executor/src/sortoperator.c
View File

@ -216,7 +216,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
return NULL;
}
doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo);
doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo, NULL);
if (blockDataGetNumOfRows(pBlock) == 0) {
continue;
}
@ -658,7 +658,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
blockDataDestroy(p);
qDebug("%s get sorted block, groupId:%0x" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
pDataBlock->info.rows);
return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
}

11
source/libs/executor/src/tfill.c
View File

@ -1501,7 +1501,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
}
doStreamFillImpl(pOperator);
doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo);
doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo, NULL);
pOperator->resultInfo.totalRows += pInfo->pRes->info.rows;
if (pInfo->pRes->info.rows > 0) {
break;
@ -1681,7 +1681,11 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
&numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
pInfo->pCondition = pPhyFillNode->node.pConditions;
pInfo->pColMatchColInfo = pColMatchColInfo;
initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols);
int32_t code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
pInfo->srcRowIndex = 0;
pOperator->name = "FillOperator";
@ -1693,7 +1697,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo,
NULL, NULL, NULL);
int32_t code = appendDownstream(pOperator, &downstream, 1);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1702,5 +1706,6 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
_error:
destroyStreamFillOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
return NULL;
}

38
source/libs/executor/src/timewindowoperator.c
View File

@ -1277,7 +1277,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL);
bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
if (!hasRemain) {
@ -1315,7 +1315,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pBlock, NULL);
doFilter(pInfo->pCondition, pBlock, NULL, NULL);
bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
if (!hasRemain) {
@ -1420,8 +1420,9 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa
SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* pGpDatas = (uint64_t*)pGpCol->pData;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
SResultRowInfo dumyInfo;
SResultRowInfo dumyInfo = {0};
dumyInfo.cur.pageId = -1;
STimeWindow win = {0};
if (IS_FINAL_OP(pInfo)) {
win.skey = startTsCols[i];
@ -2019,7 +2020,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
if (pOperator->status == OP_RES_TO_RETURN) {
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL);
bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
if (!hasRemain) {
@ -2062,7 +2063,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL);
bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
if (!hasRemain) {
@ -5251,7 +5252,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
setInputDataBlock(pOperator, pSup->pCtx, pBlock, pIaInfo->inputOrder, scanFlag, true);
doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes);
doFilter(pMiaInfo->pCondition, pRes, NULL);
doFilter(pMiaInfo->pCondition, pRes, NULL, NULL);
if (pRes->info.rows >= pOperator->resultInfo.capacity) {
break;
}
@ -5828,27 +5829,30 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
pInfo->isFinal = false;
if (pIntervalPhyNode->window.pExprs != NULL) {
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
}
pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
initResultSizeInfo(&pOperator->resultInfo, 4096);
SExprSupp* pSup = &pOperator->exprSupp;
initBasicInfo(&pInfo->binfo, pResBlock);
initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
initBasicInfo(&pInfo->binfo, pResBlock);
initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
if (pIntervalPhyNode->window.pExprs != NULL) {
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
}
pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
pInfo->invertible = false;

26
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -478,7 +478,10 @@ static int32_t tlvDecodeValueEnum(STlvDecoder* pDecoder, void* pValue, int16_t l
return code;
}
static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue) {
static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue, int32_t size) {
if (pTlv->len > size - 1) {
return TSDB_CODE_FAILED;
}
memcpy(pValue, pTlv->value, pTlv->len);
return TSDB_CODE_SUCCESS;
}
@ -919,9 +922,14 @@ static int32_t msgToDatum(STlv* pTlv, void* pObj) {
}
break;
}
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_JSON: {
if (pTlv->len <= 0 || pTlv->len > TSDB_MAX_JSON_TAG_LEN) {
code = TSDB_CODE_FAILED;
break;
}
code = tlvDecodeDynBinary(pTlv, (void**)&pNode->datum.p);
break;
}
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
// todo
@ -1097,7 +1105,7 @@ static int32_t msgToFunctionNode(STlvDecoder* pDecoder, void* pObj) {
code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node);
break;
case FUNCTION_CODE_FUNCTION_NAME:
code = tlvDecodeCStr(pTlv, pNode->functionName);
code = tlvDecodeCStr(pTlv, pNode->functionName, sizeof(pNode->functionName));
break;
case FUNCTION_CODE_FUNCTION_ID:
code = tlvDecodeI32(pTlv, &pNode->funcId);
@ -1226,10 +1234,10 @@ static int32_t msgToName(STlvDecoder* pDecoder, void* pObj) {
code = tlvDecodeI32(pTlv, &pNode->acctId);
break;
case NAME_CODE_DB_NAME:
code = tlvDecodeCStr(pTlv, pNode->dbname);
code = tlvDecodeCStr(pTlv, pNode->dbname, sizeof(pNode->dbname));
break;
case NAME_CODE_TABLE_NAME:
code = tlvDecodeCStr(pTlv, pNode->tname);
code = tlvDecodeCStr(pTlv, pNode->tname, sizeof(pNode->tname));
break;
default:
break;
@ -1538,7 +1546,7 @@ static int32_t msgToEp(STlvDecoder* pDecoder, void* pObj) {
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case EP_CODE_FQDN:
code = tlvDecodeCStr(pTlv, pNode->fqdn);
code = tlvDecodeCStr(pTlv, pNode->fqdn, sizeof(pNode->fqdn));
break;
case EP_CODE_port:
code = tlvDecodeU16(pTlv, &pNode->port);
@ -3207,7 +3215,7 @@ static int32_t msgToPhysiQueryInsertNode(STlvDecoder* pDecoder, void* pObj) {
code = tlvDecodeI8(pTlv, &pNode->tableType);
break;
case PHY_QUERY_INSERT_CODE_TABLE_NAME:
code = tlvDecodeCStr(pTlv, pNode->tableName);
code = tlvDecodeCStr(pTlv, pNode->tableName, sizeof(pNode->tableName));
break;
case PHY_QUERY_INSERT_CODE_VG_ID:
code = tlvDecodeI32(pTlv, &pNode->vgId);
@ -3284,10 +3292,10 @@ static int32_t msgToPhysiDeleteNode(STlvDecoder* pDecoder, void* pObj) {
code = tlvDecodeI8(pTlv, &pNode->tableType);
break;
case PHY_DELETER_CODE_TABLE_FNAME:
code = tlvDecodeCStr(pTlv, pNode->tableFName);
code = tlvDecodeCStr(pTlv, pNode->tableFName, sizeof(pNode->tableFName));
break;
case PHY_DELETER_CODE_TS_COL_NAME:
code = tlvDecodeCStr(pTlv, pNode->tsColName);
code = tlvDecodeCStr(pTlv, pNode->tsColName, sizeof(pNode->tsColName));
break;
case PHY_DELETER_CODE_DELETE_TIME_RANGE:
code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, &pNode->deleteTimeRange);

9
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -209,6 +209,10 @@ int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId) {
}
SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId);
if (NULL == pAllocator) {
nodesError("allocator id %" PRIx64 " weak reference failed", allocatorId);
return -1;
}
return pAllocator->self;
}
@ -1716,9 +1720,10 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
int32_t len = 0;
if ('\0' == pCol->tableAlias[0]) {
len = sprintf(name, "%s", pCol->colName);
len = snprintf(name, sizeof(name), "%s", pCol->colName);
} else {
len = snprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName);
}
len = sprintf(name, "%s.%s", pCol->tableAlias, pCol->colName);
if (NULL == taosHashGet(pCxt->pColHash, name, len)) {
pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, NULL, 0);
if (TSDB_CODE_SUCCESS == pCxt->errCode) {

2
source/libs/parser/inc/parToken.h
View File

@ -175,8 +175,6 @@ _end:
void taosCleanupKeywordsTable();
SToken tscReplaceStrToken(char **str, SToken *token, const char *newToken);
SToken taosTokenDup(SToken *pToken, char *buf, int32_t len);
#ifdef __cplusplus

22
source/libs/parser/src/parAstCreater.c
View File

@ -518,7 +518,7 @@ SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTa
if (NULL != pDbName) {
COPY_STRING_FORM_ID_TOKEN(realTable->table.dbName, pDbName);
} else {
strcpy(realTable->table.dbName, pCxt->pQueryCxt->db);
snprintf(realTable->table.dbName, sizeof(realTable->table.dbName), "%s", pCxt->pQueryCxt->db);
}
if (NULL != pTableAlias && TK_NK_NIL != pTableAlias->type) {
COPY_STRING_FORM_ID_TOKEN(realTable->table.tableAlias, pTableAlias);
@ -599,7 +599,7 @@ SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) {
state->pCol = createPrimaryKeyCol(pCxt, NULL);
if (NULL == state->pCol) {
nodesDestroyNode((SNode*)state);
CHECK_OUT_OF_MEM(state->pCol);
CHECK_OUT_OF_MEM(NULL);
}
state->pExpr = pExpr;
return (SNode*)state;
@ -613,7 +613,7 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode
interval->pCol = createPrimaryKeyCol(pCxt, NULL);
if (NULL == interval->pCol) {
nodesDestroyNode((SNode*)interval);
CHECK_OUT_OF_MEM(interval->pCol);
CHECK_OUT_OF_MEM(NULL);
}
interval->pInterval = pInterval;
interval->pOffset = pOffset;
@ -631,7 +631,7 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) {
fill->pWStartTs = nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == fill->pWStartTs) {
nodesDestroyNode((SNode*)fill);
CHECK_OUT_OF_MEM(fill->pWStartTs);
CHECK_OUT_OF_MEM(NULL);
}
strcpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstart");
return (SNode*)fill;
@ -1495,10 +1495,10 @@ SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool igno
CHECK_OUT_OF_MEM(pStmt);
pStmt->indexType = type;
pStmt->ignoreExists = ignoreExists;
strcpy(pStmt->indexDbName, ((SRealTableNode*)pIndexName)->table.dbName);
strcpy(pStmt->indexName, ((SRealTableNode*)pIndexName)->table.tableName);
strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName);
strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName);
snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", ((SRealTableNode*)pIndexName)->table.dbName);
snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName);
snprintf(pStmt->dbName, sizeof(pStmt->dbName), "%s", ((SRealTableNode*)pRealTable)->table.dbName);
snprintf(pStmt->tableName, sizeof(pStmt->tableName), "%s", ((SRealTableNode*)pRealTable)->table.tableName);
nodesDestroyNode(pIndexName);
nodesDestroyNode(pRealTable);
pStmt->pCols = pCols;
@ -1524,8 +1524,8 @@ SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode*
SDropIndexStmt* pStmt = (SDropIndexStmt*)nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->ignoreNotExists = ignoreNotExists;
strcpy(pStmt->indexDbName, ((SRealTableNode*)pIndexName)->table.dbName);
strcpy(pStmt->indexName, ((SRealTableNode*)pIndexName)->table.tableName);
snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", ((SRealTableNode*)pIndexName)->table.dbName);
snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName);
nodesDestroyNode(pIndexName);
return (SNode*)pStmt;
}
@ -1818,7 +1818,7 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb
SNode* createFuncForDelete(SAstCreateContext* pCxt, const char* pFuncName) {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
CHECK_OUT_OF_MEM(pFunc);
strcpy(pFunc->functionName, pFuncName);
snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName);
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt, NULL))) {
nodesDestroyNode((SNode*)pFunc);
CHECK_OUT_OF_MEM(NULL);

17
source/libs/parser/src/parInsert.c
View File

@ -378,7 +378,9 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time
} else if (pToken->type == TK_TODAY) {
ts = taosGetTimestampToday(timePrec);
} else if (pToken->type == TK_NK_INTEGER) {
toInteger(pToken->z, pToken->n, 10, &ts);
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &ts)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
}
} else { // parse the RFC-3339/ISO-8601 timestamp format string
if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {
return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
@ -591,8 +593,6 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
case TSDB_DATA_TYPE_BIGINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z);
} else if (!IS_VALID_BIGINT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z);
}
return func(pMsgBuf, &iv, pSchema->bytes, param);
}
@ -894,10 +894,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16
case TSDB_DATA_TYPE_BIGINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z);
} else if (!IS_VALID_BIGINT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z);
}
val->i64 = iv;
break;
}
@ -1173,7 +1170,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName*
NEXT_TOKEN(pCxt->pSql, sToken);
SName sname;
createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg);
CHECK_CODE(createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg));
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(&sname, dbFName);
strcpy(pCxt->sTableName, sname.tname);
@ -1605,7 +1602,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
if (!pCxt->pComCxt->needMultiParse) {
continue;
} else {
parserInfo("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId);
parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId);
break;
}
}
@ -1613,7 +1610,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z);
}
parserInfo("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) {
SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags));
@ -1650,7 +1647,7 @@ static int32_t parseInsertBodyAgain(SInsertParseContext* pCxt) {
pCxt->pComCxt->needMultiParse = false;
return TSDB_CODE_SUCCESS;
}
parserInfo("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
// merge according to vgId
if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) {
CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks));

25
source/libs/parser/src/parInsertData.c
View File

@ -12,7 +12,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// clang-format off
#include "parInsertData.h"
#include "catalog.h"
@ -214,16 +214,16 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq)
pBuf = pBlocks->pData + pBlocks->size;
tEncoderInit(&coder, pBuf, len);
tEncodeSVCreateTbReq(&coder, pCreateTbReq);
int32_t code = tEncodeSVCreateTbReq(&coder, pCreateTbReq);
tEncoderClear(&coder);
pBlocks->size += len;
pBlocks->createTbReqLen = len;
return TSDB_CODE_SUCCESS;
return code;
}
int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, int32_t rowSize,
STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList,
int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset,
int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList,
SVCreateTbReq* pCreateTbReq) {
*dataBlocks = NULL;
STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)id, idLen);
@ -694,9 +694,9 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
if (pBlocks->numOfRows > 0) {
STableDataBlocks* dataBuf = NULL;
pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId
int32_t ret =
getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0,
pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL);
int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId),
TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf,
pVnodeDataBlockList, NULL);
if (ret != TSDB_CODE_SUCCESS) {
tdFreeSBlockRowMerger(pBlkRowMerger);
taosHashCleanup(pVnodeDataBlockHashList);
@ -708,7 +708,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
// the maximum expanded size in byte when a row-wise data is converted to SDataRow format
int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0;
int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize +
sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + pOneTableBlock->createTbReqLen;
sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) +
pOneTableBlock->createTbReqLen;
if (dataBuf->nAllocSize < destSize) {
dataBuf->nAllocSize = (uint32_t)(destSize * 1.5);
@ -898,9 +899,7 @@ int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgI
return TSDB_CODE_SUCCESS;
}
STableMeta *qGetTableMetaInDataBlock(void* pDataBlock) {
return ((STableDataBlocks*)pDataBlock)->pTableMeta;
}
STableMeta* qGetTableMetaInDataBlock(void* pDataBlock) { return ((STableDataBlocks*)pDataBlock)->pTableMeta; }
void qFreeStmtDataBlock(void* pDataBlock) {
if (pDataBlock == NULL) {

21
source/libs/parser/src/parTokenizer.c
View File

@ -613,27 +613,6 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
return 0;
}
SToken tscReplaceStrToken(char** str, SToken* token, const char* newToken) {
char* src = *str;
size_t nsize = strlen(newToken);
int32_t size = (int32_t)strlen(*str) - token->n + (int32_t)nsize + 1;
int32_t bsize = (int32_t)((uint64_t)token->z - (uint64_t)src);
SToken ntoken;
*str = taosMemoryCalloc(1, size);
strncpy(*str, src, bsize);
strcat(*str, newToken);
strcat(*str, token->z + token->n);
ntoken.n = (uint32_t)nsize;
ntoken.z = *str + bsize;
taosMemoryFreeClear(src);
return ntoken;
}
SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) {
SToken t0 = {0};

47
source/libs/parser/src/parTranslater.c
View File

@ -292,9 +292,13 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) {
} else {
do {
SArray* pTables = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
if (NULL == pTables) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (pCxt->currLevel == currTotalLevel) {
taosArrayPush(pTables, &pTable);
if (hasSameTableAlias(pTables)) {
taosArrayDestroy(pTables);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS,
"Not unique table/alias: '%s'", ((STableNode*)pTable)->tableAlias);
}
@ -308,7 +312,7 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) {
static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) {
SFullDatabaseName name = {0};
strcpy(name.fullDbName, pFullDbName);
snprintf(name.fullDbName, sizeof(name.fullDbName), "%s", pFullDbName);
return taosHashPut(pDbs, pFullDbName, strlen(pFullDbName), &name, sizeof(SFullDatabaseName));
}
@ -540,6 +544,7 @@ static int32_t getTableIndex(STranslateContext* pCxt, const SName* pName, SArray
if (TSDB_CODE_SUCCESS == code) {
code = collectUseTable(pName, pCxt->pTables);
}
if (TSDB_CODE_SUCCESS == code) {
if (pParCxt->async) {
code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes);
} else {
@ -550,6 +555,7 @@ static int32_t getTableIndex(STranslateContext* pCxt, const SName* pName, SArray
code = catalogGetTableIndex(pParCxt->pCatalog, &conn, pName, pIndexes);
}
}
if (TSDB_CODE_SUCCESS != code) {
parserError("0x%" PRIx64 " getTableIndex error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId,
tstrerror(code), pName->dbname, pName->tname);
@ -990,9 +996,9 @@ static int32_t parseTimeFromValueNode(STranslateContext* pCxt, SValueNode* pVal)
return pCxt->errCode;
}
if (IS_UNSIGNED_NUMERIC_TYPE(pVal->node.resType.type)) {
pVal->datum.i = pVal->datum.u;
pVal->datum.i = (int64_t)pVal->datum.u;
} else if (IS_FLOAT_TYPE(pVal->node.resType.type)) {
pVal->datum.i = pVal->datum.d;
pVal->datum.i = (int64_t)pVal->datum.d;
} else if (TSDB_DATA_TYPE_BOOL == pVal->node.resType.type) {
pVal->datum.i = pVal->datum.b;
}
@ -1069,7 +1075,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal,
}
case TSDB_DATA_TYPE_BIGINT: {
code = toInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.i);
if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_BIGINT(pVal->datum.i))) {
if (strict && TSDB_CODE_SUCCESS != code) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
}
*(int64_t*)&pVal->typeData = pVal->datum.i;
@ -3430,10 +3436,10 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) {
return NULL;
}
pCol->node.resType = ((SExprNode*)pNode)->resType;
strcpy(pCol->tableAlias, pTableAlias);
strcpy(pCol->colName, ((SExprNode*)pNode)->aliasName);
strcpy(pCol->node.aliasName, pCol->colName);
strcpy(pCol->node.userAlias, ((SExprNode*)pNode)->userAlias);
snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pTableAlias);
snprintf(pCol->colName, sizeof(pCol->colName), "%s", ((SExprNode*)pNode)->aliasName);
snprintf(pCol->node.aliasName, sizeof(pCol->node.aliasName), "%s", pCol->colName);
snprintf(pCol->node.userAlias, sizeof(pCol->node.userAlias), "%s", ((SExprNode*)pNode)->userAlias);
return (SNode*)pCol;
}
@ -3740,8 +3746,8 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int32_t val, int32_t minVal,
int32_t maxVal) {
if (val >= 0 && (val < minVal || val > maxVal)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %" PRId64 " valid range: [%d, %d]", pName,
val, minVal, maxVal);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %d valid range: [%d, %d]", pName, val,
minVal, maxVal);
}
return TSDB_CODE_SUCCESS;
}
@ -3878,7 +3884,7 @@ static int32_t checkDbStrictOption(STranslateContext* pCxt, SDatabaseOptions* pO
static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t v1, int32_t v2) {
if (val >= 0 && val != v1 && val != v2) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option %s: %" PRId64 ", only %d, %d allowed", pName, val, v1, v2);
"Invalid option %s: %d, only %d, %d allowed", pName, val, v1, v2);
}
return TSDB_CODE_SUCCESS;
}
@ -4463,8 +4469,8 @@ static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, ch
nodesDestroyNode((SNode*)pSelect);
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(pTable->table.dbName, pInfo->pDbName);
strcpy(pTable->table.tableName, pInfo->pTableName);
snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName);
snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName);
TSWAP(pTable->pMeta, pInfo->pRollupTableMeta);
pSelect->pFromTable = (SNode*)pTable;
@ -4940,9 +4946,9 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt
alterReq.superUser = 0;
alterReq.enable = pStmt->enable;
alterReq.sysInfo = pStmt->sysinfo;
strcpy(alterReq.pass, pStmt->password);
snprintf(alterReq.pass, sizeof(alterReq.pass), "%s", pStmt->password);
if (NULL != pCxt->pParseCxt->db) {
strcpy(alterReq.dbname, pCxt->pParseCxt->db);
snprintf(alterReq.dbname, sizeof(alterReq.dbname), "%s", pCxt->pParseCxt->db);
}
return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &alterReq);
@ -6049,7 +6055,7 @@ static SNode* createProjectCol(const char* pProjCol) {
if (NULL == pCol) {
return NULL;
}
strcpy(pCol->colName, pProjCol);
snprintf(pCol->colName, sizeof(pCol->colName), "%s", pProjCol);
return (SNode*)pCol;
}
@ -6082,9 +6088,9 @@ static int32_t createSimpleSelectStmt(const char* pDb, const char* pTable, int32
nodesDestroyNode((SNode*)pSelect);
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(pRealTable->table.dbName, pDb);
strcpy(pRealTable->table.tableName, pTable);
strcpy(pRealTable->table.tableAlias, pTable);
snprintf(pRealTable->table.dbName, sizeof(pRealTable->table.dbName), "%s", pDb);
snprintf(pRealTable->table.tableName, sizeof(pRealTable->table.tableName), "%s", pTable);
snprintf(pRealTable->table.tableAlias, sizeof(pRealTable->table.tableAlias), "%s", pTable);
pSelect->pFromTable = (SNode*)pRealTable;
if (numOfProjs >= 0) {
@ -7125,8 +7131,9 @@ static int32_t buildUpdateOptionsReq(STranslateContext* pCxt, SAlterTableStmt* p
pReq->newComment = strdup(pStmt->pOptions->comment);
if (NULL == pReq->newComment) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} else {
pReq->newCommentLen = strlen(pReq->newComment);
}
} else {
pReq->newCommentLen = -1;
}

5
source/libs/parser/src/parUtil.c
View File

@ -381,6 +381,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi
uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue,
strerror(errno));
retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);
taosMemoryFree(tmp);
goto end;
}
val.nData = valLen;
@ -652,8 +653,8 @@ static int32_t buildCatalogReqForInsert(SParseContext* pCxt, const SParseMetaCac
}
SUserAuthInfo auth = {0};
strcpy(auth.user, pCxt->pUser);
strcpy(auth.dbFName, p->dbFName);
snprintf(auth.user, sizeof(auth.user), "%s", pCxt->pUser);
snprintf(auth.dbFName, sizeof(auth.dbFName), "%s", p->dbFName);
auth.type = AUTH_TYPE_WRITE;
taosArrayPush(pCatalogReq->pUser, &auth);

4
source/libs/planner/src/planLogicCreater.c
View File

@ -471,9 +471,9 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr)
return NULL;
}
pCol->node.resType = pExpr->resType;
strcpy(pCol->colName, pExpr->aliasName);
snprintf(pCol->colName, sizeof(pCol->colName), "%s", pExpr->aliasName);
if (NULL != pStmtName) {
strcpy(pCol->tableAlias, pStmtName);
snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pStmtName);
}
return pCol;
}

74
source/libs/planner/src/planOptimizer.c
View File

@ -1124,7 +1124,7 @@ static int32_t sortPriKeyOptGetSequencingNodes(SLogicNode* pNode, SNodeList** pS
bool notOptimize = false;
int32_t code = sortPriKeyOptGetSequencingNodesImpl(pNode, &notOptimize, pSequencingNodes);
if (TSDB_CODE_SUCCESS != code || notOptimize) {
nodesClearList(*pSequencingNodes);
NODES_CLEAR_LIST(*pSequencingNodes);
}
return code;
}
@ -1361,74 +1361,6 @@ static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo
return code;
}
static SNode* smaIndexOptCreateWStartTs() {
SFunctionNode* pWStart = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pWStart) {
return NULL;
}
strcpy(pWStart->functionName, "_wstart");
snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart);
if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pWStart, NULL, 0)) {
nodesDestroyNode((SNode*)pWStart);
return NULL;
}
return (SNode*)pWStart;
}
static int32_t smaIndexOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) {
SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
if (NULL == pMergeKey) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pMergeKey->pExpr = nodesCloneNode(pCol);
if (NULL == pMergeKey->pExpr) {
nodesDestroyNode((SNode*)pMergeKey);
return TSDB_CODE_OUT_OF_MEMORY;
}
pMergeKey->order = ORDER_ASC;
pMergeKey->nullOrder = NULL_ORDER_FIRST;
return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey);
}
static int32_t smaIndexOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) {
if (wstrartIndex < 0) {
SNode* pWStart = smaIndexOptCreateWStartTs();
if (NULL == pWStart) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = createColumnByRewriteExpr(pWStart, &pInterval->node.pTargets);
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode(pWStart);
return code;
}
wstrartIndex = LIST_LENGTH(pInterval->node.pTargets) - 1;
}
return smaIndexOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys);
}
static int32_t smaIndexOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex,
SNodeList* pSmaCols, int32_t wstrartIndex) {
SWindowLogicNode* pInterval = (SWindowLogicNode*)pScan->node.pParent;
SNodeList* pMergeTargets = nodesCloneList(pInterval->node.pTargets);
if (NULL == pMergeTargets) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SLogicNode* pSmaScan = NULL;
SLogicNode* pMerge = NULL;
SNodeList* pMergeKeys = NULL;
int32_t code = smaIndexOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys);
if (TSDB_CODE_SUCCESS == code) {
code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan);
}
if (TSDB_CODE_SUCCESS == code) {
code = smaIndexOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge);
}
if (TSDB_CODE_SUCCESS == code) {
code = smaIndexOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan);
}
return code;
}
static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex,
SNodeList* pSmaCols, int32_t wstrartIndex) {
SLogicNode* pSmaScan = NULL;
@ -1559,7 +1491,7 @@ static SNode* partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode) {
return NULL;
}
strcpy(pFunc->functionName, pFuncName);
snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName);
if (QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) {
SColumnNode* pCol = (SColumnNode*)pNode;
partTagsSetAlias(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), pCol->tableAlias, pCol->colName);
@ -2028,7 +1960,7 @@ static SNode* rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNode
if (NULL != pSelectValue) {
strcpy(pFunc->node.aliasName, pSelectValue->node.aliasName);
} else {
snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, pFunc);
snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, (void*)pFunc);
}
int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pCol));
if (TSDB_CODE_SUCCESS == code) {

18
source/libs/planner/src/planPhysiCreater.c
View File

@ -67,7 +67,7 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const S
if (NULL == pSlot) {
return NULL;
}
strcpy(pSlot->name, pName);
snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName);
pSlot->slotId = slotId;
pSlot->dataType = ((SExprNode*)pNode)->resType;
pSlot->reserve = reserve;
@ -663,13 +663,17 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc);
}
SNodeList* condCols = nodesMakeList();
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) {
code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &condCols);
SNodeList* pCondCols = nodesMakeList();
if (NULL == pCondCols) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &pCondCols);
}
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, condCols, pJoin->node.pOutputDataBlockDesc);
nodesDestroyList(condCols);
code = addDataBlockSlots(pCxt, pCondCols, pJoin->node.pOutputDataBlockDesc);
}
nodesDestroyList(pCondCols);
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) {
@ -1634,7 +1638,7 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl
pSubplan->subplanType = pLogicSubplan->subplanType;
pSubplan->level = pLogicSubplan->level;
if (NULL != pCxt->pPlanCxt->pUser) {
strcpy(pSubplan->user, pCxt->pPlanCxt->pUser);
snprintf(pSubplan->user, sizeof(pSubplan->user), "%s", pCxt->pPlanCxt->pUser);
}
return pSubplan;
}
@ -1825,7 +1829,7 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNode* pSubplan, int32_t lev
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return nodesListStrictAppend(pGroup->pNodeList, (SNode*)pSubplan);
return nodesListAppend(pGroup->pNodeList, (SNode*)pSubplan);
}
static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pParent,

20
source/libs/planner/src/planSpliter.c
View File

@ -342,7 +342,7 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) {
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(pWStart->functionName, "_wstart");
snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart);
snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, (void*)pWStart);
int32_t code = fmGetFuncInfo(pWStart, NULL, 0);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pFuncs, (SNode*)pWStart);
@ -367,7 +367,7 @@ static int32_t stbSplAppendWEnd(SWindowLogicNode* pWin, int32_t* pIndex) {
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(pWEnd->functionName, "_wend");
snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, pWEnd);
snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, (void*)pWEnd);
int32_t code = fmGetFuncInfo(pWEnd, NULL, 0);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pWin->pFuncs, (SNode*)pWEnd);
@ -389,22 +389,19 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic
SNode* pConditions = pMergeWindow->node.pConditions;
pMergeWindow->node.pConditions = NULL;
int32_t code = TSDB_CODE_SUCCESS;
SWindowLogicNode* pPartWin = (SWindowLogicNode*)nodesCloneNode((SNode*)pMergeWindow);
if (NULL == pPartWin) {
code = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_OUT_OF_MEMORY;
}
pPartWin->node.groupAction = GROUP_ACTION_KEEP;
if (TSDB_CODE_SUCCESS == code) {
pMergeWindow->node.pTargets = pTargets;
pMergeWindow->node.pConditions = pConditions;
pPartWin->node.pChildren = pChildren;
splSetParent((SLogicNode*)pPartWin);
code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs);
}
int32_t index = 0;
int32_t code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = stbSplAppendWStart(pPartWin->pFuncs, &index);
}
@ -721,15 +718,16 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO
SNode* pConditions = pMergeAgg->node.pConditions;
pMergeAgg->node.pConditions = NULL;
int32_t code = TSDB_CODE_SUCCESS;
SAggLogicNode* pPartAgg = (SAggLogicNode*)nodesCloneNode((SNode*)pMergeAgg);
if (NULL == pPartAgg) {
code = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_OUT_OF_MEMORY;
}
pPartAgg->node.groupAction = GROUP_ACTION_KEEP;
if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) {
int32_t code = TSDB_CODE_SUCCESS;
if (NULL != pGroupKeys) {
pPartAgg->pGroupKeys = pGroupKeys;
code = createColumnByRewriteExprs(pPartAgg->pGroupKeys, &pPartAgg->node.pTargets);
}

1
source/libs/qworker/inc/qwInt.h
View File

@ -123,6 +123,7 @@ typedef struct SQWTaskCtx {
int32_t execId;
int32_t level;
bool queryGotData;
bool queryRsped;
bool queryEnd;
bool queryContinue;

72
source/libs/qworker/src/qworker.c
View File

@ -10,6 +10,7 @@
#include "tdatablock.h"
#include "tmsg.h"
#include "tname.h"
#include "tglobal.h"
SQWorkerMgmt gQwMgmt = {
.lock = 0,
@ -92,6 +93,19 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
return TSDB_CODE_SUCCESS;
}
int32_t qwSendQueryRsp(QW_FPARAMS_DEF, int32_t msgType, SQWTaskCtx *ctx, int32_t rspCode, bool quickRsp) {
if ((!quickRsp) || QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy) {
if (!ctx->localExec) {
qwBuildAndSendQueryRsp(msgType, &ctx->ctrlConnInfo, rspCode, ctx);
QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, rspCode, tstrerror(rspCode));
}
ctx->queryRsped = true;
}
return TSDB_CODE_SUCCESS;
}
int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
int32_t code = 0;
bool qcontinue = true;
@ -144,7 +158,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
if (numOfResBlock == 0) {
QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds);
} else {
QW_TASK_DLOG("qExecTask done", "");
QW_TASK_DLOG("qExecTask done, useconds:%" PRIu64, useconds);
}
dsEndPut(sinkHandle, useconds);
@ -234,6 +248,10 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
int32_t code = 0;
SOutputData output = {0};
if (NULL == ctx->sinkHandle) {
return TSDB_CODE_SUCCESS;
}
*dataLen = 0;
while (true) {
@ -407,6 +425,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION);
}
if (ctx->rspCode) {
QW_TASK_ELOG("task already failed cause of %s, phase:%s", tstrerror(ctx->rspCode), qwPhaseStr(phase));
QW_ERR_JRET(ctx->rspCode);
}
if (!ctx->queryRsped) {
QW_TASK_ELOG("ready msg has not been processed, phase:%s", qwPhaseStr(phase));
QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR);
@ -419,6 +442,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
}
if (ctx->rspCode) {
QW_TASK_ELOG("task already failed cause of %s, phase:%s", tstrerror(ctx->rspCode), qwPhaseStr(phase));
QW_ERR_JRET(ctx->rspCode);
}
if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) {
QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
@ -499,23 +527,19 @@ _return:
if (TSDB_CODE_SUCCESS == code && QW_PHASE_POST_QUERY == phase) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC);
ctx->queryGotData = true;
}
if (QW_PHASE_POST_QUERY == phase && ctx) {
if (!ctx->localExec) {
if (QW_PHASE_POST_QUERY == phase && ctx && !ctx->queryRsped) {
bool rsped = false;
SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo};
qwDbgSimulateRedirect(&qwMsg, ctx, &rsped);
qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped);
if (!rsped) {
qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx);
QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code));
qwSendQueryRsp(QW_FPARAMS(), input->msgType + 1, ctx, code, false);
}
}
ctx->queryRsped = true;
}
if (ctx) {
QW_UPDATE_RSP_CODE(ctx, code);
@ -551,6 +575,7 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx));
ctx->ctrlConnInfo = qwMsg->connInfo;
ctx->phase = -1;
QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT));
@ -604,6 +629,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) {
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true);
ctx->level = plan->level;
atomic_store_ptr(&ctx->taskHandle, pTaskInfo);
atomic_store_ptr(&ctx->sinkHandle, sinkHandle);
@ -619,6 +646,31 @@ _return:
input.msgType = qwMsg->msgType;
code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL);
if (ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
void *rsp = NULL;
int32_t dataLen = 0;
SOutputData sOutput = {0};
QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput));
if (rsp) {
bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd);
qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete);
if (qComplete) {
atomic_store_8((int8_t *)&ctx->queryEnd, true);
}
qwMsg->connInfo = ctx->dataConnInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code);
rsp = NULL;
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code,
tstrerror(code), dataLen);
}
}
QW_RET(TSDB_CODE_SUCCESS);
}
@ -740,7 +792,9 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
locked = true;
// RC WARNING
if (QW_QUERY_RUNNING(ctx)) {
if (-1 == ctx->phase || false == ctx->queryGotData) {
QW_TASK_DLOG_E("task query unfinished");
} else if (QW_QUERY_RUNNING(ctx)) {
atomic_store_8((int8_t *)&ctx->queryContinue, 1);
} else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC);

8
source/libs/scheduler/src/schTask.c
View File

@ -913,7 +913,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) {
SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask));
if (SCH_IS_QUERY_JOB(pJob)) {
SCH_ERR_RET(schEnsureHbConnection(pJob, pTask));
// SCH_ERR_RET(schEnsureHbConnection(pJob, pTask));
}
SCH_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType));
@ -993,6 +993,12 @@ int32_t schLaunchTaskImpl(void *param) {
SCH_ERR_JRET(schLaunchRemoteTask(pJob, pTask));
}
#if 0
if (SCH_IS_QUERY_JOB(pJob)) {
SCH_ERR_JRET(schEnsureHbConnection(pJob, pTask));
}
#endif
_return:
if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) {

3
source/libs/scheduler/src/scheduler.c
View File

@ -155,8 +155,7 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) {
return;
}
SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", errCode:0x%x", *jobId, errCode);
SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode));
schHandleJobDrop(pJob, errCode);
schReleaseJob(*jobId);

6
source/libs/stream/src/stream.c
View File

@ -67,7 +67,11 @@ void streamSchedByTimer(void* param, void* tmrId) {
atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE);
streamTaskInput(pTask, (SStreamQueueItem*)trigger);
if (streamTaskInput(pTask, (SStreamQueueItem*)trigger) < 0) {
taosFreeQitem(trigger);
taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer);
return;
}
streamSchedExec(pTask);
}

9
source/libs/stream/src/streamMeta.c
View File

@ -28,6 +28,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
sprintf(streamPath, "%s/%s", path, "stream");
pMeta->path = strdup(streamPath);
if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db) < 0) {
taosMemoryFree(streamPath);
goto _err;
}
@ -58,7 +59,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
return pMeta;
_err:
if (pMeta->path) taosMemoryFree(pMeta->path);
taosMemoryFree(pMeta->path);
if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks);
if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb);
if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb);
@ -250,6 +251,8 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) {
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
if (pTask == NULL) {
tdbFree(pKey);
tdbFree(pVal);
return -1;
}
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
@ -257,10 +260,14 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) {
tDecoderClear(&decoder);
if (pMeta->expandFunc(pMeta->ahandle, pTask) < 0) {
tdbFree(pKey);
tdbFree(pVal);
return -1;
}
if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) {
tdbFree(pKey);
tdbFree(pVal);
return -1;
}
}

2
source/libs/stream/src/streamState.c
View File

@ -63,7 +63,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int
sprintf(statePath, "%s/%d", path, pTask->taskId);
} else {
memset(statePath, 0, 300);
strncpy(statePath, path, 300);
tstrncpy(statePath, path, 300);
}
if (tdbOpen(statePath, szPage, pages, &pState->db) < 0) {
goto _err;

9
source/libs/stream/src/streamUpdate.c
View File

@ -174,14 +174,9 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) {
if (res == TSDB_CODE_SUCCESS) {
return false;
} else {
qDebug("===stream===Update close window sbf. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64
", ts:%" PRIu64,
tableId, maxTs, *pMapMaxTs, ts);
return true;
}
}
qDebug("===stream===Update close window. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64,
tableId, maxTs, *pMapMaxTs, ts);
return true;
}
@ -203,14 +198,10 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) {
}
if (ts < pInfo->minTS) {
qDebug("===stream===Update min ts. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64,
tableId, maxTs, *pMapMaxTs, ts);
return true;
} else if (res == TSDB_CODE_SUCCESS) {
return false;
}
qDebug("===stream===Update. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId,
maxTs, *pMapMaxTs, ts);
// check from tsdb api
return true;
}

2
source/libs/sync/inc/syncInt.h
View File

@ -169,7 +169,7 @@ typedef struct SSyncNode {
} SSyncNode;
// open/close --------------
SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo);
void syncNodeStart(SSyncNode* pSyncNode);
void syncNodeStartStandBy(SSyncNode* pSyncNode);
void syncNodeClose(SSyncNode* pSyncNode);

19
source/libs/sync/src/syncMain.c
View File

@ -86,10 +86,10 @@ void syncCleanUp() {
}
}
int64_t syncOpen(const SSyncInfo* pSyncInfo) {
int64_t syncOpen(SSyncInfo* pSyncInfo) {
SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo);
if (pSyncNode == NULL) {
sError("failed to open sync node. vgId:%d", pSyncInfo->vgId);
sError("vgId:%d, failed to open sync node since %s", pSyncInfo->vgId, terrstr());
return -1;
}
@ -230,7 +230,7 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg
return ret;
}
int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) {
SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
if (pSyncNode == NULL) {
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
@ -238,6 +238,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
}
ASSERT(rid == pSyncNode->rid);
#if 0
if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) {
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR;
@ -259,6 +260,12 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
return ret;
#else
syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg);
syncNodeDoConfigChange(pSyncNode, pNewCfg, SYNC_INDEX_INVALID);
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
return 0;
#endif
}
int32_t syncLeaderTransfer(int64_t rid) {
@ -919,9 +926,7 @@ _END:
}
// open/close --------------
SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo;
SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
SSyncNode* pSyncNode = (SSyncNode*)taosMemoryCalloc(1, sizeof(SSyncNode));
if (pSyncNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -957,7 +962,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
sError("failed to open raft cfg file. path:%s", pSyncNode->configPath);
goto _error;
}
if (pSyncInfo->syncCfg.replicaNum == 0) {
pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg;
}
raftCfgClose(pSyncNode->pRaftCfg);
pSyncNode->pRaftCfg = NULL;

2
source/libs/tdb/src/db/tdbBtree.c
View File

@ -987,7 +987,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const
// fetch a new ofp and make it dirty
SPgno pgno = 0;
SPage *ofp, *nextOfp;
SPage *ofp = NULL, *nextOfp = NULL;
ret = tdbFetchOvflPage(&pgno, &ofp, pTxn, pBt);
if (ret < 0) {

6
source/libs/tdb/src/db/tdbPager.c
View File

@ -34,9 +34,9 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage);
static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage);
static FORCE_INLINE int32_t pageCmpFn(const void *lhs, const void *rhs) {
SPage *pPageL = (SPage *)(((uint8_t *)lhs) - sizeof(SRBTreeNode));
SPage *pPageR = (SPage *)(((uint8_t *)rhs) - sizeof(SRBTreeNode));
static FORCE_INLINE int32_t pageCmpFn(const SRBTreeNode *lhs, const SRBTreeNode *rhs) {
SPage *pPageL = (SPage *)(((uint8_t *)lhs) - offsetof(SPage, node));
SPage *pPageR = (SPage *)(((uint8_t *)rhs) - offsetof(SPage, node));
SPgno pgnoL = TDB_PAGE_PGNO(pPageL);
SPgno pgnoR = TDB_PAGE_PGNO(pPageR);

3
source/libs/transport/src/transCli.c
View File

@ -597,6 +597,7 @@ static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) {
static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
SCliConn* conn = handle->data;
SConnBuffer* pBuf = &conn->readBuf;
tDebug("%s conn %p alloc read buf", CONN_GET_INST_LABEL(conn), conn);
transAllocBuffer(pBuf, buf);
}
static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
@ -609,7 +610,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
if (nread > 0) {
pBuf->len += nread;
while (transReadComplete(pBuf)) {
tTrace("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn);
tDebug("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn);
if (pBuf->invalid) {
cliHandleExcept(conn);
break;

2
source/libs/transport/src/transComm.c
View File

@ -203,7 +203,7 @@ bool transReadComplete(SConnBuffer* connBuf) {
}
int transSetConnOption(uv_tcp_t* stream) {
uv_tcp_nodelay(stream, 1);
uv_tcp_nodelay(stream, 0);
int ret = uv_tcp_keepalive(stream, 5, 60);
return ret;
}

3
source/libs/transport/src/transSvr.c
View File

@ -333,6 +333,9 @@ void uvOnSendCb(uv_write_t* req, int status) {
tTrace("conn %p data already was written on stream", conn);
if (!transQueueEmpty(&conn->srvMsgs)) {
SSvrMsg* msg = transQueuePop(&conn->srvMsgs);
STraceId* trace = &msg->msg.info.traceId;
tGDebug("conn %p write data out", conn);
destroySmsg(msg);
// send cached data
if (!transQueueEmpty(&conn->srvMsgs)) {

8
source/libs/wal/src/walMeta.c
View File

@ -351,7 +351,10 @@ int walCheckAndRepairIdx(SWal* pWal) {
taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
break;
}
taosLSeekFile(pIdxFile, offset, SEEK_SET);
if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d cannot seek offset %ld when repair idx since %s", pWal->cfg.vgId, offset, terrstr());
}
int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) {
terrno = TAOS_SYSTEM_ERROR(errno);
@ -636,6 +639,5 @@ int walRemoveMeta(SWal* pWal) {
if (metaVer == -1) return 0;
char fnameStr[WAL_FILE_LEN];
walBuildMetaName(pWal, metaVer, fnameStr);
taosRemoveFile(fnameStr);
return 0;
return taosRemoveFile(fnameStr);
}

2
source/libs/wal/src/walRead.c
View File

@ -514,7 +514,6 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
pReader->pHead->head.version, ver);
pReader->curInvalid = 1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
taosThreadMutexUnlock(&pReader->mutex);
return -1;
}
@ -528,7 +527,6 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum);
pReader->curInvalid = 1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
taosThreadMutexUnlock(&pReader->mutex);
return -1;
}

14
source/libs/wal/src/walWrite.c
View File

@ -42,10 +42,20 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
SWalFileInfo *pFileInfo = taosArrayGet(pWal->fileInfoSet, i);
char fnameStr[WAL_FILE_LEN];
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
taosRemoveFile(fnameStr);
if (taosRemoveFile(fnameStr) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr());
return -1;
}
wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr);
walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
taosRemoveFile(fnameStr);
if (taosRemoveFile(fnameStr) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr());
return -1;
}
wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr);
}
}
walRemoveMeta(pWal);

12
source/util/src/trbtree.c
View File

@ -219,7 +219,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) {
while (temp != pTree->NIL) {
y = temp;
int32_t c = pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(temp));
int32_t c = pTree->cmprFn(z, temp);
if (c < 0) {
temp = temp->left;
} else if (c > 0) {
@ -232,7 +232,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) {
if (y == pTree->NIL) {
pTree->root = z;
} else if (pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(y)) < 0) {
} else if (pTree->cmprFn(z, y) < 0) {
y->left = z;
} else {
y->right = z;
@ -245,10 +245,10 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) {
tRBTreePutFix(pTree, z);
// update min/max node
if (pTree->min == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->min), RBTREE_NODE_PAYLOAD(z)) > 0) {
if (pTree->min == pTree->NIL || pTree->cmprFn(pTree->min, z) > 0) {
pTree->min = z;
}
if (pTree->max == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->max), RBTREE_NODE_PAYLOAD(z)) < 0) {
if (pTree->max == pTree->NIL || pTree->cmprFn(pTree->max, z) < 0) {
pTree->max = z;
}
pTree->n++;
@ -309,11 +309,11 @@ SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) {
return pNode;
}
SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) {
SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode) {
SRBTreeNode *pNode = pTree->root;
while (pNode != pTree->NIL) {
int32_t c = pTree->cmprFn(pKey, RBTREE_NODE_PAYLOAD(pNode));
int32_t c = pTree->cmprFn(pKeyNode, pNode);
if (c < 0) {
pNode = pNode->left;

5
source/util/src/tskiplist.c
View File

@ -145,7 +145,10 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it
tSkipListWLock(pSkipList);
void *pData = iterate(iter);
if (pData == NULL) return;
if (pData == NULL) {
tSkipListUnlock(pSkipList);
return;
}
// backward to put the first data
hasDup = tSkipListGetPosToPut(pSkipList, backward, pData);

11
tests/develop-test/5-taos-tools/taosbenchmark/default_json.py
View File

@ -32,6 +32,10 @@ class TDTestCase:
if "community" in selfPath:
projPath = selfPath[: selfPath.find("community")]
elif "src" in selfPath:
projPath = selfPath[: selfPath.find("src")]
elif "/tools/" in selfPath:
projPath = selfPath[: selfPath.find("/tools/")]
else:
projPath = selfPath[: selfPath.find("tests")]
@ -55,11 +59,10 @@ class TDTestCase:
tdLog.info("%s" % cmd)
os.system("%s" % cmd)
tdSql.execute("reset query cache")
tdSql.query("select count(*) from (select distinct(tbname) from db.stb)")
tdSql.checkData(0, 0, 10)
tdSql.query("show db.tables")
tdSql.checkRows(10)
tdSql.query("select count(*) from db.stb")
if len(tdSql.queryResult):
tdLog.exit("query result is %d" % len(tdSql.queryResult))
tdSql.checkData(0, 0, 100)
def stop(self):
tdSql.close()

2
tests/develop-test/5-taos-tools/taosbenchmark/json/default.json
View File

@ -20,6 +20,8 @@
"super_tables": [{
"name": "stb",
"childtable_prefix": "stb_",
"childtable_count": 10,
"insert_rows": 10,
"columns": [{"type": "INT"}],
"tags": [{"type": "INT"}]
}]

291
tests/script/coverage_test.sh Executable file
View File

@ -0,0 +1,291 @@
#!/bin/bash
branch=
if [ x$1 != x ];then
branch=$1
echo "Testing branch: $branch"
else
echo "Please enter branch name as a parameter"
exit 1
fi
today=`date +"%Y%m%d"`
TDENGINE_DIR=/root/pxiao/TDengine
TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log
# Color setting
RED='\033[0;31m'
GREEN='\033[1;32m'
GREEN_DARK='\033[0;32m'
GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m'
function buildTDengine {
echo "check if TDengine need build"
cd $TDENGINE_DIR
git remote prune origin > /dev/null
git remote update > /dev/null
REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch`
LOCAL_COMMIT=`git rev-parse --short @`
echo " LOCAL: $LOCAL_COMMIT"
echo "REMOTE: $REMOTE_COMMIT"
# reset counter
lcov -d . --zerocounters
if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
echo "repo up-to-date"
else
echo "repo need to pull"
fi
git reset --hard
git checkout -- .
git checkout $branch
git clean -dfx
git pull
git submodule update --init --recursive -f
[ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug
cd $TDENGINE_DIR/debug
echo "rebuild.."
LOCAL_COMMIT=`git rev-parse --short @`
rm -rf *
if [ "$branch" == "3.0" ]; then
echo "3.0 ============="
cmake -DCOVER=true -DBUILD_TEST=true ..
else
cmake -DCOVER=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null
fi
make -j4
make install
}
function runGeneralCaseOneByOne {
while read -r line; do
if [[ $line =~ ^./test.sh* ]]; then
case=`echo $line | grep sim$ | awk '{print $NF}'`
if [ -n "$case" ]; then
date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && ./test.sh -f $case > /dev/null 2>&1 && \
echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \
|| echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
fi
fi
done < $1
}
function runTestNGCaseOneByOne {
while read -r line; do
if [[ $line =~ ^taostest* ]]; then
case=`echo $line | cut -d' ' -f 3 | cut -d'=' -f 2`
yaml=`echo $line | cut -d' ' -f 2`
if [ -n "$case" ]; then
date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && taostest $yaml --case=$case --keep --disable_collection > /dev/null 2>&1 && \
echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \
|| echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
fi
fi
done < $1
}
function runTest {
echo "run Test"
if [ "$branch" == "3.0" ]; then
echo "start run unit test case ................"
echo " $TDENGINE_DIR/debug "
cd $TDENGINE_DIR/debug
ctest -j12
echo "3.0 unit test done"
fi
cd $TDENGINE_DIR/tests/script
[ -d ../../sim ] && rm -rf ../../sim
[ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT
runGeneralCaseOneByOne jenkins/basic.txt
sed -i "1i\SIM cases test result" $TDENGINE_COVERAGE_REPORT
totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l`
if [ "$totalSuccess" -gt "0" ]; then
sed -i -e "2i\ ### Total $totalSuccess SIM test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT
fi
totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l`
if [ "$totalFailed" -ne "0" ]; then
sed -i "3i\### Total $totalFailed SIM test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT
fi
sed "3G" $TDENGINE_COVERAGE_REPORT
stopTaosd
echo "run TestNG cases"
rm -rf /var/lib/taos/*
rm -rf /var/log/taos/*
nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 &
sleep 10
cd $TDENGINE_DIR/../TestNG/cases
runTestNGCaseOneByOne ../scripts/cases.txt
echo "TestNG cases done"
cd $TDENGINE_DIR/tests
rm -rf ../sim
/root/pxiao/test-all-coverage.sh full python $branch | tee -a $TDENGINE_COVERAGE_REPORT
sed -i "4i\Python cases test result" $TDENGINE_COVERAGE_REPORT
totalPySuccess=`grep 'python case(s) succeed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'`
if [ "$totalPySuccess" -gt "0" ]; then
sed -i -e "5i\ ### Total $totalPySuccess Python test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT
fi
totalPyFailed=`grep 'python case(s) failed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'`
if [ -z $totalPyFailed ]; then
sed -i "6i\\n" $TDENGINE_COVERAGE_REPORT
else
sed -i "6i\### Total $totalPyFailed Python test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT
fi
echo "### run JDBC test cases ###" | tee -a $TDENGINE_COVERAGE_REPORT
# Test Connector
stopTaosd
nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 &
sleep 10
cd $TDENGINE_DIR/src/connector/jdbc
mvn clean package > /dev/null 2>&1
mvn test > jdbc-out.log 2>&1
tail -n 20 jdbc-out.log 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
# Test C Demo
stopTaosd
$TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null &
sleep 10
yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
# Test waltest
dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'`
walDir=`find $dataDir -name "wal"|head -n1`
echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT
echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT
if [ -n "$walDir" ]; then
yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
fi
# run Unit Test
echo "Run Unit Test: utilTest, queryTest and cliTest"
#$TDENGINE_DIR/debug/build/bin/utilTest > /dev/null 2>&1 && echo "utilTest pass!" || echo "utilTest failed!"
#$TDENGINE_DIR/debug/build/bin/queryTest > /dev/null 2>&1 && echo "queryTest pass!" || echo "queryTest failed!"
#$TDENGINE_DIR/debug/build/bin/cliTest > /dev/null 2>&1 && echo "cliTest pass!" || echo "cliTest failed!"
stopTaosd
cd $TDENGINE_DIR/tests/script
find . -name '*.sql' | xargs rm -f
cd $TDENGINE_DIR/tests/pytest
find . -name '*.sql' | xargs rm -f
}
function lcovFunc {
echo "collect data by lcov"
cd $TDENGINE_DIR
# collect data
lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info
# remove exclude paths
if [ "$branch" == "3.0" ]; then
lcov --remove coverage.info \
'*/contrib/*' '*/tests/*' '*/test/*'\
'*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c'\
'*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\
'*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/tidpool.c' '*/tmempool.c'\
'*/tthread.c' '*/tversion.c'\
--rc lcov_branch_coverage=1 -o coverage.info
else
lcov --remove coverage.info \
'*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' '*/ttype.h' '*/tarithoperator.c' '*/TSDBJNIConnector.c' '*/taosdemo.c'\
--rc lcov_branch_coverage=1 -o coverage.info
fi
# generate result
echo "generate result"
lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT
# push result to coveralls.io
echo "push result to coveralls.io"
/usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT
#/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT
#cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT
cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT
}
function sendReport {
echo "send report"
receiver="develop@taosdata.com"
mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
cd $TDENGINE_DIR
sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT
BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT`
echo -e "from: <support@taosdata.com>\nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
(cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \
/usr/sbin/ssmtp "${receiver}" && echo "Report Sent!"
}
function stopTaosd {
echo "Stop taosd start"
systemctl stop taosd
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
while [ -n "$PID" ]
do
pkill -TERM -x taosd
sleep 1
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
done
echo "Stop tasod end"
}
function runTestRandomFail {
exec_random_fail_sh=$1
default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh
[ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1
dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py
default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py
[ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1
runTest NoRandomFail
}
WORK_DIR=/root/pxiao
date >> $WORK_DIR/cron.log
echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log
stopTaosd
buildTDengine
#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh
#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh
#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh
runTest
lcovFunc
#sendReport
stopTaosd
date >> $WORK_DIR/cron.log
echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log

Some files were not shown because too many files have changed in this diff Show More