xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of github.com:taosdata/TDengine into szhou/tms-wide-column

This commit is contained in:
slzhou 2023-12-18 13:27:30 +08:00
parent b309c251da 93f3a3135f
commit 0d711331b7
224 changed files with 20776 additions and 6813 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -41,6 +41,7 @@ add_subdirectory(source)
add_subdirectory(tools)
add_subdirectory(utils)
add_subdirectory(examples/c)
add_subdirectory(tests)
include(${TD_SUPPORT_DIR}/cmake.install)
# docs

BIN
deps/arm/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/darwin/arm/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/darwin/x64/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/x86/dm_static/libdmodule.a vendored
View File

Binary file not shown.

2
docs/en/07-develop/01-connect/index.md
View File

@ -177,7 +177,7 @@ Just need to add the reference to [TDengine.Connector](https://www.nuget.org/pac
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.0" />
<PackageReference Include="TDengine.Connector" Version="3.1.0" />
</ItemGroup>
</Project>

3
docs/en/07-develop/04-query-data/_cs_async.mdx
View File

@ -1,3 +0,0 @@
```csharp
{{#include docs/examples/csharp/asyncQuery/Program.cs}}
```

4
docs/en/07-develop/04-query-data/index.mdx
View File

@ -15,7 +15,6 @@ import CQuery from "./_c.mdx";
import PhpQuery from "./_php.mdx";
import PyAsync from "./_py_async.mdx";
import NodeAsync from "./_js_async.mdx";
import CsAsync from "./_cs_async.mdx";
import CAsync from "./_c_async.mdx";
## Introduction
@ -174,9 +173,6 @@ Please note that async query can only be used with a native connection.
<TabItem label="Python" value="python">
<PyAsync />
</TabItem>
<TabItem label="C#" value="csharp">
<CsAsync />
</TabItem>
<TabItem label="C" value="c">
<CAsync />
</TabItem>

52
docs/en/07-develop/07-tmq.mdx
View File

@ -248,23 +248,23 @@ function close()
<TabItem value="C#" label="C#">
```csharp
class ConsumerBuilder<TValue>
ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
virtual IConsumer Build()
Consumer(ConsumerBuilder builder)
public IConsumer<TValue> Build()
void Subscribe(IEnumerable<string> topics)
void Subscribe(string topic)
ConsumeResult Consume(int millisecondsTimeout)
ConsumeResult<TValue> Consume(int millisecondsTimeout)
List<string> Subscription()
void Unsubscribe()
void Commit(ConsumeResult consumerResult)
List<TopicPartitionOffset> Commit()
void Close()
```
@ -500,25 +500,19 @@ let consumer = taos.consumer({
<TabItem value="C#" label="C#">
```csharp
using TDengineTMQ;
// Create consumer groups on demand (GourpID) and enable automatic commits (EnableAutoCommit),
// an automatic commit interval (AutoCommitIntervalMs), and a username (TDConnectUser) and password (TDConnectPasswd)
var cfg = new ConsumerConfig
{
EnableAutoCommit = "true"
AutoCommitIntervalMs = "1000"
GourpId = "TDengine-TMQ-C#",
TDConnectUser = "root",
TDConnectPasswd = "taosdata",
AutoOffsetReset = "latest"
MsgWithTableName = "true",
TDConnectIp = "127.0.0.1",
TDConnectPort = "6030"
};
var consumer = new ConsumerBuilder(cfg).Build();
var cfg = new Dictionary<string, string>()
{
{ "group.id", "group1" },
{ "auto.offset.reset", "latest" },
{ "td.connect.ip", "127.0.0.1" },
{ "td.connect.user", "root" },
{ "td.connect.pass", "taosdata" },
{ "td.connect.port", "6030" },
{ "client.id", "tmq_example" },
{ "enable.auto.commit", "true" },
{ "msg.with.table.name", "false" },
};
var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
```
</TabItem>
@ -747,10 +741,12 @@ while(true){
## Consume data
while (true)
{
var consumerRes = consumer.Consume(100);
// process ConsumeResult
ProcessMsg(consumerRes);
consumer.Commit(consumerRes);
using (var result = consumer.Consume(500))
{
if (result == null) continue;
ProcessMsg(result);
consumer.Commit();
}
}
```

6
docs/en/12-taos-sql/05-insert.md
View File

@ -158,8 +158,8 @@ Automatically creating table and the table name is specified through the `tbname
```sql
INSERT INTO meters(tbname, location, groupId, ts, current, phase)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33)
values('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 0.32)
('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 0.33)
('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 0.33)
```

57
docs/en/14-reference/02-rest-api/02-rest-api.mdx
View File

@ -262,6 +262,63 @@ The following types may be returned:
- "INT UNSIGNED"
- "BIGINT UNSIGNED"
- "JSON"
- "VARBINARY"
- "GEOMETRY"
`VARBINARY` and `GEOMETRY` types return data as Hex string, example:
Prepare data
```bash
create database demo
use demo
create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
insert into t values(now,'\x7f8290','point(100 100)')
```
Execute query
```bash
curl --location 'http://<fqdn>:<port>/rest/sql' \
--header 'Content-Type: text/plain' \
--header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \
--data 'select * from demo.t'
```
Return results
```json
{
"code": 0,
"column_meta": [
[
"ts",
"TIMESTAMP",
8
],
[
"c1",
"VARBINARY",
20
],
[
"c2",
"GEOMETRY",
100
]
],
"data": [
[
"2023-11-01T06:28:15.210Z",
"7f8290",
"010100000000000000000059400000000000005940"
]
],
"rows": 1
}
```
- `010100000000000000000059400000000000005940` is [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/) format for `point(100 100)`
#### Errors

1313
docs/en/14-reference/03-connector/09-csharp.mdx
View File

File diff suppressed because it is too large Load Diff

3
docs/examples/csharp/.gitignore vendored
View File

@ -1,5 +1,5 @@
.vs
asyncQuery/bin
.idea
connect/bin
influxdbLine/bin
optsJSON/bin
@ -12,7 +12,6 @@ wsConnect/bin
wsInsert/bin
wsQuery/bin
wsStmt/bin
asyncQuery/obj
connect/obj
influxdbLine/obj
optsJSON/obj

111
docs/examples/csharp/asyncQuery/Program.cs
View File

@ -1,111 +0,0 @@
using System;
using System.Collections.Generic;
using TDengineDriver;
using TDengineDriver.Impl;
using System.Runtime.InteropServices;
namespace TDengineExample
{
public class AsyncQueryExample
{
static void Main()
{
IntPtr conn = GetConnection();
try
{
QueryAsyncCallback queryAsyncCallback = new QueryAsyncCallback(QueryCallback);
TDengine.QueryAsync(conn, "select * from meters", queryAsyncCallback, IntPtr.Zero);
Thread.Sleep(2000);
}
finally
{
TDengine.Close(conn);
}
}
static void QueryCallback(IntPtr param, IntPtr taosRes, int code)
{
if (code == 0 && taosRes != IntPtr.Zero)
{
FetchRawBlockAsyncCallback fetchRowAsyncCallback = new FetchRawBlockAsyncCallback(FetchRawBlockCallback);
TDengine.FetchRawBlockAsync(taosRes, fetchRowAsyncCallback, param);
}
else
{
throw new Exception($"async query data failed,code:{code},reason:{TDengine.Error(taosRes)}");
}
}
// Iteratively call this interface until "numOfRows" is no greater than 0.
static void FetchRawBlockCallback(IntPtr param, IntPtr taosRes, int numOfRows)
{
if (numOfRows > 0)
{
Console.WriteLine($"{numOfRows} rows async retrieved");
IntPtr pdata = TDengine.GetRawBlock(taosRes);
List<TDengineMeta> metaList = TDengine.FetchFields(taosRes);
List<object> dataList = LibTaos.ReadRawBlock(pdata, metaList, numOfRows);
for (int i = 0; i < dataList.Count; i++)
{
if (i != 0 && (i + 1) % metaList.Count == 0)
{
Console.WriteLine("{0}\t|", dataList[i]);
}
else
{
Console.Write("{0}\t|", dataList[i]);
}
}
Console.WriteLine("");
TDengine.FetchRawBlockAsync(taosRes, FetchRawBlockCallback, param);
}
else
{
if (numOfRows == 0)
{
Console.WriteLine("async retrieve complete.");
}
else
{
throw new Exception($"FetchRawBlockCallback callback error, error code {numOfRows}");
}
TDengine.FreeResult(taosRes);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "power";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
}
}
// //output:
// // Connect to TDengine success
// // 8 rows async retrieved
// // 1538548685500 | 11.8 | 221 | 0.28 | california.losangeles | 2 |
// // 1538548696600 | 13.4 | 223 | 0.29 | california.losangeles | 2 |
// // 1538548685000 | 10.8 | 223 | 0.29 | california.losangeles | 3 |
// // 1538548686500 | 11.5 | 221 | 0.35 | california.losangeles | 3 |
// // 1538548685000 | 10.3 | 219 | 0.31 | california.sanfrancisco | 2 |
// // 1538548695000 | 12.6 | 218 | 0.33 | california.sanfrancisco | 2 |
// // 1538548696800 | 12.3 | 221 | 0.31 | california.sanfrancisco | 2 |
// // 1538548696650 | 10.3 | 218 | 0.25 | california.sanfrancisco | 3 |
// // async retrieve complete.

15
docs/examples/csharp/asyncQuery/asyncquery.csproj
View File

@ -1,15 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<StartupObject>TDengineExample.AsyncQueryExample</StartupObject>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" />
</ItemGroup>
</Project>

21
docs/examples/csharp/connect/Program.cs
View File

@ -1,4 +1,5 @@
using TDengineDriver;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample
{
@ -7,23 +8,11 @@ namespace TDengineExample
{
static void Main(String[] args)
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
throw new Exception("Connect to TDengine failed");
Console.WriteLine("connected");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
TDengine.Close(conn);
TDengine.Cleanup();
}
}
}

2
docs/examples/csharp/connect/connect.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" />
<PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup>
</Project>

6
docs/examples/csharp/csharp.sln
View File

@ -3,8 +3,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.30114.105
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "asyncquery", "asyncQuery\asyncquery.csproj", "{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "connect", "connect\connect.csproj", "{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "influxdbline", "influxdbLine\influxdbline.csproj", "{6A24FB80-1E3C-4E2D-A5AB-914FA583874D}"
@ -38,10 +36,6 @@ Global
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Release|Any CPU.Build.0 = Release|Any CPU
{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Release|Any CPU.ActiveCfg = Release|Any CPU

58
docs/examples/csharp/influxdbLine/Program.cs
View File

@ -1,4 +1,5 @@
using TDengineDriver;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample
{
@ -6,60 +7,23 @@ namespace TDengineExample
{
static void Main()
{
IntPtr conn = GetConnection();
PrepareDatabase(conn);
var builder =
new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
client.Exec("use test");
string[] lines = {
"meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249",
"meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250",
"meters,location=California.LosAngeles,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249",
"meters,location=California.LosAngeles,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250"
};
IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_LINE_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS);
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("SchemalessInsert failed since " + TDengine.Error(res));
}
else
{
int affectedRows = TDengine.AffectRows(res);
Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows");
}
TDengine.FreeResult(res);
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void PrepareDatabase(IntPtr conn)
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to create database, reason: " + TDengine.Error(res));
}
res = TDengine.Query(conn, "USE test");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to change database, reason: " + TDengine.Error(res));
client.SchemalessInsert(lines,
TDengineSchemalessProtocol.TSDB_SML_LINE_PROTOCOL,
TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
}
}
}
}

2
docs/examples/csharp/influxdbLine/influxdbline.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" />
<PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup>
</Project>

67
docs/examples/csharp/optsJSON/Program.cs
View File

@ -1,68 +1,27 @@
using TDengineDriver;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample
{
internal class OptsJsonExample
{
static void Main()
public static void Main(string[] args)
{
IntPtr conn = GetConnection();
try
var builder =
new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
PrepareDatabase(conn);
string[] lines = { "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
client.Exec("use test");
string[] lines =
{
"[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
" {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611249, \"value\": 219, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}, " +
"{\"metric\": \"meters.current\", \"timestamp\": 1648432611250, \"value\": 12.6, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
" {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611250, \"value\": 221, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}]"
};
IntPtr res = TDengine.SchemalessInsert(conn, lines, 1, (int)TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED);
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("SchemalessInsert failed since " + TDengine.Error(res));
}
else
{
int affectedRows = TDengine.AffectRows(res);
Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows");
}
TDengine.FreeResult(res);
}
finally
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void PrepareDatabase(IntPtr conn)
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to create database, reason: " + TDengine.Error(res));
}
res = TDengine.Query(conn, "USE test");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to change database, reason: " + TDengine.Error(res));
client.SchemalessInsert(lines, TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL,
TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
}
}
}

2
docs/examples/csharp/optsJSON/optsJSON.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" />
<PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup>
</Project>

63
docs/examples/csharp/optsTelnet/Program.cs
View File

@ -1,15 +1,18 @@
using TDengineDriver;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample
{
internal class OptsTelnetExample
{
static void Main()
public static void Main(string[] args)
{
IntPtr conn = GetConnection();
try
var builder =
new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
PrepareDatabase(conn);
client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
client.Exec("USE test");
string[] lines = {
"meters.current 1648432611249 10.3 location=California.SanFrancisco groupid=2",
"meters.current 1648432611250 12.6 location=California.SanFrancisco groupid=2",
@ -20,53 +23,9 @@ namespace TDengineExample
"meters.voltage 1648432611249 221 location=California.LosAngeles groupid=3",
"meters.voltage 1648432611250 217 location=California.LosAngeles groupid=3",
};
IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED);
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("SchemalessInsert failed since " + TDengine.Error(res));
}
else
{
int affectedRows = TDengine.AffectRows(res);
Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows");
}
TDengine.FreeResult(res);
}
catch
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void PrepareDatabase(IntPtr conn)
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to create database, reason: " + TDengine.Error(res));
}
res = TDengine.Query(conn, "USE test");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to change database, reason: " + TDengine.Error(res));
client.SchemalessInsert(lines,
TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL,
TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
}
}
}

2
docs/examples/csharp/optsTelnet/optstelnet.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" />
<PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup>
</Project>

81
docs/examples/csharp/query/Program.cs
View File

@ -1,80 +1,35 @@
using TDengineDriver;
using TDengineDriver.Impl;
using System.Runtime.InteropServices;
using System.Text;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample
{
internal class QueryExample
{
static void Main()
public static void Main(string[] args)
{
var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
IntPtr conn = GetConnection();
try
{
// run query
IntPtr res = TDengine.Query(conn, "SELECT * FROM meters LIMIT 2");
if (TDengine.ErrorNo(res) != 0)
client.Exec("use power");
string query = "SELECT * FROM meters";
using (var rows = client.Query(query))
{
throw new Exception("Failed to query since: " + TDengine.Error(res));
}
// get filed count
int fieldCount = TDengine.FieldCount(res);
Console.WriteLine("fieldCount=" + fieldCount);
// print column names
List<TDengineMeta> metas = LibTaos.GetMeta(res);
for (int i = 0; i < metas.Count; i++)
while (rows.Read())
{
Console.Write(metas[i].name + "\t");
Console.WriteLine(
$"{((DateTime)rows.GetValue(0)):yyyy-MM-dd HH:mm:ss.fff}, {rows.GetValue(1)}, {rows.GetValue(2)}, {rows.GetValue(3)}, {rows.GetValue(4)}, {Encoding.UTF8.GetString((byte[])rows.GetValue(5))}");
}
Console.WriteLine();
// print values
List<Object> resData = LibTaos.GetData(res);
for (int i = 0; i < resData.Count; i++)
}
}
catch (Exception e)
{
Console.Write($"|{resData[i].ToString()} \t");
if (((i + 1) % metas.Count == 0))
{
Console.WriteLine("");
Console.WriteLine(e.ToString());
throw;
}
}
Console.WriteLine();
// Free result after use
TDengine.FreeResult(res);
}
finally
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "power";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
}
}
// output:
// Connect to TDengine success
// fieldCount=6
// ts current voltage phase location groupid
// 1648432611249 10.3 219 0.31 California.SanFrancisco 2
// 1648432611749 12.6 218 0.33 California.SanFrancisco 2

2
docs/examples/csharp/query/query.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" />
<PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup>
</Project>

86
docs/examples/csharp/sqlInsert/Program.cs
View File

@ -1,69 +1,47 @@
using TDengineDriver;
using System.Text;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample
{
internal class SQLInsertExample
{
static void Main()
public static void Main(string[] args)
{
var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
IntPtr conn = GetConnection();
try
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600");
CheckRes(conn, res, "failed to create database");
res = TDengine.Query(conn, "USE power");
CheckRes(conn, res, "failed to change database");
res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)");
CheckRes(conn, res, "failed to create stable");
var sql = "INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
"d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
"d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
"d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
res = TDengine.Query(conn, sql);
CheckRes(conn, res, "failed to insert data");
int affectedRows = TDengine.AffectRows(res);
Console.WriteLine("affectedRows " + affectedRows);
TDengine.FreeResult(res);
client.Exec("create database power");
client.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
string insertQuery =
"INSERT INTO " +
"power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " +
"VALUES " +
"('2023-10-03 14:38:05.000', 10.30000, 219, 0.31000) " +
"('2023-10-03 14:38:15.000', 12.60000, 218, 0.33000) " +
"('2023-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
"power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " +
"VALUES " +
"('2023-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
"power.d1003 USING power.meters TAGS(2,'California.LosAngeles') " +
"VALUES " +
"('2023-10-03 14:38:05.500', 11.80000, 221, 0.28000) " +
"('2023-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
"power.d1004 USING power.meters TAGS(3,'California.LosAngeles') " +
"VALUES " +
"('2023-10-03 14:38:05.000', 10.80000, 223, 0.29000) " +
"('2023-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
client.Exec(insertQuery);
}
finally
catch (Exception e)
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void CheckRes(IntPtr conn, IntPtr res, String errorMsg)
{
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception($"{errorMsg} since: {TDengine.Error(res)}");
Console.WriteLine(e.ToString());
throw;
}
}
}
}
}
// output:
// Connect to TDengine success
// affectedRows 8

2
docs/examples/csharp/sqlInsert/sqlinsert.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" />
<PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup>
</Project>

113
docs/examples/csharp/stmtInsert/Program.cs
View File

@ -1,109 +1,38 @@
using TDengineDriver;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample
{
internal class StmtInsertExample
{
private static IntPtr conn;
private static IntPtr stmt;
static void Main()
public static void Main(string[] args)
{
var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
conn = GetConnection();
try
{
PrepareSTable();
// 1. init and prepare
stmt = TDengine.StmtInit(conn);
if (stmt == IntPtr.Zero)
client.Exec($"create database power");
client.Exec(
"CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
using (var stmt = client.StmtInit())
{
throw new Exception("failed to init stmt.");
stmt.Prepare(
"Insert into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(?,?,?,?)");
var ts = new DateTime(2023, 10, 03, 14, 38, 05, 000);
stmt.BindRow(new object[] { ts, (float)10.30000, (int)219, (float)0.31000 });
stmt.AddBatch();
stmt.Exec();
var affected = stmt.Affected();
Console.WriteLine($"affected rows: {affected}");
}
int res = TDengine.StmtPrepare(stmt, "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)");
CheckStmtRes(res, "failed to prepare stmt");
// 2. bind table name and tags
TAOS_MULTI_BIND[] tags = new TAOS_MULTI_BIND[2] { TaosMultiBind.MultiBindBinary(new string[] { "California.SanFrancisco" }), TaosMultiBind.MultiBindInt(new int?[] { 2 }) };
res = TDengine.StmtSetTbnameTags(stmt, "d1001", tags);
CheckStmtRes(res, "failed to bind table name and tags");
// 3. bind values
TAOS_MULTI_BIND[] values = new TAOS_MULTI_BIND[4] {
TaosMultiBind.MultiBindTimestamp(new long[2] { 1648432611249, 1648432611749}),
TaosMultiBind.MultiBindFloat(new float?[2] { 10.3f, 12.6f}),
TaosMultiBind.MultiBindInt(new int?[2] { 219, 218}),
TaosMultiBind.MultiBindFloat(new float?[2]{ 0.31f, 0.33f})
};
res = TDengine.StmtBindParamBatch(stmt, values);
CheckStmtRes(res, "failed to bind params");
// 4. add batch
res = TDengine.StmtAddBatch(stmt);
CheckStmtRes(res, "failed to add batch");
// 5. execute
res = TDengine.StmtExecute(stmt);
CheckStmtRes(res, "failed to execute");
// 6. free
TaosMultiBind.FreeTaosBind(tags);
TaosMultiBind.FreeTaosBind(values);
}
finally
catch (Exception e)
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void PrepareSTable()
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600");
CheckResPtr(res, "failed to create database");
res = TDengine.Query(conn, "USE power");
CheckResPtr(res, "failed to change database");
res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)");
CheckResPtr(res, "failed to create stable");
}
static void CheckStmtRes(int res, string errorMsg)
{
if (res != 0)
{
Console.WriteLine(errorMsg + ", " + TDengine.StmtErrorStr(stmt));
int code = TDengine.StmtClose(stmt);
if (code != 0)
{
throw new Exception($"failed to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} ");
Console.WriteLine(e);
throw;
}
}
}
static void CheckResPtr(IntPtr res, string errorMsg)
{
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception(errorMsg + " since:" + TDengine.Error(res));
}
}
}
}

2
docs/examples/csharp/stmtInsert/stmtinsert.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" />
<PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup>
</Project>

123
docs/examples/csharp/subscribe/Program.cs
View File

@ -1,95 +1,72 @@
using System;
using TDengineTMQ;
using TDengineDriver;
using System.Runtime.InteropServices;

using TDengine.Driver;
using TDengine.Driver.Client;
using TDengine.TMQ;
namespace TMQExample
{
internal class SubscribeDemo
{
static void Main(string[] args)
public static void Main(string[] args)
{
IntPtr conn = GetConnection();
string topic = "topic_example";
//create topic
IntPtr res = TDengine.Query(conn, $"create topic if not exists {topic} as select * from meters");
if (TDengine.ErrorNo(res) != 0 )
var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
throw new Exception($"create topic failed, reason:{TDengine.Error(res)}");
}
var cfg = new ConsumerConfig
try
{
GourpId = "group_1",
TDConnectUser = "root",
TDConnectPasswd = "taosdata",
MsgWithTableName = "true",
TDConnectIp = "127.0.0.1",
client.Exec("CREATE DATABASE power");
client.Exec("USE power");
client.Exec(
"CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
client.Exec("CREATE TOPIC topic_meters as SELECT * from power.meters");
var cfg = new Dictionary<string, string>()
{
{ "group.id", "group1" },
{ "auto.offset.reset", "latest" },
{ "td.connect.ip", "127.0.0.1" },
{ "td.connect.user", "root" },
{ "td.connect.pass", "taosdata" },
{ "td.connect.port", "6030" },
{ "client.id", "tmq_example" },
{ "enable.auto.commit", "true" },
{ "msg.with.table.name", "false" },
};
// create consumer
var consumer = new ConsumerBuilder(cfg)
.Build();
// subscribe
consumer.Subscribe(topic);
// consume
for (int i = 0; i < 5; i++)
var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
consumer.Subscribe(new List<string>() { "topic_meters" });
Task.Run(InsertData);
while (true)
{
var consumeRes = consumer.Consume(300);
// print consumeResult
foreach (KeyValuePair<TopicPartition, TaosResult> kv in consumeRes.Message)
using (var cr = consumer.Consume(500))
{
Console.WriteLine("topic partitions:\n{0}", kv.Key.ToString());
kv.Value.Metas.ForEach(meta =>
if (cr == null) continue;
foreach (var message in cr.Message)
{
Console.Write("{0} {1}({2}) \t|", meta.name, meta.TypeName(), meta.size);
});
Console.WriteLine("");
kv.Value.Datas.ForEach(data =>
Console.WriteLine(
$"message {{{((DateTime)message.Value["ts"]).ToString("yyyy-MM-dd HH:mm:ss.fff")}, " +
$"{message.Value["current"]}, {message.Value["voltage"]}, {message.Value["phase"]}}}");
}
}
}
}
catch (Exception e)
{
Console.WriteLine(data.ToString());
});
Console.WriteLine(e.ToString());
throw;
}
}
}
consumer.Commit(consumeRes);
Console.WriteLine("\n================ {0} done ", i);
}
// retrieve topic list
List<string> topics = consumer.Subscription();
topics.ForEach(t => Console.WriteLine("topic name:{0}", t));
// unsubscribe
consumer.Unsubscribe();
// close consumer after use.Otherwise will lead memory leak.
consumer.Close();
TDengine.Close(conn);
}
static IntPtr GetConnection()
static void InsertData()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "power";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
throw new Exception("Connect to TDengine failed");
}
else
while (true)
{
Console.WriteLine("Connect to TDengine success");
client.Exec("INSERT into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(now,11.5,219,0.30)");
Task.Delay(1000).Wait();
}
}
return conn;
}
}

2
docs/examples/csharp/subscribe/subscribe.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" />
<PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup>
</Project>

22
docs/examples/csharp/wsConnect/Program.cs
View File

@ -1,28 +1,18 @@
using System;
using TDengineWS.Impl;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace Examples
{
public class WSConnExample
{
static int Main(string[] args)
static void Main(string[] args)
{
string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
if (wsConn == IntPtr.Zero)
var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
Console.WriteLine("get WS connection failed");
return -1;
Console.WriteLine("connected");
}
else
{
Console.WriteLine("Establish connect success.");
// close connection.
LibTaosWS.WSClose(wsConn);
}
return 0;
}
}
}

2
docs/examples/csharp/wsConnect/wsConnect.csproj
View File

@ -6,7 +6,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" />
<PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
</ItemGroup>
<Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
<ItemGroup>

77
docs/examples/csharp/wsInsert/Program.cs
View File

@ -1,61 +1,46 @@
using System;
using TDengineWS.Impl;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace Examples
{
public class WSInsertExample
{
static int Main(string[] args)
public static void Main(string[] args)
{
string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
// Assert if connection is validate
if (wsConn == IntPtr.Zero)
var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
Console.WriteLine("get WS connection failed");
return -1;
}
else
try
{
Console.WriteLine("Establish connect success.");
client.Exec("create database power");
client.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
string insertQuery =
"INSERT INTO " +
"power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " +
"VALUES " +
"('2023-10-03 14:38:05.000', 10.30000, 219, 0.31000) " +
"('2023-10-03 14:38:15.000', 12.60000, 218, 0.33000) " +
"('2023-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
"power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " +
"VALUES " +
"('2023-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
"power.d1003 USING power.meters TAGS(2,'California.LosAngeles') " +
"VALUES " +
"('2023-10-03 14:38:05.500', 11.80000, 221, 0.28000) " +
"('2023-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
"power.d1004 USING power.meters TAGS(3,'California.LosAngeles') " +
"VALUES " +
"('2023-10-03 14:38:05.000', 10.80000, 223, 0.29000) " +
"('2023-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
client.Exec(insertQuery);
}
string createTable = "CREATE STABLE test.meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);";
string insert = "INSERT INTO test.d1001 USING test.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000)" +
"test.d1002 USING test.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000)" +
"test.d1003 USING test.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
"test.d1004 USING test.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
IntPtr wsRes = LibTaosWS.WSQuery(wsConn, createTable);
ValidInsert("create table", wsRes);
LibTaosWS.WSFreeResult(wsRes);
wsRes = LibTaosWS.WSQuery(wsConn, insert);
ValidInsert("insert data", wsRes);
LibTaosWS.WSFreeResult(wsRes);
// close connection.
LibTaosWS.WSClose(wsConn);
return 0;
}
static void ValidInsert(string desc, IntPtr wsRes)
catch (Exception e)
{
int code = LibTaosWS.WSErrorNo(wsRes);
if (code != 0)
{
Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
}
else
{
Console.WriteLine("{0} success affect {2} rows, cost {1} nanoseconds", desc, LibTaosWS.WSTakeTiming(wsRes), LibTaosWS.WSAffectRows(wsRes));
Console.WriteLine(e.ToString());
throw;
}
}
}
}
}
// Establish connect success.
// create table success affect 0 rows, cost 3717542 nanoseconds
// insert data success affect 8 rows, cost 2613637 nanoseconds

2
docs/examples/csharp/wsInsert/wsInsert.csproj
View File

@ -6,7 +6,7 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" />
<PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
</ItemGroup>
<Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
<ItemGroup>

81
docs/examples/csharp/wsQuery/Program.cs
View File

@ -1,79 +1,36 @@
using System;
using TDengineWS.Impl;
using System.Collections.Generic;
using TDengineDriver;
using System.Text;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace Examples
{
public class WSQueryExample
{
static int Main(string[] args)
public static void Main(string[] args)
{
string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
if (wsConn == IntPtr.Zero)
var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
Console.WriteLine("get WS connection failed");
return -1;
try
{
client.Exec("use power");
string query = "SELECT * FROM meters";
using (var rows = client.Query(query))
{
while (rows.Read())
{
Console.WriteLine(
$"{((DateTime)rows.GetValue(0)):yyyy-MM-dd HH:mm:ss.fff}, {rows.GetValue(1)}, {rows.GetValue(2)}, {rows.GetValue(3)}, {rows.GetValue(4)}, {Encoding.UTF8.GetString((byte[])rows.GetValue(5))}");
}
else
{
Console.WriteLine("Establish connect success.");
}
string select = "select * from test.meters";
// optional:wsRes = LibTaosWS.WSQuery(wsConn, select);
IntPtr wsRes = LibTaosWS.WSQueryTimeout(wsConn, select, 1);
// Assert if query execute success.
int code = LibTaosWS.WSErrorNo(wsRes);
if (code != 0)
{
Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
LibTaosWS.WSFreeResult(wsRes);
return -1;
}
// get meta data
List<TDengineMeta> metas = LibTaosWS.WSGetFields(wsRes);
// get retrieved data
List<object> dataSet = LibTaosWS.WSGetData(wsRes);
// do something with result.
foreach (var meta in metas)
catch (Exception e)
{
Console.Write("{0} {1}({2}) \t|\t", meta.name, meta.TypeName(), meta.size);
Console.WriteLine(e.ToString());
throw;
}
Console.WriteLine("");
for (int i = 0; i < dataSet.Count;)
{
for (int j = 0; j < metas.Count; j++)
{
Console.Write("{0}\t|\t", dataSet[i]);
i++;
}
Console.WriteLine("");
}
// Free result after use.
LibTaosWS.WSFreeResult(wsRes);
// close connection.
LibTaosWS.WSClose(wsConn);
return 0;
}
}
}
// Establish connect success.
// ts TIMESTAMP(8) | current FLOAT(4) | voltage INT(4) | phase FLOAT(4) | location BINARY(64) | groupid INT(4) |
// 1538548685000 | 10.8 | 223 | 0.29 | California.LosAngeles | 3 |
// 1538548686500 | 11.5 | 221 | 0.35 | California.LosAngeles | 3 |
// 1538548685500 | 11.8 | 221 | 0.28 | California.LosAngeles | 2 |
// 1538548696600 | 13.4 | 223 | 0.29 | California.LosAngeles | 2 |
// 1538548685000 | 10.3 | 219 | 0.31 | California.SanFrancisco | 2 |
// 1538548695000 | 12.6 | 218 | 0.33 | California.SanFrancisco | 2 |
// 1538548696800 | 12.3 | 221 | 0.31 | California.SanFrancisco | 2 |
// 1538548696650 | 10.3 | 218 | 0.25 | California.SanFrancisco | 3 |

2
docs/examples/csharp/wsQuery/wsQuery.csproj
View File

@ -7,7 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" />
<PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
</ItemGroup>
<Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
<ItemGroup>

107
docs/examples/csharp/wsStmt/Program.cs
View File

@ -1,98 +1,41 @@
using System;
using TDengineWS.Impl;
using TDengineDriver;
using System.Runtime.InteropServices;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace Examples
{
public class WSStmtExample
{
static int Main(string[] args)
public static void Main(string[] args)
{
const string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
const string table = "meters";
const string database = "test";
const string childTable = "d1005";
string insert = $"insert into ? using {database}.{table} tags(?,?) values(?,?,?,?)";
const int numOfTags = 2;
const int numOfColumns = 4;
// Establish connection
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
if (wsConn == IntPtr.Zero)
var builder =
new ConnectionStringBuilder(
"protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{
Console.WriteLine($"get WS connection failed");
return -1;
try
{
client.Exec($"create database power");
client.Exec(
"CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
using (var stmt = client.StmtInit())
{
stmt.Prepare(
"Insert into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(?,?,?,?)");
var ts = new DateTime(2023, 10, 03, 14, 38, 05, 000);
stmt.BindRow(new object[] { ts, (float)10.30000, (int)219, (float)0.31000 });
stmt.AddBatch();
stmt.Exec();
var affected = stmt.Affected();
Console.WriteLine($"affected rows: {affected}");
}
else
{
Console.WriteLine("Establish connect success...");
}
// init stmt
IntPtr wsStmt = LibTaosWS.WSStmtInit(wsConn);
if (wsStmt != IntPtr.Zero)
catch (Exception e)
{
int code = LibTaosWS.WSStmtPrepare(wsStmt, insert);
ValidStmtStep(code, wsStmt, "WSStmtPrepare");
TAOS_MULTI_BIND[] wsTags = new TAOS_MULTI_BIND[] { WSMultiBind.WSBindNchar(new string[] { "California.SanDiego" }), WSMultiBind.WSBindInt(new int?[] { 4 }) };
code = LibTaosWS.WSStmtSetTbnameTags(wsStmt, $"{database}.{childTable}", wsTags, numOfTags);
ValidStmtStep(code, wsStmt, "WSStmtSetTbnameTags");
TAOS_MULTI_BIND[] data = new TAOS_MULTI_BIND[4];
data[0] = WSMultiBind.WSBindTimestamp(new long[] { 1538548687000, 1538548688000, 1538548689000, 1538548690000, 1538548691000 });
data[1] = WSMultiBind.WSBindFloat(new float?[] { 10.30F, 10.40F, 10.50F, 10.60F, 10.70F });
data[2] = WSMultiBind.WSBindInt(new int?[] { 223, 221, 222, 220, 219 });
data[3] = WSMultiBind.WSBindFloat(new float?[] { 0.31F, 0.32F, 0.33F, 0.35F, 0.28F });
code = LibTaosWS.WSStmtBindParamBatch(wsStmt, data, numOfColumns);
ValidStmtStep(code, wsStmt, "WSStmtBindParamBatch");
code = LibTaosWS.WSStmtAddBatch(wsStmt);
ValidStmtStep(code, wsStmt, "WSStmtAddBatch");
IntPtr stmtAffectRowPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(Int32)));
code = LibTaosWS.WSStmtExecute(wsStmt, stmtAffectRowPtr);
ValidStmtStep(code, wsStmt, "WSStmtExecute");
Console.WriteLine("WS STMT insert {0} rows...", Marshal.ReadInt32(stmtAffectRowPtr));
Marshal.FreeHGlobal(stmtAffectRowPtr);
LibTaosWS.WSStmtClose(wsStmt);
// Free unmanaged memory
WSMultiBind.WSFreeTaosBind(wsTags);
WSMultiBind.WSFreeTaosBind(data);
//check result with SQL "SELECT * FROM test.d1005;"
Console.WriteLine(e);
throw;
}
else
{
Console.WriteLine("Init STMT failed...");
}
// close connection.
LibTaosWS.WSClose(wsConn);
return 0;
}
static void ValidStmtStep(int code, IntPtr wsStmt, string desc)
{
if (code != 0)
{
Console.WriteLine($"{desc} failed,reason: {LibTaosWS.WSErrorStr(wsStmt)}, code: {code}");
}
else
{
Console.WriteLine("{0} success...", desc);
}
}
}
}
// WSStmtPrepare success...
// WSStmtSetTbnameTags success...
// WSStmtBindParamBatch success...
// WSStmtAddBatch success...
// WSStmtExecute success...
// WS STMT insert 5 rows...

2
docs/examples/csharp/wsStmt/wsStmt.csproj
View File

@ -7,7 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" />
<PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
</ItemGroup>
<Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
<ItemGroup>

2
docs/zh/07-develop/01-connect/index.md
View File

@ -176,7 +176,7 @@ npm install @tdengine/rest
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.0" />
<PackageReference Include="TDengine.Connector" Version="3.1.0" />
</ItemGroup>
</Project>

3
docs/zh/07-develop/04-query-data/_cs_async.mdx
View File

@ -1,3 +0,0 @@
```csharp
{{#include docs/examples/csharp/asyncQuery/Program.cs}}
```

4
docs/zh/07-develop/04-query-data/index.mdx
View File

@ -16,7 +16,6 @@ import CQuery from "./_c.mdx";
import PhpQuery from "./_php.mdx";
import PyAsync from "./_py_async.mdx";
import NodeAsync from "./_js_async.mdx";
import CsAsync from "./_cs_async.mdx";
import CAsync from "./_c_async.mdx";
## 主要查询功能
@ -175,9 +174,6 @@ Query OK, 6 rows in database (0.005515s)
<TabItem label="Python" value="python">
<PyAsync />
</TabItem>
<TabItem label="C#" value="csharp">
<CsAsync />
</TabItem>
<TabItem label="C" value="c">
<CAsync />
</TabItem>

52
docs/zh/07-develop/07-tmq.mdx
View File

@ -248,23 +248,23 @@ function close()
<TabItem value="C#" label="C#">
```csharp
class ConsumerBuilder<TValue>
ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
virtual IConsumer Build()
Consumer(ConsumerBuilder builder)
public IConsumer<TValue> Build()
void Subscribe(IEnumerable<string> topics)
void Subscribe(string topic)
ConsumeResult Consume(int millisecondsTimeout)
ConsumeResult<TValue> Consume(int millisecondsTimeout)
List<string> Subscription()
void Unsubscribe()
void Commit(ConsumeResult consumerResult)
List<TopicPartitionOffset> Commit()
void Close()
```
@ -501,25 +501,19 @@ let consumer = taos.consumer({
<TabItem value="C#" label="C#">
```csharp
using TDengineTMQ;
// 根据需要，设置消费组 (GourpId)、自动提交 (EnableAutoCommit)、
// 自动提交时间间隔 (AutoCommitIntervalMs)、用户名 (TDConnectUser)、密码 (TDConnectPasswd) 等参数
var cfg = new ConsumerConfig
{
EnableAutoCommit = "true"
AutoCommitIntervalMs = "1000"
GourpId = "TDengine-TMQ-C#",
TDConnectUser = "root",
TDConnectPasswd = "taosdata",
AutoOffsetReset = "latest"
MsgWithTableName = "true",
TDConnectIp = "127.0.0.1",
TDConnectPort = "6030"
};
var consumer = new ConsumerBuilder(cfg).Build();
var cfg = new Dictionary<string, string>()
{
{ "group.id", "group1" },
{ "auto.offset.reset", "latest" },
{ "td.connect.ip", "127.0.0.1" },
{ "td.connect.user", "root" },
{ "td.connect.pass", "taosdata" },
{ "td.connect.port", "6030" },
{ "client.id", "tmq_example" },
{ "enable.auto.commit", "true" },
{ "msg.with.table.name", "false" },
};
var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
```
</TabItem>
@ -748,10 +742,12 @@ while(true){
// 消费数据
while (true)
{
var consumerRes = consumer.Consume(100);
// process ConsumeResult
ProcessMsg(consumerRes);
consumer.Commit(consumerRes);
using (var result = consumer.Consume(500))
{
if (result == null) continue;
ProcessMsg(result);
consumer.Commit();
}
}
```

57
docs/zh/08-connector/02-rest-api.mdx
View File

@ -257,6 +257,63 @@ curl -L -u username:password -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name][?tz=timez
- "INT UNSIGNED"
- "BIGINT UNSIGNED"
- "JSON"
- "VARBINARY"
- "GEOMETRY"
`VARBINARY` 和 `GEOMETRY` 类型返回数据为 Hex 字符串，样例：
准备数据
```bash
create database demo
use demo
create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
insert into t values(now,'\x7f8290','point(100 100)')
```
执行查询
```bash
curl --location 'http://<fqdn>:<port>/rest/sql' \
--header 'Content-Type: text/plain' \
--header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \
--data 'select * from demo.t'
```
返回结果
```json
{
"code": 0,
"column_meta": [
[
"ts",
"TIMESTAMP",
8
],
[
"c1",
"VARBINARY",
20
],
[
"c2",
"GEOMETRY",
100
]
],
"data": [
[
"2023-11-01T06:28:15.210Z",
"7f8290",
"010100000000000000000059400000000000005940"
]
],
"rows": 1
}
```
- `010100000000000000000059400000000000005940` 为 `point(100 100)` 的 [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/) 格式
#### 错误

1303
docs/zh/08-connector/40-csharp.mdx
View File

File diff suppressed because it is too large Load Diff

6
docs/zh/12-taos-sql/05-insert.md
View File

@ -158,7 +158,7 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c
自动建表, 表名通过tbname列指定
```sql
INSERT INTO meters(tbname, location, groupId, ts, current, phase)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33)
values('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 0.32)
('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 0.33)
('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 0.33)
```

2
include/common/cos.h
View File

@ -34,7 +34,7 @@ extern int32_t tsS3UploadDelaySec;
int32_t s3Init();
void s3CleanUp();
int32_t s3PutObjectFromFile(const char *file, const char *object);
int32_t s3PutObjectFromFile2(const char *file, const char *object);
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp);
void s3DeleteObjectsByPrefix(const char *prefix);
void s3DeleteObjects(const char *object_name[], int nobject);
bool s3Exists(const char *object_name);

83
include/common/cos_cp.h Normal file
View File

@ -0,0 +1,83 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_COMMON_COS_CP_H_
#define _TD_COMMON_COS_CP_H_
#include "os.h"
#include "tdef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef enum {
COS_CP_TYPE_UPLOAD, // upload
COS_CP_TYPE_DOWNLOAD // download
} ECpType;
typedef struct {
int32_t index; // the index of part, start from 0
int64_t offset; // the offset point of part
int64_t size; // the size of part
int completed; // COS_TRUE completed, COS_FALSE uncompleted
char etag[128]; // the etag of part, for upload
uint64_t crc64;
} SCheckpointPart;
typedef struct {
ECpType cp_type; // 0 upload, 1 download
char md5[64]; // the md5 of checkout content
TdFilePtr thefile; // the handle of checkpoint file
char file_path[TSDB_FILENAME_LEN]; // local file path
int64_t file_size; // local file size, for upload
int32_t file_last_modified; // local file last modified time, for upload
char file_md5[64]; // md5 of the local file content, for upload, reserved
char object_name[128]; // object name
int64_t object_size; // object size, for download
char object_last_modified[64]; // object last modified time, for download
char object_etag[128]; // object etag, for download
char upload_id[128]; // upload id
int part_num; // the total number of parts
int64_t part_size; // the part size, byte
SCheckpointPart* parts; // the parts of local or object, from 0
} SCheckpoint;
int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint);
void cos_cp_close(TdFilePtr fd);
void cos_cp_remove(char const* filepath);
int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint);
int32_t cos_cp_dump(SCheckpoint* checkpoint);
void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes);
void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64);
void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
char const* upload_id, int64_t part_size);
bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime);
void cos_cp_build_download(SCheckpoint* checkpoint, char const* filepath, char const* object_name, int64_t object_size,
char const* object_lmtime, char const* object_etag, int64_t part_size);
bool cos_cp_is_valid_download(SCheckpoint* checkpoint, char const* object_name, int64_t object_size,
char const* object_lmtime, char const* object_etag);
#ifdef __cplusplus
}
#endif
#endif /*_TD_COMMON_COS_CP_H_*/

2
include/common/systable.h
View File

@ -50,6 +50,8 @@ extern "C" {
#define TSDB_INS_TABLE_STREAM_TASKS "ins_stream_tasks"
#define TSDB_INS_TABLE_USER_PRIVILEGES "ins_user_privileges"
#define TSDB_INS_TABLE_VIEWS "ins_views"
#define TSDB_INS_TABLE_COMPACTS "ins_compacts"
#define TSDB_INS_TABLE_COMPACT_DETAILS "ins_compact_details"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
#define TSDB_PERFS_TABLE_SMAS "perf_smas"

1
include/common/tcommon.h
View File

@ -253,6 +253,7 @@ typedef struct SQueryTableDataCond {
STimeWindow twindows;
int64_t startVersion;
int64_t endVersion;
bool notLoadData; // response the actual data, not only the rows in the attribute of info.row of ssdatablock
} SQueryTableDataCond;
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);

3
include/common/tglobal.h
View File

@ -109,6 +109,7 @@ extern bool tsMonitorComp;
// audit
extern bool tsEnableAudit;
extern bool tsEnableAuditCreateTable;
extern int32_t tsAuditInterval;
// telem
extern bool tsEnableTelem;
@ -195,6 +196,7 @@ extern int64_t tsWalFsyncDataSizeLimit;
// internal
extern int32_t tsTransPullupInterval;
extern int32_t tsCompactPullupInterval;
extern int32_t tsMqRebalanceInterval;
extern int32_t tsStreamCheckpointInterval;
extern float tsSinkDataRate;
@ -213,6 +215,7 @@ extern int32_t tsMaxStreamBackendCache;
extern int32_t tsPQSortMemThreshold;
extern int32_t tsResolveFQDNRetryTime;
extern bool tsExperimental;
// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,

56
include/common/tmsg.h
View File

@ -144,6 +144,8 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_STREAM_TASKS,
TSDB_MGMT_TABLE_PRIVILEGES,
TSDB_MGMT_TABLE_VIEWS,
TSDB_MGMT_TABLE_COMPACT,
TSDB_MGMT_TABLE_COMPACT_DETAIL,
TSDB_MGMT_TABLE_MAX,
} EShowType;
@ -307,6 +309,7 @@ typedef enum ENodeType {
QUERY_NODE_KILL_CONNECTION_STMT,
QUERY_NODE_KILL_QUERY_STMT,
QUERY_NODE_KILL_TRANSACTION_STMT,
QUERY_NODE_KILL_COMPACT_STMT,
QUERY_NODE_DELETE_STMT,
QUERY_NODE_INSERT_STMT,
QUERY_NODE_QUERY,
@ -353,6 +356,8 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_VNODES_STMT,
QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
QUERY_NODE_SHOW_VIEWS_STMT,
QUERY_NODE_SHOW_COMPACTS_STMT,
QUERY_NODE_SHOW_COMPACT_DETAILS_STMT,
// logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@ -1384,6 +1389,24 @@ int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
int32_t tDeserializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
void tFreeSCompactDbReq(SCompactDbReq* pReq);
typedef struct {
int32_t compactId;
int8_t bAccepted;
} SCompactDbRsp;
int32_t tSerializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp);
int32_t tDeserializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp);
typedef struct {
int32_t compactId;
int32_t sqlLen;
char* sql;
} SKillCompactReq;
int32_t tSerializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
int32_t tDeserializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
void tFreeSKillCompactReq(SKillCompactReq *pReq);
typedef struct {
char name[TSDB_FUNC_NAME_LEN];
int8_t igExists;
@ -1662,6 +1685,26 @@ int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pR
int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);
int32_t tFreeSCreateVnodeReq(SCreateVnodeReq* pReq);
typedef struct {
int32_t compactId;
int32_t vgId;
int32_t dnodeId;
} SQueryCompactProgressReq;
int32_t tSerializeSQueryCompactProgressReq(void* buf, int32_t bufLen, SQueryCompactProgressReq* pReq);
int32_t tDeserializeSQueryCompactProgressReq(void* buf, int32_t bufLen, SQueryCompactProgressReq* pReq);
typedef struct {
int32_t compactId;
int32_t vgId;
int32_t dnodeId;
int32_t numberFileset;
int32_t finished;
} SQueryCompactProgressRsp;
int32_t tSerializeSQueryCompactProgressRsp(void* buf, int32_t bufLen, SQueryCompactProgressRsp* pReq);
int32_t tDeserializeSQueryCompactProgressRsp(void* buf, int32_t bufLen, SQueryCompactProgressRsp* pReq);
typedef struct {
int32_t vgId;
int32_t dnodeId;
@ -1689,11 +1732,21 @@ typedef struct {
char db[TSDB_DB_FNAME_LEN];
int64_t compactStartTime;
STimeWindow tw;
int32_t compactId;
} SCompactVnodeReq;
int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
typedef struct {
int32_t compactId;
int32_t vgId;
int32_t dnodeId;
} SVKillCompactReq;
int32_t tSerializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq);
int32_t tDeserializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq);
typedef struct {
int32_t vgVersion;
int32_t buffer;
@ -1889,8 +1942,9 @@ typedef struct {
char db[TSDB_DB_FNAME_LEN];
char tb[TSDB_TABLE_NAME_LEN];
char user[TSDB_USER_LEN];
char filterTb[TSDB_TABLE_NAME_LEN];
char filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns
int64_t showId;
int64_t compactId; // for compact
} SRetrieveTableReq;
typedef struct SSysTableSchema {

6
include/common/tmsgdef.h
View File

@ -217,6 +217,8 @@
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_KILL_COMPACT, "kill-compact", SKillCompactReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
@ -256,7 +258,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_EXEC_RSMA, "vnode-exec-rsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp)
TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
@ -267,6 +269,8 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG)

352
include/common/ttokendef.h
View File

@ -189,184 +189,180 @@
#define TK_ALIVE 170
#define TK_VIEWS 171
#define TK_VIEW 172
#define TK_NORMAL 173
#define TK_CHILD 174
#define TK_LIKE 175
#define TK_TBNAME 176
#define TK_QTAGS 177
#define TK_AS 178
#define TK_SYSTEM 179
#define TK_INDEX 180
#define TK_FUNCTION 181
#define TK_INTERVAL 182
#define TK_COUNT 183
#define TK_LAST_ROW 184
#define TK_META 185
#define TK_ONLY 186
#define TK_TOPIC 187
#define TK_CONSUMER 188
#define TK_GROUP 189
#define TK_DESC 190
#define TK_DESCRIBE 191
#define TK_RESET 192
#define TK_QUERY 193
#define TK_CACHE 194
#define TK_EXPLAIN 195
#define TK_ANALYZE 196
#define TK_VERBOSE 197
#define TK_NK_BOOL 198
#define TK_RATIO 199
#define TK_NK_FLOAT 200
#define TK_OUTPUTTYPE 201
#define TK_AGGREGATE 202
#define TK_BUFSIZE 203
#define TK_LANGUAGE 204
#define TK_REPLACE 205
#define TK_STREAM 206
#define TK_INTO 207
#define TK_PAUSE 208
#define TK_RESUME 209
#define TK_TRIGGER 210
#define TK_AT_ONCE 211
#define TK_WINDOW_CLOSE 212
#define TK_IGNORE 213
#define TK_EXPIRED 214
#define TK_FILL_HISTORY 215
#define TK_UPDATE 216
#define TK_SUBTABLE 217
#define TK_UNTREATED 218
#define TK_KILL 219
#define TK_CONNECTION 220
#define TK_TRANSACTION 221
#define TK_BALANCE 222
#define TK_VGROUP 223
#define TK_LEADER 224
#define TK_MERGE 225
#define TK_REDISTRIBUTE 226
#define TK_SPLIT 227
#define TK_DELETE 228
#define TK_INSERT 229
#define TK_NULL 230
#define TK_NK_QUESTION 231
#define TK_NK_ALIAS 232
#define TK_NK_ARROW 233
#define TK_ROWTS 234
#define TK_QSTART 235
#define TK_QEND 236
#define TK_QDURATION 237
#define TK_WSTART 238
#define TK_WEND 239
#define TK_WDURATION 240
#define TK_IROWTS 241
#define TK_ISFILLED 242
#define TK_CAST 243
#define TK_NOW 244
#define TK_TODAY 245
#define TK_TIMEZONE 246
#define TK_CLIENT_VERSION 247
#define TK_SERVER_VERSION 248
#define TK_SERVER_STATUS 249
#define TK_CURRENT_USER 250
#define TK_CASE 251
#define TK_WHEN 252
#define TK_THEN 253
#define TK_ELSE 254
#define TK_BETWEEN 255
#define TK_IS 256
#define TK_NK_LT 257
#define TK_NK_GT 258
#define TK_NK_LE 259
#define TK_NK_GE 260
#define TK_NK_NE 261
#define TK_MATCH 262
#define TK_NMATCH 263
#define TK_CONTAINS 264
#define TK_IN 265
#define TK_JOIN 266
#define TK_INNER 267
#define TK_SELECT 268
#define TK_NK_HINT 269
#define TK_DISTINCT 270
#define TK_WHERE 271
#define TK_PARTITION 272
#define TK_BY 273
#define TK_SESSION 274
#define TK_STATE_WINDOW 275
#define TK_EVENT_WINDOW 276
#define TK_SLIDING 277
#define TK_FILL 278
#define TK_VALUE 279
#define TK_VALUE_F 280
#define TK_NONE 281
#define TK_PREV 282
#define TK_NULL_F 283
#define TK_LINEAR 284
#define TK_NEXT 285
#define TK_HAVING 286
#define TK_RANGE 287
#define TK_EVERY 288
#define TK_ORDER 289
#define TK_SLIMIT 290
#define TK_SOFFSET 291
#define TK_LIMIT 292
#define TK_OFFSET 293
#define TK_ASC 294
#define TK_NULLS 295
#define TK_ABORT 296
#define TK_AFTER 297
#define TK_ATTACH 298
#define TK_BEFORE 299
#define TK_BEGIN 300
#define TK_BITAND 301
#define TK_BITNOT 302
#define TK_BITOR 303
#define TK_BLOCKS 304
#define TK_CHANGE 305
#define TK_COMMA 306
#define TK_CONCAT 307
#define TK_CONFLICT 308
#define TK_COPY 309
#define TK_DEFERRED 310
#define TK_DELIMITERS 311
#define TK_DETACH 312
#define TK_DIVIDE 313
#define TK_DOT 314
#define TK_EACH 315
#define TK_FAIL 316
#define TK_FILE 317
#define TK_FOR 318
#define TK_GLOB 319
#define TK_ID 320
#define TK_IMMEDIATE 321
#define TK_IMPORT 322
#define TK_INITIALLY 323
#define TK_INSTEAD 324
#define TK_ISNULL 325
#define TK_KEY 326
#define TK_MODULES 327
#define TK_NK_BITNOT 328
#define TK_NK_SEMI 329
#define TK_NOTNULL 330
#define TK_OF 331
#define TK_PLUS 332
#define TK_PRIVILEGE 333
#define TK_RAISE 334
#define TK_RESTRICT 335
#define TK_ROW 336
#define TK_SEMI 337
#define TK_STAR 338
#define TK_STATEMENT 339
#define TK_STRICT 340
#define TK_STRING 341
#define TK_TIMES 342
#define TK_VALUES 343
#define TK_VARIABLE 344
#define TK_WAL 345
#define TK_COMPACTS 173
#define TK_NORMAL 174
#define TK_CHILD 175
#define TK_LIKE 176
#define TK_TBNAME 177
#define TK_QTAGS 178
#define TK_AS 179
#define TK_SYSTEM 180
#define TK_INDEX 181
#define TK_FUNCTION 182
#define TK_INTERVAL 183
#define TK_COUNT 184
#define TK_LAST_ROW 185
#define TK_META 186
#define TK_ONLY 187
#define TK_TOPIC 188
#define TK_CONSUMER 189
#define TK_GROUP 190
#define TK_DESC 191
#define TK_DESCRIBE 192
#define TK_RESET 193
#define TK_QUERY 194
#define TK_CACHE 195
#define TK_EXPLAIN 196
#define TK_ANALYZE 197
#define TK_VERBOSE 198
#define TK_NK_BOOL 199
#define TK_RATIO 200
#define TK_NK_FLOAT 201
#define TK_OUTPUTTYPE 202
#define TK_AGGREGATE 203
#define TK_BUFSIZE 204
#define TK_LANGUAGE 205
#define TK_REPLACE 206
#define TK_STREAM 207
#define TK_INTO 208
#define TK_PAUSE 209
#define TK_RESUME 210
#define TK_TRIGGER 211
#define TK_AT_ONCE 212
#define TK_WINDOW_CLOSE 213
#define TK_IGNORE 214
#define TK_EXPIRED 215
#define TK_FILL_HISTORY 216
#define TK_UPDATE 217
#define TK_SUBTABLE 218
#define TK_UNTREATED 219
#define TK_KILL 220
#define TK_CONNECTION 221
#define TK_TRANSACTION 222
#define TK_BALANCE 223
#define TK_VGROUP 224
#define TK_LEADER 225
#define TK_MERGE 226
#define TK_REDISTRIBUTE 227
#define TK_SPLIT 228
#define TK_DELETE 229
#define TK_INSERT 230
#define TK_NULL 231
#define TK_NK_QUESTION 232
#define TK_NK_ALIAS 233
#define TK_NK_ARROW 234
#define TK_ROWTS 235
#define TK_QSTART 236
#define TK_QEND 237
#define TK_QDURATION 238
#define TK_WSTART 239
#define TK_WEND 240
#define TK_WDURATION 241
#define TK_IROWTS 242
#define TK_ISFILLED 243
#define TK_CAST 244
#define TK_NOW 245
#define TK_TODAY 246
#define TK_TIMEZONE 247
#define TK_CLIENT_VERSION 248
#define TK_SERVER_VERSION 249
#define TK_SERVER_STATUS 250
#define TK_CURRENT_USER 251
#define TK_CASE 252
#define TK_WHEN 253
#define TK_THEN 254
#define TK_ELSE 255
#define TK_BETWEEN 256
#define TK_IS 257
#define TK_NK_LT 258
#define TK_NK_GT 259
#define TK_NK_LE 260
#define TK_NK_GE 261
#define TK_NK_NE 262
#define TK_MATCH 263
#define TK_NMATCH 264
#define TK_CONTAINS 265
#define TK_IN 266
#define TK_JOIN 267
#define TK_INNER 268
#define TK_SELECT 269
#define TK_NK_HINT 270
#define TK_DISTINCT 271
#define TK_WHERE 272
#define TK_PARTITION 273
#define TK_BY 274
#define TK_SESSION 275
#define TK_STATE_WINDOW 276
#define TK_EVENT_WINDOW 277
#define TK_SLIDING 278
#define TK_FILL 279
#define TK_VALUE 280
#define TK_VALUE_F 281
#define TK_NONE 282
#define TK_PREV 283
#define TK_NULL_F 284
#define TK_LINEAR 285
#define TK_NEXT 286
#define TK_HAVING 287
#define TK_RANGE 288
#define TK_EVERY 289
#define TK_ORDER 290
#define TK_SLIMIT 291
#define TK_SOFFSET 292
#define TK_LIMIT 293
#define TK_OFFSET 294
#define TK_ASC 295
#define TK_NULLS 296
#define TK_ABORT 297
#define TK_AFTER 298
#define TK_ATTACH 299
#define TK_BEFORE 300
#define TK_BEGIN 301
#define TK_BITAND 302
#define TK_BITNOT 303
#define TK_BITOR 304
#define TK_BLOCKS 305
#define TK_CHANGE 306
#define TK_COMMA 307
#define TK_CONCAT 308
#define TK_CONFLICT 309
#define TK_COPY 310
#define TK_DEFERRED 311
#define TK_DELIMITERS 312
#define TK_DETACH 313
#define TK_DIVIDE 314
#define TK_DOT 315
#define TK_EACH 316
#define TK_FAIL 317
#define TK_FILE 318
#define TK_FOR 319
#define TK_GLOB 320
#define TK_ID 321
#define TK_IMMEDIATE 322
#define TK_IMPORT 323
#define TK_INITIALLY 324
#define TK_INSTEAD 325
#define TK_ISNULL 326
#define TK_KEY 327
#define TK_MODULES 328
#define TK_NK_BITNOT 329
#define TK_NK_SEMI 330
#define TK_NOTNULL 331
#define TK_OF 332
#define TK_PLUS 333
#define TK_PRIVILEGE 334
#define TK_RAISE 335
#define TK_RESTRICT 336
#define TK_ROW 337
#define TK_SEMI 338
#define TK_STAR 339
#define TK_STATEMENT 340
#define TK_STRICT 341
#define TK_STRING 342
#define TK_TIMES 343
#define TK_VALUES 344
#define TK_VARIABLE 345
#define TK_WAL 346
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

19
include/libs/audit/audit.h
View File

@ -23,13 +23,13 @@
#include "tjson.h"
#include "tmsgcb.h"
#include "trpc.h"
#include "mnode.h"
#ifdef __cplusplus
extern "C" {
#endif
#define AUDIT_DETAIL_MAX 65472
#define AUDIT_OPERATION_LEN 20
typedef struct {
const char *server;
@ -37,13 +37,28 @@ typedef struct {
bool comp;
} SAuditCfg;
typedef struct {
int64_t curTime;
char strClusterId[TSDB_CLUSTER_ID_LEN];
char clientAddress[50];
char user[TSDB_USER_LEN];
char operation[AUDIT_OPERATION_LEN];
char target1[TSDB_DB_NAME_LEN]; //put db name
char target2[TSDB_STREAM_NAME_LEN]; //put stb name, table name, topic name, user name, stream name, use max
char* detail;
} SAuditRecord;
int32_t auditInit(const SAuditCfg *pCfg);
void auditCleanup();
void auditSend(SJson *pJson);
void auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2,
char *detail, int32_t len);
void auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2,
char *detail, int32_t len);
void auditSendRecordsInBatch();
#ifdef __cplusplus
}
#endif
#endif /*_TD_MONITOR_H_*/
#endif /*_TD_AUDIT_H_*/

32
include/libs/executor/storageapi.h
View File

@ -102,11 +102,11 @@ typedef struct SMTbCursor {
} SMTbCursor;
typedef struct SMCtbCursor {
SMeta *pMeta;
void *pCur;
SMeta* pMeta;
void* pCur;
tb_uid_t suid;
void *pKey;
void *pVal;
void* pKey;
void* pVal;
int kLen;
int vLen;
int8_t paused;
@ -153,7 +153,8 @@ typedef struct {
// clang-format off
/*-------------------------------------------------new api format---------------------------------------------------*/
typedef enum {
TSD_READER_NOTIFY_DURATION_START
TSD_READER_NOTIFY_DURATION_START,
TSD_READER_NOTIFY_NEXT_DURATION_BLOCK,
} ETsdReaderNotifyType;
typedef union {
@ -166,8 +167,7 @@ typedef void (*TsdReaderNotifyCbFn)(ETsdReaderNotifyType type, STsdReaderNotifyI
typedef struct TsdReader {
int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly,
SHashObj** pIgnoreTables);
SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables);
void (*tsdReaderClose)();
void (*tsdSetReaderTaskId)(void *pReader, const char *pId);
int32_t (*tsdSetQueryTableList)();
@ -263,21 +263,22 @@ typedef struct SStoreMeta {
void* (*storeGetIndexInfo)();
void* (*getInvertIndex)(void* pVnode);
// support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
int32_t (*getChildTableList)( void* pVnode, int64_t suid, SArray* list);
int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list);
int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList);
void* storeGetVersionRange;
void* storeGetLastTimestamp;
int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbGetTableSchema
int32_t (*getNumOfChildTables)( void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols);
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables, int64_t* numOfNormalTables);
int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols);
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
int64_t* numOfNormalTables);
int64_t (*getNumOfRowsInMem)(void* pVnode);
SMCtbCursor* (*openCtbCursor)(void *pVnode, tb_uid_t uid, int lock);
SMCtbCursor* (*openCtbCursor)(void* pVnode, tb_uid_t uid, int lock);
int32_t (*resumeCtbCursor)(SMCtbCursor* pCtbCur, int8_t first);
void (*pauseCtbCursor)(SMCtbCursor* pCtbCur);
void (*closeCtbCursor)(SMCtbCursor *pCtbCur);
void (*closeCtbCursor)(SMCtbCursor* pCtbCur);
tb_uid_t (*ctbCursorNext)(SMCtbCursor* pCur);
} SStoreMeta;
@ -366,11 +367,11 @@ typedef struct SStateStore {
TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol);
bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
bool (*isIncrementalTimeStamp)(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts);
bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
void (*updateInfoDestroy)(SUpdateInfo* pInfo);
void (*windowSBfDelete)(SUpdateInfo *pInfo, uint64_t count);
void (*windowSBfAdd)(SUpdateInfo *pInfo, uint64_t count);
void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp);
void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);
@ -397,6 +398,7 @@ typedef struct SStateStore {
void (*streamStateDestroy)(SStreamState* pState, bool remove);
int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);
void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst);
} SStateStore;
typedef struct SStorageAPI {

1
include/libs/function/function.h
View File

@ -171,6 +171,7 @@ typedef struct {
int32_t taskId;
int64_t streamId;
int64_t streamBackendRid;
int8_t dump;
} SStreamState;
typedef struct SFunctionStateStore {

13
include/libs/nodes/cmdnodes.h
View File

@ -46,6 +46,10 @@ extern "C" {
#define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
#define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE)
#define COMPACT_DB_RESULT_COLS 3
#define COMPACT_DB_RESULT_FIELD1_LEN 32
#define COMPACT_DB_RESULT_FIELD3_LEN 128
#define SHOW_ALIVE_RESULT_COLS 1
#define BIT_FLAG_MASK(n) (1 << n)
@ -335,6 +339,15 @@ typedef struct SShowTableTagsStmt {
SNodeList* pTags;
} SShowTableTagsStmt;
typedef struct SShowCompactsStmt {
ENodeType type;
} SShowCompactsStmt;
typedef struct SShowCompactDetailsStmt {
ENodeType type;
SNode* pCompactId;
} SShowCompactDetailsStmt;
typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType;
typedef struct SIndexOptions {

7
include/libs/stream/streamState.h
View File

@ -50,7 +50,7 @@ void streamStateSetNumber(SStreamState* pState, int32_t number);
int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
//session window
// session window
int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen);
int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen);
int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
@ -65,7 +65,7 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess
SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key);
//state window
// state window
int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
@ -96,6 +96,9 @@ int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char*
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal);
void streamStateReloadInfo(SStreamState* pState, TSKEY ts);
void streamStateCopyBackend(SStreamState* src, SStreamState* dst);
SStreamStateCur* createStreamStateCursor();
/***compare func **/

6
include/libs/stream/tstream.h
View File

@ -509,10 +509,7 @@ typedef struct SStreamMeta {
SArray* chkpSaved;
SArray* chkpInUse;
SRWLatch chkpDirLock;
void* qHandle;
int32_t pauseTaskNum;
void* bkdChkptMgt;
} SStreamMeta;
@ -840,11 +837,10 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int3
int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta);
SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
int32_t streamMetaReopen(SStreamMeta* pMeta);
void streamMetaClear(SStreamMeta* pMeta);
void streamMetaInitBackend(SStreamMeta* pMeta);
int32_t streamMetaCommit(SStreamMeta* pMeta);
int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta);
int32_t streamMetaReloadAllTasks(SStreamMeta* pMeta);
int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta);
void streamMetaNotifyClose(SStreamMeta* pMeta);
int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key);

4
include/libs/sync/sync.h
View File

@ -33,11 +33,12 @@ extern "C" {
#define SYNC_MAX_PROGRESS_WAIT_MS 4000
#define SYNC_MAX_START_TIME_RANGE_MS (1000 * 20)
#define SYNC_MAX_RECV_TIME_RANGE_MS 1200
#define SYNC_DEL_WAL_MS (1000 * 60)
#define SYNC_ADD_QUORUM_COUNT 3
#define SYNC_VNODE_LOG_RETENTION (TSDB_SYNC_LOG_BUFFER_RETENTION + 1)
#define SNAPSHOT_WAIT_MS 1000 * 5
#define SYNC_WAL_LOG_RETENTION_SIZE (8LL * 1024 * 1024 * 1024)
#define SYNC_MAX_RETRY_BACKOFF 5
#define SYNC_LOG_REPL_RETRY_WAIT_MS 100
#define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000
@ -219,6 +220,7 @@ typedef struct SSyncLogStore {
SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore);
SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore);
SyncIndex (*syncLogIndexRetention)(struct SSyncLogStore* pLogStore, int64_t bytes);
SyncTerm (*syncLogLastTerm)(struct SSyncLogStore* pLogStore);
int32_t (*syncLogAppendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, bool forcSync);

1
include/libs/wal/wal.h
View File

@ -225,6 +225,7 @@ bool walIsEmpty(SWal *);
int64_t walGetFirstVer(SWal *);
int64_t walGetSnapshotVer(SWal *);
int64_t walGetLastVer(SWal *);
int64_t walGetVerRetention(SWal *pWal, int64_t bytes);
int64_t walGetCommittedVer(SWal *);
int64_t walGetAppliedVer(SWal *);

7
include/os/osString.h
View File

@ -47,18 +47,13 @@ typedef int32_t TdUcs4;
#define strtof STR_TO_F_FUNC_TAOS_FORBID
#endif
#ifdef WINDOWS
#define tstrdup(str) _strdup(str)
#else
#define tstrdup(str) strdup(str)
#endif
#define tstrncpy(dst, src, size) \
do { \
strncpy((dst), (src), (size)); \
(dst)[(size)-1] = 0; \
} while (0)
char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4);
int64_t taosStr2int64(const char *str);

3
include/util/taoserror.h
View File

@ -427,6 +427,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_VIEW_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x04A0)
#define TSDB_CODE_MND_VIEW_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x04A1)
//mnode-compact
#define TSDB_CODE_MND_INVALID_COMPACT_ID TAOS_DEF_ERROR_CODE(0, 0x04B1)
// vnode
// #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x

4
source/client/inc/clientInt.h
View File

@ -205,7 +205,7 @@ typedef struct SRequestSendRecvBody {
__taos_async_fn_t queryFp;
__taos_async_fn_t fetchFp;
EQueryExecMode execMode;
void* param;
void* interParam;
SDataBuf requestMsg;
int64_t queryJob; // query job, created according to sql query DAG.
int32_t subplanNum;
@ -287,6 +287,7 @@ typedef struct SRequestObj {
typedef struct SSyncQueryParam {
tsem_t sem;
SRequestObj* pRequest;
void* userParam;
} SSyncQueryParam;
void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
@ -420,6 +421,7 @@ int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj
int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce);
void returnToUser(SRequestObj* pRequest);
void stopAllQueries(SRequestObj *pRequest);
void doRequestCallback(SRequestObj* pRequest, int32_t code);
void freeQueryParam(SSyncQueryParam* param);
#ifdef TD_ENTERPRISE

17
source/client/src/clientEnv.c
View File

@ -316,6 +316,15 @@ void *createRequest(uint64_t connId, int32_t type, int64_t reqid) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL;
}
SSyncQueryParam *interParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
if (interParam == NULL) {
doDestroyRequest(pRequest);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
tsem_init(&interParam->sem, 0, 0);
interParam->pRequest = pRequest;
pRequest->body.interParam = interParam;
pRequest->resType = RES_TYPE__QUERY;
pRequest->requestId = reqid == 0 ? generateRequestId() : reqid;
@ -437,12 +446,10 @@ void doDestroyRequest(void *p) {
deregisterRequest(pRequest);
}
if (pRequest->syncQuery) {
if (pRequest->body.param) {
tsem_destroy(&((SSyncQueryParam *)pRequest->body.param)->sem);
}
taosMemoryFree(pRequest->body.param);
if (pRequest->body.interParam) {
tsem_destroy(&((SSyncQueryParam *)pRequest->body.interParam)->sem);
}
taosMemoryFree(pRequest->body.interParam);
qDestroyQuery(pRequest->pQuery);
nodesDestroyAllocator(pRequest->allocatorRefId);

92
source/client/src/clientImpl.c
View File

@ -196,21 +196,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
(*pRequest)->sqlLen = sqlLen;
(*pRequest)->validateOnly = validateSql;
SSyncQueryParam* newpParam = NULL;
if (param == NULL) {
newpParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
if (newpParam == NULL) {
destroyRequest(*pRequest);
*pRequest = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
tsem_init(&newpParam->sem, 0, 0);
newpParam->pRequest = (*pRequest);
param = newpParam;
}
(*pRequest)->body.param = param;
((SSyncQueryParam*)(*pRequest)->body.interParam)->userParam = param;
STscObj* pTscObj = (*pRequest)->pTscObj;
int32_t err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
@ -218,7 +204,6 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
if (err) {
tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
(*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
freeQueryParam(newpParam);
destroyRequest(*pRequest);
*pRequest = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
@ -230,7 +215,6 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) {
tscError("%" PRId64 " failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
(*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
freeQueryParam(newpParam);
destroyRequest(*pRequest);
*pRequest = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
@ -336,7 +320,7 @@ static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { return pRequest->pTscOb
void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
SRetrieveTableRsp* pRsp = NULL;
if (pRequest->validateOnly) {
pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
doRequestCallback(pRequest, 0);
return;
}
@ -358,18 +342,18 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
pRequest->requestId);
}
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
}
int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
if (pRequest->validateOnly) {
pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
doRequestCallback(pRequest, 0);
return TSDB_CODE_SUCCESS;
}
// drop table if exists not_exists_table
if (NULL == pQuery->pCmdMsg) {
pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
doRequestCallback(pRequest, 0);
return TSDB_CODE_SUCCESS;
}
@ -384,7 +368,7 @@ int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
int64_t transporterId = 0;
int32_t code = asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg);
if (code) {
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
}
return code;
}
@ -913,7 +897,7 @@ void continuePostSubQuery(SRequestObj* pRequest, TAOS_ROW row) {
void returnToUser(SRequestObj* pRequest) {
if (pRequest->relation.userRefId == pRequest->self || 0 == pRequest->relation.userRefId) {
// return to client
pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
doRequestCallback(pRequest, pRequest->code);
return;
}
@ -921,7 +905,7 @@ void returnToUser(SRequestObj* pRequest) {
if (pUserReq) {
pUserReq->code = pRequest->code;
// return to client
pUserReq->body.queryFp(pUserReq->body.param, pUserReq, pUserReq->code);
doRequestCallback(pUserReq, pUserReq->code);
releaseRequest(pRequest->relation.userRefId);
return;
} else {
@ -1031,7 +1015,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
pRequest->pWrapper = NULL;
// return to client
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
}
}
@ -1186,7 +1170,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
pRequest->code = terrno;
}
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
}
// todo not to be released here
@ -1199,7 +1183,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
int32_t code = 0;
if (pRequest->parseOnly) {
pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
doRequestCallback(pRequest, 0);
return;
}
@ -1233,11 +1217,11 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
}
case QUERY_EXEC_MODE_EMPTY_RESULT:
pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
doRequestCallback(pRequest, 0);
break;
default:
tscError("0x%" PRIx64 " invalid execMode %d", pRequest->self, pQuery->execMode);
pRequest->body.queryFp(pRequest->body.param, pRequest, -1);
doRequestCallback(pRequest, -1);
break;
}
}
@ -1703,8 +1687,8 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
}
static void syncFetchFn(void* param, TAOS_RES* res, int32_t numOfRows) {
SSyncQueryParam* pParam = param;
tsem_post(&pParam->sem);
tsem_t* sem = param;
tsem_post(sem);
}
void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) {
@ -1722,10 +1706,11 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU
// convert ucs4 to native multi-bytes string
pResultInfo->convertUcs4 = convertUcs4;
SSyncQueryParam* pParam = pRequest->body.param;
taos_fetch_rows_a(pRequest, syncFetchFn, pParam);
tsem_wait(&pParam->sem);
tsem_t sem;
tsem_init(&sem, 0, 0);
taos_fetch_rows_a(pRequest, syncFetchFn, &sem);
tsem_wait(&sem);
tsem_destroy(&sem);
}
if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) {
@ -2492,6 +2477,10 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
tscDebug("taos_query start with sql:%s", sql);
SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
if (NULL == param) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
tsem_init(&param->sem, 0, 0);
taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly);
@ -2501,9 +2490,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
if (param->pRequest != NULL) {
param->pRequest->syncQuery = true;
pRequest = param->pRequest;
} else {
taosMemoryFree(param);
}
taosMemoryFree(param);
tscDebug("taos_query end with sql:%s", sql);
@ -2517,19 +2505,20 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly,
}
SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
if (param == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
tsem_init(&param->sem, 0, 0);
taosAsyncQueryImplWithReqid(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, reqid);
tsem_wait(&param->sem);
SRequestObj* pRequest = NULL;
if (param->pRequest != NULL) {
param->pRequest->syncQuery = true;
pRequest = param->pRequest;
} else {
taosMemoryFree(param);
}
taosMemoryFree(param);
return pRequest;
}
@ -2547,13 +2536,13 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
if (code != TSDB_CODE_SUCCESS) {
pRequest->code = code;
taosMemoryFreeClear(pResultInfo->pData);
pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, 0);
return;
}
if (pRequest->code != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pResultInfo->pData);
pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, 0);
return;
}
@ -2574,20 +2563,12 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
atomic_add_fetch_64((int64_t*)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen);
}
pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows);
pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, pResultInfo->numOfRows);
}
void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param) {
if (pRequest->syncQuery && pRequest->body.param != param) {
if (pRequest->body.param) {
tsem_destroy(&((SSyncQueryParam *)pRequest->body.param)->sem);
}
taosMemoryFree(pRequest->body.param);
pRequest->syncQuery = false;
}
pRequest->body.fetchFp = fp;
pRequest->body.param = param;
((SSyncQueryParam *)pRequest->body.interParam)->userParam = param;
SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
@ -2625,6 +2606,10 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param
schedulerFetchRows(pRequest->body.queryJob, &req);
}
void doRequestCallback(SRequestObj* pRequest, int32_t code) {
pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code);
}
int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_SUCCESS;
@ -2633,4 +2618,3 @@ int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool pa
#endif
}

12
source/client/src/clientMain.c
View File

@ -1095,7 +1095,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c
pRequest->pWrapper = NULL;
terrno = code;
pRequest->code = code;
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
}
}
@ -1112,7 +1112,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest)
pRequest->pWrapper = NULL;
terrno = code;
pRequest->code = code;
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
}
}
@ -1210,7 +1210,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
terrno = code;
pRequest->code = code;
tscDebug("call sync query cb with code: %s", tstrerror(code));
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
return;
}
@ -1242,7 +1242,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
terrno = code;
pRequest->code = code;
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
}
}
@ -1545,12 +1545,12 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, pRequest->body.param, NULL);
code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, pRequest->body.interParam, NULL);
if (code) {
goto _return;
}
SSyncQueryParam *pParam = pRequest->body.param;
SSyncQueryParam *pParam = pRequest->body.interParam;
tsem_wait(&pParam->sem);
_return:

132
source/client/src/clientMsgHandler.c
View File

@ -16,6 +16,7 @@
#include "catalog.h"
#include "clientInt.h"
#include "clientLog.h"
#include "cmdnodes.h"
#include "os.h"
#include "query.h"
#include "systable.h"
@ -41,7 +42,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pEpSet);
taosMemoryFree(pMsg->pData);
if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
} else {
tsem_post(&pRequest->body.rspSem);
}
@ -199,7 +200,7 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
}
if (pRequest->body.queryFp) {
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
} else {
tsem_post(&pRequest->body.rspSem);
}
@ -235,7 +236,8 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
setErrno(pRequest, code);
if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
doRequestCallback(pRequest, pRequest->code);
} else {
tsem_post(&pRequest->body.rspSem);
}
@ -299,7 +301,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pEpSet);
if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
doRequestCallback(pRequest, pRequest->code);
} else {
tsem_post(&pRequest->body.rspSem);
}
@ -343,7 +345,7 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) {
}
}
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
} else {
tsem_post(&pRequest->body.rspSem);
}
@ -380,7 +382,7 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pEpSet);
if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
} else {
tsem_post(&pRequest->body.rspSem);
}
@ -420,7 +422,7 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) {
}
}
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
} else {
tsem_post(&pRequest->body.rspSem);
}
@ -534,7 +536,119 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pEpSet);
if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
doRequestCallback(pRequest, code);
} else {
tsem_post(&pRequest->body.rspSem);
}
return code;
}
static int32_t buildCompactDbBlock(SCompactDbRsp* pRsp, SSDataBlock** block) {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->info.hasVarCol = true;
pBlock->pDataBlock = taosArrayInit(COMPACT_DB_RESULT_COLS, sizeof(SColumnInfoData));
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = COMPACT_DB_RESULT_FIELD1_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_INT;
infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = COMPACT_DB_RESULT_FIELD3_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
blockDataEnsureCapacity(pBlock, 1);
SColumnInfoData* pResultCol = taosArrayGet(pBlock->pDataBlock, 0);
SColumnInfoData* pIdCol = taosArrayGet(pBlock->pDataBlock, 1);
SColumnInfoData* pReasonCol = taosArrayGet(pBlock->pDataBlock, 2);
char result[COMPACT_DB_RESULT_FIELD1_LEN] = {0};
char reason[COMPACT_DB_RESULT_FIELD3_LEN] = {0};
if (pRsp->bAccepted) {
STR_TO_VARSTR(result, "accepted");
colDataSetVal(pResultCol, 0, result, false);
colDataSetVal(pIdCol, 0, (void*)&pRsp->compactId, false);
STR_TO_VARSTR(reason, "success");
colDataSetVal(pReasonCol, 0, reason, false);
} else {
STR_TO_VARSTR(result, "rejected");
colDataSetVal(pResultCol, 0, result, false);
colDataSetNULL(pIdCol, 0);
STR_TO_VARSTR(reason, "compaction is ongoing");
colDataSetVal(pReasonCol, 0, reason, false);
}
pBlock->info.rows = 1;
*block = pBlock;
return TSDB_CODE_SUCCESS;
}
static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetrieveTableRsp** pRsp) {
SSDataBlock* pBlock = NULL;
int32_t code = buildCompactDbBlock(pCompactDb, &pBlock);
if (code) {
return code;
}
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
*pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) {
blockDataDestroy(pBlock);
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pRsp)->useconds = 0;
(*pRsp)->completed = 1;
(*pRsp)->precision = 0;
(*pRsp)->compressed = 0;
(*pRsp)->compLen = 0;
(*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
(*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS);
int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS);
blockDataDestroy(pBlock);
if (len != rspSize - sizeof(SRetrieveTableRsp)) {
uError("buildRetriveTableRspForCompactDb error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len,
(uint64_t)(rspSize - sizeof(SRetrieveTableRsp)));
return TSDB_CODE_TSC_INVALID_INPUT;
}
return TSDB_CODE_SUCCESS;
}
int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
SRequestObj* pRequest = param;
if (code != TSDB_CODE_SUCCESS) {
setErrno(pRequest, code);
} else {
SCompactDbRsp rsp = {0};
SRetrieveTableRsp* pRes = NULL;
code = tDeserializeSCompactDbRsp(pMsg->pData, pMsg->len, &rsp);
if (TSDB_CODE_SUCCESS == code) {
code = buildRetriveTableRspForCompactDb(&rsp, &pRes);
}
if (TSDB_CODE_SUCCESS == code) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true);
}
if (code != 0) {
taosMemoryFree(pRes);
}
}
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code);
} else {
tsem_post(&pRequest->body.rspSem);
}
@ -557,6 +671,8 @@ __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
return processAlterStbRsp;
case TDMT_MND_SHOW_VARIABLES:
return processShowVariablesRsp;
case TDMT_MND_COMPACT_DB:
return processCompactDbRsp;
default:
return genericRspCallback;
}

426
source/common/src/cos.c
View File

@ -1,6 +1,8 @@
#define ALLOW_FORBID_FUNC
#include "cos.h"
#include "cos_cp.h"
#include "tdef.h"
extern char tsS3Endpoint[];
extern char tsS3AccessKeyId[];
@ -86,7 +88,7 @@ typedef struct {
char err_msg[128];
S3Status status;
uint64_t content_length;
char * buf;
char *buf;
int64_t buf_pos;
} TS3SizeCBD;
@ -270,7 +272,7 @@ typedef struct list_parts_callback_data {
typedef struct MultipartPartData {
put_object_callback_data put_object_data;
int seq;
UploadManager * manager;
UploadManager *manager;
} MultipartPartData;
static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) {
@ -317,12 +319,23 @@ S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properti
MultipartPartData *data = (MultipartPartData *)callbackData;
int seq = data->seq;
const char * etag = properties->eTag;
const char *etag = properties->eTag;
data->manager->etags[seq - 1] = strdup(etag);
data->manager->next_etags_pos = seq;
return S3StatusOK;
}
S3Status MultipartResponseProperiesCallbackWithCp(const S3ResponseProperties *properties, void *callbackData) {
responsePropertiesCallbackNull(properties, callbackData);
MultipartPartData *data = (MultipartPartData *)callbackData;
int seq = data->seq;
const char *etag = properties->eTag;
data->manager->etags[seq - 1] = strdup(etag);
// data->manager->next_etags_pos = seq;
return S3StatusOK;
}
static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) {
UploadManager *manager = (UploadManager *)callbackData;
int ret = 0;
@ -446,77 +459,36 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan
return 0;
}
*/
int32_t s3PutObjectFromFile2(const char *file, const char *object) {
static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char const *object_name, int64_t size,
S3PutProperties *put_prop, put_object_callback_data *data) {
int32_t code = 0;
const char *key = object;
// const char *uploadId = 0;
const char * filename = 0;
uint64_t contentLength = 0;
const char * cacheControl = 0, *contentType = 0, *md5 = 0;
const char * contentDispositionFilename = 0, *contentEncoding = 0;
int64_t expires = -1;
S3CannedAcl cannedAcl = S3CannedAclPrivate;
int metaPropertiesCount = 0;
S3NameValue metaProperties[S3_MAX_METADATA_COUNT];
char useServerSideEncryption = 0;
put_object_callback_data data = {0};
// int noStatus = 0;
// data.infile = 0;
// data.gb = 0;
// data.infileFD = NULL;
// data.noStatus = noStatus;
// uError("ERROR: %s stat file %s: ", __func__, file);
if (taosStatFile(file, &contentLength, NULL, NULL) < 0) {
uError("ERROR: %s Failed to stat file %s: ", __func__, file);
code = TAOS_SYSTEM_ERROR(errno);
return code;
}
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
uError("ERROR: %s Failed to open file %s: ", __func__, file);
code = TAOS_SYSTEM_ERROR(errno);
return code;
}
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
contentLength;
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
0, awsRegionG};
S3PutProperties putProperties = {contentType, md5,
cacheControl, contentDispositionFilename,
contentEncoding, expires,
cannedAcl, metaPropertiesCount,
metaProperties, useServerSideEncryption};
if (contentLength <= MULTIPART_CHUNK_SIZE) {
S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
&putObjectDataCallback};
do {
S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data);
} while (S3_status_is_retryable(data.status) && should_retry());
S3_put_object(bucket_context, object_name, size, put_prop, 0, 0, &putObjectHandler, data);
} while (S3_status_is_retryable(data->status) && should_retry());
if (data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
if (data->status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
} else if (data.contentLength) {
uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
(unsigned long long)data.contentLength);
} else if (data->contentLength) {
uError("%s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data->contentLength);
code = TAOS_SYSTEM_ERROR(EIO);
}
} else {
return code;
}
static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name,
int64_t contentLength, S3PutProperties *put_prop,
put_object_callback_data *data) {
int32_t code = 0;
uint64_t totalContentLength = contentLength;
uint64_t todoContentLength = contentLength;
UploadManager manager = {0};
// manager.upload_id = 0;
// manager.gb = 0;
// div round up
int seq;
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3;
int totalSeq = (contentLength + chunk_size - 1) / chunk_size;
const int max_part_num = 10000;
@ -540,22 +512,8 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
manager.next_etags_pos = 0;
/*
if (uploadId) {
manager.upload_id = strdup(uploadId);
manager.remaining = contentLength;
if (!try_get_parts_info(tsS3BucketName, key, &manager)) {
fseek(data.infile, -(manager.remaining), 2);
taosLSeekFile(data.infileFD, -(manager.remaining), SEEK_END);
contentLength = manager.remaining;
goto upload;
} else {
goto clean;
}
}
*/
do {
S3_initiate_multipart(&bucketContext, key, 0, &handler, 0, timeoutMsG, &manager);
S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.upload_id == 0 || manager.status != S3StatusOK) {
@ -564,13 +522,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
goto clean;
}
upload:
upload:
todoContentLength -= chunk_size * manager.next_etags_pos;
for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
for (int seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
partData.manager = &manager;
partData.seq = seq;
if (partData.put_object_data.gb == NULL) {
partData.put_object_data = data;
partData.put_object_data = *data;
}
partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength);
// printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);
@ -578,9 +536,9 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
partData.put_object_data.originalContentLength = partContentLength;
partData.put_object_data.totalContentLength = todoContentLength;
partData.put_object_data.totalOriginalContentLength = totalContentLength;
putProperties.md5 = 0;
put_prop->md5 = 0;
do {
S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id,
S3_upload_part(bucket_context, object_name, put_prop, &putObjectHandler, seq, manager.upload_id,
partContentLength, 0, timeoutMsG, &partData);
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) {
@ -612,7 +570,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
manager.remaining = size;
do {
S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, 0,
S3_complete_multipart_upload(bucket_context, object_name, &commit_handler, manager.upload_id, manager.remaining, 0,
timeoutMsG, &manager);
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) {
@ -621,7 +579,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
goto clean;
}
clean:
clean:
if (manager.upload_id) {
taosMemoryFree(manager.upload_id);
}
@ -630,6 +588,237 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
}
growbuffer_destroy(manager.gb);
taosMemoryFree(manager.etags);
return code;
}
static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime,
char const *object_name, int64_t contentLength, S3PutProperties *put_prop,
put_object_callback_data *data) {
int32_t code = 0;
uint64_t totalContentLength = contentLength;
// uint64_t todoContentLength = contentLength;
UploadManager manager = {0};
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3;
int totalSeq = (contentLength + chunk_size - 1) / chunk_size;
const int max_part_num = 10000;
if (totalSeq > max_part_num) {
chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
totalSeq = (contentLength + chunk_size - 1) / chunk_size;
}
bool need_init_upload = true;
char file_cp_path[TSDB_FILENAME_LEN];
snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file);
SCheckpoint cp = {0};
cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
if (taosCheckExistFile(file_cp_path)) {
if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) {
manager.upload_id = strdup(cp.upload_id);
need_init_upload = false;
} else {
cos_cp_remove(file_cp_path);
}
}
if (need_init_upload) {
S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
&initial_multipart_callback};
do {
S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size);
}
if (cos_cp_open(file_cp_path, &cp)) {
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
int part_num = 0;
int64_t consume_bytes = 0;
// SCheckpointPart *parts = taosMemoryCalloc(cp.part_num, sizeof(SCheckpointPart));
// cos_cp_get_undo_parts(&cp, &part_num, parts, &consume_bytes);
MultipartPartData partData;
memset(&partData, 0, sizeof(MultipartPartData));
int partContentLength = 0;
S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallbackWithCp, &responseCompleteCallback},
&putObjectDataCallback};
S3MultipartCommitHandler commit_handler = {
{&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
manager.next_etags_pos = 0;
upload:
// todoContentLength -= chunk_size * manager.next_etags_pos;
for (int i = 0; i < cp.part_num; ++i) {
if (cp.parts[i].completed) {
continue;
}
if (i > 0 && cp.parts[i - 1].completed) {
if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto clean;
}
}
int seq = cp.parts[i].index + 1;
partData.manager = &manager;
partData.seq = seq;
if (partData.put_object_data.gb == NULL) {
partData.put_object_data = *data;
}
partContentLength = cp.parts[i].size;
partData.put_object_data.contentLength = partContentLength;
partData.put_object_data.originalContentLength = partContentLength;
// partData.put_object_data.totalContentLength = todoContentLength;
partData.put_object_data.totalOriginalContentLength = totalContentLength;
put_prop->md5 = 0;
do {
S3_upload_part(bucket_context, object_name, put_prop, &putObjectHandler, seq, manager.upload_id,
partContentLength, 0, timeoutMsG, &partData);
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
//(void)cos_cp_dump(&cp);
goto clean;
}
if (!manager.etags[seq - 1]) {
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0);
(void)cos_cp_dump(&cp);
contentLength -= chunk_size;
// todoContentLength -= chunk_size;
}
cos_cp_close(cp.thefile);
cp.thefile = 0;
int size = 0;
size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
char buf[256];
int n;
for (int i = 0; i < cp.part_num; ++i) {
n = snprintf(buf, sizeof(buf),
"<Part><PartNumber>%d</PartNumber>"
"<ETag>%s</ETag></Part>",
// i + 1, manager.etags[i]);
cp.parts[i].index + 1, cp.parts[i].etag);
size += growbuffer_append(&(manager.gb), buf, n);
}
size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
manager.remaining = size;
do {
S3_complete_multipart_upload(bucket_context, object_name, &commit_handler, manager.upload_id, manager.remaining, 0,
timeoutMsG, &manager);
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
cos_cp_remove(file_cp_path);
clean:
/*
if (parts) {
taosMemoryFree(parts);
}
*/
if (cp.thefile) {
cos_cp_close(cp.thefile);
}
if (cp.parts) {
taosMemoryFree(cp.parts);
}
if (manager.upload_id) {
taosMemoryFree(manager.upload_id);
}
for (int i = 0; i < cp.part_num; ++i) {
if (manager.etags[i]) {
taosMemoryFree(manager.etags[i]);
}
}
taosMemoryFree(manager.etags);
growbuffer_destroy(manager.gb);
return code;
}
int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) {
int32_t code = 0;
int32_t lmtime = 0;
const char *filename = 0;
uint64_t contentLength = 0;
const char *cacheControl = 0, *contentType = 0, *md5 = 0;
const char *contentDispositionFilename = 0, *contentEncoding = 0;
int64_t expires = -1;
S3CannedAcl cannedAcl = S3CannedAclPrivate;
int metaPropertiesCount = 0;
S3NameValue metaProperties[S3_MAX_METADATA_COUNT];
char useServerSideEncryption = 0;
put_object_callback_data data = {0};
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code;
}
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code;
}
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
contentLength;
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
0, awsRegionG};
S3PutProperties putProperties = {contentType, md5,
cacheControl, contentDispositionFilename,
contentEncoding, expires,
cannedAcl, metaPropertiesCount,
metaProperties, useServerSideEncryption};
if (contentLength <= MULTIPART_CHUNK_SIZE) {
code = s3PutObjectFromFileSimple(&bucketContext, object_name, contentLength, &putProperties, &data);
} else {
if (withcp) {
code = s3PutObjectFromFileWithCp(&bucketContext, file, lmtime, object_name, contentLength, &putProperties, &data);
} else {
code = s3PutObjectFromFileWithoutCp(&bucketContext, object_name, contentLength, &putProperties, &data);
}
}
if (data.infileFD) {
@ -648,7 +837,7 @@ typedef struct list_bucket_callback_data {
char nextMarker[1024];
int keyCount;
int allDetails;
SArray * objectArray;
SArray *objectArray;
} list_bucket_callback_data;
static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount,
@ -697,7 +886,7 @@ static SArray *getListByPrefix(const char *prefix) {
S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
&listBucketCallback};
const char * marker = 0, *delimiter = 0;
const char *marker = 0, *delimiter = 0;
int maxkeys = 0, allDetails = 0;
list_bucket_callback_data data;
data.objectArray = taosArrayInit(32, sizeof(void *));
@ -858,7 +1047,7 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
if (objectArray == NULL) return -1;
for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) {
char * object = taosArrayGetP(objectArray, i);
char *object = taosArrayGetP(objectArray, i);
const char *tmp = strchr(object, '/');
tmp = (tmp == NULL) ? object : tmp + 1;
char fileName[PATH_MAX] = {0};
@ -949,12 +1138,12 @@ static void s3InitRequestOptions(cos_request_options_t *options, int is_cname) {
int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
int32_t code = 0;
cos_pool_t * p = NULL;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t * s = NULL;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket, object, file;
cos_table_t * resp_headers;
cos_table_t *resp_headers;
// int traffic_limit = 0;
cos_pool_create(&p, NULL);
@ -983,18 +1172,19 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
return code;
}
int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) {
int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) {
int32_t code = 0;
cos_pool_t * p = NULL;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t * s = NULL;
cos_request_options_t * options = NULL;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket, object, file;
cos_table_t * resp_headers;
cos_table_t *resp_headers;
int traffic_limit = 0;
cos_table_t * headers = NULL;
cos_table_t *headers = NULL;
cos_resumable_clt_params_t *clt_params = NULL;
(void)withcp;
cos_pool_create(&p, NULL);
options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname);
@ -1025,11 +1215,11 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) {
}
void s3DeleteObjectsByPrefix(const char *prefix_str) {
cos_pool_t * p = NULL;
cos_pool_t *p = NULL;
cos_request_options_t *options = NULL;
int is_cname = 0;
cos_string_t bucket;
cos_status_t * s = NULL;
cos_status_t *s = NULL;
cos_string_t prefix;
cos_pool_create(&p, NULL);
@ -1044,10 +1234,10 @@ void s3DeleteObjectsByPrefix(const char *prefix_str) {
}
void s3DeleteObjects(const char *object_name[], int nobject) {
cos_pool_t * p = NULL;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_string_t bucket;
cos_table_t * resp_headers = NULL;
cos_table_t *resp_headers = NULL;
cos_request_options_t *options = NULL;
cos_list_t object_list;
cos_list_t deleted_object_list;
@ -1081,14 +1271,14 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
bool s3Exists(const char *object_name) {
bool ret = false;
cos_pool_t * p = NULL;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t * s = NULL;
cos_request_options_t * options = NULL;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket;
cos_string_t object;
cos_table_t * resp_headers;
cos_table_t * headers = NULL;
cos_table_t *resp_headers;
cos_table_t *headers = NULL;
cos_object_exist_status_e object_exist;
cos_pool_create(&p, NULL);
@ -1115,15 +1305,15 @@ bool s3Exists(const char *object_name) {
bool s3Get(const char *object_name, const char *path) {
bool ret = false;
cos_pool_t * p = NULL;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t * s = NULL;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket;
cos_string_t object;
cos_string_t file;
cos_table_t * resp_headers = NULL;
cos_table_t * headers = NULL;
cos_table_t *resp_headers = NULL;
cos_table_t *headers = NULL;
int traffic_limit = 0;
//创建内存池
@ -1159,15 +1349,15 @@ bool s3Get(const char *object_name, const char *path) {
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) {
(void)check;
int32_t code = 0;
cos_pool_t * p = NULL;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t * s = NULL;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket;
cos_string_t object;
cos_table_t * resp_headers;
cos_table_t * headers = NULL;
cos_buf_t * content = NULL;
cos_table_t *resp_headers;
cos_table_t *headers = NULL;
cos_buf_t *content = NULL;
// cos_string_t file;
// int traffic_limit = 0;
char range_buf[64];
@ -1261,7 +1451,7 @@ void s3EvictCache(const char *path, long object_size) {
terrno = TAOS_SYSTEM_ERROR(errno);
vError("failed to open %s since %s", dir_name, terrstr());
}
SArray * evict_files = taosArrayInit(16, sizeof(SEvictFile));
SArray *evict_files = taosArrayInit(16, sizeof(SEvictFile));
tdbDirEntryPtr pDirEntry;
while ((pDirEntry = taosReadDir(pDir)) != NULL) {
char *name = taosGetDirEntryName(pDirEntry);
@ -1303,13 +1493,13 @@ void s3EvictCache(const char *path, long object_size) {
long s3Size(const char *object_name) {
long size = 0;
cos_pool_t * p = NULL;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t * s = NULL;
cos_status_t *s = NULL;
cos_request_options_t *options = NULL;
cos_string_t bucket;
cos_string_t object;
cos_table_t * resp_headers = NULL;
cos_table_t *resp_headers = NULL;
//创建内存池
cos_pool_create(&p, NULL);
@ -1344,7 +1534,7 @@ long s3Size(const char *object_name) {
int32_t s3Init() { return 0; }
void s3CleanUp() {}
int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; }
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
void s3DeleteObjectsByPrefix(const char *prefix) {}
void s3DeleteObjects(const char *object_name[], int nobject) {}
bool s3Exists(const char *object_name) { return false; }

417
source/common/src/cos_cp.c Normal file
View File

@ -0,0 +1,417 @@
#define ALLOW_FORBID_FUNC
#include "cos_cp.h"
#include "cJSON.h"
#include "tutil.h"
int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) {
int32_t code = 0;
TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH);
if (!fd) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open %s", __func__, cp_path);
return code;
}
checkpoint->thefile = fd;
return code;
}
void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); }
void cos_cp_remove(char const* filepath) { taosRemoveFile(filepath); }
static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
int32_t code = 0;
cJSON const* item2 = NULL;
cJSON* json = cJSON_Parse(cp_body);
if (NULL == json) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to parse json", __func__);
goto _exit;
}
cJSON const* item = cJSON_GetObjectItem(json, "ver");
if (!cJSON_IsNumber(item) || item->valuedouble != 1) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to parse json ver: %f", __func__, item->valuedouble);
goto _exit;
}
item = cJSON_GetObjectItem(json, "type");
if (!cJSON_IsNumber(item)) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to parse json", __func__);
goto _exit;
}
cp->cp_type = item->valuedouble;
item = cJSON_GetObjectItem(json, "md5");
if (cJSON_IsString(item)) {
memcpy(cp->md5, item->valuestring, strlen(item->valuestring));
}
item = cJSON_GetObjectItem(json, "upload_id");
if (cJSON_IsString(item)) {
strncpy(cp->upload_id, item->valuestring, 128);
}
item2 = cJSON_GetObjectItem(json, "file");
if (cJSON_IsObject(item2)) {
item = cJSON_GetObjectItem(item2, "size");
if (cJSON_IsNumber(item)) {
cp->file_size = item->valuedouble;
}
item = cJSON_GetObjectItem(item2, "lastmodified");
if (cJSON_IsNumber(item)) {
cp->file_last_modified = item->valuedouble;
}
item = cJSON_GetObjectItem(item2, "path");
if (cJSON_IsString(item)) {
strncpy(cp->file_path, item->valuestring, TSDB_FILENAME_LEN);
}
item = cJSON_GetObjectItem(item2, "file_md5");
if (cJSON_IsString(item)) {
strncpy(cp->file_md5, item->valuestring, 64);
}
}
item2 = cJSON_GetObjectItem(json, "object");
if (cJSON_IsObject(item2)) {
item = cJSON_GetObjectItem(item2, "object_size");
if (cJSON_IsNumber(item)) {
cp->object_size = item->valuedouble;
}
item = cJSON_GetObjectItem(item2, "object_name");
if (cJSON_IsString(item)) {
strncpy(cp->object_name, item->valuestring, 128);
}
item = cJSON_GetObjectItem(item2, "object_last_modified");
if (cJSON_IsString(item)) {
strncpy(cp->object_last_modified, item->valuestring, 64);
}
item = cJSON_GetObjectItem(item2, "object_etag");
if (cJSON_IsString(item)) {
strncpy(cp->object_etag, item->valuestring, 128);
}
}
item2 = cJSON_GetObjectItem(json, "cpparts");
if (cJSON_IsObject(item2)) {
item = cJSON_GetObjectItem(item2, "number");
if (cJSON_IsNumber(item)) {
cp->part_num = item->valuedouble;
}
item = cJSON_GetObjectItem(item2, "size");
if (cJSON_IsNumber(item)) {
cp->part_size = item->valuedouble;
}
item2 = cJSON_GetObjectItem(item2, "parts");
if (cJSON_IsArray(item2) && cp->part_num > 0) {
cJSON_ArrayForEach(item, item2) {
cJSON const* item3 = cJSON_GetObjectItem(item, "index");
int32_t index = 0;
if (cJSON_IsNumber(item3)) {
index = item3->valuedouble;
cp->parts[index].index = index;
}
item3 = cJSON_GetObjectItem(item, "offset");
if (cJSON_IsNumber(item3)) {
cp->parts[index].offset = item3->valuedouble;
}
item3 = cJSON_GetObjectItem(item, "size");
if (cJSON_IsNumber(item3)) {
cp->parts[index].size = item3->valuedouble;
}
item3 = cJSON_GetObjectItem(item, "completed");
if (cJSON_IsNumber(item3)) {
cp->parts[index].completed = item3->valuedouble;
}
item3 = cJSON_GetObjectItem(item, "crc64");
if (cJSON_IsNumber(item3)) {
cp->parts[index].crc64 = item3->valuedouble;
}
item3 = cJSON_GetObjectItem(item, "etag");
if (cJSON_IsString(item3)) {
strncpy(cp->parts[index].etag, item3->valuestring, 128);
}
}
}
}
_exit:
if (json) cJSON_Delete(json);
if (cp_body) taosMemoryFree(cp_body);
return code;
}
int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
int32_t code = 0;
TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ);
if (!fd) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open %s", __func__, filepath);
goto _exit;
}
int64_t size = -1;
code = taosStatFile(filepath, &size, NULL, NULL);
if (code) {
uError("ERROR: %s Failed to stat %s", __func__, filepath);
goto _exit;
}
char* cp_body = taosMemoryMalloc(size + 1);
int64_t n = taosReadFile(fd, cp_body, size);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to read %s", __func__, filepath);
goto _exit;
} else if (n != size) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to read %s %" PRId64 "/%" PRId64, __func__, filepath, n, size);
goto _exit;
}
taosCloseFile(&fd);
cp_body[size] = '\0';
return cos_cp_parse_body(cp_body, checkpoint);
_exit:
if (fd) {
taosCloseFile(&fd);
}
if (cp_body) {
taosMemoryFree(cp_body);
}
return code;
}
static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) {
int32_t code = 0;
char* data = cJSON_PrintUnformatted(json);
if (NULL == data) {
return TSDB_CODE_OUT_OF_MEMORY;
}
TdFilePtr fp = checkpoint->thefile;
if (taosFtruncateFile(fp, 0) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
if (taosLSeekFile(fp, 0, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
if (taosWriteFile(fp, data, strlen(data)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
_exit:
taosMemoryFree(data);
return code;
}
int32_t cos_cp_dump(SCheckpoint* cp) {
int32_t code = 0;
int32_t lino = 0;
cJSON* ojson = NULL;
cJSON* json = cJSON_CreateObject();
if (!json) return TSDB_CODE_OUT_OF_MEMORY;
if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (COS_CP_TYPE_UPLOAD == cp->cp_type) {
ojson = cJSON_AddObjectToObject(json, "file");
if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
} else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) {
ojson = cJSON_AddObjectToObject(json, "object");
if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
}
ojson = cJSON_AddObjectToObject(json, "cpparts");
if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts");
if (!ajson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
for (int i = 0; i < cp->part_num; ++i) {
cJSON* item = cJSON_CreateObject();
if (!item) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
cJSON_AddItemToArray(ajson, item);
if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
}
code = cos_cp_save_json(json, cp);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
cJSON_Delete(json);
return code;
}
void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {}
void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64) {
checkpoint->parts[part_index].completed = 1;
strncpy(checkpoint->parts[part_index].etag, etag, 128);
checkpoint->parts[part_index].crc64 = crc64;
}
void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
char const* upload_id, int64_t part_size) {
int i = 0;
checkpoint->cp_type = COS_CP_TYPE_UPLOAD;
strncpy(checkpoint->file_path, filepath, TSDB_FILENAME_LEN);
checkpoint->file_size = size;
checkpoint->file_last_modified = mtime;
strncpy(checkpoint->upload_id, upload_id, 128);
checkpoint->part_size = part_size;
for (; i * part_size < size; i++) {
checkpoint->parts[i].index = i;
checkpoint->parts[i].offset = i * part_size;
checkpoint->parts[i].size = TMIN(part_size, (size - i * part_size));
checkpoint->parts[i].completed = 0;
checkpoint->parts[i].etag[0] = '\0';
}
checkpoint->part_num = i;
}
static bool cos_cp_verify_md5(SCheckpoint* cp) { return true; }
bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime) {
if (cos_cp_verify_md5(checkpoint) && checkpoint->file_size == size && checkpoint->file_last_modified == mtime) {
return true;
}
return false;
}

16
source/common/src/systable.c
View File

@ -334,6 +334,20 @@ static const SSysDbTableSchema userViewsSchema[] = {
// {.name = "column_list", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysDbTableSchema userCompactsSchema[] = {
{.name = "compact_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysDbTableSchema userCompactsDetailSchema[] = {
{.name = "compact_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "number_fileset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
@ -362,6 +376,8 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true},
{TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), true},
{TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false},
{TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false},
{TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false},
};
static const SSysDbTableSchema connectionsSchema[] = {

21
source/common/src/tglobal.c
View File

@ -97,6 +97,7 @@ bool tsMonitorComp = false;
// audit
bool tsEnableAudit = true;
bool tsEnableAuditCreateTable = true;
int32_t tsAuditInterval = 5000;
// telem
#ifdef TD_ENTERPRISE
@ -247,6 +248,7 @@ int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch
// internal
int32_t tsTransPullupInterval = 2;
int32_t tsCompactPullupInterval = 10;
int32_t tsMqRebalanceInterval = 2;
int32_t tsStreamCheckpointInterval = 60;
float tsSinkDataRate = 2.0;
@ -280,6 +282,8 @@ int32_t tsS3BlockCacheSize = 16; // number of blocks
int32_t tsS3PageCacheSize = 4096; // number of pages
int32_t tsS3UploadDelaySec = 60 * 60 * 24;
bool tsExperimental = true;
#ifndef _STORAGE
int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
@ -527,6 +531,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
}
if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0)
return -1;
if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
return 0;
}
@ -686,6 +691,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1;
if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER) != 0) return -1;
@ -704,6 +711,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0)
return -1;
if (cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0)
return -1;
if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0)
return -1;
@ -789,6 +799,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
return -1;
if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
GRANT_CFG_ADD;
return 0;
}
@ -1074,6 +1086,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32;
tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
return 0;
}
@ -1137,6 +1151,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsEnableAudit = cfgGetItem(pCfg, "audit")->bval;
tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval;
tsAuditInterval = cfgGetItem(pCfg, "auditInterval")->i32;
tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval;
@ -1152,6 +1167,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32;
tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
tsCompactPullupInterval = cfgGetItem(pCfg, "compactPullupInterval")->i32;
tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32;
tsTtlPushIntervalSec = cfgGetItem(pCfg, "ttlPushInterval")->i32;
@ -1206,6 +1222,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32;
tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32;
tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
GRANT_CFG_GET;
return 0;
}
@ -1466,6 +1484,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
{"timeseriesThreshold", &tsTimeSeriesThreshold},
{"tmqMaxTopicNum", &tmqMaxTopicNum},
{"transPullupInterval", &tsTransPullupInterval},
{"compactPullupInterval", &tsCompactPullupInterval},
{"trimVDbIntervalSec", &tsTrimVDbIntervalSec},
{"ttlBatchDropNum", &tsTtlBatchDropNum},
{"ttlFlushThreshold", &tsTtlFlushThreshold},
@ -1475,6 +1494,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
{"s3PageCacheSize", &tsS3PageCacheSize},
{"s3UploadDelaySec", &tsS3UploadDelaySec},
{"supportVnodes", &tsNumOfSupportVnodes},
{"experimental", &tsExperimental}
};
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
@ -1698,6 +1718,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
{"shellActivityTimer", &tsShellActivityTimer},
{"slowLogThreshold", &tsSlowLogThreshold},
{"useAdapter", &tsUseAdapter},
{"experimental", &tsExperimental}
};
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {

197
source/common/src/tmsg.c
View File

@ -1827,7 +1827,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
char *tb = taosHashIterate(pRsp->readTbs, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen);
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1842,7 +1842,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->writeTbs, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen);
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1857,7 +1857,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->alterTbs, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen);
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1872,7 +1872,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->readViews, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen);
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1887,7 +1887,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->writeViews, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen);
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1902,7 +1902,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->alterViews, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen);
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1917,7 +1917,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL);
while (useDb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(useDb, &keyLen);
void *key = taosHashGetKey(useDb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -3424,6 +3424,66 @@ int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq
void tFreeSCompactDbReq(SCompactDbReq *pReq) { FREESQL(); }
int32_t tSerializeSCompactDbRsp(void *buf, int32_t bufLen, SCompactDbRsp *pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->compactId) < 0) return -1;
if (tEncodeI8(&encoder, pRsp->bAccepted) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCompactDbRsp(void *buf, int32_t bufLen, SCompactDbRsp *pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->compactId) < 0) return -1;
if (tDecodeI8(&decoder, &pRsp->bAccepted) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSKillCompactReq(void *buf, int32_t bufLen, SKillCompactReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
ENCODESQL();
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSKillCompactReq(void *buf, int32_t bufLen, SKillCompactReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
DECODESQL();
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSKillCompactReq(SKillCompactReq *pReq) { FREESQL(); }
int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) {
if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1;
@ -4259,6 +4319,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->filterTb) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
if (tEncodeI64(&encoder, pReq->compactId) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -4276,6 +4337,11 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->filterTb) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI64(&decoder, &pReq->compactId) < 0) return -1;
} else {
pReq->compactId = -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -5051,6 +5117,75 @@ int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) {
return 0;
}
int32_t tSerializeSQueryCompactProgressReq(void *buf, int32_t bufLen, SQueryCompactProgressReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSQueryCompactProgressReq(void *buf, int32_t bufLen, SQueryCompactProgressReq *pReq) {
int32_t headLen = sizeof(SMsgHead);
SDecoder decoder = {0};
tDecoderInit(&decoder, ((uint8_t *)buf) + headLen, bufLen - headLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCompactProgressRsp *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numberFileset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->finished) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCompactProgressRsp *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->finished) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
@ -5139,6 +5274,8 @@ int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *
if (tEncodeI64(&encoder, pReq->tw.skey) < 0) return -1;
if (tEncodeI64(&encoder, pReq->tw.ekey) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -5165,6 +5302,42 @@ int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq
if (tDecodeI64(&decoder, &pReq->tw.ekey) < 0) return -1;
}
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -8169,7 +8342,7 @@ int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
for (int32_t i = 0; i < pRsp->blockNum; i++) {
int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i);
void * data = taosArrayGetP(pRsp->blockData, i);
void *data = taosArrayGetP(pRsp->blockData, i);
if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1;
if (pRsp->withSchema) {
SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i);
@ -8202,7 +8375,7 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
}
for (int32_t i = 0; i < pRsp->blockNum; i++) {
void * data;
void *data;
uint64_t bLen;
if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1;
taosArrayPush(pRsp->blockData, &data);
@ -8248,7 +8421,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) {
if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1;
if (pRsp->createTableNum) {
for (int32_t i = 0; i < pRsp->createTableNum; i++) {
void * createTableReq = taosArrayGetP(pRsp->createTableReq, i);
void *createTableReq = taosArrayGetP(pRsp->createTableReq, i);
int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i);
if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1;
}
@ -8264,7 +8437,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) {
pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t));
pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *));
for (int32_t i = 0; i < pRsp->createTableNum; i++) {
void * pCreate = NULL;
void *pCreate = NULL;
uint64_t len;
if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1;
int32_t l = (int32_t)len;
@ -8566,7 +8739,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
taosArrayDestroy(pTbData->aCol);
} else {
int32_t nRow = TARRAY_SIZE(pTbData->aRowP);
SRow ** rows = (SRow **)TARRAY_DATA(pTbData->aRowP);
SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP);
for (int32_t i = 0; i < nRow; ++i) {
tRowDestroy(rows[i]);

4
source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
View File

@ -30,12 +30,14 @@ typedef struct SDnodeMgmt {
TdThread statusThread;
TdThread notifyThread;
TdThread monitorThread;
TdThread auditThread;
TdThread crashReportThread;
SSingleWorker mgmtWorker;
ProcessCreateNodeFp processCreateNodeFp;
ProcessAlterNodeTypeFp processAlterNodeTypeFp;
ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp;
SendAuditRecordsFp sendAuditRecordsFp;
GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp;
@ -62,7 +64,9 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt);
int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt);
void dmStopNotifyThread(SDnodeMgmt *pMgmt);
int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt);
int32_t dmStartAuditThread(SDnodeMgmt *pMgmt);
void dmStopMonitorThread(SDnodeMgmt *pMgmt);
void dmStopAuditThread(SDnodeMgmt *pMgmt);
int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt);
void dmStopCrashReportThread(SDnodeMgmt *pMgmt);
int32_t dmStartWorker(SDnodeMgmt *pMgmt);

5
source/dnode/mgmt/mgmt_dnode/src/dmInt.c
View File

@ -29,6 +29,9 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
if (dmStartMonitorThread(pMgmt) != 0) {
return -1;
}
if (dmStartAuditThread(pMgmt) != 0) {
return -1;
}
if (dmStartCrashReportThread(pMgmt) != 0) {
return -1;
}
@ -38,6 +41,7 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
static void dmStopMgmt(SDnodeMgmt *pMgmt) {
pMgmt->pData->stopped = true;
dmStopMonitorThread(pMgmt);
dmStopAuditThread(pMgmt);
dmStopStatusThread(pMgmt);
#if defined(TD_ENTERPRISE)
dmStopNotifyThread(pMgmt);
@ -60,6 +64,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->processAlterNodeTypeFp = pInput->processAlterNodeTypeFp;
pMgmt->processDropNodeFp = pInput->processDropNodeFp;
pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp;
pMgmt->sendAuditRecordsFp = pInput->sendAuditRecordFp;
pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp;
pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp;
pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;

42
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -99,6 +99,27 @@ static void *dmMonitorThreadFp(void *param) {
return NULL;
}
static void *dmAuditThreadFp(void *param) {
SDnodeMgmt *pMgmt = param;
int64_t lastTime = taosGetTimestampMs();
setThreadName("dnode-audit");
while (1) {
taosMsleep(100);
if (pMgmt->pData->dropped || pMgmt->pData->stopped) break;
int64_t curTime = taosGetTimestampMs();
if (curTime < lastTime) lastTime = curTime;
float interval = curTime - lastTime;
if (interval >= tsAuditInterval) {
(*pMgmt->sendAuditRecordsFp)();
lastTime = curTime;
}
}
return NULL;
}
static void *dmCrashReportThreadFp(void *param) {
SDnodeMgmt *pMgmt = param;
int64_t lastTime = taosGetTimestampMs();
@ -218,6 +239,20 @@ int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
return 0;
}
int32_t dmStartAuditThread(SDnodeMgmt *pMgmt) {
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->auditThread, &thAttr, dmAuditThreadFp, pMgmt) != 0) {
dError("failed to create audit thread since %s", strerror(errno));
return -1;
}
taosThreadAttrDestroy(&thAttr);
tmsgReportStartup("dnode-audit", "initialized");
return 0;
}
void dmStopMonitorThread(SDnodeMgmt *pMgmt) {
if (taosCheckPthreadValid(pMgmt->monitorThread)) {
taosThreadJoin(pMgmt->monitorThread, NULL);
@ -225,6 +260,13 @@ void dmStopMonitorThread(SDnodeMgmt *pMgmt) {
}
}
void dmStopAuditThread(SDnodeMgmt *pMgmt) {
if (taosCheckPthreadValid(pMgmt->auditThread)) {
taosThreadJoin(pMgmt->auditThread, NULL);
taosThreadClear(&pMgmt->auditThread);
}
}
int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) {
if (!tsEnableCrashReport) {
return 0;

3
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -192,6 +192,8 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
@ -221,6 +223,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -818,6 +818,8 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

25
source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
View File

@ -70,6 +70,21 @@ typedef struct SUdfdData {
int32_t dnodeId;
} SUdfdData;
#ifndef TD_MODULE_OPTIMIZE
typedef struct SDnode {
int8_t once;
bool stop;
EDndRunStatus status;
SStartupInfo startup;
SDnodeData data;
SUdfdData udfdData;
TdThreadMutex mutex;
TdFilePtr lockfile;
STfs *pTfs;
SMgmtWrapper wrappers[NODE_END];
SDnodeTrans trans;
} SDnode;
#else
typedef struct SDnode {
int8_t once;
bool stop;
@ -83,6 +98,7 @@ typedef struct SDnode {
STfs *pTfs;
SMgmtWrapper wrappers[NODE_END];
} SDnode;
#endif
// dmEnv.c
SDnode *dmInstance();
@ -97,8 +113,12 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper);
void dmReleaseWrapper(SMgmtWrapper *pWrapper);
int32_t dmInitVars(SDnode *pDnode);
void dmClearVars(SDnode *pDnode);
#ifdef TD_MODULE_OPTIMIZE
int32_t dmInitModule(SDnode *pDnode, SMgmtWrapper *wrappers);
bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper);
#else
int32_t dmInitModule(SDnode *pDnode);
#endif
SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);
void dmSetStatus(SDnode *pDnode, EDndRunStatus stype);
void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg);
@ -119,11 +139,16 @@ int32_t dmInitStatusClient(SDnode *pDnode);
void dmCleanupClient(SDnode *pDnode);
void dmCleanupStatusClient(SDnode *pDnode);
SMsgCb dmGetMsgcb(SDnode *pDnode);
#ifdef TD_MODULE_OPTIMIZE
int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers);
#else
int32_t dmInitMsgHandle(SDnode *pDnode);
#endif
int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
// dmMonitor.c
void dmSendMonitorReport();
void dmSendAuditRecords();
void dmGetVnodeLoads(SMonVloadInfo *pInfo);
void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo);
void dmGetMnodeLoads(SMonMloadInfo *pInfo);

2
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -189,6 +189,7 @@ void dmCleanup() {
if (dmCheckRepeatCleanup(pDnode) != 0) return;
dmCleanupDnode(pDnode);
monCleanup();
auditCleanup();
syncCleanUp();
walCleanUp();
udfcClose();
@ -396,6 +397,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) {
.processAlterNodeTypeFp = dmProcessAlterNodeTypeReq,
.processDropNodeFp = dmProcessDropNodeReq,
.sendMonitorReportFp = dmSendMonitorReport,
.sendAuditRecordFp = auditSendRecordsInBatch,
.getVnodeLoadsFp = dmGetVnodeLoads,
.getVnodeLoadsLiteFp = dmGetVnodeLoadsLite,
.getMnodeLoadsFp = dmGetMnodeLoads,

24
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -24,6 +24,22 @@
#include "tglobal.h"
#endif
#ifndef TD_MODULE_OPTIMIZE
static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
bool required = false;
int32_t code = (*pWrapper->func.requiredFp)(&input, &required);
if (!required) {
dDebug("node:%s, does not require startup", pWrapper->name);
} else {
dDebug("node:%s, required to startup", pWrapper->name);
}
return required;
}
#endif
int32_t dmInitDnode(SDnode *pDnode) {
dDebug("start to create dnode");
int32_t code = -1;
@ -65,11 +81,15 @@ int32_t dmInitDnode(SDnode *pDnode) {
if (pDnode->lockfile == NULL) {
goto _OVER;
}
#ifdef TD_MODULE_OPTIMIZE
if (dmInitModule(pDnode, pDnode->wrappers) != 0) {
goto _OVER;
}
#else
if (dmInitModule(pDnode) != 0) {
goto _OVER;
}
#endif
indexInit(tsNumOfCommitThreads);
streamMetaInit();

6
source/dnode/mgmt/node_mgmt/src/dmMonitor.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE
#include "dmMgmt.h"
#include "dmNodes.h"
#include "audit.h"
static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
pInfo->protocol = 1;
@ -108,6 +109,11 @@ void dmSendMonitorReport() {
monSendReport();
}
//Todo: put this in seperate file in the future
void dmSendAuditRecords() {
auditSendRecordsInBatch();
}
void dmGetVnodeLoads(SMonVloadInfo *pInfo) {
SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = &pDnode->wrappers[VNODE];

28
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -251,6 +251,7 @@ _OVER:
dmReleaseWrapper(pWrapper);
}
#ifdef TD_MODULE_OPTIMIZE
int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) {
SDnodeTrans *pTrans = &pDnode->trans;
@ -276,6 +277,33 @@ int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) {
return 0;
}
#else
int32_t dmInitMsgHandle(SDnode *pDnode) {
SDnodeTrans *pTrans = &pDnode->trans;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
SArray *pArray = (*pWrapper->func.getHandlesFp)();
if (pArray == NULL) return -1;
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
SMgmtHandle *pMgmt = taosArrayGet(pArray, i);
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
if (pMgmt->needCheckVgId) {
pHandle->needCheckVgId = pMgmt->needCheckVgId;
}
if (!pMgmt->needCheckVgId) {
pHandle->defaultNtype = ntype;
}
pWrapper->msgFps[TMSG_INDEX(pMgmt->msgType)] = pMgmt->msgFp;
}
taosArrayDestroy(pArray);
}
return 0;
}
#endif
static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
SDnode *pDnode = dmInstance();

2
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -86,6 +86,7 @@ typedef enum {
typedef int32_t (*ProcessCreateNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg);
typedef int32_t (*ProcessDropNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg);
typedef void (*SendMonitorReportFp)();
typedef void (*SendAuditRecordsFp)();
typedef void (*GetVnodeLoadsFp)(SMonVloadInfo *pInfo);
typedef void (*GetMnodeLoadsFp)(SMonMloadInfo *pInfo);
typedef void (*GetQnodeLoadsFp)(SQnodeLoad *pInfo);
@ -120,6 +121,7 @@ typedef struct {
ProcessAlterNodeTypeFp processAlterNodeTypeFp;
ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp;
SendAuditRecordsFp sendAuditRecordFp;
GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp;

9
source/dnode/mgmt/test/qnode/dqnode.cpp
View File

@ -37,6 +37,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION);
rpcFreeCont(pRsp->pCont);
}
{
@ -50,6 +51,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
}
{
@ -63,6 +65,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED);
rpcFreeCont(pRsp->pCont);
}
test.Restart();
@ -78,6 +81,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED);
rpcFreeCont(pRsp->pCont);
}
}
@ -94,6 +98,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION);
rpcFreeCont(pRsp->pCont);
}
#endif
@ -108,6 +113,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
}
{
@ -121,6 +127,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED);
rpcFreeCont(pRsp->pCont);
}
test.Restart();
@ -136,6 +143,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED);
rpcFreeCont(pRsp->pCont);
}
{
@ -149,5 +157,6 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
}
}

9
source/dnode/mgmt/test/snode/dsnode.cpp
View File

@ -37,6 +37,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION);
rpcFreeCont(pRsp->pCont);
}
{
@ -50,6 +51,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
}
{
@ -63,6 +65,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED);
rpcFreeCont(pRsp->pCont);
}
test.Restart();
@ -78,6 +81,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED);
rpcFreeCont(pRsp->pCont);
}
}
@ -94,6 +98,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION);
rpcFreeCont(pRsp->pCont);
}
#endif
@ -108,6 +113,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
}
{
@ -121,6 +127,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED);
rpcFreeCont(pRsp->pCont);
}
test.Restart();
@ -136,6 +143,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED);
rpcFreeCont(pRsp->pCont);
}
{
@ -149,5 +157,6 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
}
}

56
source/dnode/mnode/impl/inc/mndCompact.h Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_COMPACT_H_
#define _TD_MND_COMPACT_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitCompact(SMnode *pMnode);
void mndCleanupCompact(SMnode *pMnode);
void tFreeCompactObj(SCompactObj *pCompact);
int32_t tSerializeSCompactObj(void *buf, int32_t bufLen, const SCompactObj *pObj);
int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj);
SSdbRaw* mndCompactActionEncode(SCompactObj *pCompact);
SSdbRow* mndCompactActionDecode(SSdbRaw *pRaw);
int32_t mndCompactActionInsert(SSdb *pSdb, SCompactObj *pCompact);
int32_t mndCompactActionDelete(SSdb *pSdb, SCompactObj *pCompact);
int32_t mndCompactActionUpdate(SSdb *pSdb, SCompactObj *pOldCompact, SCompactObj *pNewCompact);
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SDbObj *pDb, SCompactDbRsp *rsp);
int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
int32_t mndProcessKillCompactReq(SRpcMsg *pReq);
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq);
SCompactObj *mndAcquireCompact(SMnode *pMnode, int64_t compactId);
void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact);
void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_COMPACT_H_*/

48
source/dnode/mnode/impl/inc/mndCompactDetail.h Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_COMPACT_DETAIL_H_
#define _TD_MND_COMPACT_DETAIL_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitCompactDetail(SMnode *pMnode);
void mndCleanupCompactDetail(SMnode *pMnode);
void tFreeCompactDetailObj(SCompactDetailObj *pCompact);
int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDetailObj *pObj);
int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailObj *pObj);
SSdbRaw* mndCompactDetailActionEncode(SCompactDetailObj *pCompact);
SSdbRow* mndCompactDetailActionDecode(SSdbRaw *pRaw);
int32_t mndCompactDetailActionInsert(SSdb *pSdb, SCompactDetailObj *pCompact);
int32_t mndCompactDetailActionDelete(SSdb *pSdb, SCompactDetailObj *pCompact);
int32_t mndCompactDetailActionUpdate(SSdb *pSdb, SCompactDetailObj *pOldCompact, SCompactDetailObj *pNewCompact);
int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup,
SVnodeGid *pVgid, int32_t index);
int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_COMPACT_DETAIL_H_*/

19
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -741,6 +741,25 @@ int32_t tEncodeSViewObj(SEncoder* pEncoder, const SViewObj* pObj);
int32_t tDecodeSViewObj(SDecoder* pDecoder, SViewObj* pObj, int32_t sver);
void tFreeSViewObj(SViewObj* pObj);
typedef struct {
int32_t compactDetailId;
int32_t compactId;
int32_t vgId;
int32_t dnodeId;
int32_t numberFileset;
int32_t finished;
int64_t startTime;
int32_t newNumberFileset;
int32_t newFinished;
} SCompactDetailObj;
typedef struct {
int32_t compactId;
char dbname[TSDB_TABLE_FNAME_LEN];
int64_t startTime;
SArray* compactDetail;
} SCompactObj;
#ifdef __cplusplus
}
#endif

15
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -24,8 +24,9 @@ extern "C" {
typedef struct SStreamTransInfo {
int64_t startTime;
int32_t transId;
int64_t streamUid;
const char *name;
int32_t transId;
} SStreamTransInfo;
// time to generated the checkpoint, if now() - checkpointTs >= tsCheckpointInterval, this checkpoint will be discard
@ -52,6 +53,14 @@ typedef struct SStreamExecInfo {
TdThreadMutex lock;
} SStreamExecInfo;
#define MND_STREAM_CREATE_NAME "stream-create"
#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint"
#define MND_STREAM_PAUSE_NAME "stream-pause"
#define MND_STREAM_RESUME_NAME "stream-resume"
#define MND_STREAM_DROP_NAME "stream-drop"
#define MND_STREAM_TASK_RESET_NAME "stream-task-reset"
#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update"
extern SStreamExecInfo execInfo;
int32_t mndInitStream(SMnode *pMnode);
@ -61,9 +70,9 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pName, const char *pSrcDb, const char *pDstDb);
int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamUid);
int32_t mndAddtoCheckpointWaitingList(SStreamObj *pStream, int64_t checkpointId);
bool streamTransConflictOtherTrans(SMnode *pMnode, const char *pSrcDb, const char *pDstDb, bool lock);
bool streamTransConflictOtherTrans(SMnode *pMnode, int64_t streamUid, const char *pTransName, bool lock);
// for sma
// TODO refactor

748
source/dnode/mnode/impl/src/mndCompact.c Normal file
View File

@ -0,0 +1,748 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndCompact.h"
#include "mndTrans.h"
#include "mndShow.h"
#include "mndDb.h"
#include "mndCompactDetail.h"
#include "mndVgroup.h"
#include "tmsgcb.h"
#include "mndDnode.h"
#include "tmisce.h"
#include "audit.h"
#include "mndPrivilege.h"
#include "mndTrans.h"
#define MND_COMPACT_VER_NUMBER 1
static int32_t mndProcessCompactTimer(SRpcMsg *pReq);
int32_t mndInitCompact(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_COMPACT, mndRetrieveCompact);
mndSetMsgHandle(pMnode, TDMT_MND_KILL_COMPACT, mndProcessKillCompactReq);
mndSetMsgHandle(pMnode, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mndProcessQueryCompactRsp);
mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_TIMER, mndProcessCompactTimer);
mndSetMsgHandle(pMnode, TDMT_VND_KILL_COMPACT_RSP, mndTransProcessRsp);
SSdbTable table = {
.sdbType = SDB_COMPACT,
.keyType = SDB_KEY_INT32,
.encodeFp = (SdbEncodeFp)mndCompactActionEncode,
.decodeFp = (SdbDecodeFp)mndCompactActionDecode,
.insertFp = (SdbInsertFp)mndCompactActionInsert,
.updateFp = (SdbUpdateFp)mndCompactActionUpdate,
.deleteFp = (SdbDeleteFp)mndCompactActionDelete,
};
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupCompact(SMnode *pMnode) {
mDebug("mnd compact cleanup");
}
void tFreeCompactObj(SCompactObj *pCompact) {
}
int32_t tSerializeSCompactObj(void *buf, int32_t bufLen, const SCompactObj *pObj) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pObj->compactId) < 0) return -1;
if (tEncodeCStr(&encoder, pObj->dbname) < 0) return -1;
if (tEncodeI64(&encoder, pObj->startTime) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj) {
int8_t ex = 0;
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->compactId) < 0) return -1;
if (tDecodeCStrTo(&decoder, pObj->dbname) < 0) return -1;
if (tDecodeI64(&decoder, &pObj->startTime) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
SSdbRaw *mndCompactActionEncode(SCompactObj *pCompact) {
terrno = TSDB_CODE_SUCCESS;
void *buf = NULL;
SSdbRaw *pRaw = NULL;
int32_t tlen = tSerializeSCompactObj(NULL, 0, pCompact);
if (tlen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t size = sizeof(int32_t) + tlen;
pRaw = sdbAllocRaw(SDB_COMPACT, MND_COMPACT_VER_NUMBER, size);
if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
buf = taosMemoryMalloc(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
tlen = tSerializeSCompactObj(buf, tlen, pCompact);
if (tlen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, OVER);
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OVER);
SDB_SET_DATALEN(pRaw, dataPos, OVER);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("compact:%" PRId32 ", failed to encode to raw:%p since %s", pCompact->compactId, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("compact:%" PRId32 ", encode to raw:%p, row:%p", pCompact->compactId, pRaw, pCompact);
return pRaw;
}
SSdbRow *mndCompactActionDecode(SSdbRaw *pRaw) {
SSdbRow *pRow = NULL;
SCompactObj *pCompact = NULL;
void *buf = NULL;
terrno = TSDB_CODE_SUCCESS;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) {
goto OVER;
}
if (sver != MND_COMPACT_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("compact read invalid ver, data ver: %d, curr ver: %d", sver, MND_COMPACT_VER_NUMBER);
goto OVER;
}
pRow = sdbAllocRow(sizeof(SCompactObj));
if (pRow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
pCompact = sdbGetRowObj(pRow);
if (pCompact == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t tlen;
int32_t dataPos = 0;
SDB_GET_INT32(pRaw, dataPos, &tlen, OVER);
buf = taosMemoryMalloc(tlen + 1);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER);
if (tDeserializeSCompactObj(buf, tlen, pCompact) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
//taosInitRWLatch(&pView->lock);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("compact:%" PRId32 ", failed to decode from raw:%p since %s", pCompact->compactId, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
mTrace("compact:%" PRId32 ", decode from raw:%p, row:%p", pCompact->compactId, pRaw, pCompact);
return pRow;
}
int32_t mndCompactActionInsert(SSdb *pSdb, SCompactObj *pCompact) {
mTrace("compact:%" PRId32 ", perform insert action", pCompact->compactId);
return 0;
}
int32_t mndCompactActionDelete(SSdb *pSdb, SCompactObj *pCompact) {
mTrace("compact:%" PRId32 ", perform insert action", pCompact->compactId);
tFreeCompactObj(pCompact);
return 0;
}
int32_t mndCompactActionUpdate(SSdb *pSdb, SCompactObj *pOldCompact, SCompactObj *pNewCompact) {
mTrace("compact:%" PRId32 ", perform update action, old row:%p new row:%p",
pOldCompact->compactId, pOldCompact, pNewCompact);
return 0;
}
SCompactObj *mndAcquireCompact(SMnode *pMnode, int64_t compactId) {
SSdb *pSdb = pMnode->pSdb;
SCompactObj *pCompact = sdbAcquire(pSdb, SDB_COMPACT, &compactId);
if (pCompact == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_SUCCESS;
}
return pCompact;
}
void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pCompact);
}
//compact db
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SDbObj *pDb, SCompactDbRsp *rsp){
pCompact->compactId = tGenIdPI32();
strcpy(pCompact->dbname, pDb->name);
pCompact->startTime = taosGetTimestampMs();
SSdbRaw *pVgRaw = mndCompactActionEncode(pCompact);
if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
}
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
rsp->compactId = pCompact->compactId;
return 0;
}
//retrieve compact
int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows){
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SCompactObj *pCompact = NULL;
char *sep = NULL;
SDbObj *pDb = NULL;
if (strlen(pShow->db) > 0) {
sep = strchr(pShow->db, '.');
if (sep && ((0 == strcmp(sep + 1, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcmp(sep + 1, TSDB_PERFORMANCE_SCHEMA_DB))))) {
sep++;
} else {
pDb = mndAcquireDb(pMnode, pShow->db);
if (pDb == NULL) return terrno;
}
}
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_COMPACT, pShow->pIter, (void **)&pCompact);
if (pShow->pIter == NULL) break;
SColumnInfoData *pColInfo;
SName n;
int32_t cols = 0;
char tmpBuf[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompact->compactId, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
if (pDb != NULL || !IS_SYS_DBNAME(pCompact->dbname)) {
SName name = {0};
tNameFromString(&name, pCompact->dbname, T_NAME_ACCT | T_NAME_DB);
tNameGetDbName(&name, varDataVal(tmpBuf));
} else {
strncpy(varDataVal(tmpBuf), pCompact->dbname, strlen(pCompact->dbname) + 1);
}
varDataSetLen(tmpBuf, strlen(varDataVal(tmpBuf)));
colDataSetVal(pColInfo, numOfRows, (const char *)tmpBuf, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompact->startTime, false);
numOfRows++;
sdbRelease(pSdb, pCompact);
}
pShow->numOfRows += numOfRows;
mndReleaseDb(pMnode, pDb);
return numOfRows;
}
//kill compact
static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pContLen,
int32_t compactId, int32_t dnodeid) {
SVKillCompactReq req = {0};
req.compactId = compactId;
req.vgId = pVgroup->vgId;
req.dnodeId = dnodeid;
mInfo("vgId:%d, build compact vnode config req", pVgroup->vgId);
int32_t contLen = tSerializeSVKillCompactReq(NULL, 0, &req);
if (contLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
contLen += sizeof(SMsgHead);
void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
SMsgHead *pHead = pReq;
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req);
*pContLen = contLen;
return pReq;
}
static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup,
int32_t compactId, int32_t dnodeid) {
STransAction action = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeid);
if (pDnode == NULL) return -1;
action.epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode);
int32_t contLen = 0;
void *pReq = mndBuildKillCompactReq(pMnode, pVgroup, &contLen, compactId, dnodeid);
if (pReq == NULL) return -1;
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_VND_KILL_COMPACT;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
return -1;
}
return 0;
}
static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "kill-compact");
if (pTrans == NULL) {
mError("compact:%" PRId32 ", failed to drop since %s" , pCompact->compactId, terrstr());
return -1;
}
mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId);
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == pCompact->compactId) {
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pDetail->vgId);
if(pVgroup == NULL){
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
if(mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId) != 0){
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndReleaseVgroup(pMnode, pVgroup);
/*
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
*/
}
sdbRelease(pMnode->pSdb, pDetail);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
return 0;
}
int32_t mndProcessKillCompactReq(SRpcMsg *pReq){
SKillCompactReq killCompactReq = {0};
if (tDeserializeSKillCompactReq(pReq->pCont, pReq->contLen, &killCompactReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mInfo("start to kill compact:%" PRId32, killCompactReq.compactId);
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SCompactObj *pCompact = mndAcquireCompact(pMnode, killCompactReq.compactId);
if(pCompact == NULL){
terrno = TSDB_CODE_MND_INVALID_COMPACT_ID;
tFreeSKillCompactReq(&killCompactReq);
return -1;
}
if (0 != mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB)) {
goto _OVER;
}
if (mndKillCompact(pMnode, pReq, pCompact) < 0) {
goto _OVER;
}
code = TSDB_CODE_ACTION_IN_PROGRESS;
char obj[10] = {0};
sprintf(obj, "%d", pCompact->compactId);
auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen);
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to kill compact %" PRId32 " since %s", killCompactReq.compactId, terrstr());
}
tFreeSKillCompactReq(&killCompactReq);
sdbRelease(pMnode->pSdb, pCompact);
return code;
}
//update progress
static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t compactId, SQueryCompactProgressRsp* rsp) {
void* pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == compactId && pDetail->vgId == rsp->vgId && pDetail->dnodeId == rsp->dnodeId) {
pDetail->newNumberFileset = rsp->numberFileset;
pDetail->newFinished = rsp->finished;
sdbRelease(pMnode->pSdb, pDetail);
return 0;
}
sdbRelease(pMnode->pSdb, pDetail);
}
return -1;
}
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){
SQueryCompactProgressRsp req = {0};
if (tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mDebug("compact:%d, receive query response, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
if(mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req) != 0){
mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
return -1;
}
return 0;
}
//timer
void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact){
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == pCompact->compactId) {
SEpSet epSet = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDetail->dnodeId);
if(pDnode == NULL) break;
addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port);
mndReleaseDnode(pMnode, pDnode);
SQueryCompactProgressReq req;
req.compactId = pDetail->compactId;
req.vgId = pDetail->vgId;
req.dnodeId = pDetail->dnodeId;
int32_t contLen = tSerializeSQueryCompactProgressReq(NULL, 0, &req);
if (contLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
continue;
}
contLen += sizeof(SMsgHead);
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
sdbCancelFetch(pMnode->pSdb, pDetail);
sdbRelease(pMnode->pSdb, pDetail);
continue;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pDetail->vgId);
tSerializeSQueryCompactProgressReq((char *)pHead + sizeof(SMsgHead), contLen - sizeof(SMsgHead), &req);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS,
.contLen = contLen};
//rpcMsg.pCont = rpcMallocCont(contLen);
//if (rpcMsg.pCont == NULL) {
// return;
//}
//memcpy(rpcMsg.pCont, pHead, contLen);
rpcMsg.pCont = pHead;
char detail[1024] = {0};
int32_t len = snprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", TMSG_INFO(TDMT_VND_QUERY_COMPACT_PROGRESS),
epSet.numOfEps, epSet.inUse);
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
len += snprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, epSet.eps[i].fqdn,
epSet.eps[i].port);
}
mDebug("compact:%d, send update progress msg to %s", pDetail->compactId, detail);
tmsgSendReq(&epSet, &rpcMsg);
}
sdbRelease(pMnode->pSdb, pDetail);
}
}
static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
bool needSave = false;
void* pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == compactId) {
mDebug("compact:%d, check save progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
if(pDetail->numberFileset < pDetail->newNumberFileset || pDetail->finished < pDetail->newFinished)
needSave = true;
}
sdbRelease(pMnode->pSdb, pDetail);
}
if(!needSave) {
mDebug("compact:%" PRId32 ", no need to save" , compactId);
return 0;
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-compact-progress");
if (pTrans == NULL) {
mError("trans:%" PRId32 ", failed to create since %s" , pTrans->id, terrstr());
return -1;
}
mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id);
SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId);
pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == compactId) {
mInfo("compact:%d, trans:%d, check compact progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
pDetail->numberFileset = pDetail->newNumberFileset;
pDetail->finished = pDetail->newFinished;
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
}
sdbRelease(pMnode->pSdb, pDetail);
}
bool allFinished = true;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if(pDetail->compactId == compactId){
mInfo("compact:%d, trans:%d, check compact finished, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished);
if(pDetail->numberFileset == -1 && pDetail->finished == -1){
allFinished = false;
sdbRelease(pMnode->pSdb, pDetail);
break;
}
if (pDetail->numberFileset != -1 && pDetail->finished != -1 &&
pDetail->numberFileset != pDetail->finished) {
allFinished = false;
sdbRelease(pMnode->pSdb, pDetail);
break;
}
}
sdbRelease(pMnode->pSdb, pDetail);
}
if(allFinished){
mInfo("compact:%d, all finished", pCompact->compactId);
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == pCompact->compactId) {
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
}
sdbRelease(pMnode->pSdb, pDetail);
}
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("compact:%d, trans:%d, failed to prepare since %s", compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
sdbRelease(pMnode->pSdb, pCompact);
return -1;
}
sdbRelease(pMnode->pSdb, pCompact);
mndTransDrop(pTrans);
return 0;
}
void mndCompactPullup(SMnode *pMnode) {
SSdb *pSdb = pMnode->pSdb;
SArray *pArray = taosArrayInit(sdbGetSize(pSdb, SDB_COMPACT), sizeof(int32_t));
if (pArray == NULL) return;
void *pIter = NULL;
while (1) {
SCompactObj *pCompact = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT, pIter, (void **)&pCompact);
if (pIter == NULL) break;
taosArrayPush(pArray, &pCompact->compactId);
sdbRelease(pSdb, pCompact);
}
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
mInfo("begin to pull up");
int32_t *pCompactId = taosArrayGet(pArray, i);
SCompactObj *pCompact = mndAcquireCompact(pMnode, *pCompactId);
if (pCompact != NULL) {
mInfo("compact:%d, begin to pull up", pCompact->compactId);
mndCompactSendProgressReq(pMnode, pCompact);
mndSaveCompactProgress(pMnode, pCompact->compactId);
}
mndReleaseCompact(pMnode, pCompact);
}
taosArrayDestroy(pArray);
}
static int32_t mndProcessCompactTimer(SRpcMsg *pReq) {
mTrace("start to process compact timer");
mndCompactPullup(pReq->info.node);
return 0;
}

300
source/dnode/mnode/impl/src/mndCompactDetail.c Normal file
View File

@ -0,0 +1,300 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndCompactDetail.h"
#include "mndTrans.h"
#include "mndShow.h"
#include "mndDb.h"
#define MND_COMPACT_VER_NUMBER 1
int32_t mndInitCompactDetail(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_COMPACT_DETAIL, mndRetrieveCompactDetail);
SSdbTable table = {
.sdbType = SDB_COMPACT_DETAIL,
.keyType = SDB_KEY_INT64,
.encodeFp = (SdbEncodeFp)mndCompactDetailActionEncode,
.decodeFp = (SdbDecodeFp)mndCompactDetailActionDecode,
.insertFp = (SdbInsertFp)mndCompactDetailActionInsert,
.updateFp = (SdbUpdateFp)mndCompactDetailActionUpdate,
.deleteFp = (SdbDeleteFp)mndCompactDetailActionDelete,
};
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupCompactDetail(SMnode *pMnode) {
mDebug("mnd compact detail cleanup");
}
int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows){
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SCompactDetailObj *pCompactDetail = NULL;
char *sep = NULL;
SDbObj *pDb = NULL;
if (strlen(pShow->db) > 0) {
sep = strchr(pShow->db, '.');
if (sep && ((0 == strcmp(sep + 1, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcmp(sep + 1, TSDB_PERFORMANCE_SCHEMA_DB))))) {
sep++;
} else {
pDb = mndAcquireDb(pMnode, pShow->db);
if (pDb == NULL) return terrno;
}
}
while(numOfRows < rows){
pShow->pIter = sdbFetch(pSdb, SDB_COMPACT_DETAIL, pShow->pIter, (void **)&pCompactDetail);
if (pShow->pIter == NULL) break;
SColumnInfoData *pColInfo;
SName n;
int32_t cols = 0;
char tmpBuf[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->compactId, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->vgId, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->dnodeId, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->numberFileset, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->finished, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->startTime, false);
numOfRows++;
sdbRelease(pSdb, pCompactDetail);
}
pShow->numOfRows += numOfRows;
mndReleaseDb(pMnode, pDb);
return numOfRows;
}
void tFreeCompactDetailObj(SCompactDetailObj *pCompact) {
}
int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDetailObj *pObj) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pObj->compactDetailId) < 0) return -1;
if (tEncodeI32(&encoder, pObj->compactId) < 0) return -1;
if (tEncodeI32(&encoder, pObj->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pObj->dnodeId) < 0) return -1;
if (tEncodeI32(&encoder, pObj->numberFileset) < 0) return -1;
if (tEncodeI32(&encoder, pObj->finished) < 0) return -1;
if (tEncodeI64(&encoder, pObj->startTime) < 0) return -1;
if (tEncodeI32(&encoder, pObj->newNumberFileset) < 0) return -1;
if (tEncodeI32(&encoder, pObj->newFinished) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailObj *pObj) {
int8_t ex = 0;
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->compactDetailId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->dnodeId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->numberFileset) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->finished) < 0) return -1;
if (tDecodeI64(&decoder, &pObj->startTime) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->newNumberFileset) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->newFinished) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
SSdbRaw *mndCompactDetailActionEncode(SCompactDetailObj *pCompact) {
terrno = TSDB_CODE_SUCCESS;
void *buf = NULL;
SSdbRaw *pRaw = NULL;
int32_t tlen = tSerializeSCompactDetailObj(NULL, 0, pCompact);
if (tlen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t size = sizeof(int32_t) + tlen;
pRaw = sdbAllocRaw(SDB_COMPACT_DETAIL, MND_COMPACT_VER_NUMBER, size);
if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
buf = taosMemoryMalloc(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
tlen = tSerializeSCompactDetailObj(buf, tlen, pCompact);
if (tlen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, OVER);
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OVER);
SDB_SET_DATALEN(pRaw, dataPos, OVER);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("compact detail:%" PRId32 ", failed to encode to raw:%p since %s", pCompact->compactId, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("compact detail:%" PRId32 ", encode to raw:%p, row:%p", pCompact->compactId, pRaw, pCompact);
return pRaw;
}
SSdbRow *mndCompactDetailActionDecode(SSdbRaw *pRaw) {
SSdbRow *pRow = NULL;
SCompactDetailObj *pCompact = NULL;
void *buf = NULL;
terrno = TSDB_CODE_SUCCESS;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) {
goto OVER;
}
if (sver != MND_COMPACT_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("compact detail read invalid ver, data ver: %d, curr ver: %d", sver, MND_COMPACT_VER_NUMBER);
goto OVER;
}
pRow = sdbAllocRow(sizeof(SCompactObj));
if (pRow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
pCompact = sdbGetRowObj(pRow);
if (pCompact == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t tlen;
int32_t dataPos = 0;
SDB_GET_INT32(pRaw, dataPos, &tlen, OVER);
buf = taosMemoryMalloc(tlen + 1);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER);
if (tDeserializeSCompactDetailObj(buf, tlen, pCompact) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
//taosInitRWLatch(&pView->lock);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("compact detail:%" PRId32 ", failed to decode from raw:%p since %s", pCompact->compactId, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
mTrace("compact detail:%" PRId32 ", decode from raw:%p, row:%p", pCompact->compactId, pRaw, pCompact);
return pRow;
}
int32_t mndCompactDetailActionInsert(SSdb *pSdb, SCompactDetailObj *pCompact) {
mTrace("compact detail:%" PRId32 ", perform insert action", pCompact->compactId);
return 0;
}
int32_t mndCompactDetailActionDelete(SSdb *pSdb, SCompactDetailObj *pCompact) {
mTrace("compact detail:%" PRId32 ", perform insert action", pCompact->compactId);
tFreeCompactDetailObj(pCompact);
return 0;
}
int32_t mndCompactDetailActionUpdate(SSdb *pSdb, SCompactDetailObj *pOldCompact, SCompactDetailObj *pNewCompact) {
mTrace("compact detail:%" PRId32 ", perform update action, old row:%p new row:%p",
pOldCompact->compactId, pOldCompact, pNewCompact);
pOldCompact->numberFileset = pNewCompact->numberFileset;
pOldCompact->finished = pNewCompact->finished;
return 0;
}
int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup,
SVnodeGid *pVgid, int32_t index){
SCompactDetailObj compactDetail = {0};
compactDetail.compactDetailId = index;
compactDetail.compactId = pCompact->compactId;
compactDetail.vgId = pVgroup->vgId;
compactDetail.dnodeId = pVgid->dnodeId;
compactDetail.startTime = taosGetTimestampMs();
compactDetail.numberFileset = -1;
compactDetail.finished = -1;
compactDetail.newNumberFileset = -1;
compactDetail.newFinished = -1;
mInfo("compact:%d, add compact detail to trans, index:%d, vgId:%d, dnodeId:%d",
compactDetail.compactId, compactDetail.compactDetailId, compactDetail.vgId, compactDetail.dnodeId);
SSdbRaw *pVgRaw = mndCompactDetailActionEncode(&compactDetail);
if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
}
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
return 0;
}

21
source/dnode/mnode/impl/src/mndMain.c
View File

@ -42,6 +42,8 @@
#include "mndUser.h"
#include "mndVgroup.h"
#include "mndView.h"
#include "mndCompact.h"
#include "mndCompactDetail.h"
static inline int32_t mndAcquireRpc(SMnode *pMnode) {
int32_t code = 0;
@ -112,6 +114,16 @@ static void mndPullupTrans(SMnode *pMnode) {
}
}
static void mndPullupCompacts(SMnode *pMnode) {
mTrace("pullup compact timer msg");
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
if (pReq != NULL) {
SRpcMsg rpcMsg = {.msgType = TDMT_MND_COMPACT_TIMER, .pCont = pReq, .contLen = contLen};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
}
static void mndPullupTtl(SMnode *pMnode) {
mTrace("pullup ttl");
int32_t contLen = 0;
@ -287,6 +299,10 @@ static void *mndThreadFp(void *param) {
mndPullupTrans(pMnode);
}
if (sec % tsCompactPullupInterval == 0) {
mndPullupCompacts(pMnode);
}
if (sec % tsMqRebalanceInterval == 0) {
mndCalMqRebalance(pMnode);
}
@ -462,6 +478,8 @@ static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-compact", mndInitCompact, mndCleanupCompact) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-compact-detail", mndInitCompactDetail, mndCleanupCompactDetail) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1;
@ -693,7 +711,8 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
_OVER:
if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER ||
pMsg->msgType == TDMT_MND_TRIM_DB_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) {
pMsg->msgType == TDMT_MND_TRIM_DB_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER ||
pMsg->msgType == TDMT_MND_COMPACT_TIMER) {
mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
pMnode->stopped, state.restored, syncStr(state.state));
return -1;

7
source/dnode/mnode/impl/src/mndShow.c
View File

@ -17,6 +17,7 @@
#include "mndShow.h"
#include "mndPrivilege.h"
#include "systable.h"
#include "mndUser.h"
#define SHOW_STEP_SIZE 100
#define SHOW_COLS_STEP_SIZE 4096
@ -118,6 +119,10 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_PRIVILEGES;
} else if (strncasecmp(name, TSDB_INS_TABLE_VIEWS, len) == 0) {
type = TSDB_MGMT_TABLE_VIEWS;
} else if (strncasecmp(name, TSDB_INS_TABLE_COMPACTS, len) == 0) {
type = TSDB_MGMT_TABLE_COMPACT;
} else if (strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, len) == 0) {
type = TSDB_MGMT_TABLE_COMPACT_DETAIL;
} else {
mError("invalid show name:%s len:%d", name, len);
}
@ -206,7 +211,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
return -1;
}
mDebug("mndProcessRetrieveSysTableReq tb:%s", retrieveReq.tb);
mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb);
if (retrieveReq.showId == 0) {
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb));

72
source/dnode/mnode/impl/src/mndStream.c
View File

@ -16,12 +16,9 @@
#include "mndStream.h"
#include "audit.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndPrivilege.h"
#include "mndScheduler.h"
#include "mndShow.h"
#include "mndSnode.h"
#include "mndStb.h"
#include "mndTrans.h"
#include "mndVgroup.h"
@ -34,13 +31,6 @@
#define MND_STREAM_RESERVE_SIZE 64
#define MND_STREAM_MAX_NUM 60
#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint"
#define MND_STREAM_PAUSE_NAME "stream-pause"
#define MND_STREAM_RESUME_NAME "stream-resume"
#define MND_STREAM_DROP_NAME "stream-drop"
#define MND_STREAM_TASK_RESET_NAME "stream-task-reset"
#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update"
typedef struct SNodeEntry {
int32_t nodeId;
bool stageUpdated; // the stage has been updated due to the leader/follower change or node reboot.
@ -705,7 +695,7 @@ static int32_t mndPersistTaskDropReq(SMnode *pMnode, STrans *pTrans, SStreamTask
STransAction action = {0};
SEpSet epset = {0};
if(pTask->info.nodeId == SNODE_HANDLE){
if (pTask->info.nodeId == SNODE_HANDLE) {
SSnodeObj *pObj = NULL;
void *pIter = NULL;
while (1) {
@ -717,10 +707,16 @@ static int32_t mndPersistTaskDropReq(SMnode *pMnode, STrans *pTrans, SStreamTask
addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port);
sdbRelease(pMnode->pSdb, pObj);
}
}else{
} else {
SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId);
if (pVgObj != NULL) {
epset = mndGetVgroupEpset(pMnode, pVgObj);
mndReleaseVgroup(pMnode, pVgObj);
} else {
mDebug("orphaned task:0x%x need to be dropped, nodeId:%d, no redo action", pTask->id.taskId, pTask->info.nodeId);
taosMemoryFree(pReq);
return 0;
}
}
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
@ -833,7 +829,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER;
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stream");
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, MND_STREAM_CREATE_NAME);
if (pTrans == NULL) {
mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
goto _OVER;
@ -841,11 +837,12 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
mInfo("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name);
mndTransSetDbName(pTrans, createStreamReq.sourceDB, streamObj.targetDb);
mndTransSetDbName(pTrans, createStreamReq.sourceDB, streamObj.targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
mndTransDrop(pTrans);
goto _OVER;
}
// create stb for stream
if (createStreamReq.createStb == STREAM_CREATE_STABLE_TRUE &&
mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) {
@ -932,7 +929,8 @@ int64_t mndStreamGenChkpId(SMnode *pMnode) {
if (pIter == NULL) break;
maxChkpId = TMAX(maxChkpId, pStream->checkpointId);
mDebug("stream %p checkpoint %" PRId64 "", pStream, pStream->checkpointId);
mDebug("stream:%p, %s id:%" PRIx64 "checkpoint %" PRId64 "", pStream, pStream->name, pStream->uid,
pStream->checkpointId);
sdbRelease(pSdb, pStream);
}
@ -1022,7 +1020,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
return -1;
}
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true);
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, true);
if (conflict) {
mndAddtoCheckpointWaitingList(pStream, checkpointId);
mWarn("checkpoint conflict with other trans in %s, ignore the checkpoint for stream:%s %" PRIx64, pStream->sourceDb,
@ -1030,14 +1028,14 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
return -1;
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, MND_STREAM_CHECKPOINT_NAME);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, MND_STREAM_CHECKPOINT_NAME);
if (pTrans == NULL) {
return -1;
}
mndStreamRegisterTrans(pTrans, MND_STREAM_CHECKPOINT_NAME, pStream->sourceDb, pStream->targetDb);
mndStreamRegisterTrans(pTrans, MND_STREAM_CHECKPOINT_NAME, pStream->uid);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTrancCheckConflict(pMnode, pTrans) != 0) {
mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId,
tstrerror(TSDB_CODE_MND_TRANS_CONFLICT));
@ -1374,14 +1372,14 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
}
// check if it is conflict with other trans in both sourceDb and targetDb.
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true);
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_DROP_NAME, true);
if (conflict) {
sdbRelease(pMnode->pSdb, pStream);
tFreeMDropStreamReq(&dropReq);
return -1;
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, MND_STREAM_DROP_NAME);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, MND_STREAM_DROP_NAME);
if (pTrans == NULL) {
mError("stream:%s, failed to drop since %s", dropReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream);
@ -1391,7 +1389,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
mInfo("trans:%d used to drop stream:%s", pTrans->id, dropReq.name);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
@ -1399,7 +1397,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
return -1;
}
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->sourceDb, pStream->targetDb);
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->uid);
// drop all tasks
if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
@ -1657,6 +1655,7 @@ static void setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDat
STaskStatusEntry *pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
if (pe == NULL) {
mError("task:0x%" PRIx64 " not exists in vnode, no valid status/stage info", id.taskId);
return;
}
@ -1857,7 +1856,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
}
// check if it is conflict with other trans in both sourceDb and targetDb.
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true);
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_PAUSE_NAME, true);
if (conflict) {
sdbRelease(pMnode->pSdb, pStream);
return -1;
@ -1869,7 +1868,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
return -1;
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "pause-stream");
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, MND_STREAM_PAUSE_NAME);
if (pTrans == NULL) {
mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream);
@ -1878,14 +1877,14 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
mInfo("trans:%d, used to pause stream:%s", pTrans->id, pauseReq.name);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
}
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->sourceDb, pStream->targetDb);
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->uid);
// if nodeUpdate happened, not send pause trans
if (mndPauseAllStreamTasks(pMnode, pTrans, pStream) < 0) {
@ -1992,13 +1991,13 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
}
// check if it is conflict with other trans in both sourceDb and targetDb.
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true);
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_RESUME_NAME, true);
if (conflict) {
sdbRelease(pMnode->pSdb, pStream);
return -1;
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, MND_STREAM_RESUME_NAME);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, MND_STREAM_RESUME_NAME);
if (pTrans == NULL) {
mError("stream:%s, failed to resume stream since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream);
@ -2007,14 +2006,14 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
mInfo("trans:%d used to resume stream:%s", pTrans->id, pauseReq.name);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
}
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->sourceDb, pStream->targetDb);
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->uid);
// resume all tasks
if (mndResumeAllStreamTasks(pTrans, pMnode, pStream, pauseReq.igUntreated) < 0) {
@ -2651,7 +2650,7 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
}
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char *pMsg) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, name);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, name);
if (pTrans == NULL) {
mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -2660,7 +2659,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const
mDebug("s-task:0x%" PRIx64 " start to build trans %s", pStream->uid, pMsg);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
terrno = TSDB_CODE_MND_TRANS_CONFLICT;
mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno));
@ -2753,7 +2752,10 @@ int32_t killActiveCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t le
STrans *pTrans = mndAcquireTrans(pMnode, pTransInfo->transId);
if (pTrans != NULL) {
mInfo("kill checkpoint transId:%d in Db:%s", pTransInfo->transId, pDBName);
char* pDupDBName = strndup(pDBName, len);
mInfo("kill checkpoint transId:%d in Db:%s", pTransInfo->transId, pDupDBName);
taosMemoryFree(pDupDBName);
mndKillTrans(pMnode, pTrans);
mndReleaseTrans(pMnode, pTrans);
}
@ -2779,7 +2781,7 @@ static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) {
break;
}
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, false);
bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_TASK_RESET_NAME, false);
if (conflict) {
mError("stream:%s other trans exists in DB:%s & %s failed to start reset-status trans", pStream->name,
pStream->sourceDb, pStream->targetDb);

43
source/dnode/mnode/impl/src/mndStreamTrans.c
View File

@ -23,21 +23,17 @@ typedef struct SKeyInfo {
static int32_t clearFinishedTrans(SMnode* pMnode);
int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pName, const char* pSrcDb, const char* pDstDb) {
SStreamTransInfo info = {.transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pName};
taosHashPut(execInfo.transMgmt.pDBTrans, pSrcDb, strlen(pSrcDb), &info, sizeof(SStreamTransInfo));
if (strcmp(pSrcDb, pDstDb) != 0) {
taosHashPut(execInfo.transMgmt.pDBTrans, pDstDb, strlen(pDstDb), &info, sizeof(SStreamTransInfo));
}
int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamUid) {
SStreamTransInfo info = {
.transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pTransName, .streamUid = streamUid};
taosHashPut(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid), &info, sizeof(SStreamTransInfo));
return 0;
}
int32_t clearFinishedTrans(SMnode* pMnode) {
size_t keyLen = 0;
SArray* pList = taosArrayInit(4, sizeof(SKeyInfo));
void* pIter = NULL;
SArray* pList = taosArrayInit(4, sizeof(SKeyInfo));
while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) {
SStreamTransInfo* pEntry = (SStreamTransInfo*)pIter;
@ -48,8 +44,7 @@ int32_t clearFinishedTrans(SMnode* pMnode) {
void* pKey = taosHashGetKey(pEntry, &keyLen);
// key is the name of src/dst db name
SKeyInfo info = {.pKey = pKey, .keyLen = keyLen};
mDebug("transId:%d %s startTs:%" PRId64 "cleared due to finished", pEntry->transId, pEntry->name,
mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name,
pEntry->startTime);
taosArrayPush(pList, &info);
} else {
@ -70,7 +65,7 @@ int32_t clearFinishedTrans(SMnode* pMnode) {
return 0;
}
bool streamTransConflictOtherTrans(SMnode* pMnode, const char* pSrcDb, const char* pDstDb, bool lock) {
bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char* pTransName, bool lock) {
if (lock) {
taosThreadMutexLock(&execInfo.lock);
}
@ -85,23 +80,27 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, const char* pSrcDb, const cha
clearFinishedTrans(pMnode);
SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, pSrcDb, strlen(pSrcDb));
SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid));
if (pEntry != NULL) {
SStreamTransInfo tInfo = *pEntry;
if (lock) {
taosThreadMutexUnlock(&execInfo.lock);
}
mWarn("conflict with other transId:%d in Db:%s, trans:%s", pEntry->transId, pSrcDb, pEntry->name);
return true;
}
pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, pDstDb, strlen(pDstDb));
if (pEntry != NULL) {
if (lock) {
taosThreadMutexUnlock(&execInfo.lock);
}
mWarn("conflict with other transId:%d in Db:%s, trans:%s", pEntry->transId, pSrcDb, pEntry->name);
if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0) {
if (strcmp(pTransName, MND_STREAM_DROP_NAME) != 0) {
mWarn("conflict with other transId:%d streamUid:%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid,
tInfo.name);
return true;
}
} else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) ||
(strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0)) {
mWarn("conflict with other transId:%d streamUid:%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid,
tInfo.name);
return true;
}
}
if (lock) {
taosThreadMutexUnlock(&execInfo.lock);

3
source/dnode/mnode/impl/test/acct/acct.cpp
View File

@ -33,6 +33,7 @@ TEST_F(MndTestAcct, 01_Create_Acct) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
rpcFreeCont(pRsp->pCont);
}
TEST_F(MndTestAcct, 02_Alter_Acct) {
@ -43,6 +44,7 @@ TEST_F(MndTestAcct, 02_Alter_Acct) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
rpcFreeCont(pRsp->pCont);
}
TEST_F(MndTestAcct, 03_Drop_Acct) {
@ -53,4 +55,5 @@ TEST_F(MndTestAcct, 03_Drop_Acct) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
rpcFreeCont(pRsp->pCont);
}

5
source/dnode/mnode/impl/test/sma/sma.cpp
View File

@ -253,12 +253,14 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
pReq = BuildCreateDbReq(dbname, &contLen);
pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
}
{
pReq = BuildCreateBSmaStbReq(stbname, &contLen);
pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 1);
}
@ -269,12 +271,14 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
pReq = BuildCreateBSmaStbReq(stbname, &contLen);
pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_ALREADY_EXIST);
rpcFreeCont(pRsp->pCont);
}
{
pReq = BuildDropStbReq(stbname, &contLen);
pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 0);
}
@ -283,5 +287,6 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
pReq = BuildDropStbReq(stbname, &contLen);
pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
}
}

Some files were not shown because too many files have changed in this diff Show More