xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

other: merge 3.0

This commit is contained in:
Haojun Liao 2023-12-18 23:16:37 +08:00
parent e734569de0 4cac96d2df
commit ec3183b6a8
209 changed files with 19990 additions and 6069 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -41,6 +41,7 @@ add_subdirectory(source)
add_subdirectory(tools) add_subdirectory(tools)
add_subdirectory(utils) add_subdirectory(utils)
add_subdirectory(examples/c) add_subdirectory(examples/c)
add_subdirectory(tests)
include(${TD_SUPPORT_DIR}/cmake.install) include(${TD_SUPPORT_DIR}/cmake.install)
# docs # docs

BIN
deps/arm/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/darwin/arm/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/darwin/x64/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/x86/dm_static/libdmodule.a vendored
View File

Binary file not shown.

2
docs/en/07-develop/01-connect/index.md
View File

@ -177,7 +177,7 @@ Just need to add the reference to [TDengine.Connector](https://www.nuget.org/pac
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.0" /> <PackageReference Include="TDengine.Connector" Version="3.1.0" />
</ItemGroup> </ItemGroup>
</Project> </Project>

3
docs/en/07-develop/04-query-data/_cs_async.mdx
View File

@ -1,3 +0,0 @@
```csharp
{{#include docs/examples/csharp/asyncQuery/Program.cs}}
```

4
docs/en/07-develop/04-query-data/index.mdx
View File

@ -15,7 +15,6 @@ import CQuery from "./_c.mdx";
import PhpQuery from "./_php.mdx"; import PhpQuery from "./_php.mdx";
import PyAsync from "./_py_async.mdx"; import PyAsync from "./_py_async.mdx";
import NodeAsync from "./_js_async.mdx"; import NodeAsync from "./_js_async.mdx";
import CsAsync from "./_cs_async.mdx";
import CAsync from "./_c_async.mdx"; import CAsync from "./_c_async.mdx";
## Introduction ## Introduction
@ -174,9 +173,6 @@ Please note that async query can only be used with a native connection.
<TabItem label="Python" value="python"> <TabItem label="Python" value="python">
<PyAsync /> <PyAsync />
</TabItem> </TabItem>
<TabItem label="C#" value="csharp">
<CsAsync />
</TabItem>
<TabItem label="C" value="c"> <TabItem label="C" value="c">
<CAsync /> <CAsync />
</TabItem> </TabItem>

52
docs/en/07-develop/07-tmq.mdx
View File

@ -248,23 +248,23 @@ function close()
<TabItem value="C#" label="C#"> <TabItem value="C#" label="C#">
```csharp ```csharp
class ConsumerBuilder<TValue>
ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config) ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
virtual IConsumer Build() public IConsumer<TValue> Build()
Consumer(ConsumerBuilder builder)
void Subscribe(IEnumerable<string> topics) void Subscribe(IEnumerable<string> topics)
void Subscribe(string topic) void Subscribe(string topic)
ConsumeResult Consume(int millisecondsTimeout) ConsumeResult<TValue> Consume(int millisecondsTimeout)
List<string> Subscription() List<string> Subscription()
void Unsubscribe() void Unsubscribe()
void Commit(ConsumeResult consumerResult) List<TopicPartitionOffset> Commit()
void Close() void Close()
``` ```
@ -500,25 +500,19 @@ let consumer = taos.consumer({
<TabItem value="C#" label="C#"> <TabItem value="C#" label="C#">
```csharp ```csharp
using TDengineTMQ; var cfg = new Dictionary<string, string>()
{
// Create consumer groups on demand (GourpID) and enable automatic commits (EnableAutoCommit), { "group.id", "group1" },
// an automatic commit interval (AutoCommitIntervalMs), and a username (TDConnectUser) and password (TDConnectPasswd) { "auto.offset.reset", "latest" },
var cfg = new ConsumerConfig { "td.connect.ip", "127.0.0.1" },
{ { "td.connect.user", "root" },
EnableAutoCommit = "true" { "td.connect.pass", "taosdata" },
AutoCommitIntervalMs = "1000" { "td.connect.port", "6030" },
GourpId = "TDengine-TMQ-C#", { "client.id", "tmq_example" },
TDConnectUser = "root", { "enable.auto.commit", "true" },
TDConnectPasswd = "taosdata", { "msg.with.table.name", "false" },
AutoOffsetReset = "latest" };
MsgWithTableName = "true", var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
TDConnectIp = "127.0.0.1",
TDConnectPort = "6030"
};
var consumer = new ConsumerBuilder(cfg).Build();
``` ```
</TabItem> </TabItem>
@ -747,10 +741,12 @@ while(true){
## Consume data ## Consume data
while (true) while (true)
{ {
var consumerRes = consumer.Consume(100); using (var result = consumer.Consume(500))
// process ConsumeResult {
ProcessMsg(consumerRes); if (result == null) continue;
consumer.Commit(consumerRes); ProcessMsg(result);
consumer.Commit();
}
} }
``` ```

6
docs/en/12-taos-sql/05-insert.md
View File

@ -158,8 +158,8 @@ Automatically creating table and the table name is specified through the `tbname
```sql ```sql
INSERT INTO meters(tbname, location, groupId, ts, current, phase) INSERT INTO meters(tbname, location, groupId, ts, current, phase)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 0.32)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 0.33)
values('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 0.33)
``` ```

57
docs/en/14-reference/02-rest-api/02-rest-api.mdx
View File

@ -262,6 +262,63 @@ The following types may be returned:
- "INT UNSIGNED" - "INT UNSIGNED"
- "BIGINT UNSIGNED" - "BIGINT UNSIGNED"
- "JSON" - "JSON"
- "VARBINARY"
- "GEOMETRY"
`VARBINARY` and `GEOMETRY` types return data as Hex string, example:
Prepare data
```bash
create database demo
use demo
create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
insert into t values(now,'\x7f8290','point(100 100)')
```
Execute query
```bash
curl --location 'http://<fqdn>:<port>/rest/sql' \
--header 'Content-Type: text/plain' \
--header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \
--data 'select * from demo.t'
```
Return results
```json
{
"code": 0,
"column_meta": [
[
"ts",
"TIMESTAMP",
8
],
[
"c1",
"VARBINARY",
20
],
[
"c2",
"GEOMETRY",
100
]
],
"data": [
[
"2023-11-01T06:28:15.210Z",
"7f8290",
"010100000000000000000059400000000000005940"
]
],
"rows": 1
}
```
- `010100000000000000000059400000000000005940` is [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/) format for `point(100 100)`
#### Errors #### Errors

1317
docs/en/14-reference/03-connector/09-csharp.mdx
View File

File diff suppressed because it is too large Load Diff

3
docs/examples/csharp/.gitignore vendored
View File

@ -1,5 +1,5 @@
.vs .vs
asyncQuery/bin .idea
connect/bin connect/bin
influxdbLine/bin influxdbLine/bin
optsJSON/bin optsJSON/bin
@ -12,7 +12,6 @@ wsConnect/bin
wsInsert/bin wsInsert/bin
wsQuery/bin wsQuery/bin
wsStmt/bin wsStmt/bin
asyncQuery/obj
connect/obj connect/obj
influxdbLine/obj influxdbLine/obj
optsJSON/obj optsJSON/obj

111
docs/examples/csharp/asyncQuery/Program.cs
View File

@ -1,111 +0,0 @@
using System;
using System.Collections.Generic;
using TDengineDriver;
using TDengineDriver.Impl;
using System.Runtime.InteropServices;
namespace TDengineExample
{
public class AsyncQueryExample
{
static void Main()
{
IntPtr conn = GetConnection();
try
{
QueryAsyncCallback queryAsyncCallback = new QueryAsyncCallback(QueryCallback);
TDengine.QueryAsync(conn, "select * from meters", queryAsyncCallback, IntPtr.Zero);
Thread.Sleep(2000);
}
finally
{
TDengine.Close(conn);
}
}
static void QueryCallback(IntPtr param, IntPtr taosRes, int code)
{
if (code == 0 && taosRes != IntPtr.Zero)
{
FetchRawBlockAsyncCallback fetchRowAsyncCallback = new FetchRawBlockAsyncCallback(FetchRawBlockCallback);
TDengine.FetchRawBlockAsync(taosRes, fetchRowAsyncCallback, param);
}
else
{
throw new Exception($"async query data failed,code:{code},reason:{TDengine.Error(taosRes)}");
}
}
// Iteratively call this interface until "numOfRows" is no greater than 0.
static void FetchRawBlockCallback(IntPtr param, IntPtr taosRes, int numOfRows)
{
if (numOfRows > 0)
{
Console.WriteLine($"{numOfRows} rows async retrieved");
IntPtr pdata = TDengine.GetRawBlock(taosRes);
List<TDengineMeta> metaList = TDengine.FetchFields(taosRes);
List<object> dataList = LibTaos.ReadRawBlock(pdata, metaList, numOfRows);
for (int i = 0; i < dataList.Count; i++)
{
if (i != 0 && (i + 1) % metaList.Count == 0)
{
Console.WriteLine("{0}\t|", dataList[i]);
}
else
{
Console.Write("{0}\t|", dataList[i]);
}
}
Console.WriteLine("");
TDengine.FetchRawBlockAsync(taosRes, FetchRawBlockCallback, param);
}
else
{
if (numOfRows == 0)
{
Console.WriteLine("async retrieve complete.");
}
else
{
throw new Exception($"FetchRawBlockCallback callback error, error code {numOfRows}");
}
TDengine.FreeResult(taosRes);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "power";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
}
}
// //output:
// // Connect to TDengine success
// // 8 rows async retrieved
// // 1538548685500 | 11.8 | 221 | 0.28 | california.losangeles | 2 |
// // 1538548696600 | 13.4 | 223 | 0.29 | california.losangeles | 2 |
// // 1538548685000 | 10.8 | 223 | 0.29 | california.losangeles | 3 |
// // 1538548686500 | 11.5 | 221 | 0.35 | california.losangeles | 3 |
// // 1538548685000 | 10.3 | 219 | 0.31 | california.sanfrancisco | 2 |
// // 1538548695000 | 12.6 | 218 | 0.33 | california.sanfrancisco | 2 |
// // 1538548696800 | 12.3 | 221 | 0.31 | california.sanfrancisco | 2 |
// // 1538548696650 | 10.3 | 218 | 0.25 | california.sanfrancisco | 3 |
// // async retrieve complete.

15
docs/examples/csharp/asyncQuery/asyncquery.csproj
View File

@ -1,15 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<StartupObject>TDengineExample.AsyncQueryExample</StartupObject>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" />
</ItemGroup>
</Project>

21
docs/examples/csharp/connect/Program.cs
View File

@ -1,4 +1,5 @@
using TDengineDriver; using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
@ -7,23 +8,11 @@ namespace TDengineExample
{ {
static void Main(String[] args) static void Main(String[] args)
{ {
string host = "localhost"; var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
short port = 6030; using (var client = DbDriver.Open(builder))
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{ {
throw new Exception("Connect to TDengine failed"); Console.WriteLine("connected");
} }
else
{
Console.WriteLine("Connect to TDengine success");
}
TDengine.Close(conn);
TDengine.Cleanup();
} }
} }
} }

2
docs/examples/csharp/connect/connect.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

6
docs/examples/csharp/csharp.sln
View File

@ -3,8 +3,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16 # Visual Studio Version 16
VisualStudioVersion = 16.0.30114.105 VisualStudioVersion = 16.0.30114.105
MinimumVisualStudioVersion = 10.0.40219.1 MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "asyncquery", "asyncQuery\asyncquery.csproj", "{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "connect", "connect\connect.csproj", "{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "connect", "connect\connect.csproj", "{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "influxdbline", "influxdbLine\influxdbline.csproj", "{6A24FB80-1E3C-4E2D-A5AB-914FA583874D}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "influxdbline", "influxdbLine\influxdbline.csproj", "{6A24FB80-1E3C-4E2D-A5AB-914FA583874D}"
@ -38,10 +36,6 @@ Global
HideSolutionNode = FALSE HideSolutionNode = FALSE
EndGlobalSection EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution GlobalSection(ProjectConfigurationPlatforms) = postSolution
{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Release|Any CPU.Build.0 = Release|Any CPU
{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.Build.0 = Debug|Any CPU {CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Release|Any CPU.ActiveCfg = Release|Any CPU {CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Release|Any CPU.ActiveCfg = Release|Any CPU

68
docs/examples/csharp/influxdbLine/Program.cs
View File

@ -1,4 +1,5 @@
using TDengineDriver; using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
@ -6,60 +7,23 @@ namespace TDengineExample
{ {
static void Main() static void Main()
{ {
IntPtr conn = GetConnection(); var builder =
PrepareDatabase(conn); new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
string[] lines = { using (var client = DbDriver.Open(builder))
"meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249",
"meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250",
"meters,location=California.LosAngeles,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249",
"meters,location=California.LosAngeles,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250"
};
IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_LINE_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS);
if (TDengine.ErrorNo(res) != 0)
{ {
throw new Exception("SchemalessInsert failed since " + TDengine.Error(res)); client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
} client.Exec("use test");
else string[] lines = {
{ "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249",
int affectedRows = TDengine.AffectRows(res); "meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250",
Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows"); "meters,location=California.LosAngeles,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249",
} "meters,location=California.LosAngeles,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250"
TDengine.FreeResult(res); };
client.SchemalessInsert(lines,
} TDengineSchemalessProtocol.TSDB_SML_LINE_PROTOCOL,
static IntPtr GetConnection() TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void PrepareDatabase(IntPtr conn)
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to create database, reason: " + TDengine.Error(res));
}
res = TDengine.Query(conn, "USE test");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to change database, reason: " + TDengine.Error(res));
} }
} }
} }
} }

2
docs/examples/csharp/influxdbLine/influxdbline.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

75
docs/examples/csharp/optsJSON/Program.cs
View File

@ -1,69 +1,28 @@
using TDengineDriver; using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
internal class OptsJsonExample internal class OptsJsonExample
{ {
static void Main() public static void Main(string[] args)
{ {
IntPtr conn = GetConnection(); var builder =
try new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{ {
PrepareDatabase(conn); client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
string[] lines = { "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," + client.Exec("use test");
" {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611249, \"value\": 219, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}, " + string[] lines =
"{\"metric\": \"meters.current\", \"timestamp\": 1648432611250, \"value\": 12.6, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
" {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611250, \"value\": 221, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}]"
};
IntPtr res = TDengine.SchemalessInsert(conn, lines, 1, (int)TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED);
if (TDengine.ErrorNo(res) != 0)
{ {
throw new Exception("SchemalessInsert failed since " + TDengine.Error(res)); "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
} " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611249, \"value\": 219, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}, " +
else "{\"metric\": \"meters.current\", \"timestamp\": 1648432611250, \"value\": 12.6, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
{ " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611250, \"value\": 221, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}]"
int affectedRows = TDengine.AffectRows(res); };
Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows"); client.SchemalessInsert(lines, TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL,
} TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
TDengine.FreeResult(res);
}
finally
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void PrepareDatabase(IntPtr conn)
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to create database, reason: " + TDengine.Error(res));
}
res = TDengine.Query(conn, "USE test");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to change database, reason: " + TDengine.Error(res));
} }
} }
} }
} }

2
docs/examples/csharp/optsJSON/optsJSON.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

81
docs/examples/csharp/optsTelnet/Program.cs
View File

@ -1,72 +1,31 @@
using TDengineDriver; using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
internal class OptsTelnetExample internal class OptsTelnetExample
{ {
static void Main() public static void Main(string[] args)
{ {
IntPtr conn = GetConnection(); var builder =
try new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
using (var client = DbDriver.Open(builder))
{ {
PrepareDatabase(conn); client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
client.Exec("USE test");
string[] lines = { string[] lines = {
"meters.current 1648432611249 10.3 location=California.SanFrancisco groupid=2", "meters.current 1648432611249 10.3 location=California.SanFrancisco groupid=2",
"meters.current 1648432611250 12.6 location=California.SanFrancisco groupid=2", "meters.current 1648432611250 12.6 location=California.SanFrancisco groupid=2",
"meters.current 1648432611249 10.8 location=California.LosAngeles groupid=3", "meters.current 1648432611249 10.8 location=California.LosAngeles groupid=3",
"meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3", "meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3",
"meters.voltage 1648432611249 219 location=California.SanFrancisco groupid=2", "meters.voltage 1648432611249 219 location=California.SanFrancisco groupid=2",
"meters.voltage 1648432611250 218 location=California.SanFrancisco groupid=2", "meters.voltage 1648432611250 218 location=California.SanFrancisco groupid=2",
"meters.voltage 1648432611249 221 location=California.LosAngeles groupid=3", "meters.voltage 1648432611249 221 location=California.LosAngeles groupid=3",
"meters.voltage 1648432611250 217 location=California.LosAngeles groupid=3", "meters.voltage 1648432611250 217 location=California.LosAngeles groupid=3",
}; };
IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED); client.SchemalessInsert(lines,
if (TDengine.ErrorNo(res) != 0) TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL,
{ TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
throw new Exception("SchemalessInsert failed since " + TDengine.Error(res));
}
else
{
int affectedRows = TDengine.AffectRows(res);
Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows");
}
TDengine.FreeResult(res);
}
catch
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void PrepareDatabase(IntPtr conn)
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to create database, reason: " + TDengine.Error(res));
}
res = TDengine.Query(conn, "USE test");
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception("failed to change database, reason: " + TDengine.Error(res));
} }
} }
} }

2
docs/examples/csharp/optsTelnet/optstelnet.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

87
docs/examples/csharp/query/Program.cs
View File

@ -1,80 +1,35 @@
using TDengineDriver; using System.Text;
using TDengineDriver.Impl; using TDengine.Driver;
using System.Runtime.InteropServices; using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
internal class QueryExample internal class QueryExample
{ {
static void Main() public static void Main(string[] args)
{ {
IntPtr conn = GetConnection(); var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
try using (var client = DbDriver.Open(builder))
{ {
// run query try
IntPtr res = TDengine.Query(conn, "SELECT * FROM meters LIMIT 2");
if (TDengine.ErrorNo(res) != 0)
{ {
throw new Exception("Failed to query since: " + TDengine.Error(res)); client.Exec("use power");
} string query = "SELECT * FROM meters";
using (var rows = client.Query(query))
// get filed count
int fieldCount = TDengine.FieldCount(res);
Console.WriteLine("fieldCount=" + fieldCount);
// print column names
List<TDengineMeta> metas = LibTaos.GetMeta(res);
for (int i = 0; i < metas.Count; i++)
{
Console.Write(metas[i].name + "\t");
}
Console.WriteLine();
// print values
List<Object> resData = LibTaos.GetData(res);
for (int i = 0; i < resData.Count; i++)
{
Console.Write($"|{resData[i].ToString()} \t");
if (((i + 1) % metas.Count == 0))
{ {
Console.WriteLine(""); while (rows.Read())
{
Console.WriteLine(
$"{((DateTime)rows.GetValue(0)):yyyy-MM-dd HH:mm:ss.fff}, {rows.GetValue(1)}, {rows.GetValue(2)}, {rows.GetValue(3)}, {rows.GetValue(4)}, {Encoding.UTF8.GetString((byte[])rows.GetValue(5))}");
}
} }
} }
Console.WriteLine(); catch (Exception e)
{
// Free result after use Console.WriteLine(e.ToString());
TDengine.FreeResult(res); throw;
}
} }
finally
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "power";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
} }
} }
} }
// output:
// Connect to TDengine success
// fieldCount=6
// ts current voltage phase location groupid
// 1648432611249 10.3 219 0.31 California.SanFrancisco 2
// 1648432611749 12.6 218 0.33 California.SanFrancisco 2

2
docs/examples/csharp/query/query.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

92
docs/examples/csharp/sqlInsert/Program.cs
View File

@ -1,69 +1,47 @@
using TDengineDriver; using System.Text;
using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
internal class SQLInsertExample internal class SQLInsertExample
{ {
static void Main() public static void Main(string[] args)
{ {
IntPtr conn = GetConnection(); var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
try using (var client = DbDriver.Open(builder))
{ {
IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600"); try
CheckRes(conn, res, "failed to create database"); {
res = TDengine.Query(conn, "USE power"); client.Exec("create database power");
CheckRes(conn, res, "failed to change database"); client.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)"); string insertQuery =
CheckRes(conn, res, "failed to create stable"); "INSERT INTO " +
var sql = "INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) " + "power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " +
"d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) " + "VALUES " +
"d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " + "('2023-10-03 14:38:05.000', 10.30000, 219, 0.31000) " +
"d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)"; "('2023-10-03 14:38:15.000', 12.60000, 218, 0.33000) " +
res = TDengine.Query(conn, sql); "('2023-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
CheckRes(conn, res, "failed to insert data"); "power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " +
int affectedRows = TDengine.AffectRows(res); "VALUES " +
Console.WriteLine("affectedRows " + affectedRows); "('2023-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
TDengine.FreeResult(res); "power.d1003 USING power.meters TAGS(2,'California.LosAngeles') " +
} "VALUES " +
finally "('2023-10-03 14:38:05.500', 11.80000, 221, 0.28000) " +
{ "('2023-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
TDengine.Close(conn); "power.d1004 USING power.meters TAGS(3,'California.LosAngeles') " +
} "VALUES " +
"('2023-10-03 14:38:05.000', 10.80000, 223, 0.29000) " +
} "('2023-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
client.Exec(insertQuery);
static IntPtr GetConnection() }
{ catch (Exception e)
string host = "localhost"; {
short port = 6030; Console.WriteLine(e.ToString());
string username = "root"; throw;
string password = "taosdata"; }
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void CheckRes(IntPtr conn, IntPtr res, String errorMsg)
{
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception($"{errorMsg} since: {TDengine.Error(res)}");
} }
} }
} }
} }
// output:
// Connect to TDengine success
// affectedRows 8

2
docs/examples/csharp/sqlInsert/sqlinsert.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

119
docs/examples/csharp/stmtInsert/Program.cs
View File

@ -1,109 +1,38 @@
using TDengineDriver; using TDengine.Driver;
using TDengine.Driver.Client;
namespace TDengineExample namespace TDengineExample
{ {
internal class StmtInsertExample internal class StmtInsertExample
{ {
private static IntPtr conn; public static void Main(string[] args)
private static IntPtr stmt;
static void Main()
{ {
conn = GetConnection(); var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
try using (var client = DbDriver.Open(builder))
{ {
PrepareSTable(); try
// 1. init and prepare
stmt = TDengine.StmtInit(conn);
if (stmt == IntPtr.Zero)
{ {
throw new Exception("failed to init stmt."); client.Exec($"create database power");
client.Exec(
"CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
using (var stmt = client.StmtInit())
{
stmt.Prepare(
"Insert into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(?,?,?,?)");
var ts = new DateTime(2023, 10, 03, 14, 38, 05, 000);
stmt.BindRow(new object[] { ts, (float)10.30000, (int)219, (float)0.31000 });
stmt.AddBatch();
stmt.Exec();
var affected = stmt.Affected();
Console.WriteLine($"affected rows: {affected}");
}
} }
int res = TDengine.StmtPrepare(stmt, "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)"); catch (Exception e)
CheckStmtRes(res, "failed to prepare stmt");
// 2. bind table name and tags
TAOS_MULTI_BIND[] tags = new TAOS_MULTI_BIND[2] { TaosMultiBind.MultiBindBinary(new string[] { "California.SanFrancisco" }), TaosMultiBind.MultiBindInt(new int?[] { 2 }) };
res = TDengine.StmtSetTbnameTags(stmt, "d1001", tags);
CheckStmtRes(res, "failed to bind table name and tags");
// 3. bind values
TAOS_MULTI_BIND[] values = new TAOS_MULTI_BIND[4] {
TaosMultiBind.MultiBindTimestamp(new long[2] { 1648432611249, 1648432611749}),
TaosMultiBind.MultiBindFloat(new float?[2] { 10.3f, 12.6f}),
TaosMultiBind.MultiBindInt(new int?[2] { 219, 218}),
TaosMultiBind.MultiBindFloat(new float?[2]{ 0.31f, 0.33f})
};
res = TDengine.StmtBindParamBatch(stmt, values);
CheckStmtRes(res, "failed to bind params");
// 4. add batch
res = TDengine.StmtAddBatch(stmt);
CheckStmtRes(res, "failed to add batch");
// 5. execute
res = TDengine.StmtExecute(stmt);
CheckStmtRes(res, "failed to execute");
// 6. free
TaosMultiBind.FreeTaosBind(tags);
TaosMultiBind.FreeTaosBind(values);
}
finally
{
TDengine.Close(conn);
}
}
static IntPtr GetConnection()
{
string host = "localhost";
short port = 6030;
string username = "root";
string password = "taosdata";
string dbname = "";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{
throw new Exception("Connect to TDengine failed");
}
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
}
static void PrepareSTable()
{
IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600");
CheckResPtr(res, "failed to create database");
res = TDengine.Query(conn, "USE power");
CheckResPtr(res, "failed to change database");
res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)");
CheckResPtr(res, "failed to create stable");
}
static void CheckStmtRes(int res, string errorMsg)
{
if (res != 0)
{
Console.WriteLine(errorMsg + ", " + TDengine.StmtErrorStr(stmt));
int code = TDengine.StmtClose(stmt);
if (code != 0)
{ {
throw new Exception($"failed to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); Console.WriteLine(e);
throw;
} }
} }
} }
static void CheckResPtr(IntPtr res, string errorMsg)
{
if (TDengine.ErrorNo(res) != 0)
{
throw new Exception(errorMsg + " since:" + TDengine.Error(res));
}
}
} }
} }

2
docs/examples/csharp/stmtInsert/stmtinsert.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

127
docs/examples/csharp/subscribe/Program.cs
View File

@ -1,95 +1,72 @@
using System; 
using TDengineTMQ; using TDengine.Driver;
using TDengineDriver; using TDengine.Driver.Client;
using System.Runtime.InteropServices; using TDengine.TMQ;
namespace TMQExample namespace TMQExample
{ {
internal class SubscribeDemo internal class SubscribeDemo
{ {
static void Main(string[] args) public static void Main(string[] args)
{ {
IntPtr conn = GetConnection(); var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
string topic = "topic_example"; using (var client = DbDriver.Open(builder))
//create topic
IntPtr res = TDengine.Query(conn, $"create topic if not exists {topic} as select * from meters");
if (TDengine.ErrorNo(res) != 0 )
{ {
throw new Exception($"create topic failed, reason:{TDengine.Error(res)}"); try
}
var cfg = new ConsumerConfig
{
GourpId = "group_1",
TDConnectUser = "root",
TDConnectPasswd = "taosdata",
MsgWithTableName = "true",
TDConnectIp = "127.0.0.1",
};
// create consumer
var consumer = new ConsumerBuilder(cfg)
.Build();
// subscribe
consumer.Subscribe(topic);
// consume
for (int i = 0; i < 5; i++)
{
var consumeRes = consumer.Consume(300);
// print consumeResult
foreach (KeyValuePair<TopicPartition, TaosResult> kv in consumeRes.Message)
{ {
Console.WriteLine("topic partitions:\n{0}", kv.Key.ToString()); client.Exec("CREATE DATABASE power");
client.Exec("USE power");
kv.Value.Metas.ForEach(meta => client.Exec(
"CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
client.Exec("CREATE TOPIC topic_meters as SELECT * from power.meters");
var cfg = new Dictionary<string, string>()
{ {
Console.Write("{0} {1}({2}) \t|", meta.name, meta.TypeName(), meta.size); { "group.id", "group1" },
}); { "auto.offset.reset", "latest" },
Console.WriteLine(""); { "td.connect.ip", "127.0.0.1" },
kv.Value.Datas.ForEach(data => { "td.connect.user", "root" },
{ "td.connect.pass", "taosdata" },
{ "td.connect.port", "6030" },
{ "client.id", "tmq_example" },
{ "enable.auto.commit", "true" },
{ "msg.with.table.name", "false" },
};
var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
consumer.Subscribe(new List<string>() { "topic_meters" });
Task.Run(InsertData);
while (true)
{ {
Console.WriteLine(data.ToString()); using (var cr = consumer.Consume(500))
}); {
if (cr == null) continue;
foreach (var message in cr.Message)
{
Console.WriteLine(
$"message {{{((DateTime)message.Value["ts"]).ToString("yyyy-MM-dd HH:mm:ss.fff")}, " +
$"{message.Value["current"]}, {message.Value["voltage"]}, {message.Value["phase"]}}}");
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
throw;
} }
consumer.Commit(consumeRes);
Console.WriteLine("\n================ {0} done ", i);
} }
// retrieve topic list
List<string> topics = consumer.Subscription();
topics.ForEach(t => Console.WriteLine("topic name:{0}", t));
// unsubscribe
consumer.Unsubscribe();
// close consumer after use.Otherwise will lead memory leak.
consumer.Close();
TDengine.Close(conn);
} }
static IntPtr GetConnection() static void InsertData()
{ {
string host = "localhost"; var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
short port = 6030; using (var client = DbDriver.Open(builder))
string username = "root";
string password = "taosdata";
string dbname = "power";
var conn = TDengine.Connect(host, username, password, dbname, port);
if (conn == IntPtr.Zero)
{ {
throw new Exception("Connect to TDengine failed"); while (true)
{
client.Exec("INSERT into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(now,11.5,219,0.30)");
Task.Delay(1000).Wait();
}
} }
else
{
Console.WriteLine("Connect to TDengine success");
}
return conn;
} }
} }

2
docs/examples/csharp/subscribe/subscribe.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" />
</ItemGroup> </ItemGroup>
</Project> </Project>

22
docs/examples/csharp/wsConnect/Program.cs
View File

@ -1,28 +1,18 @@
using System; using System;
using TDengineWS.Impl; using TDengine.Driver;
using TDengine.Driver.Client;
namespace Examples namespace Examples
{ {
public class WSConnExample public class WSConnExample
{ {
static int Main(string[] args) static void Main(string[] args)
{ {
string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); using (var client = DbDriver.Open(builder))
if (wsConn == IntPtr.Zero)
{ {
Console.WriteLine("get WS connection failed"); Console.WriteLine("connected");
return -1;
} }
else
{
Console.WriteLine("Establish connect success.");
// close connection.
LibTaosWS.WSClose(wsConn);
}
return 0;
} }
} }
} }

2
docs/examples/csharp/wsConnect/wsConnect.csproj
View File

@ -6,7 +6,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
</ItemGroup> </ItemGroup>
<Target Name="copyDLLDependency" BeforeTargets="BeforeBuild"> <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
<ItemGroup> <ItemGroup>

83
docs/examples/csharp/wsInsert/Program.cs
View File

@ -1,61 +1,46 @@
using System; using System;
using TDengineWS.Impl; using TDengine.Driver;
using TDengine.Driver.Client;
namespace Examples namespace Examples
{ {
public class WSInsertExample public class WSInsertExample
{ {
static int Main(string[] args) public static void Main(string[] args)
{ {
string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); using (var client = DbDriver.Open(builder))
// Assert if connection is validate
if (wsConn == IntPtr.Zero)
{ {
Console.WriteLine("get WS connection failed"); try
return -1; {
} client.Exec("create database power");
else client.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
{ string insertQuery =
Console.WriteLine("Establish connect success."); "INSERT INTO " +
} "power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " +
"VALUES " +
string createTable = "CREATE STABLE test.meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);"; "('2023-10-03 14:38:05.000', 10.30000, 219, 0.31000) " +
string insert = "INSERT INTO test.d1001 USING test.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000)" + "('2023-10-03 14:38:15.000', 12.60000, 218, 0.33000) " +
"test.d1002 USING test.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000)" + "('2023-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
"test.d1003 USING test.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " + "power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " +
"test.d1004 USING test.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)"; "VALUES " +
"('2023-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
IntPtr wsRes = LibTaosWS.WSQuery(wsConn, createTable); "power.d1003 USING power.meters TAGS(2,'California.LosAngeles') " +
ValidInsert("create table", wsRes); "VALUES " +
LibTaosWS.WSFreeResult(wsRes); "('2023-10-03 14:38:05.500', 11.80000, 221, 0.28000) " +
"('2023-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
wsRes = LibTaosWS.WSQuery(wsConn, insert); "power.d1004 USING power.meters TAGS(3,'California.LosAngeles') " +
ValidInsert("insert data", wsRes); "VALUES " +
LibTaosWS.WSFreeResult(wsRes); "('2023-10-03 14:38:05.000', 10.80000, 223, 0.29000) " +
"('2023-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
// close connection. client.Exec(insertQuery);
LibTaosWS.WSClose(wsConn); }
catch (Exception e)
return 0; {
} Console.WriteLine(e.ToString());
throw;
static void ValidInsert(string desc, IntPtr wsRes) }
{
int code = LibTaosWS.WSErrorNo(wsRes);
if (code != 0)
{
Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
}
else
{
Console.WriteLine("{0} success affect {2} rows, cost {1} nanoseconds", desc, LibTaosWS.WSTakeTiming(wsRes), LibTaosWS.WSAffectRows(wsRes));
} }
} }
} }
} }
// Establish connect success.
// create table success affect 0 rows, cost 3717542 nanoseconds
// insert data success affect 8 rows, cost 2613637 nanoseconds

2
docs/examples/csharp/wsInsert/wsInsert.csproj
View File

@ -6,7 +6,7 @@
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
</ItemGroup> </ItemGroup>
<Target Name="copyDLLDependency" BeforeTargets="BeforeBuild"> <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
<ItemGroup> <ItemGroup>

89
docs/examples/csharp/wsQuery/Program.cs
View File

@ -1,79 +1,36 @@
using System; using System;
using TDengineWS.Impl; using System.Text;
using System.Collections.Generic; using TDengine.Driver;
using TDengineDriver; using TDengine.Driver.Client;
namespace Examples namespace Examples
{ {
public class WSQueryExample public class WSQueryExample
{ {
static int Main(string[] args) public static void Main(string[] args)
{ {
string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); using (var client = DbDriver.Open(builder))
if (wsConn == IntPtr.Zero)
{ {
Console.WriteLine("get WS connection failed"); try
return -1;
}
else
{
Console.WriteLine("Establish connect success.");
}
string select = "select * from test.meters";
// optional:wsRes = LibTaosWS.WSQuery(wsConn, select);
IntPtr wsRes = LibTaosWS.WSQueryTimeout(wsConn, select, 1);
// Assert if query execute success.
int code = LibTaosWS.WSErrorNo(wsRes);
if (code != 0)
{
Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
LibTaosWS.WSFreeResult(wsRes);
return -1;
}
// get meta data
List<TDengineMeta> metas = LibTaosWS.WSGetFields(wsRes);
// get retrieved data
List<object> dataSet = LibTaosWS.WSGetData(wsRes);
// do something with result.
foreach (var meta in metas)
{
Console.Write("{0} {1}({2}) \t|\t", meta.name, meta.TypeName(), meta.size);
}
Console.WriteLine("");
for (int i = 0; i < dataSet.Count;)
{
for (int j = 0; j < metas.Count; j++)
{ {
Console.Write("{0}\t|\t", dataSet[i]); client.Exec("use power");
i++; string query = "SELECT * FROM meters";
using (var rows = client.Query(query))
{
while (rows.Read())
{
Console.WriteLine(
$"{((DateTime)rows.GetValue(0)):yyyy-MM-dd HH:mm:ss.fff}, {rows.GetValue(1)}, {rows.GetValue(2)}, {rows.GetValue(3)}, {rows.GetValue(4)}, {Encoding.UTF8.GetString((byte[])rows.GetValue(5))}");
}
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
throw;
} }
Console.WriteLine("");
} }
// Free result after use.
LibTaosWS.WSFreeResult(wsRes);
// close connection.
LibTaosWS.WSClose(wsConn);
return 0;
} }
} }
} }
// Establish connect success.
// ts TIMESTAMP(8) | current FLOAT(4) | voltage INT(4) | phase FLOAT(4) | location BINARY(64) | groupid INT(4) |
// 1538548685000 | 10.8 | 223 | 0.29 | California.LosAngeles | 3 |
// 1538548686500 | 11.5 | 221 | 0.35 | California.LosAngeles | 3 |
// 1538548685500 | 11.8 | 221 | 0.28 | California.LosAngeles | 2 |
// 1538548696600 | 13.4 | 223 | 0.29 | California.LosAngeles | 2 |
// 1538548685000 | 10.3 | 219 | 0.31 | California.SanFrancisco | 2 |
// 1538548695000 | 12.6 | 218 | 0.33 | California.SanFrancisco | 2 |
// 1538548696800 | 12.3 | 221 | 0.31 | California.SanFrancisco | 2 |
// 1538548696650 | 10.3 | 218 | 0.25 | California.SanFrancisco | 3 |

2
docs/examples/csharp/wsQuery/wsQuery.csproj
View File

@ -7,7 +7,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
</ItemGroup> </ItemGroup>
<Target Name="copyDLLDependency" BeforeTargets="BeforeBuild"> <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
<ItemGroup> <ItemGroup>

117
docs/examples/csharp/wsStmt/Program.cs
View File

@ -1,98 +1,41 @@
using System; using System;
using TDengineWS.Impl; using TDengine.Driver;
using TDengineDriver; using TDengine.Driver.Client;
using System.Runtime.InteropServices;
namespace Examples namespace Examples
{ {
public class WSStmtExample public class WSStmtExample
{ {
static int Main(string[] args) public static void Main(string[] args)
{ {
const string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; var builder =
const string table = "meters"; new ConnectionStringBuilder(
const string database = "test"; "protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
const string childTable = "d1005"; using (var client = DbDriver.Open(builder))
string insert = $"insert into ? using {database}.{table} tags(?,?) values(?,?,?,?)";
const int numOfTags = 2;
const int numOfColumns = 4;
// Establish connection
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
if (wsConn == IntPtr.Zero)
{ {
Console.WriteLine($"get WS connection failed"); try
return -1; {
} client.Exec($"create database power");
else client.Exec(
{ "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
Console.WriteLine("Establish connect success..."); using (var stmt = client.StmtInit())
} {
stmt.Prepare(
// init stmt "Insert into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(?,?,?,?)");
IntPtr wsStmt = LibTaosWS.WSStmtInit(wsConn); var ts = new DateTime(2023, 10, 03, 14, 38, 05, 000);
if (wsStmt != IntPtr.Zero) stmt.BindRow(new object[] { ts, (float)10.30000, (int)219, (float)0.31000 });
{ stmt.AddBatch();
int code = LibTaosWS.WSStmtPrepare(wsStmt, insert); stmt.Exec();
ValidStmtStep(code, wsStmt, "WSStmtPrepare"); var affected = stmt.Affected();
Console.WriteLine($"affected rows: {affected}");
TAOS_MULTI_BIND[] wsTags = new TAOS_MULTI_BIND[] { WSMultiBind.WSBindNchar(new string[] { "California.SanDiego" }), WSMultiBind.WSBindInt(new int?[] { 4 }) }; }
code = LibTaosWS.WSStmtSetTbnameTags(wsStmt, $"{database}.{childTable}", wsTags, numOfTags); }
ValidStmtStep(code, wsStmt, "WSStmtSetTbnameTags"); catch (Exception e)
{
TAOS_MULTI_BIND[] data = new TAOS_MULTI_BIND[4]; Console.WriteLine(e);
data[0] = WSMultiBind.WSBindTimestamp(new long[] { 1538548687000, 1538548688000, 1538548689000, 1538548690000, 1538548691000 }); throw;
data[1] = WSMultiBind.WSBindFloat(new float?[] { 10.30F, 10.40F, 10.50F, 10.60F, 10.70F }); }
data[2] = WSMultiBind.WSBindInt(new int?[] { 223, 221, 222, 220, 219 });
data[3] = WSMultiBind.WSBindFloat(new float?[] { 0.31F, 0.32F, 0.33F, 0.35F, 0.28F });
code = LibTaosWS.WSStmtBindParamBatch(wsStmt, data, numOfColumns);
ValidStmtStep(code, wsStmt, "WSStmtBindParamBatch");
code = LibTaosWS.WSStmtAddBatch(wsStmt);
ValidStmtStep(code, wsStmt, "WSStmtAddBatch");
IntPtr stmtAffectRowPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(Int32)));
code = LibTaosWS.WSStmtExecute(wsStmt, stmtAffectRowPtr);
ValidStmtStep(code, wsStmt, "WSStmtExecute");
Console.WriteLine("WS STMT insert {0} rows...", Marshal.ReadInt32(stmtAffectRowPtr));
Marshal.FreeHGlobal(stmtAffectRowPtr);
LibTaosWS.WSStmtClose(wsStmt);
// Free unmanaged memory
WSMultiBind.WSFreeTaosBind(wsTags);
WSMultiBind.WSFreeTaosBind(data);
//check result with SQL "SELECT * FROM test.d1005;"
}
else
{
Console.WriteLine("Init STMT failed...");
}
// close connection.
LibTaosWS.WSClose(wsConn);
return 0;
}
static void ValidStmtStep(int code, IntPtr wsStmt, string desc)
{
if (code != 0)
{
Console.WriteLine($"{desc} failed,reason: {LibTaosWS.WSErrorStr(wsStmt)}, code: {code}");
}
else
{
Console.WriteLine("{0} success...", desc);
} }
} }
} }
} }
// WSStmtPrepare success...
// WSStmtSetTbnameTags success...
// WSStmtBindParamBatch success...
// WSStmtAddBatch success...
// WSStmtExecute success...
// WS STMT insert 5 rows...

2
docs/examples/csharp/wsStmt/wsStmt.csproj
View File

@ -7,7 +7,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" /> <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
</ItemGroup> </ItemGroup>
<Target Name="copyDLLDependency" BeforeTargets="BeforeBuild"> <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
<ItemGroup> <ItemGroup>

2
docs/zh/07-develop/01-connect/index.md
View File

@ -176,7 +176,7 @@ npm install @tdengine/rest
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.0" /> <PackageReference Include="TDengine.Connector" Version="3.1.0" />
</ItemGroup> </ItemGroup>
</Project> </Project>

3
docs/zh/07-develop/04-query-data/_cs_async.mdx
View File

@ -1,3 +0,0 @@
```csharp
{{#include docs/examples/csharp/asyncQuery/Program.cs}}
```

4
docs/zh/07-develop/04-query-data/index.mdx
View File

@ -16,7 +16,6 @@ import CQuery from "./_c.mdx";
import PhpQuery from "./_php.mdx"; import PhpQuery from "./_php.mdx";
import PyAsync from "./_py_async.mdx"; import PyAsync from "./_py_async.mdx";
import NodeAsync from "./_js_async.mdx"; import NodeAsync from "./_js_async.mdx";
import CsAsync from "./_cs_async.mdx";
import CAsync from "./_c_async.mdx"; import CAsync from "./_c_async.mdx";
## 主要查询功能 ## 主要查询功能
@ -175,9 +174,6 @@ Query OK, 6 rows in database (0.005515s)
<TabItem label="Python" value="python"> <TabItem label="Python" value="python">
<PyAsync /> <PyAsync />
</TabItem> </TabItem>
<TabItem label="C#" value="csharp">
<CsAsync />
</TabItem>
<TabItem label="C" value="c"> <TabItem label="C" value="c">
<CAsync /> <CAsync />
</TabItem> </TabItem>

52
docs/zh/07-develop/07-tmq.mdx
View File

@ -248,23 +248,23 @@ function close()
<TabItem value="C#" label="C#"> <TabItem value="C#" label="C#">
```csharp ```csharp
class ConsumerBuilder<TValue>
ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config) ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
virtual IConsumer Build() public IConsumer<TValue> Build()
Consumer(ConsumerBuilder builder)
void Subscribe(IEnumerable<string> topics) void Subscribe(IEnumerable<string> topics)
void Subscribe(string topic) void Subscribe(string topic)
ConsumeResult Consume(int millisecondsTimeout) ConsumeResult<TValue> Consume(int millisecondsTimeout)
List<string> Subscription() List<string> Subscription()
void Unsubscribe() void Unsubscribe()
void Commit(ConsumeResult consumerResult) List<TopicPartitionOffset> Commit()
void Close() void Close()
``` ```
@ -501,25 +501,19 @@ let consumer = taos.consumer({
<TabItem value="C#" label="C#"> <TabItem value="C#" label="C#">
```csharp ```csharp
using TDengineTMQ; var cfg = new Dictionary<string, string>()
{
// 根据需要，设置消费组 (GourpId)、自动提交 (EnableAutoCommit)、 { "group.id", "group1" },
// 自动提交时间间隔 (AutoCommitIntervalMs)、用户名 (TDConnectUser)、密码 (TDConnectPasswd) 等参数 { "auto.offset.reset", "latest" },
var cfg = new ConsumerConfig { "td.connect.ip", "127.0.0.1" },
{ { "td.connect.user", "root" },
EnableAutoCommit = "true" { "td.connect.pass", "taosdata" },
AutoCommitIntervalMs = "1000" { "td.connect.port", "6030" },
GourpId = "TDengine-TMQ-C#", { "client.id", "tmq_example" },
TDConnectUser = "root", { "enable.auto.commit", "true" },
TDConnectPasswd = "taosdata", { "msg.with.table.name", "false" },
AutoOffsetReset = "latest" };
MsgWithTableName = "true", var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
TDConnectIp = "127.0.0.1",
TDConnectPort = "6030"
};
var consumer = new ConsumerBuilder(cfg).Build();
``` ```
</TabItem> </TabItem>
@ -748,10 +742,12 @@ while(true){
// 消费数据 // 消费数据
while (true) while (true)
{ {
var consumerRes = consumer.Consume(100); using (var result = consumer.Consume(500))
// process ConsumeResult {
ProcessMsg(consumerRes); if (result == null) continue;
consumer.Commit(consumerRes); ProcessMsg(result);
consumer.Commit();
}
} }
``` ```

57
docs/zh/08-connector/02-rest-api.mdx
View File

@ -257,6 +257,63 @@ curl -L -u username:password -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name][?tz=timez
- "INT UNSIGNED" - "INT UNSIGNED"
- "BIGINT UNSIGNED" - "BIGINT UNSIGNED"
- "JSON" - "JSON"
- "VARBINARY"
- "GEOMETRY"
`VARBINARY` 和 `GEOMETRY` 类型返回数据为 Hex 字符串，样例：
准备数据
```bash
create database demo
use demo
create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
insert into t values(now,'\x7f8290','point(100 100)')
```
执行查询
```bash
curl --location 'http://<fqdn>:<port>/rest/sql' \
--header 'Content-Type: text/plain' \
--header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \
--data 'select * from demo.t'
```
返回结果
```json
{
"code": 0,
"column_meta": [
[
"ts",
"TIMESTAMP",
8
],
[
"c1",
"VARBINARY",
20
],
[
"c2",
"GEOMETRY",
100
]
],
"data": [
[
"2023-11-01T06:28:15.210Z",
"7f8290",
"010100000000000000000059400000000000005940"
]
],
"rows": 1
}
```
- `010100000000000000000059400000000000005940` 为 `point(100 100)` 的 [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/) 格式
#### 错误 #### 错误

1303
docs/zh/08-connector/40-csharp.mdx
View File

File diff suppressed because it is too large Load Diff

6
docs/zh/12-taos-sql/05-insert.md
View File

@ -158,7 +158,7 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c
自动建表, 表名通过tbname列指定 自动建表, 表名通过tbname列指定
```sql ```sql
INSERT INTO meters(tbname, location, groupId, ts, current, phase) INSERT INTO meters(tbname, location, groupId, ts, current, phase)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 0.32)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 0.33)
values('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 0.33)
``` ```

2
include/common/cos.h
View File

@ -34,7 +34,7 @@ extern int32_t tsS3UploadDelaySec;
int32_t s3Init(); int32_t s3Init();
void s3CleanUp(); void s3CleanUp();
int32_t s3PutObjectFromFile(const char *file, const char *object); int32_t s3PutObjectFromFile(const char *file, const char *object);
int32_t s3PutObjectFromFile2(const char *file, const char *object); int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp);
void s3DeleteObjectsByPrefix(const char *prefix); void s3DeleteObjectsByPrefix(const char *prefix);
void s3DeleteObjects(const char *object_name[], int nobject); void s3DeleteObjects(const char *object_name[], int nobject);
bool s3Exists(const char *object_name); bool s3Exists(const char *object_name);

83
include/common/cos_cp.h Normal file
View File

@ -0,0 +1,83 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_COMMON_COS_CP_H_
#define _TD_COMMON_COS_CP_H_
#include "os.h"
#include "tdef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef enum {
COS_CP_TYPE_UPLOAD, // upload
COS_CP_TYPE_DOWNLOAD // download
} ECpType;
typedef struct {
int32_t index; // the index of part, start from 0
int64_t offset; // the offset point of part
int64_t size; // the size of part
int completed; // COS_TRUE completed, COS_FALSE uncompleted
char etag[128]; // the etag of part, for upload
uint64_t crc64;
} SCheckpointPart;
typedef struct {
ECpType cp_type; // 0 upload, 1 download
char md5[64]; // the md5 of checkout content
TdFilePtr thefile; // the handle of checkpoint file
char file_path[TSDB_FILENAME_LEN]; // local file path
int64_t file_size; // local file size, for upload
int32_t file_last_modified; // local file last modified time, for upload
char file_md5[64]; // md5 of the local file content, for upload, reserved
char object_name[128]; // object name
int64_t object_size; // object size, for download
char object_last_modified[64]; // object last modified time, for download
char object_etag[128]; // object etag, for download
char upload_id[128]; // upload id
int part_num; // the total number of parts
int64_t part_size; // the part size, byte
SCheckpointPart* parts; // the parts of local or object, from 0
} SCheckpoint;
int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint);
void cos_cp_close(TdFilePtr fd);
void cos_cp_remove(char const* filepath);
int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint);
int32_t cos_cp_dump(SCheckpoint* checkpoint);
void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes);
void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64);
void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
char const* upload_id, int64_t part_size);
bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime);
void cos_cp_build_download(SCheckpoint* checkpoint, char const* filepath, char const* object_name, int64_t object_size,
char const* object_lmtime, char const* object_etag, int64_t part_size);
bool cos_cp_is_valid_download(SCheckpoint* checkpoint, char const* object_name, int64_t object_size,
char const* object_lmtime, char const* object_etag);
#ifdef __cplusplus
}
#endif
#endif /*_TD_COMMON_COS_CP_H_*/

2
include/common/systable.h
View File

@ -50,6 +50,8 @@ extern "C" {
#define TSDB_INS_TABLE_STREAM_TASKS "ins_stream_tasks" #define TSDB_INS_TABLE_STREAM_TASKS "ins_stream_tasks"
#define TSDB_INS_TABLE_USER_PRIVILEGES "ins_user_privileges" #define TSDB_INS_TABLE_USER_PRIVILEGES "ins_user_privileges"
#define TSDB_INS_TABLE_VIEWS "ins_views" #define TSDB_INS_TABLE_VIEWS "ins_views"
#define TSDB_INS_TABLE_COMPACTS "ins_compacts"
#define TSDB_INS_TABLE_COMPACT_DETAILS "ins_compact_details"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" #define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
#define TSDB_PERFS_TABLE_SMAS "perf_smas" #define TSDB_PERFS_TABLE_SMAS "perf_smas"

7
include/common/tglobal.h
View File

@ -107,8 +107,9 @@ extern int32_t tsMonitorMaxLogs;
extern bool tsMonitorComp; extern bool tsMonitorComp;
// audit // audit
extern bool tsEnableAudit; extern bool tsEnableAudit;
extern bool tsEnableAuditCreateTable; extern bool tsEnableAuditCreateTable;
extern int32_t tsAuditInterval;
// telem // telem
extern bool tsEnableTelem; extern bool tsEnableTelem;
@ -195,6 +196,7 @@ extern int64_t tsWalFsyncDataSizeLimit;
// internal // internal
extern int32_t tsTransPullupInterval; extern int32_t tsTransPullupInterval;
extern int32_t tsCompactPullupInterval;
extern int32_t tsMqRebalanceInterval; extern int32_t tsMqRebalanceInterval;
extern int32_t tsStreamCheckpointInterval; extern int32_t tsStreamCheckpointInterval;
extern float tsSinkDataRate; extern float tsSinkDataRate;
@ -213,6 +215,7 @@ extern int32_t tsMaxStreamBackendCache;
extern int32_t tsPQSortMemThreshold; extern int32_t tsPQSortMemThreshold;
extern int32_t tsResolveFQDNRetryTime; extern int32_t tsResolveFQDNRetryTime;
extern bool tsExperimental;
// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,

56
include/common/tmsg.h
View File

@ -144,6 +144,8 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_STREAM_TASKS, TSDB_MGMT_TABLE_STREAM_TASKS,
TSDB_MGMT_TABLE_PRIVILEGES, TSDB_MGMT_TABLE_PRIVILEGES,
TSDB_MGMT_TABLE_VIEWS, TSDB_MGMT_TABLE_VIEWS,
TSDB_MGMT_TABLE_COMPACT,
TSDB_MGMT_TABLE_COMPACT_DETAIL,
TSDB_MGMT_TABLE_MAX, TSDB_MGMT_TABLE_MAX,
} EShowType; } EShowType;
@ -307,6 +309,7 @@ typedef enum ENodeType {
QUERY_NODE_KILL_CONNECTION_STMT, QUERY_NODE_KILL_CONNECTION_STMT,
QUERY_NODE_KILL_QUERY_STMT, QUERY_NODE_KILL_QUERY_STMT,
QUERY_NODE_KILL_TRANSACTION_STMT, QUERY_NODE_KILL_TRANSACTION_STMT,
QUERY_NODE_KILL_COMPACT_STMT,
QUERY_NODE_DELETE_STMT, QUERY_NODE_DELETE_STMT,
QUERY_NODE_INSERT_STMT, QUERY_NODE_INSERT_STMT,
QUERY_NODE_QUERY, QUERY_NODE_QUERY,
@ -353,6 +356,8 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_VNODES_STMT, QUERY_NODE_SHOW_VNODES_STMT,
QUERY_NODE_SHOW_USER_PRIVILEGES_STMT, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
QUERY_NODE_SHOW_VIEWS_STMT, QUERY_NODE_SHOW_VIEWS_STMT,
QUERY_NODE_SHOW_COMPACTS_STMT,
QUERY_NODE_SHOW_COMPACT_DETAILS_STMT,
// logic plan node // logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN = 1000, QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@ -1384,6 +1389,24 @@ int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
int32_t tDeserializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq); int32_t tDeserializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
void tFreeSCompactDbReq(SCompactDbReq* pReq); void tFreeSCompactDbReq(SCompactDbReq* pReq);
typedef struct {
int32_t compactId;
int8_t bAccepted;
} SCompactDbRsp;
int32_t tSerializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp);
int32_t tDeserializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp);
typedef struct {
int32_t compactId;
int32_t sqlLen;
char* sql;
} SKillCompactReq;
int32_t tSerializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
int32_t tDeserializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
void tFreeSKillCompactReq(SKillCompactReq *pReq);
typedef struct { typedef struct {
char name[TSDB_FUNC_NAME_LEN]; char name[TSDB_FUNC_NAME_LEN];
int8_t igExists; int8_t igExists;
@ -1662,6 +1685,26 @@ int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pR
int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);
int32_t tFreeSCreateVnodeReq(SCreateVnodeReq* pReq); int32_t tFreeSCreateVnodeReq(SCreateVnodeReq* pReq);
typedef struct {
int32_t compactId;
int32_t vgId;
int32_t dnodeId;
} SQueryCompactProgressReq;
int32_t tSerializeSQueryCompactProgressReq(void* buf, int32_t bufLen, SQueryCompactProgressReq* pReq);
int32_t tDeserializeSQueryCompactProgressReq(void* buf, int32_t bufLen, SQueryCompactProgressReq* pReq);
typedef struct {
int32_t compactId;
int32_t vgId;
int32_t dnodeId;
int32_t numberFileset;
int32_t finished;
} SQueryCompactProgressRsp;
int32_t tSerializeSQueryCompactProgressRsp(void* buf, int32_t bufLen, SQueryCompactProgressRsp* pReq);
int32_t tDeserializeSQueryCompactProgressRsp(void* buf, int32_t bufLen, SQueryCompactProgressRsp* pReq);
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
int32_t dnodeId; int32_t dnodeId;
@ -1689,11 +1732,21 @@ typedef struct {
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
int64_t compactStartTime; int64_t compactStartTime;
STimeWindow tw; STimeWindow tw;
int32_t compactId;
} SCompactVnodeReq; } SCompactVnodeReq;
int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq); int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq); int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
typedef struct {
int32_t compactId;
int32_t vgId;
int32_t dnodeId;
} SVKillCompactReq;
int32_t tSerializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq);
int32_t tDeserializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq);
typedef struct { typedef struct {
int32_t vgVersion; int32_t vgVersion;
int32_t buffer; int32_t buffer;
@ -1889,8 +1942,9 @@ typedef struct {
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
char tb[TSDB_TABLE_NAME_LEN]; char tb[TSDB_TABLE_NAME_LEN];
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char filterTb[TSDB_TABLE_NAME_LEN]; char filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns
int64_t showId; int64_t showId;
int64_t compactId; // for compact
} SRetrieveTableReq; } SRetrieveTableReq;
typedef struct SSysTableSchema { typedef struct SSysTableSchema {

6
include/common/tmsgdef.h
View File

@ -217,6 +217,8 @@
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_KILL_COMPACT, "kill-compact", SKillCompactReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
@ -256,7 +258,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_EXEC_RSMA, "vnode-exec-rsma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_EXEC_RSMA, "vnode-exec-rsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp)
TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
@ -267,6 +269,8 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG) TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG)

352
include/common/ttokendef.h
View File

@ -189,184 +189,180 @@
#define TK_ALIVE 170 #define TK_ALIVE 170
#define TK_VIEWS 171 #define TK_VIEWS 171
#define TK_VIEW 172 #define TK_VIEW 172
#define TK_NORMAL 173 #define TK_COMPACTS 173
#define TK_CHILD 174 #define TK_NORMAL 174
#define TK_LIKE 175 #define TK_CHILD 175
#define TK_TBNAME 176 #define TK_LIKE 176
#define TK_QTAGS 177 #define TK_TBNAME 177
#define TK_AS 178 #define TK_QTAGS 178
#define TK_SYSTEM 179 #define TK_AS 179
#define TK_INDEX 180 #define TK_SYSTEM 180
#define TK_FUNCTION 181 #define TK_INDEX 181
#define TK_INTERVAL 182 #define TK_FUNCTION 182
#define TK_COUNT 183 #define TK_INTERVAL 183
#define TK_LAST_ROW 184 #define TK_COUNT 184
#define TK_META 185 #define TK_LAST_ROW 185
#define TK_ONLY 186 #define TK_META 186
#define TK_TOPIC 187 #define TK_ONLY 187
#define TK_CONSUMER 188 #define TK_TOPIC 188
#define TK_GROUP 189 #define TK_CONSUMER 189
#define TK_DESC 190 #define TK_GROUP 190
#define TK_DESCRIBE 191 #define TK_DESC 191
#define TK_RESET 192 #define TK_DESCRIBE 192
#define TK_QUERY 193 #define TK_RESET 193
#define TK_CACHE 194 #define TK_QUERY 194
#define TK_EXPLAIN 195 #define TK_CACHE 195
#define TK_ANALYZE 196 #define TK_EXPLAIN 196
#define TK_VERBOSE 197 #define TK_ANALYZE 197
#define TK_NK_BOOL 198 #define TK_VERBOSE 198
#define TK_RATIO 199 #define TK_NK_BOOL 199
#define TK_NK_FLOAT 200 #define TK_RATIO 200
#define TK_OUTPUTTYPE 201 #define TK_NK_FLOAT 201
#define TK_AGGREGATE 202 #define TK_OUTPUTTYPE 202
#define TK_BUFSIZE 203 #define TK_AGGREGATE 203
#define TK_LANGUAGE 204 #define TK_BUFSIZE 204
#define TK_REPLACE 205 #define TK_LANGUAGE 205
#define TK_STREAM 206 #define TK_REPLACE 206
#define TK_INTO 207 #define TK_STREAM 207
#define TK_PAUSE 208 #define TK_INTO 208
#define TK_RESUME 209 #define TK_PAUSE 209
#define TK_TRIGGER 210 #define TK_RESUME 210
#define TK_AT_ONCE 211 #define TK_TRIGGER 211
#define TK_WINDOW_CLOSE 212 #define TK_AT_ONCE 212
#define TK_IGNORE 213 #define TK_WINDOW_CLOSE 213
#define TK_EXPIRED 214 #define TK_IGNORE 214
#define TK_FILL_HISTORY 215 #define TK_EXPIRED 215
#define TK_UPDATE 216 #define TK_FILL_HISTORY 216
#define TK_SUBTABLE 217 #define TK_UPDATE 217
#define TK_UNTREATED 218 #define TK_SUBTABLE 218
#define TK_KILL 219 #define TK_UNTREATED 219
#define TK_CONNECTION 220 #define TK_KILL 220
#define TK_TRANSACTION 221 #define TK_CONNECTION 221
#define TK_BALANCE 222 #define TK_TRANSACTION 222
#define TK_VGROUP 223 #define TK_BALANCE 223
#define TK_LEADER 224 #define TK_VGROUP 224
#define TK_MERGE 225 #define TK_LEADER 225
#define TK_REDISTRIBUTE 226 #define TK_MERGE 226
#define TK_SPLIT 227 #define TK_REDISTRIBUTE 227
#define TK_DELETE 228 #define TK_SPLIT 228
#define TK_INSERT 229 #define TK_DELETE 229
#define TK_NULL 230 #define TK_INSERT 230
#define TK_NK_QUESTION 231 #define TK_NULL 231
#define TK_NK_ALIAS 232 #define TK_NK_QUESTION 232
#define TK_NK_ARROW 233 #define TK_NK_ALIAS 233
#define TK_ROWTS 234 #define TK_NK_ARROW 234
#define TK_QSTART 235 #define TK_ROWTS 235
#define TK_QEND 236 #define TK_QSTART 236
#define TK_QDURATION 237 #define TK_QEND 237
#define TK_WSTART 238 #define TK_QDURATION 238
#define TK_WEND 239 #define TK_WSTART 239
#define TK_WDURATION 240 #define TK_WEND 240
#define TK_IROWTS 241 #define TK_WDURATION 241
#define TK_ISFILLED 242 #define TK_IROWTS 242
#define TK_CAST 243 #define TK_ISFILLED 243
#define TK_NOW 244 #define TK_CAST 244
#define TK_TODAY 245 #define TK_NOW 245
#define TK_TIMEZONE 246 #define TK_TODAY 246
#define TK_CLIENT_VERSION 247 #define TK_TIMEZONE 247
#define TK_SERVER_VERSION 248 #define TK_CLIENT_VERSION 248
#define TK_SERVER_STATUS 249 #define TK_SERVER_VERSION 249
#define TK_CURRENT_USER 250 #define TK_SERVER_STATUS 250
#define TK_CASE 251 #define TK_CURRENT_USER 251
#define TK_WHEN 252 #define TK_CASE 252
#define TK_THEN 253 #define TK_WHEN 253
#define TK_ELSE 254 #define TK_THEN 254
#define TK_BETWEEN 255 #define TK_ELSE 255
#define TK_IS 256 #define TK_BETWEEN 256
#define TK_NK_LT 257 #define TK_IS 257
#define TK_NK_GT 258 #define TK_NK_LT 258
#define TK_NK_LE 259 #define TK_NK_GT 259
#define TK_NK_GE 260 #define TK_NK_LE 260
#define TK_NK_NE 261 #define TK_NK_GE 261
#define TK_MATCH 262 #define TK_NK_NE 262
#define TK_NMATCH 263 #define TK_MATCH 263
#define TK_CONTAINS 264 #define TK_NMATCH 264
#define TK_IN 265 #define TK_CONTAINS 265
#define TK_JOIN 266 #define TK_IN 266
#define TK_INNER 267 #define TK_JOIN 267
#define TK_SELECT 268 #define TK_INNER 268
#define TK_NK_HINT 269 #define TK_SELECT 269
#define TK_DISTINCT 270 #define TK_NK_HINT 270
#define TK_WHERE 271 #define TK_DISTINCT 271
#define TK_PARTITION 272 #define TK_WHERE 272
#define TK_BY 273 #define TK_PARTITION 273
#define TK_SESSION 274 #define TK_BY 274
#define TK_STATE_WINDOW 275 #define TK_SESSION 275
#define TK_EVENT_WINDOW 276 #define TK_STATE_WINDOW 276
#define TK_SLIDING 277 #define TK_EVENT_WINDOW 277
#define TK_FILL 278 #define TK_SLIDING 278
#define TK_VALUE 279 #define TK_FILL 279
#define TK_VALUE_F 280 #define TK_VALUE 280
#define TK_NONE 281 #define TK_VALUE_F 281
#define TK_PREV 282 #define TK_NONE 282
#define TK_NULL_F 283 #define TK_PREV 283
#define TK_LINEAR 284 #define TK_NULL_F 284
#define TK_NEXT 285 #define TK_LINEAR 285
#define TK_HAVING 286 #define TK_NEXT 286
#define TK_RANGE 287 #define TK_HAVING 287
#define TK_EVERY 288 #define TK_RANGE 288
#define TK_ORDER 289 #define TK_EVERY 289
#define TK_SLIMIT 290 #define TK_ORDER 290
#define TK_SOFFSET 291 #define TK_SLIMIT 291
#define TK_LIMIT 292 #define TK_SOFFSET 292
#define TK_OFFSET 293 #define TK_LIMIT 293
#define TK_ASC 294 #define TK_OFFSET 294
#define TK_NULLS 295 #define TK_ASC 295
#define TK_ABORT 296 #define TK_NULLS 296
#define TK_AFTER 297 #define TK_ABORT 297
#define TK_ATTACH 298 #define TK_AFTER 298
#define TK_BEFORE 299 #define TK_ATTACH 299
#define TK_BEGIN 300 #define TK_BEFORE 300
#define TK_BITAND 301 #define TK_BEGIN 301
#define TK_BITNOT 302 #define TK_BITAND 302
#define TK_BITOR 303 #define TK_BITNOT 303
#define TK_BLOCKS 304 #define TK_BITOR 304
#define TK_CHANGE 305 #define TK_BLOCKS 305
#define TK_COMMA 306 #define TK_CHANGE 306
#define TK_CONCAT 307 #define TK_COMMA 307
#define TK_CONFLICT 308 #define TK_CONCAT 308
#define TK_COPY 309 #define TK_CONFLICT 309
#define TK_DEFERRED 310 #define TK_COPY 310
#define TK_DELIMITERS 311 #define TK_DEFERRED 311
#define TK_DETACH 312 #define TK_DELIMITERS 312
#define TK_DIVIDE 313 #define TK_DETACH 313
#define TK_DOT 314 #define TK_DIVIDE 314
#define TK_EACH 315 #define TK_DOT 315
#define TK_FAIL 316 #define TK_EACH 316
#define TK_FILE 317 #define TK_FAIL 317
#define TK_FOR 318 #define TK_FILE 318
#define TK_GLOB 319 #define TK_FOR 319
#define TK_ID 320 #define TK_GLOB 320
#define TK_IMMEDIATE 321 #define TK_ID 321
#define TK_IMPORT 322 #define TK_IMMEDIATE 322
#define TK_INITIALLY 323 #define TK_IMPORT 323
#define TK_INSTEAD 324 #define TK_INITIALLY 324
#define TK_ISNULL 325 #define TK_INSTEAD 325
#define TK_KEY 326 #define TK_ISNULL 326
#define TK_MODULES 327 #define TK_KEY 327
#define TK_NK_BITNOT 328 #define TK_MODULES 328
#define TK_NK_SEMI 329 #define TK_NK_BITNOT 329
#define TK_NOTNULL 330 #define TK_NK_SEMI 330
#define TK_OF 331 #define TK_NOTNULL 331
#define TK_PLUS 332 #define TK_OF 332
#define TK_PRIVILEGE 333 #define TK_PLUS 333
#define TK_RAISE 334 #define TK_PRIVILEGE 334
#define TK_RESTRICT 335 #define TK_RAISE 335
#define TK_ROW 336 #define TK_RESTRICT 336
#define TK_SEMI 337 #define TK_ROW 337
#define TK_STAR 338 #define TK_SEMI 338
#define TK_STATEMENT 339 #define TK_STAR 339
#define TK_STRICT 340 #define TK_STATEMENT 340
#define TK_STRING 341 #define TK_STRICT 341
#define TK_TIMES 342 #define TK_STRING 342
#define TK_VALUES 343 #define TK_TIMES 343
#define TK_VARIABLE 344 #define TK_VALUES 344
#define TK_WAL 345 #define TK_VARIABLE 345
#define TK_WAL 346
#define TK_NK_SPACE 600 #define TK_NK_SPACE 600
#define TK_NK_COMMENT 601 #define TK_NK_COMMENT 601

19
include/libs/audit/audit.h
View File

@ -23,13 +23,13 @@
#include "tjson.h" #include "tjson.h"
#include "tmsgcb.h" #include "tmsgcb.h"
#include "trpc.h" #include "trpc.h"
#include "mnode.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#define AUDIT_DETAIL_MAX 65472 #define AUDIT_DETAIL_MAX 65472
#define AUDIT_OPERATION_LEN 20
typedef struct { typedef struct {
const char *server; const char *server;
@ -37,13 +37,28 @@ typedef struct {
bool comp; bool comp;
} SAuditCfg; } SAuditCfg;
typedef struct {
int64_t curTime;
char strClusterId[TSDB_CLUSTER_ID_LEN];
char clientAddress[50];
char user[TSDB_USER_LEN];
char operation[AUDIT_OPERATION_LEN];
char target1[TSDB_DB_NAME_LEN]; //put db name
char target2[TSDB_STREAM_NAME_LEN]; //put stb name, table name, topic name, user name, stream name, use max
char* detail;
} SAuditRecord;
int32_t auditInit(const SAuditCfg *pCfg); int32_t auditInit(const SAuditCfg *pCfg);
void auditCleanup();
void auditSend(SJson *pJson); void auditSend(SJson *pJson);
void auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, void auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2,
char *detail, int32_t len); char *detail, int32_t len);
void auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2,
char *detail, int32_t len);
void auditSendRecordsInBatch();
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif /*_TD_MONITOR_H_*/ #endif /*_TD_AUDIT_H_*/

47
include/libs/executor/storageapi.h
View File

@ -38,8 +38,8 @@ extern "C" {
#define META_READER_NOLOCK 0x1 #define META_READER_NOLOCK 0x1
#define STREAM_STATE_BUFF_HASH 1 #define STREAM_STATE_BUFF_HASH 1
#define STREAM_STATE_BUFF_SORT 2 #define STREAM_STATE_BUFF_SORT 2
typedef struct SMeta SMeta; typedef struct SMeta SMeta;
typedef TSKEY (*GetTsFun)(void*); typedef TSKEY (*GetTsFun)(void*);
@ -102,14 +102,14 @@ typedef struct SMTbCursor {
} SMTbCursor; } SMTbCursor;
typedef struct SMCtbCursor { typedef struct SMCtbCursor {
SMeta *pMeta; SMeta* pMeta;
void *pCur; void* pCur;
tb_uid_t suid; tb_uid_t suid;
void *pKey; void* pKey;
void *pVal; void* pVal;
int kLen; int kLen;
int vLen; int vLen;
int8_t paused; int8_t paused;
int lock; int lock;
} SMCtbCursor; } SMCtbCursor;
@ -153,7 +153,8 @@ typedef struct {
// clang-format off // clang-format off
/*-------------------------------------------------new api format---------------------------------------------------*/ /*-------------------------------------------------new api format---------------------------------------------------*/
typedef enum { typedef enum {
TSD_READER_NOTIFY_DURATION_START TSD_READER_NOTIFY_DURATION_START,
TSD_READER_NOTIFY_NEXT_DURATION_BLOCK,
} ETsdReaderNotifyType; } ETsdReaderNotifyType;
typedef union { typedef union {
@ -262,22 +263,23 @@ typedef struct SStoreMeta {
void* (*storeGetIndexInfo)(); void* (*storeGetIndexInfo)();
void* (*getInvertIndex)(void* pVnode); void* (*getInvertIndex)(void* pVnode);
// support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
int32_t (*getChildTableList)( void* pVnode, int64_t suid, SArray* list); int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list);
int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList);
void* storeGetVersionRange; void* storeGetVersionRange;
void* storeGetLastTimestamp; void* storeGetLastTimestamp;
int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbGetTableSchema int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbGetTableSchema
int32_t (*getNumOfChildTables)( void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols); int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols);
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables, int64_t* numOfNormalTables); void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
int64_t* numOfNormalTables);
int64_t (*getNumOfRowsInMem)(void* pVnode); int64_t (*getNumOfRowsInMem)(void* pVnode);
SMCtbCursor* (*openCtbCursor)(void *pVnode, tb_uid_t uid, int lock); SMCtbCursor* (*openCtbCursor)(void* pVnode, tb_uid_t uid, int lock);
int32_t (*resumeCtbCursor)(SMCtbCursor* pCtbCur, int8_t first); int32_t (*resumeCtbCursor)(SMCtbCursor* pCtbCur, int8_t first);
void (*pauseCtbCursor)(SMCtbCursor* pCtbCur); void (*pauseCtbCursor)(SMCtbCursor* pCtbCur);
void (*closeCtbCursor)(SMCtbCursor *pCtbCur); void (*closeCtbCursor)(SMCtbCursor* pCtbCur);
tb_uid_t (*ctbCursorNext)(SMCtbCursor* pCur); tb_uid_t (*ctbCursorNext)(SMCtbCursor* pCur);
} SStoreMeta; } SStoreMeta;
typedef struct SStoreMetaReader { typedef struct SStoreMetaReader {
@ -362,14 +364,14 @@ typedef struct SStateStore {
const SSessionKey* pKey, void** pVal, int32_t* pVLen); const SSessionKey* pKey, void** pVal, int32_t* pVLen);
SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp); SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp);
TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol); TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol);
bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
bool (*isIncrementalTimeStamp)(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts); bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
void (*updateInfoDestroy)(SUpdateInfo* pInfo); void (*updateInfoDestroy)(SUpdateInfo* pInfo);
void (*windowSBfDelete)(SUpdateInfo *pInfo, uint64_t count); void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
void (*windowSBfAdd)(SUpdateInfo *pInfo, uint64_t count); void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp); SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp);
void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);
@ -396,6 +398,7 @@ typedef struct SStateStore {
void (*streamStateDestroy)(SStreamState* pState, bool remove); void (*streamStateDestroy)(SStreamState* pState, bool remove);
int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts); void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);
void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst);
} SStateStore; } SStateStore;
typedef struct SStorageAPI { typedef struct SStorageAPI {

1
include/libs/function/function.h
View File

@ -171,6 +171,7 @@ typedef struct {
int32_t taskId; int32_t taskId;
int64_t streamId; int64_t streamId;
int64_t streamBackendRid; int64_t streamBackendRid;
int8_t dump;
} SStreamState; } SStreamState;
typedef struct SFunctionStateStore { typedef struct SFunctionStateStore {

13
include/libs/nodes/cmdnodes.h
View File

@ -46,6 +46,10 @@ extern "C" {
#define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
#define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE)
#define COMPACT_DB_RESULT_COLS 3
#define COMPACT_DB_RESULT_FIELD1_LEN 32
#define COMPACT_DB_RESULT_FIELD3_LEN 128
#define SHOW_ALIVE_RESULT_COLS 1 #define SHOW_ALIVE_RESULT_COLS 1
#define BIT_FLAG_MASK(n) (1 << n) #define BIT_FLAG_MASK(n) (1 << n)
@ -335,6 +339,15 @@ typedef struct SShowTableTagsStmt {
SNodeList* pTags; SNodeList* pTags;
} SShowTableTagsStmt; } SShowTableTagsStmt;
typedef struct SShowCompactsStmt {
ENodeType type;
} SShowCompactsStmt;
typedef struct SShowCompactDetailsStmt {
ENodeType type;
SNode* pCompactId;
} SShowCompactDetailsStmt;
typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType; typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType;
typedef struct SIndexOptions { typedef struct SIndexOptions {

7
include/libs/stream/streamState.h
View File

@ -50,7 +50,7 @@ void streamStateSetNumber(SStreamState* pState, int32_t number);
int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
//session window // session window
int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen);
int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen);
int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
@ -65,7 +65,7 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess
SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key); SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key); SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key);
//state window // state window
int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
@ -96,6 +96,9 @@ int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char*
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal); int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal);
void streamStateReloadInfo(SStreamState* pState, TSKEY ts); void streamStateReloadInfo(SStreamState* pState, TSKEY ts);
void streamStateCopyBackend(SStreamState* src, SStreamState* dst);
SStreamStateCur* createStreamStateCursor(); SStreamStateCur* createStreamStateCursor();
/***compare func **/ /***compare func **/

4
include/libs/sync/sync.h
View File

@ -33,11 +33,12 @@ extern "C" {
#define SYNC_MAX_PROGRESS_WAIT_MS 4000 #define SYNC_MAX_PROGRESS_WAIT_MS 4000
#define SYNC_MAX_START_TIME_RANGE_MS (1000 * 20) #define SYNC_MAX_START_TIME_RANGE_MS (1000 * 20)
#define SYNC_MAX_RECV_TIME_RANGE_MS 1200 #define SYNC_MAX_RECV_TIME_RANGE_MS 1200
#define SYNC_DEL_WAL_MS (1000 * 60)
#define SYNC_ADD_QUORUM_COUNT 3 #define SYNC_ADD_QUORUM_COUNT 3
#define SYNC_VNODE_LOG_RETENTION (TSDB_SYNC_LOG_BUFFER_RETENTION + 1) #define SYNC_VNODE_LOG_RETENTION (TSDB_SYNC_LOG_BUFFER_RETENTION + 1)
#define SNAPSHOT_WAIT_MS 1000 * 5 #define SNAPSHOT_WAIT_MS 1000 * 5
#define SYNC_WAL_LOG_RETENTION_SIZE (8LL * 1024 * 1024 * 1024)
#define SYNC_MAX_RETRY_BACKOFF 5 #define SYNC_MAX_RETRY_BACKOFF 5
#define SYNC_LOG_REPL_RETRY_WAIT_MS 100 #define SYNC_LOG_REPL_RETRY_WAIT_MS 100
#define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000
@ -219,6 +220,7 @@ typedef struct SSyncLogStore {
SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore);
SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore);
SyncIndex (*syncLogIndexRetention)(struct SSyncLogStore* pLogStore, int64_t bytes);
SyncTerm (*syncLogLastTerm)(struct SSyncLogStore* pLogStore); SyncTerm (*syncLogLastTerm)(struct SSyncLogStore* pLogStore);
int32_t (*syncLogAppendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, bool forcSync); int32_t (*syncLogAppendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, bool forcSync);

1
include/libs/wal/wal.h
View File

@ -225,6 +225,7 @@ bool walIsEmpty(SWal *);
int64_t walGetFirstVer(SWal *); int64_t walGetFirstVer(SWal *);
int64_t walGetSnapshotVer(SWal *); int64_t walGetSnapshotVer(SWal *);
int64_t walGetLastVer(SWal *); int64_t walGetLastVer(SWal *);
int64_t walGetVerRetention(SWal *pWal, int64_t bytes);
int64_t walGetCommittedVer(SWal *); int64_t walGetCommittedVer(SWal *);
int64_t walGetAppliedVer(SWal *); int64_t walGetAppliedVer(SWal *);

7
include/os/osString.h
View File

@ -47,18 +47,13 @@ typedef int32_t TdUcs4;
#define strtof STR_TO_F_FUNC_TAOS_FORBID #define strtof STR_TO_F_FUNC_TAOS_FORBID
#endif #endif
#ifdef WINDOWS
#define tstrdup(str) _strdup(str)
#else
#define tstrdup(str) strdup(str)
#endif
#define tstrncpy(dst, src, size) \ #define tstrncpy(dst, src, size) \
do { \ do { \
strncpy((dst), (src), (size)); \ strncpy((dst), (src), (size)); \
(dst)[(size)-1] = 0; \ (dst)[(size)-1] = 0; \
} while (0) } while (0)
char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4); int32_t taosUcs4len(TdUcs4 *ucs4);
int64_t taosStr2int64(const char *str); int64_t taosStr2int64(const char *str);

3
include/util/taoserror.h
View File

@ -427,6 +427,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_VIEW_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x04A0) #define TSDB_CODE_MND_VIEW_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x04A0)
#define TSDB_CODE_MND_VIEW_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x04A1) #define TSDB_CODE_MND_VIEW_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x04A1)
//mnode-compact
#define TSDB_CODE_MND_INVALID_COMPACT_ID TAOS_DEF_ERROR_CODE(0, 0x04B1)
// vnode // vnode
// #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x

4
source/client/inc/clientInt.h
View File

@ -205,7 +205,7 @@ typedef struct SRequestSendRecvBody {
__taos_async_fn_t queryFp; __taos_async_fn_t queryFp;
__taos_async_fn_t fetchFp; __taos_async_fn_t fetchFp;
EQueryExecMode execMode; EQueryExecMode execMode;
void* param; void* interParam;
SDataBuf requestMsg; SDataBuf requestMsg;
int64_t queryJob; // query job, created according to sql query DAG. int64_t queryJob; // query job, created according to sql query DAG.
int32_t subplanNum; int32_t subplanNum;
@ -287,6 +287,7 @@ typedef struct SRequestObj {
typedef struct SSyncQueryParam { typedef struct SSyncQueryParam {
tsem_t sem; tsem_t sem;
SRequestObj* pRequest; SRequestObj* pRequest;
void* userParam;
} SSyncQueryParam; } SSyncQueryParam;
void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
@ -420,6 +421,7 @@ int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj
int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce); int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce);
void returnToUser(SRequestObj* pRequest); void returnToUser(SRequestObj* pRequest);
void stopAllQueries(SRequestObj *pRequest); void stopAllQueries(SRequestObj *pRequest);
void doRequestCallback(SRequestObj* pRequest, int32_t code);
void freeQueryParam(SSyncQueryParam* param); void freeQueryParam(SSyncQueryParam* param);
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE

17
source/client/src/clientEnv.c
View File

@ -316,6 +316,15 @@ void *createRequest(uint64_t connId, int32_t type, int64_t reqid) {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL; return NULL;
} }
SSyncQueryParam *interParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
if (interParam == NULL) {
doDestroyRequest(pRequest);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
tsem_init(&interParam->sem, 0, 0);
interParam->pRequest = pRequest;
pRequest->body.interParam = interParam;
pRequest->resType = RES_TYPE__QUERY; pRequest->resType = RES_TYPE__QUERY;
pRequest->requestId = reqid == 0 ? generateRequestId() : reqid; pRequest->requestId = reqid == 0 ? generateRequestId() : reqid;
@ -437,12 +446,10 @@ void doDestroyRequest(void *p) {
deregisterRequest(pRequest); deregisterRequest(pRequest);
} }
if (pRequest->syncQuery) { if (pRequest->body.interParam) {
if (pRequest->body.param) { tsem_destroy(&((SSyncQueryParam *)pRequest->body.interParam)->sem);
tsem_destroy(&((SSyncQueryParam *)pRequest->body.param)->sem);
}
taosMemoryFree(pRequest->body.param);
} }
taosMemoryFree(pRequest->body.interParam);
qDestroyQuery(pRequest->pQuery); qDestroyQuery(pRequest->pQuery);
nodesDestroyAllocator(pRequest->allocatorRefId); nodesDestroyAllocator(pRequest->allocatorRefId);

92
source/client/src/clientImpl.c
View File

@ -196,21 +196,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
(*pRequest)->sqlLen = sqlLen; (*pRequest)->sqlLen = sqlLen;
(*pRequest)->validateOnly = validateSql; (*pRequest)->validateOnly = validateSql;
SSyncQueryParam* newpParam = NULL; ((SSyncQueryParam*)(*pRequest)->body.interParam)->userParam = param;
if (param == NULL) {
newpParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
if (newpParam == NULL) {
destroyRequest(*pRequest);
*pRequest = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
tsem_init(&newpParam->sem, 0, 0);
newpParam->pRequest = (*pRequest);
param = newpParam;
}
(*pRequest)->body.param = param;
STscObj* pTscObj = (*pRequest)->pTscObj; STscObj* pTscObj = (*pRequest)->pTscObj;
int32_t err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, int32_t err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
@ -218,7 +204,6 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
if (err) { if (err) {
tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s", tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
(*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
freeQueryParam(newpParam);
destroyRequest(*pRequest); destroyRequest(*pRequest);
*pRequest = NULL; *pRequest = NULL;
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -230,7 +215,6 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) { nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) {
tscError("%" PRId64 " failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s", tscError("%" PRId64 " failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
(*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
freeQueryParam(newpParam);
destroyRequest(*pRequest); destroyRequest(*pRequest);
*pRequest = NULL; *pRequest = NULL;
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -336,7 +320,7 @@ static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { return pRequest->pTscOb
void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
SRetrieveTableRsp* pRsp = NULL; SRetrieveTableRsp* pRsp = NULL;
if (pRequest->validateOnly) { if (pRequest->validateOnly) {
pRequest->body.queryFp(pRequest->body.param, pRequest, 0); doRequestCallback(pRequest, 0);
return; return;
} }
@ -358,18 +342,18 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
pRequest->requestId); pRequest->requestId);
} }
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} }
int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
if (pRequest->validateOnly) { if (pRequest->validateOnly) {
pRequest->body.queryFp(pRequest->body.param, pRequest, 0); doRequestCallback(pRequest, 0);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// drop table if exists not_exists_table // drop table if exists not_exists_table
if (NULL == pQuery->pCmdMsg) { if (NULL == pQuery->pCmdMsg) {
pRequest->body.queryFp(pRequest->body.param, pRequest, 0); doRequestCallback(pRequest, 0);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -384,7 +368,7 @@ int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
int64_t transporterId = 0; int64_t transporterId = 0;
int32_t code = asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg); int32_t code = asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg);
if (code) { if (code) {
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} }
return code; return code;
} }
@ -913,7 +897,7 @@ void continuePostSubQuery(SRequestObj* pRequest, TAOS_ROW row) {
void returnToUser(SRequestObj* pRequest) { void returnToUser(SRequestObj* pRequest) {
if (pRequest->relation.userRefId == pRequest->self || 0 == pRequest->relation.userRefId) { if (pRequest->relation.userRefId == pRequest->self || 0 == pRequest->relation.userRefId) {
// return to client // return to client
pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code); doRequestCallback(pRequest, pRequest->code);
return; return;
} }
@ -921,7 +905,7 @@ void returnToUser(SRequestObj* pRequest) {
if (pUserReq) { if (pUserReq) {
pUserReq->code = pRequest->code; pUserReq->code = pRequest->code;
// return to client // return to client
pUserReq->body.queryFp(pUserReq->body.param, pUserReq, pUserReq->code); doRequestCallback(pUserReq, pUserReq->code);
releaseRequest(pRequest->relation.userRefId); releaseRequest(pRequest->relation.userRefId);
return; return;
} else { } else {
@ -1031,7 +1015,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
pRequest->pWrapper = NULL; pRequest->pWrapper = NULL;
// return to client // return to client
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} }
} }
@ -1186,7 +1170,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
pRequest->code = terrno; pRequest->code = terrno;
} }
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} }
// todo not to be released here // todo not to be released here
@ -1199,7 +1183,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
int32_t code = 0; int32_t code = 0;
if (pRequest->parseOnly) { if (pRequest->parseOnly) {
pRequest->body.queryFp(pRequest->body.param, pRequest, 0); doRequestCallback(pRequest, 0);
return; return;
} }
@ -1233,11 +1217,11 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
} }
case QUERY_EXEC_MODE_EMPTY_RESULT: case QUERY_EXEC_MODE_EMPTY_RESULT:
pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
pRequest->body.queryFp(pRequest->body.param, pRequest, 0); doRequestCallback(pRequest, 0);
break; break;
default: default:
tscError("0x%" PRIx64 " invalid execMode %d", pRequest->self, pQuery->execMode); tscError("0x%" PRIx64 " invalid execMode %d", pRequest->self, pQuery->execMode);
pRequest->body.queryFp(pRequest->body.param, pRequest, -1); doRequestCallback(pRequest, -1);
break; break;
} }
} }
@ -1703,8 +1687,8 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
} }
static void syncFetchFn(void* param, TAOS_RES* res, int32_t numOfRows) { static void syncFetchFn(void* param, TAOS_RES* res, int32_t numOfRows) {
SSyncQueryParam* pParam = param; tsem_t* sem = param;
tsem_post(&pParam->sem); tsem_post(sem);
} }
void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) { void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) {
@ -1722,10 +1706,11 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU
// convert ucs4 to native multi-bytes string // convert ucs4 to native multi-bytes string
pResultInfo->convertUcs4 = convertUcs4; pResultInfo->convertUcs4 = convertUcs4;
tsem_t sem;
SSyncQueryParam* pParam = pRequest->body.param; tsem_init(&sem, 0, 0);
taos_fetch_rows_a(pRequest, syncFetchFn, pParam); taos_fetch_rows_a(pRequest, syncFetchFn, &sem);
tsem_wait(&pParam->sem); tsem_wait(&sem);
tsem_destroy(&sem);
} }
if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) { if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) {
@ -2492,6 +2477,10 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
tscDebug("taos_query start with sql:%s", sql); tscDebug("taos_query start with sql:%s", sql);
SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
if (NULL == param) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
tsem_init(&param->sem, 0, 0); tsem_init(&param->sem, 0, 0);
taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly); taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly);
@ -2501,9 +2490,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
if (param->pRequest != NULL) { if (param->pRequest != NULL) {
param->pRequest->syncQuery = true; param->pRequest->syncQuery = true;
pRequest = param->pRequest; pRequest = param->pRequest;
} else {
taosMemoryFree(param);
} }
taosMemoryFree(param);
tscDebug("taos_query end with sql:%s", sql); tscDebug("taos_query end with sql:%s", sql);
@ -2517,19 +2505,20 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly,
} }
SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
if (param == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
tsem_init(&param->sem, 0, 0); tsem_init(&param->sem, 0, 0);
taosAsyncQueryImplWithReqid(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, reqid); taosAsyncQueryImplWithReqid(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, reqid);
tsem_wait(&param->sem); tsem_wait(&param->sem);
SRequestObj* pRequest = NULL; SRequestObj* pRequest = NULL;
if (param->pRequest != NULL) { if (param->pRequest != NULL) {
param->pRequest->syncQuery = true; param->pRequest->syncQuery = true;
pRequest = param->pRequest; pRequest = param->pRequest;
} else {
taosMemoryFree(param);
} }
taosMemoryFree(param);
return pRequest; return pRequest;
} }
@ -2547,13 +2536,13 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pRequest->code = code; pRequest->code = code;
taosMemoryFreeClear(pResultInfo->pData); taosMemoryFreeClear(pResultInfo->pData);
pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, 0);
return; return;
} }
if (pRequest->code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pResultInfo->pData); taosMemoryFreeClear(pResultInfo->pData);
pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, 0);
return; return;
} }
@ -2574,20 +2563,12 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
atomic_add_fetch_64((int64_t*)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen); atomic_add_fetch_64((int64_t*)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen);
} }
pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, pResultInfo->numOfRows);
} }
void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param) { void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param) {
if (pRequest->syncQuery && pRequest->body.param != param) {
if (pRequest->body.param) {
tsem_destroy(&((SSyncQueryParam *)pRequest->body.param)->sem);
}
taosMemoryFree(pRequest->body.param);
pRequest->syncQuery = false;
}
pRequest->body.fetchFp = fp; pRequest->body.fetchFp = fp;
pRequest->body.param = param; ((SSyncQueryParam *)pRequest->body.interParam)->userParam = param;
SReqResultInfo* pResultInfo = &pRequest->body.resInfo; SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
@ -2625,6 +2606,10 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param
schedulerFetchRows(pRequest->body.queryJob, &req); schedulerFetchRows(pRequest->body.queryJob, &req);
} }
void doRequestCallback(SRequestObj* pRequest, int32_t code) {
pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code);
}
int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes) { int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes) {
#ifndef TD_ENTERPRISE #ifndef TD_ENTERPRISE
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -2633,4 +2618,3 @@ int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool pa
#endif #endif
} }

12
source/client/src/clientMain.c
View File

@ -1095,7 +1095,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c
pRequest->pWrapper = NULL; pRequest->pWrapper = NULL;
terrno = code; terrno = code;
pRequest->code = code; pRequest->code = code;
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} }
} }
@ -1112,7 +1112,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest)
pRequest->pWrapper = NULL; pRequest->pWrapper = NULL;
terrno = code; terrno = code;
pRequest->code = code; pRequest->code = code;
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} }
} }
@ -1210,7 +1210,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
terrno = code; terrno = code;
pRequest->code = code; pRequest->code = code;
tscDebug("call sync query cb with code: %s", tstrerror(code)); tscDebug("call sync query cb with code: %s", tstrerror(code));
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
return; return;
} }
@ -1242,7 +1242,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
terrno = code; terrno = code;
pRequest->code = code; pRequest->code = code;
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} }
} }
@ -1545,12 +1545,12 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, pRequest->body.param, NULL); code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, pRequest->body.interParam, NULL);
if (code) { if (code) {
goto _return; goto _return;
} }
SSyncQueryParam *pParam = pRequest->body.param; SSyncQueryParam *pParam = pRequest->body.interParam;
tsem_wait(&pParam->sem); tsem_wait(&pParam->sem);
_return: _return:

132
source/client/src/clientMsgHandler.c
View File

@ -16,6 +16,7 @@
#include "catalog.h" #include "catalog.h"
#include "clientInt.h" #include "clientInt.h"
#include "clientLog.h" #include "clientLog.h"
#include "cmdnodes.h"
#include "os.h" #include "os.h"
#include "query.h" #include "query.h"
#include "systable.h" #include "systable.h"
@ -41,7 +42,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
if (pRequest->body.queryFp != NULL) { if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} else { } else {
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
} }
@ -199,7 +200,7 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
} }
if (pRequest->body.queryFp) { if (pRequest->body.queryFp) {
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} else { } else {
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
} }
@ -235,7 +236,8 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
setErrno(pRequest, code); setErrno(pRequest, code);
if (pRequest->body.queryFp != NULL) { if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code); doRequestCallback(pRequest, pRequest->code);
} else { } else {
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
} }
@ -299,7 +301,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);
if (pRequest->body.queryFp != NULL) { if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code); doRequestCallback(pRequest, pRequest->code);
} else { } else {
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
} }
@ -343,7 +345,7 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) {
} }
} }
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} else { } else {
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
} }
@ -380,7 +382,7 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);
if (pRequest->body.queryFp != NULL) { if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} else { } else {
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
} }
@ -420,7 +422,7 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) {
} }
} }
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} else { } else {
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
} }
@ -534,13 +536,125 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);
if (pRequest->body.queryFp != NULL) { if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(pRequest->body.param, pRequest, code); doRequestCallback(pRequest, code);
} else { } else {
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
} }
return code; return code;
} }
static int32_t buildCompactDbBlock(SCompactDbRsp* pRsp, SSDataBlock** block) {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->info.hasVarCol = true;
pBlock->pDataBlock = taosArrayInit(COMPACT_DB_RESULT_COLS, sizeof(SColumnInfoData));
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = COMPACT_DB_RESULT_FIELD1_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_INT;
infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = COMPACT_DB_RESULT_FIELD3_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
blockDataEnsureCapacity(pBlock, 1);
SColumnInfoData* pResultCol = taosArrayGet(pBlock->pDataBlock, 0);
SColumnInfoData* pIdCol = taosArrayGet(pBlock->pDataBlock, 1);
SColumnInfoData* pReasonCol = taosArrayGet(pBlock->pDataBlock, 2);
char result[COMPACT_DB_RESULT_FIELD1_LEN] = {0};
char reason[COMPACT_DB_RESULT_FIELD3_LEN] = {0};
if (pRsp->bAccepted) {
STR_TO_VARSTR(result, "accepted");
colDataSetVal(pResultCol, 0, result, false);
colDataSetVal(pIdCol, 0, (void*)&pRsp->compactId, false);
STR_TO_VARSTR(reason, "success");
colDataSetVal(pReasonCol, 0, reason, false);
} else {
STR_TO_VARSTR(result, "rejected");
colDataSetVal(pResultCol, 0, result, false);
colDataSetNULL(pIdCol, 0);
STR_TO_VARSTR(reason, "compaction is ongoing");
colDataSetVal(pReasonCol, 0, reason, false);
}
pBlock->info.rows = 1;
*block = pBlock;
return TSDB_CODE_SUCCESS;
}
static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetrieveTableRsp** pRsp) {
SSDataBlock* pBlock = NULL;
int32_t code = buildCompactDbBlock(pCompactDb, &pBlock);
if (code) {
return code;
}
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
*pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) {
blockDataDestroy(pBlock);
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pRsp)->useconds = 0;
(*pRsp)->completed = 1;
(*pRsp)->precision = 0;
(*pRsp)->compressed = 0;
(*pRsp)->compLen = 0;
(*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
(*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS);
int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS);
blockDataDestroy(pBlock);
if (len != rspSize - sizeof(SRetrieveTableRsp)) {
uError("buildRetriveTableRspForCompactDb error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len,
(uint64_t)(rspSize - sizeof(SRetrieveTableRsp)));
return TSDB_CODE_TSC_INVALID_INPUT;
}
return TSDB_CODE_SUCCESS;
}
int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
SRequestObj* pRequest = param;
if (code != TSDB_CODE_SUCCESS) {
setErrno(pRequest, code);
} else {
SCompactDbRsp rsp = {0};
SRetrieveTableRsp* pRes = NULL;
code = tDeserializeSCompactDbRsp(pMsg->pData, pMsg->len, &rsp);
if (TSDB_CODE_SUCCESS == code) {
code = buildRetriveTableRspForCompactDb(&rsp, &pRes);
}
if (TSDB_CODE_SUCCESS == code) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true);
}
if (code != 0) {
taosMemoryFree(pRes);
}
}
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code);
} else {
tsem_post(&pRequest->body.rspSem);
}
return code;
}
__async_send_cb_fn_t getMsgRspHandle(int32_t msgType) { __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
switch (msgType) { switch (msgType) {
case TDMT_MND_CONNECT: case TDMT_MND_CONNECT:
@ -557,6 +671,8 @@ __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
return processAlterStbRsp; return processAlterStbRsp;
case TDMT_MND_SHOW_VARIABLES: case TDMT_MND_SHOW_VARIABLES:
return processShowVariablesRsp; return processShowVariablesRsp;
case TDMT_MND_COMPACT_DB:
return processCompactDbRsp;
default: default:
return genericRspCallback; return genericRspCallback;
} }

578
source/common/src/cos.c
View File

@ -1,6 +1,8 @@
#define ALLOW_FORBID_FUNC #define ALLOW_FORBID_FUNC
#include "cos.h" #include "cos.h"
#include "cos_cp.h"
#include "tdef.h"
extern char tsS3Endpoint[]; extern char tsS3Endpoint[];
extern char tsS3AccessKeyId[]; extern char tsS3AccessKeyId[];
@ -86,7 +88,7 @@ typedef struct {
char err_msg[128]; char err_msg[128];
S3Status status; S3Status status;
uint64_t content_length; uint64_t content_length;
char * buf; char *buf;
int64_t buf_pos; int64_t buf_pos;
} TS3SizeCBD; } TS3SizeCBD;
@ -270,7 +272,7 @@ typedef struct list_parts_callback_data {
typedef struct MultipartPartData { typedef struct MultipartPartData {
put_object_callback_data put_object_data; put_object_callback_data put_object_data;
int seq; int seq;
UploadManager * manager; UploadManager *manager;
} MultipartPartData; } MultipartPartData;
static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) {
@ -317,12 +319,23 @@ S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properti
MultipartPartData *data = (MultipartPartData *)callbackData; MultipartPartData *data = (MultipartPartData *)callbackData;
int seq = data->seq; int seq = data->seq;
const char * etag = properties->eTag; const char *etag = properties->eTag;
data->manager->etags[seq - 1] = strdup(etag); data->manager->etags[seq - 1] = strdup(etag);
data->manager->next_etags_pos = seq; data->manager->next_etags_pos = seq;
return S3StatusOK; return S3StatusOK;
} }
S3Status MultipartResponseProperiesCallbackWithCp(const S3ResponseProperties *properties, void *callbackData) {
responsePropertiesCallbackNull(properties, callbackData);
MultipartPartData *data = (MultipartPartData *)callbackData;
int seq = data->seq;
const char *etag = properties->eTag;
data->manager->etags[seq - 1] = strdup(etag);
// data->manager->next_etags_pos = seq;
return S3StatusOK;
}
static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) { static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) {
UploadManager *manager = (UploadManager *)callbackData; UploadManager *manager = (UploadManager *)callbackData;
int ret = 0; int ret = 0;
@ -446,37 +459,343 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan
return 0; return 0;
} }
*/ */
int32_t s3PutObjectFromFile2(const char *file, const char *object) {
int32_t code = 0; static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char const *object_name, int64_t size,
const char *key = object; S3PutProperties *put_prop, put_object_callback_data *data) {
// const char *uploadId = 0; int32_t code = 0;
const char * filename = 0; S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
&putObjectDataCallback};
do {
S3_put_object(bucket_context, object_name, size, put_prop, 0, 0, &putObjectHandler, data);
} while (S3_status_is_retryable(data->status) && should_retry());
if (data->status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
} else if (data->contentLength) {
uError("%s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data->contentLength);
code = TAOS_SYSTEM_ERROR(EIO);
}
return code;
}
static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name,
int64_t contentLength, S3PutProperties *put_prop,
put_object_callback_data *data) {
int32_t code = 0;
uint64_t totalContentLength = contentLength;
uint64_t todoContentLength = contentLength;
UploadManager manager = {0};
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3;
int totalSeq = (contentLength + chunk_size - 1) / chunk_size;
const int max_part_num = 10000;
if (totalSeq > max_part_num) {
chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
totalSeq = (contentLength + chunk_size - 1) / chunk_size;
}
MultipartPartData partData;
memset(&partData, 0, sizeof(MultipartPartData));
int partContentLength = 0;
S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
&initial_multipart_callback};
S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback},
&putObjectDataCallback};
S3MultipartCommitHandler commit_handler = {
{&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
manager.next_etags_pos = 0;
do {
S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
upload:
todoContentLength -= chunk_size * manager.next_etags_pos;
for (int seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
partData.manager = &manager;
partData.seq = seq;
if (partData.put_object_data.gb == NULL) {
partData.put_object_data = *data;
}
partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength);
// printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);
partData.put_object_data.contentLength = partContentLength;
partData.put_object_data.originalContentLength = partContentLength;
partData.put_object_data.totalContentLength = todoContentLength;
partData.put_object_data.totalOriginalContentLength = totalContentLength;
put_prop->md5 = 0;
do {
S3_upload_part(bucket_context, object_name, put_prop, &putObjectHandler, seq, manager.upload_id,
partContentLength, 0, timeoutMsG, &partData);
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
contentLength -= chunk_size;
todoContentLength -= chunk_size;
}
int i;
int size = 0;
size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
char buf[256];
int n;
for (i = 0; i < totalSeq; i++) {
if (!manager.etags[i]) {
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
n = snprintf(buf, sizeof(buf),
"<Part><PartNumber>%d</PartNumber>"
"<ETag>%s</ETag></Part>",
i + 1, manager.etags[i]);
size += growbuffer_append(&(manager.gb), buf, n);
}
size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
manager.remaining = size;
do {
S3_complete_multipart_upload(bucket_context, object_name, &commit_handler, manager.upload_id, manager.remaining, 0,
timeoutMsG, &manager);
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
clean:
if (manager.upload_id) {
taosMemoryFree(manager.upload_id);
}
for (i = 0; i < manager.next_etags_pos; i++) {
taosMemoryFree(manager.etags[i]);
}
growbuffer_destroy(manager.gb);
taosMemoryFree(manager.etags);
return code;
}
static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime,
char const *object_name, int64_t contentLength, S3PutProperties *put_prop,
put_object_callback_data *data) {
int32_t code = 0;
uint64_t totalContentLength = contentLength;
// uint64_t todoContentLength = contentLength;
UploadManager manager = {0};
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3;
int totalSeq = (contentLength + chunk_size - 1) / chunk_size;
const int max_part_num = 10000;
if (totalSeq > max_part_num) {
chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
totalSeq = (contentLength + chunk_size - 1) / chunk_size;
}
bool need_init_upload = true;
char file_cp_path[TSDB_FILENAME_LEN];
snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file);
SCheckpoint cp = {0};
cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
if (taosCheckExistFile(file_cp_path)) {
if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) {
manager.upload_id = strdup(cp.upload_id);
need_init_upload = false;
} else {
cos_cp_remove(file_cp_path);
}
}
if (need_init_upload) {
S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
&initial_multipart_callback};
do {
S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size);
}
if (cos_cp_open(file_cp_path, &cp)) {
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
int part_num = 0;
int64_t consume_bytes = 0;
// SCheckpointPart *parts = taosMemoryCalloc(cp.part_num, sizeof(SCheckpointPart));
// cos_cp_get_undo_parts(&cp, &part_num, parts, &consume_bytes);
MultipartPartData partData;
memset(&partData, 0, sizeof(MultipartPartData));
int partContentLength = 0;
S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallbackWithCp, &responseCompleteCallback},
&putObjectDataCallback};
S3MultipartCommitHandler commit_handler = {
{&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
manager.next_etags_pos = 0;
upload:
// todoContentLength -= chunk_size * manager.next_etags_pos;
for (int i = 0; i < cp.part_num; ++i) {
if (cp.parts[i].completed) {
continue;
}
if (i > 0 && cp.parts[i - 1].completed) {
if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto clean;
}
}
int seq = cp.parts[i].index + 1;
partData.manager = &manager;
partData.seq = seq;
if (partData.put_object_data.gb == NULL) {
partData.put_object_data = *data;
}
partContentLength = cp.parts[i].size;
partData.put_object_data.contentLength = partContentLength;
partData.put_object_data.originalContentLength = partContentLength;
// partData.put_object_data.totalContentLength = todoContentLength;
partData.put_object_data.totalOriginalContentLength = totalContentLength;
put_prop->md5 = 0;
do {
S3_upload_part(bucket_context, object_name, put_prop, &putObjectHandler, seq, manager.upload_id,
partContentLength, 0, timeoutMsG, &partData);
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
//(void)cos_cp_dump(&cp);
goto clean;
}
if (!manager.etags[seq - 1]) {
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0);
(void)cos_cp_dump(&cp);
contentLength -= chunk_size;
// todoContentLength -= chunk_size;
}
cos_cp_close(cp.thefile);
cp.thefile = 0;
int size = 0;
size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
char buf[256];
int n;
for (int i = 0; i < cp.part_num; ++i) {
n = snprintf(buf, sizeof(buf),
"<Part><PartNumber>%d</PartNumber>"
"<ETag>%s</ETag></Part>",
// i + 1, manager.etags[i]);
cp.parts[i].index + 1, cp.parts[i].etag);
size += growbuffer_append(&(manager.gb), buf, n);
}
size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
manager.remaining = size;
do {
S3_complete_multipart_upload(bucket_context, object_name, &commit_handler, manager.upload_id, manager.remaining, 0,
timeoutMsG, &manager);
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
cos_cp_remove(file_cp_path);
clean:
/*
if (parts) {
taosMemoryFree(parts);
}
*/
if (cp.thefile) {
cos_cp_close(cp.thefile);
}
if (cp.parts) {
taosMemoryFree(cp.parts);
}
if (manager.upload_id) {
taosMemoryFree(manager.upload_id);
}
for (int i = 0; i < cp.part_num; ++i) {
if (manager.etags[i]) {
taosMemoryFree(manager.etags[i]);
}
}
taosMemoryFree(manager.etags);
growbuffer_destroy(manager.gb);
return code;
}
int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) {
int32_t code = 0;
int32_t lmtime = 0;
const char *filename = 0;
uint64_t contentLength = 0; uint64_t contentLength = 0;
const char * cacheControl = 0, *contentType = 0, *md5 = 0; const char *cacheControl = 0, *contentType = 0, *md5 = 0;
const char * contentDispositionFilename = 0, *contentEncoding = 0; const char *contentDispositionFilename = 0, *contentEncoding = 0;
int64_t expires = -1; int64_t expires = -1;
S3CannedAcl cannedAcl = S3CannedAclPrivate; S3CannedAcl cannedAcl = S3CannedAclPrivate;
int metaPropertiesCount = 0; int metaPropertiesCount = 0;
S3NameValue metaProperties[S3_MAX_METADATA_COUNT]; S3NameValue metaProperties[S3_MAX_METADATA_COUNT];
char useServerSideEncryption = 0; char useServerSideEncryption = 0;
put_object_callback_data data = {0}; put_object_callback_data data = {0};
// int noStatus = 0;
// data.infile = 0; if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
// data.gb = 0;
// data.infileFD = NULL;
// data.noStatus = noStatus;
// uError("ERROR: %s stat file %s: ", __func__, file);
if (taosStatFile(file, &contentLength, NULL, NULL) < 0) {
uError("ERROR: %s Failed to stat file %s: ", __func__, file);
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code; return code;
} }
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
uError("ERROR: %s Failed to open file %s: ", __func__, file);
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code; return code;
} }
@ -493,143 +812,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
metaProperties, useServerSideEncryption}; metaProperties, useServerSideEncryption};
if (contentLength <= MULTIPART_CHUNK_SIZE) { if (contentLength <= MULTIPART_CHUNK_SIZE) {
S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, code = s3PutObjectFromFileSimple(&bucketContext, object_name, contentLength, &putProperties, &data);
&putObjectDataCallback};
do {
S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data);
} while (S3_status_is_retryable(data.status) && should_retry());
if (data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
} else if (data.contentLength) {
uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
(unsigned long long)data.contentLength);
code = TAOS_SYSTEM_ERROR(EIO);
}
} else { } else {
uint64_t totalContentLength = contentLength; if (withcp) {
uint64_t todoContentLength = contentLength; code = s3PutObjectFromFileWithCp(&bucketContext, file, lmtime, object_name, contentLength, &putProperties, &data);
UploadManager manager = {0}; } else {
// manager.upload_id = 0; code = s3PutObjectFromFileWithoutCp(&bucketContext, object_name, contentLength, &putProperties, &data);
// manager.gb = 0;
// div round up
int seq;
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3;
int totalSeq = (contentLength + chunk_size - 1) / chunk_size;
const int max_part_num = 10000;
if (totalSeq > max_part_num) {
chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
totalSeq = (contentLength + chunk_size - 1) / chunk_size;
} }
MultipartPartData partData;
memset(&partData, 0, sizeof(MultipartPartData));
int partContentLength = 0;
S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
&initial_multipart_callback};
S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback},
&putObjectDataCallback};
S3MultipartCommitHandler commit_handler = {
{&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
manager.next_etags_pos = 0;
/*
if (uploadId) {
manager.upload_id = strdup(uploadId);
manager.remaining = contentLength;
if (!try_get_parts_info(tsS3BucketName, key, &manager)) {
fseek(data.infile, -(manager.remaining), 2);
taosLSeekFile(data.infileFD, -(manager.remaining), SEEK_END);
contentLength = manager.remaining;
goto upload;
} else {
goto clean;
}
}
*/
do {
S3_initiate_multipart(&bucketContext, key, 0, &handler, 0, timeoutMsG, &manager);
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
upload:
todoContentLength -= chunk_size * manager.next_etags_pos;
for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
partData.manager = &manager;
partData.seq = seq;
if (partData.put_object_data.gb == NULL) {
partData.put_object_data = data;
}
partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength);
// printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);
partData.put_object_data.contentLength = partContentLength;
partData.put_object_data.originalContentLength = partContentLength;
partData.put_object_data.totalContentLength = todoContentLength;
partData.put_object_data.totalOriginalContentLength = totalContentLength;
putProperties.md5 = 0;
do {
S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id,
partContentLength, 0, timeoutMsG, &partData);
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
contentLength -= chunk_size;
todoContentLength -= chunk_size;
}
int i;
int size = 0;
size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
char buf[256];
int n;
for (i = 0; i < totalSeq; i++) {
if (!manager.etags[i]) {
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
n = snprintf(buf, sizeof(buf),
"<Part><PartNumber>%d</PartNumber>"
"<ETag>%s</ETag></Part>",
i + 1, manager.etags[i]);
size += growbuffer_append(&(manager.gb), buf, n);
}
size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
manager.remaining = size;
do {
S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, 0,
timeoutMsG, &manager);
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
clean:
if (manager.upload_id) {
taosMemoryFree(manager.upload_id);
}
for (i = 0; i < manager.next_etags_pos; i++) {
taosMemoryFree(manager.etags[i]);
}
growbuffer_destroy(manager.gb);
taosMemoryFree(manager.etags);
} }
if (data.infileFD) { if (data.infileFD) {
@ -648,7 +837,7 @@ typedef struct list_bucket_callback_data {
char nextMarker[1024]; char nextMarker[1024];
int keyCount; int keyCount;
int allDetails; int allDetails;
SArray * objectArray; SArray *objectArray;
} list_bucket_callback_data; } list_bucket_callback_data;
static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount, static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount,
@ -693,11 +882,11 @@ static void s3FreeObjectKey(void *pItem) {
static SArray *getListByPrefix(const char *prefix) { static SArray *getListByPrefix(const char *prefix) {
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
0, awsRegionG}; 0, awsRegionG};
S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
&listBucketCallback}; &listBucketCallback};
const char * marker = 0, *delimiter = 0; const char *marker = 0, *delimiter = 0;
int maxkeys = 0, allDetails = 0; int maxkeys = 0, allDetails = 0;
list_bucket_callback_data data; list_bucket_callback_data data;
data.objectArray = taosArrayInit(32, sizeof(void *)); data.objectArray = taosArrayInit(32, sizeof(void *));
@ -738,7 +927,7 @@ static SArray *getListByPrefix(const char *prefix) {
void s3DeleteObjects(const char *object_name[], int nobject) { void s3DeleteObjects(const char *object_name[], int nobject) {
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
0, awsRegionG}; 0, awsRegionG};
S3ResponseHandler responseHandler = {0, &responseCompleteCallback}; S3ResponseHandler responseHandler = {0, &responseCompleteCallback};
for (int i = 0; i < nobject; ++i) { for (int i = 0; i < nobject; ++i) {
@ -789,7 +978,7 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
const char *ifMatch = 0, *ifNotMatch = 0; const char *ifMatch = 0, *ifNotMatch = 0;
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
0, awsRegionG}; 0, awsRegionG};
S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch};
S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
&getObjectDataCallback}; &getObjectDataCallback};
@ -827,7 +1016,7 @@ int32_t s3GetObjectToFile(const char *object_name, char *fileName) {
const char *ifMatch = 0, *ifNotMatch = 0; const char *ifMatch = 0, *ifNotMatch = 0;
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
0, awsRegionG}; 0, awsRegionG};
S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch};
S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
&getObjectCallback}; &getObjectCallback};
@ -858,7 +1047,7 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
if (objectArray == NULL) return -1; if (objectArray == NULL) return -1;
for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) { for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) {
char * object = taosArrayGetP(objectArray, i); char *object = taosArrayGetP(objectArray, i);
const char *tmp = strchr(object, '/'); const char *tmp = strchr(object, '/');
tmp = (tmp == NULL) ? object : tmp + 1; tmp = (tmp == NULL) ? object : tmp + 1;
char fileName[PATH_MAX] = {0}; char fileName[PATH_MAX] = {0};
@ -949,12 +1138,12 @@ static void s3InitRequestOptions(cos_request_options_t *options, int is_cname) {
int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) { int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
int32_t code = 0; int32_t code = 0;
cos_pool_t * p = NULL; cos_pool_t *p = NULL;
int is_cname = 0; int is_cname = 0;
cos_status_t * s = NULL; cos_status_t *s = NULL;
cos_request_options_t *options = NULL; cos_request_options_t *options = NULL;
cos_string_t bucket, object, file; cos_string_t bucket, object, file;
cos_table_t * resp_headers; cos_table_t *resp_headers;
// int traffic_limit = 0; // int traffic_limit = 0;
cos_pool_create(&p, NULL); cos_pool_create(&p, NULL);
@ -983,18 +1172,19 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
return code; return code;
} }
int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) { int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) {
int32_t code = 0; int32_t code = 0;
cos_pool_t * p = NULL; cos_pool_t *p = NULL;
int is_cname = 0; int is_cname = 0;
cos_status_t * s = NULL; cos_status_t *s = NULL;
cos_request_options_t * options = NULL; cos_request_options_t *options = NULL;
cos_string_t bucket, object, file; cos_string_t bucket, object, file;
cos_table_t * resp_headers; cos_table_t *resp_headers;
int traffic_limit = 0; int traffic_limit = 0;
cos_table_t * headers = NULL; cos_table_t *headers = NULL;
cos_resumable_clt_params_t *clt_params = NULL; cos_resumable_clt_params_t *clt_params = NULL;
(void)withcp;
cos_pool_create(&p, NULL); cos_pool_create(&p, NULL);
options = cos_request_options_create(p); options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname); s3InitRequestOptions(options, is_cname);
@ -1025,11 +1215,11 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) {
} }
void s3DeleteObjectsByPrefix(const char *prefix_str) { void s3DeleteObjectsByPrefix(const char *prefix_str) {
cos_pool_t * p = NULL; cos_pool_t *p = NULL;
cos_request_options_t *options = NULL; cos_request_options_t *options = NULL;
int is_cname = 0; int is_cname = 0;
cos_string_t bucket; cos_string_t bucket;
cos_status_t * s = NULL; cos_status_t *s = NULL;
cos_string_t prefix; cos_string_t prefix;
cos_pool_create(&p, NULL); cos_pool_create(&p, NULL);
@ -1044,10 +1234,10 @@ void s3DeleteObjectsByPrefix(const char *prefix_str) {
} }
void s3DeleteObjects(const char *object_name[], int nobject) { void s3DeleteObjects(const char *object_name[], int nobject) {
cos_pool_t * p = NULL; cos_pool_t *p = NULL;
int is_cname = 0; int is_cname = 0;
cos_string_t bucket; cos_string_t bucket;
cos_table_t * resp_headers = NULL; cos_table_t *resp_headers = NULL;
cos_request_options_t *options = NULL; cos_request_options_t *options = NULL;
cos_list_t object_list; cos_list_t object_list;
cos_list_t deleted_object_list; cos_list_t deleted_object_list;
@ -1081,14 +1271,14 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
bool s3Exists(const char *object_name) { bool s3Exists(const char *object_name) {
bool ret = false; bool ret = false;
cos_pool_t * p = NULL; cos_pool_t *p = NULL;
int is_cname = 0; int is_cname = 0;
cos_status_t * s = NULL; cos_status_t *s = NULL;
cos_request_options_t * options = NULL; cos_request_options_t *options = NULL;
cos_string_t bucket; cos_string_t bucket;
cos_string_t object; cos_string_t object;
cos_table_t * resp_headers; cos_table_t *resp_headers;
cos_table_t * headers = NULL; cos_table_t *headers = NULL;
cos_object_exist_status_e object_exist; cos_object_exist_status_e object_exist;
cos_pool_create(&p, NULL); cos_pool_create(&p, NULL);
@ -1115,15 +1305,15 @@ bool s3Exists(const char *object_name) {
bool s3Get(const char *object_name, const char *path) { bool s3Get(const char *object_name, const char *path) {
bool ret = false; bool ret = false;
cos_pool_t * p = NULL; cos_pool_t *p = NULL;
int is_cname = 0; int is_cname = 0;
cos_status_t * s = NULL; cos_status_t *s = NULL;
cos_request_options_t *options = NULL; cos_request_options_t *options = NULL;
cos_string_t bucket; cos_string_t bucket;
cos_string_t object; cos_string_t object;
cos_string_t file; cos_string_t file;
cos_table_t * resp_headers = NULL; cos_table_t *resp_headers = NULL;
cos_table_t * headers = NULL; cos_table_t *headers = NULL;
int traffic_limit = 0; int traffic_limit = 0;
//创建内存池 //创建内存池
@ -1159,15 +1349,15 @@ bool s3Get(const char *object_name, const char *path) {
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) { int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) {
(void)check; (void)check;
int32_t code = 0; int32_t code = 0;
cos_pool_t * p = NULL; cos_pool_t *p = NULL;
int is_cname = 0; int is_cname = 0;
cos_status_t * s = NULL; cos_status_t *s = NULL;
cos_request_options_t *options = NULL; cos_request_options_t *options = NULL;
cos_string_t bucket; cos_string_t bucket;
cos_string_t object; cos_string_t object;
cos_table_t * resp_headers; cos_table_t *resp_headers;
cos_table_t * headers = NULL; cos_table_t *headers = NULL;
cos_buf_t * content = NULL; cos_buf_t *content = NULL;
// cos_string_t file; // cos_string_t file;
// int traffic_limit = 0; // int traffic_limit = 0;
char range_buf[64]; char range_buf[64];
@ -1261,7 +1451,7 @@ void s3EvictCache(const char *path, long object_size) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
vError("failed to open %s since %s", dir_name, terrstr()); vError("failed to open %s since %s", dir_name, terrstr());
} }
SArray * evict_files = taosArrayInit(16, sizeof(SEvictFile)); SArray *evict_files = taosArrayInit(16, sizeof(SEvictFile));
tdbDirEntryPtr pDirEntry; tdbDirEntryPtr pDirEntry;
while ((pDirEntry = taosReadDir(pDir)) != NULL) { while ((pDirEntry = taosReadDir(pDir)) != NULL) {
char *name = taosGetDirEntryName(pDirEntry); char *name = taosGetDirEntryName(pDirEntry);
@ -1303,13 +1493,13 @@ void s3EvictCache(const char *path, long object_size) {
long s3Size(const char *object_name) { long s3Size(const char *object_name) {
long size = 0; long size = 0;
cos_pool_t * p = NULL; cos_pool_t *p = NULL;
int is_cname = 0; int is_cname = 0;
cos_status_t * s = NULL; cos_status_t *s = NULL;
cos_request_options_t *options = NULL; cos_request_options_t *options = NULL;
cos_string_t bucket; cos_string_t bucket;
cos_string_t object; cos_string_t object;
cos_table_t * resp_headers = NULL; cos_table_t *resp_headers = NULL;
//创建内存池 //创建内存池
cos_pool_create(&p, NULL); cos_pool_create(&p, NULL);
@ -1344,7 +1534,7 @@ long s3Size(const char *object_name) {
int32_t s3Init() { return 0; } int32_t s3Init() { return 0; }
void s3CleanUp() {} void s3CleanUp() {}
int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; } int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; } int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
void s3DeleteObjectsByPrefix(const char *prefix) {} void s3DeleteObjectsByPrefix(const char *prefix) {}
void s3DeleteObjects(const char *object_name[], int nobject) {} void s3DeleteObjects(const char *object_name[], int nobject) {}
bool s3Exists(const char *object_name) { return false; } bool s3Exists(const char *object_name) { return false; }

417
source/common/src/cos_cp.c Normal file
View File

@ -0,0 +1,417 @@
#define ALLOW_FORBID_FUNC
#include "cos_cp.h"
#include "cJSON.h"
#include "tutil.h"
int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) {
int32_t code = 0;
TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH);
if (!fd) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open %s", __func__, cp_path);
return code;
}
checkpoint->thefile = fd;
return code;
}
void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); }
void cos_cp_remove(char const* filepath) { taosRemoveFile(filepath); }
static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
int32_t code = 0;
cJSON const* item2 = NULL;
cJSON* json = cJSON_Parse(cp_body);
if (NULL == json) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to parse json", __func__);
goto _exit;
}
cJSON const* item = cJSON_GetObjectItem(json, "ver");
if (!cJSON_IsNumber(item) || item->valuedouble != 1) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to parse json ver: %f", __func__, item->valuedouble);
goto _exit;
}
item = cJSON_GetObjectItem(json, "type");
if (!cJSON_IsNumber(item)) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to parse json", __func__);
goto _exit;
}
cp->cp_type = item->valuedouble;
item = cJSON_GetObjectItem(json, "md5");
if (cJSON_IsString(item)) {
memcpy(cp->md5, item->valuestring, strlen(item->valuestring));
}
item = cJSON_GetObjectItem(json, "upload_id");
if (cJSON_IsString(item)) {
strncpy(cp->upload_id, item->valuestring, 128);
}
item2 = cJSON_GetObjectItem(json, "file");
if (cJSON_IsObject(item2)) {
item = cJSON_GetObjectItem(item2, "size");
if (cJSON_IsNumber(item)) {
cp->file_size = item->valuedouble;
}
item = cJSON_GetObjectItem(item2, "lastmodified");
if (cJSON_IsNumber(item)) {
cp->file_last_modified = item->valuedouble;
}
item = cJSON_GetObjectItem(item2, "path");
if (cJSON_IsString(item)) {
strncpy(cp->file_path, item->valuestring, TSDB_FILENAME_LEN);
}
item = cJSON_GetObjectItem(item2, "file_md5");
if (cJSON_IsString(item)) {
strncpy(cp->file_md5, item->valuestring, 64);
}
}
item2 = cJSON_GetObjectItem(json, "object");
if (cJSON_IsObject(item2)) {
item = cJSON_GetObjectItem(item2, "object_size");
if (cJSON_IsNumber(item)) {
cp->object_size = item->valuedouble;
}
item = cJSON_GetObjectItem(item2, "object_name");
if (cJSON_IsString(item)) {
strncpy(cp->object_name, item->valuestring, 128);
}
item = cJSON_GetObjectItem(item2, "object_last_modified");
if (cJSON_IsString(item)) {
strncpy(cp->object_last_modified, item->valuestring, 64);
}
item = cJSON_GetObjectItem(item2, "object_etag");
if (cJSON_IsString(item)) {
strncpy(cp->object_etag, item->valuestring, 128);
}
}
item2 = cJSON_GetObjectItem(json, "cpparts");
if (cJSON_IsObject(item2)) {
item = cJSON_GetObjectItem(item2, "number");
if (cJSON_IsNumber(item)) {
cp->part_num = item->valuedouble;
}
item = cJSON_GetObjectItem(item2, "size");
if (cJSON_IsNumber(item)) {
cp->part_size = item->valuedouble;
}
item2 = cJSON_GetObjectItem(item2, "parts");
if (cJSON_IsArray(item2) && cp->part_num > 0) {
cJSON_ArrayForEach(item, item2) {
cJSON const* item3 = cJSON_GetObjectItem(item, "index");
int32_t index = 0;
if (cJSON_IsNumber(item3)) {
index = item3->valuedouble;
cp->parts[index].index = index;
}
item3 = cJSON_GetObjectItem(item, "offset");
if (cJSON_IsNumber(item3)) {
cp->parts[index].offset = item3->valuedouble;
}
item3 = cJSON_GetObjectItem(item, "size");
if (cJSON_IsNumber(item3)) {
cp->parts[index].size = item3->valuedouble;
}
item3 = cJSON_GetObjectItem(item, "completed");
if (cJSON_IsNumber(item3)) {
cp->parts[index].completed = item3->valuedouble;
}
item3 = cJSON_GetObjectItem(item, "crc64");
if (cJSON_IsNumber(item3)) {
cp->parts[index].crc64 = item3->valuedouble;
}
item3 = cJSON_GetObjectItem(item, "etag");
if (cJSON_IsString(item3)) {
strncpy(cp->parts[index].etag, item3->valuestring, 128);
}
}
}
}
_exit:
if (json) cJSON_Delete(json);
if (cp_body) taosMemoryFree(cp_body);
return code;
}
int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
int32_t code = 0;
TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ);
if (!fd) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open %s", __func__, filepath);
goto _exit;
}
int64_t size = -1;
code = taosStatFile(filepath, &size, NULL, NULL);
if (code) {
uError("ERROR: %s Failed to stat %s", __func__, filepath);
goto _exit;
}
char* cp_body = taosMemoryMalloc(size + 1);
int64_t n = taosReadFile(fd, cp_body, size);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to read %s", __func__, filepath);
goto _exit;
} else if (n != size) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to read %s %" PRId64 "/%" PRId64, __func__, filepath, n, size);
goto _exit;
}
taosCloseFile(&fd);
cp_body[size] = '\0';
return cos_cp_parse_body(cp_body, checkpoint);
_exit:
if (fd) {
taosCloseFile(&fd);
}
if (cp_body) {
taosMemoryFree(cp_body);
}
return code;
}
static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) {
int32_t code = 0;
char* data = cJSON_PrintUnformatted(json);
if (NULL == data) {
return TSDB_CODE_OUT_OF_MEMORY;
}
TdFilePtr fp = checkpoint->thefile;
if (taosFtruncateFile(fp, 0) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
if (taosLSeekFile(fp, 0, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
if (taosWriteFile(fp, data, strlen(data)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
_exit:
taosMemoryFree(data);
return code;
}
int32_t cos_cp_dump(SCheckpoint* cp) {
int32_t code = 0;
int32_t lino = 0;
cJSON* ojson = NULL;
cJSON* json = cJSON_CreateObject();
if (!json) return TSDB_CODE_OUT_OF_MEMORY;
if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (COS_CP_TYPE_UPLOAD == cp->cp_type) {
ojson = cJSON_AddObjectToObject(json, "file");
if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
} else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) {
ojson = cJSON_AddObjectToObject(json, "object");
if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
}
ojson = cJSON_AddObjectToObject(json, "cpparts");
if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts");
if (!ajson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
for (int i = 0; i < cp->part_num; ++i) {
cJSON* item = cJSON_CreateObject();
if (!item) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
cJSON_AddItemToArray(ajson, item);
if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
}
code = cos_cp_save_json(json, cp);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
cJSON_Delete(json);
return code;
}
void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {}
void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64) {
checkpoint->parts[part_index].completed = 1;
strncpy(checkpoint->parts[part_index].etag, etag, 128);
checkpoint->parts[part_index].crc64 = crc64;
}
void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
char const* upload_id, int64_t part_size) {
int i = 0;
checkpoint->cp_type = COS_CP_TYPE_UPLOAD;
strncpy(checkpoint->file_path, filepath, TSDB_FILENAME_LEN);
checkpoint->file_size = size;
checkpoint->file_last_modified = mtime;
strncpy(checkpoint->upload_id, upload_id, 128);
checkpoint->part_size = part_size;
for (; i * part_size < size; i++) {
checkpoint->parts[i].index = i;
checkpoint->parts[i].offset = i * part_size;
checkpoint->parts[i].size = TMIN(part_size, (size - i * part_size));
checkpoint->parts[i].completed = 0;
checkpoint->parts[i].etag[0] = '\0';
}
checkpoint->part_num = i;
}
static bool cos_cp_verify_md5(SCheckpoint* cp) { return true; }
bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime) {
if (cos_cp_verify_md5(checkpoint) && checkpoint->file_size == size && checkpoint->file_last_modified == mtime) {
return true;
}
return false;
}

16
source/common/src/systable.c
View File

@ -334,6 +334,20 @@ static const SSysDbTableSchema userViewsSchema[] = {
// {.name = "column_list", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, // {.name = "column_list", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
}; };
static const SSysDbTableSchema userCompactsSchema[] = {
{.name = "compact_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysDbTableSchema userCompactsDetailSchema[] = {
{.name = "compact_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "number_fileset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysTableMeta infosMeta[] = { static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true}, {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
@ -362,6 +376,8 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true}, {TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true},
{TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), true}, {TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), true},
{TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false}, {TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false},
{TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false},
{TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false},
}; };
static const SSysDbTableSchema connectionsSchema[] = { static const SSysDbTableSchema connectionsSchema[] = {

25
source/common/src/tglobal.c
View File

@ -95,8 +95,9 @@ int32_t tsMonitorMaxLogs = 100;
bool tsMonitorComp = false; bool tsMonitorComp = false;
// audit // audit
bool tsEnableAudit = true; bool tsEnableAudit = true;
bool tsEnableAuditCreateTable = true; bool tsEnableAuditCreateTable = true;
int32_t tsAuditInterval = 5000;
// telem // telem
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
@ -247,6 +248,7 @@ int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch
// internal // internal
int32_t tsTransPullupInterval = 2; int32_t tsTransPullupInterval = 2;
int32_t tsCompactPullupInterval = 10;
int32_t tsMqRebalanceInterval = 2; int32_t tsMqRebalanceInterval = 2;
int32_t tsStreamCheckpointInterval = 60; int32_t tsStreamCheckpointInterval = 60;
float tsSinkDataRate = 2.0; float tsSinkDataRate = 2.0;
@ -280,6 +282,8 @@ int32_t tsS3BlockCacheSize = 16; // number of blocks
int32_t tsS3PageCacheSize = 4096; // number of pages int32_t tsS3PageCacheSize = 4096; // number of pages
int32_t tsS3UploadDelaySec = 60 * 60 * 24; int32_t tsS3UploadDelaySec = 60 * 60 * 24;
bool tsExperimental = true;
#ifndef _STORAGE #ifndef _STORAGE
int32_t taosSetTfsCfg(SConfig *pCfg) { int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
@ -527,6 +531,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
} }
if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0)
return -1; return -1;
if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
return 0; return 0;
} }
@ -686,6 +691,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1;
if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER) != 0) return -1;
@ -704,6 +711,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0) 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0)
return -1;
if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0) 0)
return -1; return -1;
@ -789,6 +799,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
return -1; return -1;
if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
GRANT_CFG_ADD; GRANT_CFG_ADD;
return 0; return 0;
} }
@ -1074,6 +1086,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32; tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32;
tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32; tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
return 0; return 0;
} }
@ -1137,6 +1151,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsEnableAudit = cfgGetItem(pCfg, "audit")->bval; tsEnableAudit = cfgGetItem(pCfg, "audit")->bval;
tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval; tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval;
tsAuditInterval = cfgGetItem(pCfg, "auditInterval")->i32;
tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval;
@ -1152,6 +1167,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32; tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32;
tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
tsCompactPullupInterval = cfgGetItem(pCfg, "compactPullupInterval")->i32;
tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32;
tsTtlPushIntervalSec = cfgGetItem(pCfg, "ttlPushInterval")->i32; tsTtlPushIntervalSec = cfgGetItem(pCfg, "ttlPushInterval")->i32;
@ -1206,6 +1222,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32; tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32;
tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32; tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32;
tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
GRANT_CFG_GET; GRANT_CFG_GET;
return 0; return 0;
} }
@ -1466,6 +1484,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
{"timeseriesThreshold", &tsTimeSeriesThreshold}, {"timeseriesThreshold", &tsTimeSeriesThreshold},
{"tmqMaxTopicNum", &tmqMaxTopicNum}, {"tmqMaxTopicNum", &tmqMaxTopicNum},
{"transPullupInterval", &tsTransPullupInterval}, {"transPullupInterval", &tsTransPullupInterval},
{"compactPullupInterval", &tsCompactPullupInterval},
{"trimVDbIntervalSec", &tsTrimVDbIntervalSec}, {"trimVDbIntervalSec", &tsTrimVDbIntervalSec},
{"ttlBatchDropNum", &tsTtlBatchDropNum}, {"ttlBatchDropNum", &tsTtlBatchDropNum},
{"ttlFlushThreshold", &tsTtlFlushThreshold}, {"ttlFlushThreshold", &tsTtlFlushThreshold},
@ -1475,6 +1494,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
{"s3PageCacheSize", &tsS3PageCacheSize}, {"s3PageCacheSize", &tsS3PageCacheSize},
{"s3UploadDelaySec", &tsS3UploadDelaySec}, {"s3UploadDelaySec", &tsS3UploadDelaySec},
{"supportVnodes", &tsNumOfSupportVnodes}, {"supportVnodes", &tsNumOfSupportVnodes},
{"experimental", &tsExperimental}
}; };
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
@ -1698,6 +1718,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
{"shellActivityTimer", &tsShellActivityTimer}, {"shellActivityTimer", &tsShellActivityTimer},
{"slowLogThreshold", &tsSlowLogThreshold}, {"slowLogThreshold", &tsSlowLogThreshold},
{"useAdapter", &tsUseAdapter}, {"useAdapter", &tsUseAdapter},
{"experimental", &tsExperimental}
}; };
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {

197
source/common/src/tmsg.c
View File

@ -1827,7 +1827,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
char *tb = taosHashIterate(pRsp->readTbs, NULL); char *tb = taosHashIterate(pRsp->readTbs, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1842,7 +1842,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->writeTbs, NULL); tb = taosHashIterate(pRsp->writeTbs, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1857,7 +1857,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->alterTbs, NULL); tb = taosHashIterate(pRsp->alterTbs, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1872,7 +1872,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->readViews, NULL); tb = taosHashIterate(pRsp->readViews, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1887,7 +1887,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->writeViews, NULL); tb = taosHashIterate(pRsp->writeViews, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1902,7 +1902,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->alterViews, NULL); tb = taosHashIterate(pRsp->alterViews, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1917,7 +1917,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL); int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL);
while (useDb != NULL) { while (useDb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(useDb, &keyLen); void *key = taosHashGetKey(useDb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -3424,6 +3424,66 @@ int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq
void tFreeSCompactDbReq(SCompactDbReq *pReq) { FREESQL(); } void tFreeSCompactDbReq(SCompactDbReq *pReq) { FREESQL(); }
int32_t tSerializeSCompactDbRsp(void *buf, int32_t bufLen, SCompactDbRsp *pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->compactId) < 0) return -1;
if (tEncodeI8(&encoder, pRsp->bAccepted) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCompactDbRsp(void *buf, int32_t bufLen, SCompactDbRsp *pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->compactId) < 0) return -1;
if (tDecodeI8(&decoder, &pRsp->bAccepted) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSKillCompactReq(void *buf, int32_t bufLen, SKillCompactReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
ENCODESQL();
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSKillCompactReq(void *buf, int32_t bufLen, SKillCompactReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
DECODESQL();
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSKillCompactReq(SKillCompactReq *pReq) { FREESQL(); }
int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) { int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) {
if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1; if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1; if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1;
@ -4259,6 +4319,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1; if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->filterTb) < 0) return -1; if (tEncodeCStr(&encoder, pReq->filterTb) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
if (tEncodeI64(&encoder, pReq->compactId) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
@ -4276,6 +4337,11 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->filterTb) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->filterTb) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI64(&decoder, &pReq->compactId) < 0) return -1;
} else {
pReq->compactId = -1;
}
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);
@ -5051,6 +5117,75 @@ int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) {
return 0; return 0;
} }
int32_t tSerializeSQueryCompactProgressReq(void *buf, int32_t bufLen, SQueryCompactProgressReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSQueryCompactProgressReq(void *buf, int32_t bufLen, SQueryCompactProgressReq *pReq) {
int32_t headLen = sizeof(SMsgHead);
SDecoder decoder = {0};
tDecoderInit(&decoder, ((uint8_t *)buf) + headLen, bufLen - headLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCompactProgressRsp *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numberFileset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->finished) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCompactProgressRsp *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->finished) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) { int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) {
SEncoder encoder = {0}; SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen); tEncoderInit(&encoder, buf, bufLen);
@ -5139,6 +5274,8 @@ int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *
if (tEncodeI64(&encoder, pReq->tw.skey) < 0) return -1; if (tEncodeI64(&encoder, pReq->tw.skey) < 0) return -1;
if (tEncodeI64(&encoder, pReq->tw.ekey) < 0) return -1; if (tEncodeI64(&encoder, pReq->tw.ekey) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
@ -5165,6 +5302,42 @@ int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq
if (tDecodeI64(&decoder, &pReq->tw.ekey) < 0) return -1; if (tDecodeI64(&decoder, &pReq->tw.ekey) < 0) return -1;
} }
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);
return 0; return 0;
@ -8169,7 +8342,7 @@ int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
for (int32_t i = 0; i < pRsp->blockNum; i++) { for (int32_t i = 0; i < pRsp->blockNum; i++) {
int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i); int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i);
void * data = taosArrayGetP(pRsp->blockData, i); void *data = taosArrayGetP(pRsp->blockData, i);
if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1; if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1;
if (pRsp->withSchema) { if (pRsp->withSchema) {
SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i); SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i);
@ -8202,7 +8375,7 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
} }
for (int32_t i = 0; i < pRsp->blockNum; i++) { for (int32_t i = 0; i < pRsp->blockNum; i++) {
void * data; void *data;
uint64_t bLen; uint64_t bLen;
if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1; if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1;
taosArrayPush(pRsp->blockData, &data); taosArrayPush(pRsp->blockData, &data);
@ -8248,7 +8421,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) {
if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1;
if (pRsp->createTableNum) { if (pRsp->createTableNum) {
for (int32_t i = 0; i < pRsp->createTableNum; i++) { for (int32_t i = 0; i < pRsp->createTableNum; i++) {
void * createTableReq = taosArrayGetP(pRsp->createTableReq, i); void *createTableReq = taosArrayGetP(pRsp->createTableReq, i);
int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i); int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i);
if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1; if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1;
} }
@ -8264,7 +8437,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) {
pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t)); pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t));
pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *)); pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *));
for (int32_t i = 0; i < pRsp->createTableNum; i++) { for (int32_t i = 0; i < pRsp->createTableNum; i++) {
void * pCreate = NULL; void *pCreate = NULL;
uint64_t len; uint64_t len;
if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1; if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1;
int32_t l = (int32_t)len; int32_t l = (int32_t)len;
@ -8566,7 +8739,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
taosArrayDestroy(pTbData->aCol); taosArrayDestroy(pTbData->aCol);
} else { } else {
int32_t nRow = TARRAY_SIZE(pTbData->aRowP); int32_t nRow = TARRAY_SIZE(pTbData->aRowP);
SRow ** rows = (SRow **)TARRAY_DATA(pTbData->aRowP); SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP);
for (int32_t i = 0; i < nRow; ++i) { for (int32_t i = 0; i < nRow; ++i) {
tRowDestroy(rows[i]); tRowDestroy(rows[i]);

4
source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
View File

@ -30,12 +30,14 @@ typedef struct SDnodeMgmt {
TdThread statusThread; TdThread statusThread;
TdThread notifyThread; TdThread notifyThread;
TdThread monitorThread; TdThread monitorThread;
TdThread auditThread;
TdThread crashReportThread; TdThread crashReportThread;
SSingleWorker mgmtWorker; SSingleWorker mgmtWorker;
ProcessCreateNodeFp processCreateNodeFp; ProcessCreateNodeFp processCreateNodeFp;
ProcessAlterNodeTypeFp processAlterNodeTypeFp; ProcessAlterNodeTypeFp processAlterNodeTypeFp;
ProcessDropNodeFp processDropNodeFp; ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp; SendMonitorReportFp sendMonitorReportFp;
SendAuditRecordsFp sendAuditRecordsFp;
GetVnodeLoadsFp getVnodeLoadsFp; GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp; GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp; GetMnodeLoadsFp getMnodeLoadsFp;
@ -62,7 +64,9 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt);
int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt); int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt);
void dmStopNotifyThread(SDnodeMgmt *pMgmt); void dmStopNotifyThread(SDnodeMgmt *pMgmt);
int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt); int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt);
int32_t dmStartAuditThread(SDnodeMgmt *pMgmt);
void dmStopMonitorThread(SDnodeMgmt *pMgmt); void dmStopMonitorThread(SDnodeMgmt *pMgmt);
void dmStopAuditThread(SDnodeMgmt *pMgmt);
int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt); int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt);
void dmStopCrashReportThread(SDnodeMgmt *pMgmt); void dmStopCrashReportThread(SDnodeMgmt *pMgmt);
int32_t dmStartWorker(SDnodeMgmt *pMgmt); int32_t dmStartWorker(SDnodeMgmt *pMgmt);

5
source/dnode/mgmt/mgmt_dnode/src/dmInt.c
View File

@ -29,6 +29,9 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
if (dmStartMonitorThread(pMgmt) != 0) { if (dmStartMonitorThread(pMgmt) != 0) {
return -1; return -1;
} }
if (dmStartAuditThread(pMgmt) != 0) {
return -1;
}
if (dmStartCrashReportThread(pMgmt) != 0) { if (dmStartCrashReportThread(pMgmt) != 0) {
return -1; return -1;
} }
@ -38,6 +41,7 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
static void dmStopMgmt(SDnodeMgmt *pMgmt) { static void dmStopMgmt(SDnodeMgmt *pMgmt) {
pMgmt->pData->stopped = true; pMgmt->pData->stopped = true;
dmStopMonitorThread(pMgmt); dmStopMonitorThread(pMgmt);
dmStopAuditThread(pMgmt);
dmStopStatusThread(pMgmt); dmStopStatusThread(pMgmt);
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
dmStopNotifyThread(pMgmt); dmStopNotifyThread(pMgmt);
@ -60,6 +64,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->processAlterNodeTypeFp = pInput->processAlterNodeTypeFp; pMgmt->processAlterNodeTypeFp = pInput->processAlterNodeTypeFp;
pMgmt->processDropNodeFp = pInput->processDropNodeFp; pMgmt->processDropNodeFp = pInput->processDropNodeFp;
pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp; pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp;
pMgmt->sendAuditRecordsFp = pInput->sendAuditRecordFp;
pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp; pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp;
pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp; pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp;
pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp; pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;

42
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -99,6 +99,27 @@ static void *dmMonitorThreadFp(void *param) {
return NULL; return NULL;
} }
static void *dmAuditThreadFp(void *param) {
SDnodeMgmt *pMgmt = param;
int64_t lastTime = taosGetTimestampMs();
setThreadName("dnode-audit");
while (1) {
taosMsleep(100);
if (pMgmt->pData->dropped || pMgmt->pData->stopped) break;
int64_t curTime = taosGetTimestampMs();
if (curTime < lastTime) lastTime = curTime;
float interval = curTime - lastTime;
if (interval >= tsAuditInterval) {
(*pMgmt->sendAuditRecordsFp)();
lastTime = curTime;
}
}
return NULL;
}
static void *dmCrashReportThreadFp(void *param) { static void *dmCrashReportThreadFp(void *param) {
SDnodeMgmt *pMgmt = param; SDnodeMgmt *pMgmt = param;
int64_t lastTime = taosGetTimestampMs(); int64_t lastTime = taosGetTimestampMs();
@ -218,6 +239,20 @@ int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
return 0; return 0;
} }
int32_t dmStartAuditThread(SDnodeMgmt *pMgmt) {
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->auditThread, &thAttr, dmAuditThreadFp, pMgmt) != 0) {
dError("failed to create audit thread since %s", strerror(errno));
return -1;
}
taosThreadAttrDestroy(&thAttr);
tmsgReportStartup("dnode-audit", "initialized");
return 0;
}
void dmStopMonitorThread(SDnodeMgmt *pMgmt) { void dmStopMonitorThread(SDnodeMgmt *pMgmt) {
if (taosCheckPthreadValid(pMgmt->monitorThread)) { if (taosCheckPthreadValid(pMgmt->monitorThread)) {
taosThreadJoin(pMgmt->monitorThread, NULL); taosThreadJoin(pMgmt->monitorThread, NULL);
@ -225,6 +260,13 @@ void dmStopMonitorThread(SDnodeMgmt *pMgmt) {
} }
} }
void dmStopAuditThread(SDnodeMgmt *pMgmt) {
if (taosCheckPthreadValid(pMgmt->auditThread)) {
taosThreadJoin(pMgmt->auditThread, NULL);
taosThreadClear(&pMgmt->auditThread);
}
}
int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) { int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) {
if (!tsEnableCrashReport) { if (!tsEnableCrashReport) {
return 0; return 0;

3
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -192,6 +192,8 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
@ -221,6 +223,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -818,6 +818,8 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

29
source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
View File

@ -70,6 +70,21 @@ typedef struct SUdfdData {
int32_t dnodeId; int32_t dnodeId;
} SUdfdData; } SUdfdData;
#ifndef TD_MODULE_OPTIMIZE
typedef struct SDnode {
int8_t once;
bool stop;
EDndRunStatus status;
SStartupInfo startup;
SDnodeData data;
SUdfdData udfdData;
TdThreadMutex mutex;
TdFilePtr lockfile;
STfs *pTfs;
SMgmtWrapper wrappers[NODE_END];
SDnodeTrans trans;
} SDnode;
#else
typedef struct SDnode { typedef struct SDnode {
int8_t once; int8_t once;
bool stop; bool stop;
@ -83,6 +98,7 @@ typedef struct SDnode {
STfs *pTfs; STfs *pTfs;
SMgmtWrapper wrappers[NODE_END]; SMgmtWrapper wrappers[NODE_END];
} SDnode; } SDnode;
#endif
// dmEnv.c // dmEnv.c
SDnode *dmInstance(); SDnode *dmInstance();
@ -97,8 +113,12 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper);
void dmReleaseWrapper(SMgmtWrapper *pWrapper); void dmReleaseWrapper(SMgmtWrapper *pWrapper);
int32_t dmInitVars(SDnode *pDnode); int32_t dmInitVars(SDnode *pDnode);
void dmClearVars(SDnode *pDnode); void dmClearVars(SDnode *pDnode);
int32_t dmInitModule(SDnode *pDnode, SMgmtWrapper *wrappers); #ifdef TD_MODULE_OPTIMIZE
bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper); int32_t dmInitModule(SDnode *pDnode, SMgmtWrapper *wrappers);
bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper);
#else
int32_t dmInitModule(SDnode *pDnode);
#endif
SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper); SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);
void dmSetStatus(SDnode *pDnode, EDndRunStatus stype); void dmSetStatus(SDnode *pDnode, EDndRunStatus stype);
void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg); void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg);
@ -119,11 +139,16 @@ int32_t dmInitStatusClient(SDnode *pDnode);
void dmCleanupClient(SDnode *pDnode); void dmCleanupClient(SDnode *pDnode);
void dmCleanupStatusClient(SDnode *pDnode); void dmCleanupStatusClient(SDnode *pDnode);
SMsgCb dmGetMsgcb(SDnode *pDnode); SMsgCb dmGetMsgcb(SDnode *pDnode);
#ifdef TD_MODULE_OPTIMIZE
int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers); int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers);
#else
int32_t dmInitMsgHandle(SDnode *pDnode);
#endif
int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
// dmMonitor.c // dmMonitor.c
void dmSendMonitorReport(); void dmSendMonitorReport();
void dmSendAuditRecords();
void dmGetVnodeLoads(SMonVloadInfo *pInfo); void dmGetVnodeLoads(SMonVloadInfo *pInfo);
void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo); void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo);
void dmGetMnodeLoads(SMonMloadInfo *pInfo); void dmGetMnodeLoads(SMonMloadInfo *pInfo);

2
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -189,6 +189,7 @@ void dmCleanup() {
if (dmCheckRepeatCleanup(pDnode) != 0) return; if (dmCheckRepeatCleanup(pDnode) != 0) return;
dmCleanupDnode(pDnode); dmCleanupDnode(pDnode);
monCleanup(); monCleanup();
auditCleanup();
syncCleanUp(); syncCleanUp();
walCleanUp(); walCleanUp();
udfcClose(); udfcClose();
@ -396,6 +397,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) {
.processAlterNodeTypeFp = dmProcessAlterNodeTypeReq, .processAlterNodeTypeFp = dmProcessAlterNodeTypeReq,
.processDropNodeFp = dmProcessDropNodeReq, .processDropNodeFp = dmProcessDropNodeReq,
.sendMonitorReportFp = dmSendMonitorReport, .sendMonitorReportFp = dmSendMonitorReport,
.sendAuditRecordFp = auditSendRecordsInBatch,
.getVnodeLoadsFp = dmGetVnodeLoads, .getVnodeLoadsFp = dmGetVnodeLoads,
.getVnodeLoadsLiteFp = dmGetVnodeLoadsLite, .getVnodeLoadsLiteFp = dmGetVnodeLoadsLite,
.getMnodeLoadsFp = dmGetMnodeLoads, .getMnodeLoadsFp = dmGetMnodeLoads,

24
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -24,6 +24,22 @@
#include "tglobal.h" #include "tglobal.h"
#endif #endif
#ifndef TD_MODULE_OPTIMIZE
static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
bool required = false;
int32_t code = (*pWrapper->func.requiredFp)(&input, &required);
if (!required) {
dDebug("node:%s, does not require startup", pWrapper->name);
} else {
dDebug("node:%s, required to startup", pWrapper->name);
}
return required;
}
#endif
int32_t dmInitDnode(SDnode *pDnode) { int32_t dmInitDnode(SDnode *pDnode) {
dDebug("start to create dnode"); dDebug("start to create dnode");
int32_t code = -1; int32_t code = -1;
@ -65,11 +81,15 @@ int32_t dmInitDnode(SDnode *pDnode) {
if (pDnode->lockfile == NULL) { if (pDnode->lockfile == NULL) {
goto _OVER; goto _OVER;
} }
#ifdef TD_MODULE_OPTIMIZE
if (dmInitModule(pDnode, pDnode->wrappers) != 0) { if (dmInitModule(pDnode, pDnode->wrappers) != 0) {
goto _OVER; goto _OVER;
} }
#else
if (dmInitModule(pDnode) != 0) {
goto _OVER;
}
#endif
indexInit(tsNumOfCommitThreads); indexInit(tsNumOfCommitThreads);
streamMetaInit(); streamMetaInit();

6
source/dnode/mgmt/node_mgmt/src/dmMonitor.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "dmMgmt.h" #include "dmMgmt.h"
#include "dmNodes.h" #include "dmNodes.h"
#include "audit.h"
static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
pInfo->protocol = 1; pInfo->protocol = 1;
@ -108,6 +109,11 @@ void dmSendMonitorReport() {
monSendReport(); monSendReport();
} }
//Todo: put this in seperate file in the future
void dmSendAuditRecords() {
auditSendRecordsInBatch();
}
void dmGetVnodeLoads(SMonVloadInfo *pInfo) { void dmGetVnodeLoads(SMonVloadInfo *pInfo) {
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = &pDnode->wrappers[VNODE]; SMgmtWrapper *pWrapper = &pDnode->wrappers[VNODE];

28
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -251,6 +251,7 @@ _OVER:
dmReleaseWrapper(pWrapper); dmReleaseWrapper(pWrapper);
} }
#ifdef TD_MODULE_OPTIMIZE
int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) { int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) {
SDnodeTrans *pTrans = &pDnode->trans; SDnodeTrans *pTrans = &pDnode->trans;
@ -276,6 +277,33 @@ int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) {
return 0; return 0;
} }
#else
int32_t dmInitMsgHandle(SDnode *pDnode) {
SDnodeTrans *pTrans = &pDnode->trans;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
SArray *pArray = (*pWrapper->func.getHandlesFp)();
if (pArray == NULL) return -1;
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
SMgmtHandle *pMgmt = taosArrayGet(pArray, i);
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
if (pMgmt->needCheckVgId) {
pHandle->needCheckVgId = pMgmt->needCheckVgId;
}
if (!pMgmt->needCheckVgId) {
pHandle->defaultNtype = ntype;
}
pWrapper->msgFps[TMSG_INDEX(pMgmt->msgType)] = pMgmt->msgFp;
}
taosArrayDestroy(pArray);
}
return 0;
}
#endif
static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();

2
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -86,6 +86,7 @@ typedef enum {
typedef int32_t (*ProcessCreateNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg); typedef int32_t (*ProcessCreateNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg);
typedef int32_t (*ProcessDropNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg); typedef int32_t (*ProcessDropNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg);
typedef void (*SendMonitorReportFp)(); typedef void (*SendMonitorReportFp)();
typedef void (*SendAuditRecordsFp)();
typedef void (*GetVnodeLoadsFp)(SMonVloadInfo *pInfo); typedef void (*GetVnodeLoadsFp)(SMonVloadInfo *pInfo);
typedef void (*GetMnodeLoadsFp)(SMonMloadInfo *pInfo); typedef void (*GetMnodeLoadsFp)(SMonMloadInfo *pInfo);
typedef void (*GetQnodeLoadsFp)(SQnodeLoad *pInfo); typedef void (*GetQnodeLoadsFp)(SQnodeLoad *pInfo);
@ -120,6 +121,7 @@ typedef struct {
ProcessAlterNodeTypeFp processAlterNodeTypeFp; ProcessAlterNodeTypeFp processAlterNodeTypeFp;
ProcessDropNodeFp processDropNodeFp; ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp; SendMonitorReportFp sendMonitorReportFp;
SendAuditRecordsFp sendAuditRecordFp;
GetVnodeLoadsFp getVnodeLoadsFp; GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp; GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp; GetMnodeLoadsFp getMnodeLoadsFp;

9
source/dnode/mgmt/test/qnode/dqnode.cpp
View File

@ -37,6 +37,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -50,6 +51,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -63,6 +65,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED); ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED);
rpcFreeCont(pRsp->pCont);
} }
test.Restart(); test.Restart();
@ -78,6 +81,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED); ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED);
rpcFreeCont(pRsp->pCont);
} }
} }
@ -94,6 +98,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION);
rpcFreeCont(pRsp->pCont);
} }
#endif #endif
@ -108,6 +113,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -121,6 +127,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED); ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED);
rpcFreeCont(pRsp->pCont);
} }
test.Restart(); test.Restart();
@ -136,6 +143,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED); ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -149,5 +157,6 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
} }

9
source/dnode/mgmt/test/snode/dsnode.cpp
View File

@ -37,6 +37,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -50,6 +51,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -63,6 +65,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED); ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED);
rpcFreeCont(pRsp->pCont);
} }
test.Restart(); test.Restart();
@ -78,6 +81,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED); ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED);
rpcFreeCont(pRsp->pCont);
} }
} }
@ -94,6 +98,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION);
rpcFreeCont(pRsp->pCont);
} }
#endif #endif
@ -108,6 +113,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -121,6 +127,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED); ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED);
rpcFreeCont(pRsp->pCont);
} }
test.Restart(); test.Restart();
@ -136,6 +143,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED); ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -149,5 +157,6 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
} }

56
source/dnode/mnode/impl/inc/mndCompact.h Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_COMPACT_H_
#define _TD_MND_COMPACT_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitCompact(SMnode *pMnode);
void mndCleanupCompact(SMnode *pMnode);
void tFreeCompactObj(SCompactObj *pCompact);
int32_t tSerializeSCompactObj(void *buf, int32_t bufLen, const SCompactObj *pObj);
int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj);
SSdbRaw* mndCompactActionEncode(SCompactObj *pCompact);
SSdbRow* mndCompactActionDecode(SSdbRaw *pRaw);
int32_t mndCompactActionInsert(SSdb *pSdb, SCompactObj *pCompact);
int32_t mndCompactActionDelete(SSdb *pSdb, SCompactObj *pCompact);
int32_t mndCompactActionUpdate(SSdb *pSdb, SCompactObj *pOldCompact, SCompactObj *pNewCompact);
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SDbObj *pDb, SCompactDbRsp *rsp);
int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
int32_t mndProcessKillCompactReq(SRpcMsg *pReq);
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq);
SCompactObj *mndAcquireCompact(SMnode *pMnode, int64_t compactId);
void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact);
void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_COMPACT_H_*/

48
source/dnode/mnode/impl/inc/mndCompactDetail.h Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_COMPACT_DETAIL_H_
#define _TD_MND_COMPACT_DETAIL_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitCompactDetail(SMnode *pMnode);
void mndCleanupCompactDetail(SMnode *pMnode);
void tFreeCompactDetailObj(SCompactDetailObj *pCompact);
int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDetailObj *pObj);
int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailObj *pObj);
SSdbRaw* mndCompactDetailActionEncode(SCompactDetailObj *pCompact);
SSdbRow* mndCompactDetailActionDecode(SSdbRaw *pRaw);
int32_t mndCompactDetailActionInsert(SSdb *pSdb, SCompactDetailObj *pCompact);
int32_t mndCompactDetailActionDelete(SSdb *pSdb, SCompactDetailObj *pCompact);
int32_t mndCompactDetailActionUpdate(SSdb *pSdb, SCompactDetailObj *pOldCompact, SCompactDetailObj *pNewCompact);
int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup,
SVnodeGid *pVgid, int32_t index);
int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_COMPACT_DETAIL_H_*/

19
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -741,6 +741,25 @@ int32_t tEncodeSViewObj(SEncoder* pEncoder, const SViewObj* pObj);
int32_t tDecodeSViewObj(SDecoder* pDecoder, SViewObj* pObj, int32_t sver); int32_t tDecodeSViewObj(SDecoder* pDecoder, SViewObj* pObj, int32_t sver);
void tFreeSViewObj(SViewObj* pObj); void tFreeSViewObj(SViewObj* pObj);
typedef struct {
int32_t compactDetailId;
int32_t compactId;
int32_t vgId;
int32_t dnodeId;
int32_t numberFileset;
int32_t finished;
int64_t startTime;
int32_t newNumberFileset;
int32_t newFinished;
} SCompactDetailObj;
typedef struct {
int32_t compactId;
char dbname[TSDB_TABLE_FNAME_LEN];
int64_t startTime;
SArray* compactDetail;
} SCompactObj;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

748
source/dnode/mnode/impl/src/mndCompact.c Normal file
View File

@ -0,0 +1,748 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndCompact.h"
#include "mndTrans.h"
#include "mndShow.h"
#include "mndDb.h"
#include "mndCompactDetail.h"
#include "mndVgroup.h"
#include "tmsgcb.h"
#include "mndDnode.h"
#include "tmisce.h"
#include "audit.h"
#include "mndPrivilege.h"
#include "mndTrans.h"
#define MND_COMPACT_VER_NUMBER 1
static int32_t mndProcessCompactTimer(SRpcMsg *pReq);
int32_t mndInitCompact(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_COMPACT, mndRetrieveCompact);
mndSetMsgHandle(pMnode, TDMT_MND_KILL_COMPACT, mndProcessKillCompactReq);
mndSetMsgHandle(pMnode, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mndProcessQueryCompactRsp);
mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_TIMER, mndProcessCompactTimer);
mndSetMsgHandle(pMnode, TDMT_VND_KILL_COMPACT_RSP, mndTransProcessRsp);
SSdbTable table = {
.sdbType = SDB_COMPACT,
.keyType = SDB_KEY_INT32,
.encodeFp = (SdbEncodeFp)mndCompactActionEncode,
.decodeFp = (SdbDecodeFp)mndCompactActionDecode,
.insertFp = (SdbInsertFp)mndCompactActionInsert,
.updateFp = (SdbUpdateFp)mndCompactActionUpdate,
.deleteFp = (SdbDeleteFp)mndCompactActionDelete,
};
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupCompact(SMnode *pMnode) {
mDebug("mnd compact cleanup");
}
void tFreeCompactObj(SCompactObj *pCompact) {
}
int32_t tSerializeSCompactObj(void *buf, int32_t bufLen, const SCompactObj *pObj) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pObj->compactId) < 0) return -1;
if (tEncodeCStr(&encoder, pObj->dbname) < 0) return -1;
if (tEncodeI64(&encoder, pObj->startTime) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj) {
int8_t ex = 0;
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->compactId) < 0) return -1;
if (tDecodeCStrTo(&decoder, pObj->dbname) < 0) return -1;
if (tDecodeI64(&decoder, &pObj->startTime) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
SSdbRaw *mndCompactActionEncode(SCompactObj *pCompact) {
terrno = TSDB_CODE_SUCCESS;
void *buf = NULL;
SSdbRaw *pRaw = NULL;
int32_t tlen = tSerializeSCompactObj(NULL, 0, pCompact);
if (tlen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t size = sizeof(int32_t) + tlen;
pRaw = sdbAllocRaw(SDB_COMPACT, MND_COMPACT_VER_NUMBER, size);
if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
buf = taosMemoryMalloc(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
tlen = tSerializeSCompactObj(buf, tlen, pCompact);
if (tlen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, OVER);
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OVER);
SDB_SET_DATALEN(pRaw, dataPos, OVER);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("compact:%" PRId32 ", failed to encode to raw:%p since %s", pCompact->compactId, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("compact:%" PRId32 ", encode to raw:%p, row:%p", pCompact->compactId, pRaw, pCompact);
return pRaw;
}
SSdbRow *mndCompactActionDecode(SSdbRaw *pRaw) {
SSdbRow *pRow = NULL;
SCompactObj *pCompact = NULL;
void *buf = NULL;
terrno = TSDB_CODE_SUCCESS;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) {
goto OVER;
}
if (sver != MND_COMPACT_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("compact read invalid ver, data ver: %d, curr ver: %d", sver, MND_COMPACT_VER_NUMBER);
goto OVER;
}
pRow = sdbAllocRow(sizeof(SCompactObj));
if (pRow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
pCompact = sdbGetRowObj(pRow);
if (pCompact == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t tlen;
int32_t dataPos = 0;
SDB_GET_INT32(pRaw, dataPos, &tlen, OVER);
buf = taosMemoryMalloc(tlen + 1);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER);
if (tDeserializeSCompactObj(buf, tlen, pCompact) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
//taosInitRWLatch(&pView->lock);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("compact:%" PRId32 ", failed to decode from raw:%p since %s", pCompact->compactId, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
mTrace("compact:%" PRId32 ", decode from raw:%p, row:%p", pCompact->compactId, pRaw, pCompact);
return pRow;
}
int32_t mndCompactActionInsert(SSdb *pSdb, SCompactObj *pCompact) {
mTrace("compact:%" PRId32 ", perform insert action", pCompact->compactId);
return 0;
}
int32_t mndCompactActionDelete(SSdb *pSdb, SCompactObj *pCompact) {
mTrace("compact:%" PRId32 ", perform insert action", pCompact->compactId);
tFreeCompactObj(pCompact);
return 0;
}
int32_t mndCompactActionUpdate(SSdb *pSdb, SCompactObj *pOldCompact, SCompactObj *pNewCompact) {
mTrace("compact:%" PRId32 ", perform update action, old row:%p new row:%p",
pOldCompact->compactId, pOldCompact, pNewCompact);
return 0;
}
SCompactObj *mndAcquireCompact(SMnode *pMnode, int64_t compactId) {
SSdb *pSdb = pMnode->pSdb;
SCompactObj *pCompact = sdbAcquire(pSdb, SDB_COMPACT, &compactId);
if (pCompact == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_SUCCESS;
}
return pCompact;
}
void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pCompact);
}
//compact db
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SDbObj *pDb, SCompactDbRsp *rsp){
pCompact->compactId = tGenIdPI32();
strcpy(pCompact->dbname, pDb->name);
pCompact->startTime = taosGetTimestampMs();
SSdbRaw *pVgRaw = mndCompactActionEncode(pCompact);
if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
}
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
rsp->compactId = pCompact->compactId;
return 0;
}
//retrieve compact
int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows){
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SCompactObj *pCompact = NULL;
char *sep = NULL;
SDbObj *pDb = NULL;
if (strlen(pShow->db) > 0) {
sep = strchr(pShow->db, '.');
if (sep && ((0 == strcmp(sep + 1, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcmp(sep + 1, TSDB_PERFORMANCE_SCHEMA_DB))))) {
sep++;
} else {
pDb = mndAcquireDb(pMnode, pShow->db);
if (pDb == NULL) return terrno;
}
}
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_COMPACT, pShow->pIter, (void **)&pCompact);
if (pShow->pIter == NULL) break;
SColumnInfoData *pColInfo;
SName n;
int32_t cols = 0;
char tmpBuf[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompact->compactId, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
if (pDb != NULL || !IS_SYS_DBNAME(pCompact->dbname)) {
SName name = {0};
tNameFromString(&name, pCompact->dbname, T_NAME_ACCT | T_NAME_DB);
tNameGetDbName(&name, varDataVal(tmpBuf));
} else {
strncpy(varDataVal(tmpBuf), pCompact->dbname, strlen(pCompact->dbname) + 1);
}
varDataSetLen(tmpBuf, strlen(varDataVal(tmpBuf)));
colDataSetVal(pColInfo, numOfRows, (const char *)tmpBuf, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompact->startTime, false);
numOfRows++;
sdbRelease(pSdb, pCompact);
}
pShow->numOfRows += numOfRows;
mndReleaseDb(pMnode, pDb);
return numOfRows;
}
//kill compact
static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pContLen,
int32_t compactId, int32_t dnodeid) {
SVKillCompactReq req = {0};
req.compactId = compactId;
req.vgId = pVgroup->vgId;
req.dnodeId = dnodeid;
mInfo("vgId:%d, build compact vnode config req", pVgroup->vgId);
int32_t contLen = tSerializeSVKillCompactReq(NULL, 0, &req);
if (contLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
contLen += sizeof(SMsgHead);
void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
SMsgHead *pHead = pReq;
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req);
*pContLen = contLen;
return pReq;
}
static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup,
int32_t compactId, int32_t dnodeid) {
STransAction action = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeid);
if (pDnode == NULL) return -1;
action.epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode);
int32_t contLen = 0;
void *pReq = mndBuildKillCompactReq(pMnode, pVgroup, &contLen, compactId, dnodeid);
if (pReq == NULL) return -1;
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_VND_KILL_COMPACT;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
return -1;
}
return 0;
}
static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "kill-compact");
if (pTrans == NULL) {
mError("compact:%" PRId32 ", failed to drop since %s" , pCompact->compactId, terrstr());
return -1;
}
mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId);
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == pCompact->compactId) {
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pDetail->vgId);
if(pVgroup == NULL){
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
if(mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId) != 0){
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndReleaseVgroup(pMnode, pVgroup);
/*
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
*/
}
sdbRelease(pMnode->pSdb, pDetail);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
return 0;
}
int32_t mndProcessKillCompactReq(SRpcMsg *pReq){
SKillCompactReq killCompactReq = {0};
if (tDeserializeSKillCompactReq(pReq->pCont, pReq->contLen, &killCompactReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mInfo("start to kill compact:%" PRId32, killCompactReq.compactId);
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SCompactObj *pCompact = mndAcquireCompact(pMnode, killCompactReq.compactId);
if(pCompact == NULL){
terrno = TSDB_CODE_MND_INVALID_COMPACT_ID;
tFreeSKillCompactReq(&killCompactReq);
return -1;
}
if (0 != mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB)) {
goto _OVER;
}
if (mndKillCompact(pMnode, pReq, pCompact) < 0) {
goto _OVER;
}
code = TSDB_CODE_ACTION_IN_PROGRESS;
char obj[10] = {0};
sprintf(obj, "%d", pCompact->compactId);
auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen);
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to kill compact %" PRId32 " since %s", killCompactReq.compactId, terrstr());
}
tFreeSKillCompactReq(&killCompactReq);
sdbRelease(pMnode->pSdb, pCompact);
return code;
}
//update progress
static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t compactId, SQueryCompactProgressRsp* rsp) {
void* pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == compactId && pDetail->vgId == rsp->vgId && pDetail->dnodeId == rsp->dnodeId) {
pDetail->newNumberFileset = rsp->numberFileset;
pDetail->newFinished = rsp->finished;
sdbRelease(pMnode->pSdb, pDetail);
return 0;
}
sdbRelease(pMnode->pSdb, pDetail);
}
return -1;
}
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){
SQueryCompactProgressRsp req = {0};
if (tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mDebug("compact:%d, receive query response, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
if(mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req) != 0){
mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
return -1;
}
return 0;
}
//timer
void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact){
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == pCompact->compactId) {
SEpSet epSet = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDetail->dnodeId);
if(pDnode == NULL) break;
addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port);
mndReleaseDnode(pMnode, pDnode);
SQueryCompactProgressReq req;
req.compactId = pDetail->compactId;
req.vgId = pDetail->vgId;
req.dnodeId = pDetail->dnodeId;
int32_t contLen = tSerializeSQueryCompactProgressReq(NULL, 0, &req);
if (contLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
continue;
}
contLen += sizeof(SMsgHead);
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
sdbCancelFetch(pMnode->pSdb, pDetail);
sdbRelease(pMnode->pSdb, pDetail);
continue;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pDetail->vgId);
tSerializeSQueryCompactProgressReq((char *)pHead + sizeof(SMsgHead), contLen - sizeof(SMsgHead), &req);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS,
.contLen = contLen};
//rpcMsg.pCont = rpcMallocCont(contLen);
//if (rpcMsg.pCont == NULL) {
// return;
//}
//memcpy(rpcMsg.pCont, pHead, contLen);
rpcMsg.pCont = pHead;
char detail[1024] = {0};
int32_t len = snprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", TMSG_INFO(TDMT_VND_QUERY_COMPACT_PROGRESS),
epSet.numOfEps, epSet.inUse);
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
len += snprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, epSet.eps[i].fqdn,
epSet.eps[i].port);
}
mDebug("compact:%d, send update progress msg to %s", pDetail->compactId, detail);
tmsgSendReq(&epSet, &rpcMsg);
}
sdbRelease(pMnode->pSdb, pDetail);
}
}
static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
bool needSave = false;
void* pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == compactId) {
mDebug("compact:%d, check save progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
if(pDetail->numberFileset < pDetail->newNumberFileset || pDetail->finished < pDetail->newFinished)
needSave = true;
}
sdbRelease(pMnode->pSdb, pDetail);
}
if(!needSave) {
mDebug("compact:%" PRId32 ", no need to save" , compactId);
return 0;
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-compact-progress");
if (pTrans == NULL) {
mError("trans:%" PRId32 ", failed to create since %s" , pTrans->id, terrstr());
return -1;
}
mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id);
SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId);
pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == compactId) {
mInfo("compact:%d, trans:%d, check compact progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
pDetail->numberFileset = pDetail->newNumberFileset;
pDetail->finished = pDetail->newFinished;
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
}
sdbRelease(pMnode->pSdb, pDetail);
}
bool allFinished = true;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if(pDetail->compactId == compactId){
mInfo("compact:%d, trans:%d, check compact finished, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished);
if(pDetail->numberFileset == -1 && pDetail->finished == -1){
allFinished = false;
sdbRelease(pMnode->pSdb, pDetail);
break;
}
if (pDetail->numberFileset != -1 && pDetail->finished != -1 &&
pDetail->numberFileset != pDetail->finished) {
allFinished = false;
sdbRelease(pMnode->pSdb, pDetail);
break;
}
}
sdbRelease(pMnode->pSdb, pDetail);
}
if(allFinished){
mInfo("compact:%d, all finished", pCompact->compactId);
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == pCompact->compactId) {
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
}
sdbRelease(pMnode->pSdb, pDetail);
}
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("compact:%d, trans:%d, failed to prepare since %s", compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
sdbRelease(pMnode->pSdb, pCompact);
return -1;
}
sdbRelease(pMnode->pSdb, pCompact);
mndTransDrop(pTrans);
return 0;
}
void mndCompactPullup(SMnode *pMnode) {
SSdb *pSdb = pMnode->pSdb;
SArray *pArray = taosArrayInit(sdbGetSize(pSdb, SDB_COMPACT), sizeof(int32_t));
if (pArray == NULL) return;
void *pIter = NULL;
while (1) {
SCompactObj *pCompact = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT, pIter, (void **)&pCompact);
if (pIter == NULL) break;
taosArrayPush(pArray, &pCompact->compactId);
sdbRelease(pSdb, pCompact);
}
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
mInfo("begin to pull up");
int32_t *pCompactId = taosArrayGet(pArray, i);
SCompactObj *pCompact = mndAcquireCompact(pMnode, *pCompactId);
if (pCompact != NULL) {
mInfo("compact:%d, begin to pull up", pCompact->compactId);
mndCompactSendProgressReq(pMnode, pCompact);
mndSaveCompactProgress(pMnode, pCompact->compactId);
}
mndReleaseCompact(pMnode, pCompact);
}
taosArrayDestroy(pArray);
}
static int32_t mndProcessCompactTimer(SRpcMsg *pReq) {
mTrace("start to process compact timer");
mndCompactPullup(pReq->info.node);
return 0;
}

300
source/dnode/mnode/impl/src/mndCompactDetail.c Normal file
View File

@ -0,0 +1,300 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndCompactDetail.h"
#include "mndTrans.h"
#include "mndShow.h"
#include "mndDb.h"
#define MND_COMPACT_VER_NUMBER 1
int32_t mndInitCompactDetail(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_COMPACT_DETAIL, mndRetrieveCompactDetail);
SSdbTable table = {
.sdbType = SDB_COMPACT_DETAIL,
.keyType = SDB_KEY_INT64,
.encodeFp = (SdbEncodeFp)mndCompactDetailActionEncode,
.decodeFp = (SdbDecodeFp)mndCompactDetailActionDecode,
.insertFp = (SdbInsertFp)mndCompactDetailActionInsert,
.updateFp = (SdbUpdateFp)mndCompactDetailActionUpdate,
.deleteFp = (SdbDeleteFp)mndCompactDetailActionDelete,
};
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupCompactDetail(SMnode *pMnode) {
mDebug("mnd compact detail cleanup");
}
int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows){
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SCompactDetailObj *pCompactDetail = NULL;
char *sep = NULL;
SDbObj *pDb = NULL;
if (strlen(pShow->db) > 0) {
sep = strchr(pShow->db, '.');
if (sep && ((0 == strcmp(sep + 1, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcmp(sep + 1, TSDB_PERFORMANCE_SCHEMA_DB))))) {
sep++;
} else {
pDb = mndAcquireDb(pMnode, pShow->db);
if (pDb == NULL) return terrno;
}
}
while(numOfRows < rows){
pShow->pIter = sdbFetch(pSdb, SDB_COMPACT_DETAIL, pShow->pIter, (void **)&pCompactDetail);
if (pShow->pIter == NULL) break;
SColumnInfoData *pColInfo;
SName n;
int32_t cols = 0;
char tmpBuf[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->compactId, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->vgId, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->dnodeId, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->numberFileset, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->finished, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->startTime, false);
numOfRows++;
sdbRelease(pSdb, pCompactDetail);
}
pShow->numOfRows += numOfRows;
mndReleaseDb(pMnode, pDb);
return numOfRows;
}
void tFreeCompactDetailObj(SCompactDetailObj *pCompact) {
}
int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDetailObj *pObj) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pObj->compactDetailId) < 0) return -1;
if (tEncodeI32(&encoder, pObj->compactId) < 0) return -1;
if (tEncodeI32(&encoder, pObj->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pObj->dnodeId) < 0) return -1;
if (tEncodeI32(&encoder, pObj->numberFileset) < 0) return -1;
if (tEncodeI32(&encoder, pObj->finished) < 0) return -1;
if (tEncodeI64(&encoder, pObj->startTime) < 0) return -1;
if (tEncodeI32(&encoder, pObj->newNumberFileset) < 0) return -1;
if (tEncodeI32(&encoder, pObj->newFinished) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailObj *pObj) {
int8_t ex = 0;
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->compactDetailId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->dnodeId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->numberFileset) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->finished) < 0) return -1;
if (tDecodeI64(&decoder, &pObj->startTime) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->newNumberFileset) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->newFinished) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
SSdbRaw *mndCompactDetailActionEncode(SCompactDetailObj *pCompact) {
terrno = TSDB_CODE_SUCCESS;
void *buf = NULL;
SSdbRaw *pRaw = NULL;
int32_t tlen = tSerializeSCompactDetailObj(NULL, 0, pCompact);
if (tlen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t size = sizeof(int32_t) + tlen;
pRaw = sdbAllocRaw(SDB_COMPACT_DETAIL, MND_COMPACT_VER_NUMBER, size);
if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
buf = taosMemoryMalloc(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
tlen = tSerializeSCompactDetailObj(buf, tlen, pCompact);
if (tlen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, OVER);
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OVER);
SDB_SET_DATALEN(pRaw, dataPos, OVER);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("compact detail:%" PRId32 ", failed to encode to raw:%p since %s", pCompact->compactId, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("compact detail:%" PRId32 ", encode to raw:%p, row:%p", pCompact->compactId, pRaw, pCompact);
return pRaw;
}
SSdbRow *mndCompactDetailActionDecode(SSdbRaw *pRaw) {
SSdbRow *pRow = NULL;
SCompactDetailObj *pCompact = NULL;
void *buf = NULL;
terrno = TSDB_CODE_SUCCESS;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) {
goto OVER;
}
if (sver != MND_COMPACT_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("compact detail read invalid ver, data ver: %d, curr ver: %d", sver, MND_COMPACT_VER_NUMBER);
goto OVER;
}
pRow = sdbAllocRow(sizeof(SCompactObj));
if (pRow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
pCompact = sdbGetRowObj(pRow);
if (pCompact == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t tlen;
int32_t dataPos = 0;
SDB_GET_INT32(pRaw, dataPos, &tlen, OVER);
buf = taosMemoryMalloc(tlen + 1);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER);
if (tDeserializeSCompactDetailObj(buf, tlen, pCompact) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
//taosInitRWLatch(&pView->lock);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("compact detail:%" PRId32 ", failed to decode from raw:%p since %s", pCompact->compactId, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
mTrace("compact detail:%" PRId32 ", decode from raw:%p, row:%p", pCompact->compactId, pRaw, pCompact);
return pRow;
}
int32_t mndCompactDetailActionInsert(SSdb *pSdb, SCompactDetailObj *pCompact) {
mTrace("compact detail:%" PRId32 ", perform insert action", pCompact->compactId);
return 0;
}
int32_t mndCompactDetailActionDelete(SSdb *pSdb, SCompactDetailObj *pCompact) {
mTrace("compact detail:%" PRId32 ", perform insert action", pCompact->compactId);
tFreeCompactDetailObj(pCompact);
return 0;
}
int32_t mndCompactDetailActionUpdate(SSdb *pSdb, SCompactDetailObj *pOldCompact, SCompactDetailObj *pNewCompact) {
mTrace("compact detail:%" PRId32 ", perform update action, old row:%p new row:%p",
pOldCompact->compactId, pOldCompact, pNewCompact);
pOldCompact->numberFileset = pNewCompact->numberFileset;
pOldCompact->finished = pNewCompact->finished;
return 0;
}
int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup,
SVnodeGid *pVgid, int32_t index){
SCompactDetailObj compactDetail = {0};
compactDetail.compactDetailId = index;
compactDetail.compactId = pCompact->compactId;
compactDetail.vgId = pVgroup->vgId;
compactDetail.dnodeId = pVgid->dnodeId;
compactDetail.startTime = taosGetTimestampMs();
compactDetail.numberFileset = -1;
compactDetail.finished = -1;
compactDetail.newNumberFileset = -1;
compactDetail.newFinished = -1;
mInfo("compact:%d, add compact detail to trans, index:%d, vgId:%d, dnodeId:%d",
compactDetail.compactId, compactDetail.compactDetailId, compactDetail.vgId, compactDetail.dnodeId);
SSdbRaw *pVgRaw = mndCompactDetailActionEncode(&compactDetail);
if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
}
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
return 0;
}

21
source/dnode/mnode/impl/src/mndMain.c
View File

@ -42,6 +42,8 @@
#include "mndUser.h" #include "mndUser.h"
#include "mndVgroup.h" #include "mndVgroup.h"
#include "mndView.h" #include "mndView.h"
#include "mndCompact.h"
#include "mndCompactDetail.h"
static inline int32_t mndAcquireRpc(SMnode *pMnode) { static inline int32_t mndAcquireRpc(SMnode *pMnode) {
int32_t code = 0; int32_t code = 0;
@ -112,6 +114,16 @@ static void mndPullupTrans(SMnode *pMnode) {
} }
} }
static void mndPullupCompacts(SMnode *pMnode) {
mTrace("pullup compact timer msg");
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
if (pReq != NULL) {
SRpcMsg rpcMsg = {.msgType = TDMT_MND_COMPACT_TIMER, .pCont = pReq, .contLen = contLen};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
}
static void mndPullupTtl(SMnode *pMnode) { static void mndPullupTtl(SMnode *pMnode) {
mTrace("pullup ttl"); mTrace("pullup ttl");
int32_t contLen = 0; int32_t contLen = 0;
@ -287,6 +299,10 @@ static void *mndThreadFp(void *param) {
mndPullupTrans(pMnode); mndPullupTrans(pMnode);
} }
if (sec % tsCompactPullupInterval == 0) {
mndPullupCompacts(pMnode);
}
if (sec % tsMqRebalanceInterval == 0) { if (sec % tsMqRebalanceInterval == 0) {
mndCalMqRebalance(pMnode); mndCalMqRebalance(pMnode);
} }
@ -462,6 +478,8 @@ static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView) != 0) return -1; if (mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-compact", mndInitCompact, mndCleanupCompact) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-compact-detail", mndInitCompactDetail, mndCleanupCompactDetail) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1; if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1; if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1;
@ -693,7 +711,8 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
_OVER: _OVER:
if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER ||
pMsg->msgType == TDMT_MND_TRIM_DB_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { pMsg->msgType == TDMT_MND_TRIM_DB_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER ||
pMsg->msgType == TDMT_MND_COMPACT_TIMER) {
mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
pMnode->stopped, state.restored, syncStr(state.state)); pMnode->stopped, state.restored, syncStr(state.state));
return -1; return -1;

7
source/dnode/mnode/impl/src/mndShow.c
View File

@ -17,6 +17,7 @@
#include "mndShow.h" #include "mndShow.h"
#include "mndPrivilege.h" #include "mndPrivilege.h"
#include "systable.h" #include "systable.h"
#include "mndUser.h"
#define SHOW_STEP_SIZE 100 #define SHOW_STEP_SIZE 100
#define SHOW_COLS_STEP_SIZE 4096 #define SHOW_COLS_STEP_SIZE 4096
@ -118,6 +119,10 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_PRIVILEGES; type = TSDB_MGMT_TABLE_PRIVILEGES;
} else if (strncasecmp(name, TSDB_INS_TABLE_VIEWS, len) == 0) { } else if (strncasecmp(name, TSDB_INS_TABLE_VIEWS, len) == 0) {
type = TSDB_MGMT_TABLE_VIEWS; type = TSDB_MGMT_TABLE_VIEWS;
} else if (strncasecmp(name, TSDB_INS_TABLE_COMPACTS, len) == 0) {
type = TSDB_MGMT_TABLE_COMPACT;
} else if (strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, len) == 0) {
type = TSDB_MGMT_TABLE_COMPACT_DETAIL;
} else { } else {
mError("invalid show name:%s len:%d", name, len); mError("invalid show name:%s len:%d", name, len);
} }
@ -206,7 +211,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
return -1; return -1;
} }
mDebug("mndProcessRetrieveSysTableReq tb:%s", retrieveReq.tb); mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb);
if (retrieveReq.showId == 0) { if (retrieveReq.showId == 0) {
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb)); STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb));

3
source/dnode/mnode/impl/test/acct/acct.cpp
View File

@ -33,6 +33,7 @@ TEST_F(MndTestAcct, 01_Create_Acct) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
rpcFreeCont(pRsp->pCont);
} }
TEST_F(MndTestAcct, 02_Alter_Acct) { TEST_F(MndTestAcct, 02_Alter_Acct) {
@ -43,6 +44,7 @@ TEST_F(MndTestAcct, 02_Alter_Acct) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
rpcFreeCont(pRsp->pCont);
} }
TEST_F(MndTestAcct, 03_Drop_Acct) { TEST_F(MndTestAcct, 03_Drop_Acct) {
@ -53,4 +55,5 @@ TEST_F(MndTestAcct, 03_Drop_Acct) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
rpcFreeCont(pRsp->pCont);
} }

5
source/dnode/mnode/impl/test/sma/sma.cpp
View File

@ -253,12 +253,14 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
pReq = BuildCreateDbReq(dbname, &contLen); pReq = BuildCreateDbReq(dbname, &contLen);
pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
pReq = BuildCreateBSmaStbReq(stbname, &contLen); pReq = BuildCreateBSmaStbReq(stbname, &contLen);
pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 1); EXPECT_EQ(test.GetShowRows(), 1);
} }
@ -269,12 +271,14 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
pReq = BuildCreateBSmaStbReq(stbname, &contLen); pReq = BuildCreateBSmaStbReq(stbname, &contLen);
pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_ALREADY_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_ALREADY_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
pReq = BuildDropStbReq(stbname, &contLen); pReq = BuildDropStbReq(stbname, &contLen);
pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 0); EXPECT_EQ(test.GetShowRows(), 0);
} }
@ -283,5 +287,6 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
pReq = BuildDropStbReq(stbname, &contLen); pReq = BuildDropStbReq(stbname, &contLen);
pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
} }
} }

66
source/dnode/mnode/impl/test/stb/stb.cpp
View File

@ -155,6 +155,7 @@ void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagnam
tSerializeSMAlterStbReq(pHead, contLen, &req); tSerializeSMAlterStbReq(pHead, contLen, &req);
*pContLen = contLen; *pContLen = contLen;
taosArrayDestroy(req.pFields);
return pHead; return pHead;
} }
@ -176,6 +177,7 @@ void* MndTestStb::BuildAlterStbDropTagReq(const char* stbname, const char* tagna
tSerializeSMAlterStbReq(pHead, contLen, &req); tSerializeSMAlterStbReq(pHead, contLen, &req);
*pContLen = contLen; *pContLen = contLen;
taosArrayDestroy(req.pFields);
return pHead; return pHead;
} }
@ -204,6 +206,7 @@ void* MndTestStb::BuildAlterStbUpdateTagNameReq(const char* stbname, const char*
tSerializeSMAlterStbReq(pHead, contLen, &req); tSerializeSMAlterStbReq(pHead, contLen, &req);
*pContLen = contLen; *pContLen = contLen;
taosArrayDestroy(req.pFields);
return pHead; return pHead;
} }
@ -226,6 +229,7 @@ void* MndTestStb::BuildAlterStbUpdateTagBytesReq(const char* stbname, const char
tSerializeSMAlterStbReq(pHead, contLen, &req); tSerializeSMAlterStbReq(pHead, contLen, &req);
*pContLen = contLen; *pContLen = contLen;
taosArrayDestroy(req.pFields);
return pHead; return pHead;
} }
@ -247,6 +251,7 @@ void* MndTestStb::BuildAlterStbAddColumnReq(const char* stbname, const char* col
tSerializeSMAlterStbReq(pHead, contLen, &req); tSerializeSMAlterStbReq(pHead, contLen, &req);
*pContLen = contLen; *pContLen = contLen;
taosArrayDestroy(req.pFields);
return pHead; return pHead;
} }
@ -268,6 +273,7 @@ void* MndTestStb::BuildAlterStbDropColumnReq(const char* stbname, const char* co
tSerializeSMAlterStbReq(pHead, contLen, &req); tSerializeSMAlterStbReq(pHead, contLen, &req);
*pContLen = contLen; *pContLen = contLen;
taosArrayDestroy(req.pFields);
return pHead; return pHead;
} }
@ -290,6 +296,7 @@ void* MndTestStb::BuildAlterStbUpdateColumnBytesReq(const char* stbname, const c
tSerializeSMAlterStbReq(pHead, contLen, &req); tSerializeSMAlterStbReq(pHead, contLen, &req);
*pContLen = contLen; *pContLen = contLen;
taosArrayDestroy(req.pFields);
return pHead; return pHead;
} }
@ -303,6 +310,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -311,6 +319,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -334,6 +343,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
STableMetaRsp metaRsp = {0}; STableMetaRsp metaRsp = {0};
tDeserializeSTableMetaRsp(pMsg->pCont, pMsg->contLen, &metaRsp); tDeserializeSTableMetaRsp(pMsg->pCont, pMsg->contLen, &metaRsp);
rpcFreeCont(pMsg->pCont);
EXPECT_STREQ(metaRsp.dbFName, dbname); EXPECT_STREQ(metaRsp.dbFName, dbname);
EXPECT_STREQ(metaRsp.tbName, "stb"); EXPECT_STREQ(metaRsp.tbName, "stb");
@ -410,6 +420,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_STB, pHead, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_STB, pHead, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -423,6 +434,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
} }
@ -436,6 +448,7 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -443,30 +456,35 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbAddTagReq("1.d3.stb", "tag4", &contLen); void* pReq = BuildAlterStbAddTagReq("1.d3.stb", "tag4", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DB_NOT_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DB_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbAddTagReq("1.d2.stb3", "tag4", &contLen); void* pReq = BuildAlterStbAddTagReq("1.d2.stb3", "tag4", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbAddTagReq(stbname, "tag3", &contLen); void* pReq = BuildAlterStbAddTagReq(stbname, "tag3", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbAddTagReq(stbname, "col1", &contLen); void* pReq = BuildAlterStbAddTagReq(stbname, "col1", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_ALREADY_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_ALREADY_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -474,6 +492,7 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
} }
@ -483,6 +502,7 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
} }
@ -495,18 +515,21 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildCreateStbReq(stbname, &contLen); void* pReq = BuildCreateStbReq(stbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbDropTagReq(stbname, "tag5", &contLen); void* pReq = BuildAlterStbDropTagReq(stbname, "tag5", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -514,6 +537,7 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 1); EXPECT_EQ(test.GetShowRows(), 1);
@ -524,6 +548,7 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
} }
@ -536,41 +561,48 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildCreateStbReq(stbname, &contLen); void* pReq = BuildCreateStbReq(stbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag5", "tag6", &contLen); void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag5", "tag6", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "col1", "tag6", &contLen); void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "col1", "tag6", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "col1", &contLen); void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "col1", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_ALREADY_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_ALREADY_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "tag2", &contLen); void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "tag2", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "tag2", &contLen); void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "tag2", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -578,6 +610,7 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 1); EXPECT_EQ(test.GetShowRows(), 1);
@ -588,6 +621,7 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
} }
@ -600,36 +634,42 @@ TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildCreateStbReq(stbname, &contLen); void* pReq = BuildCreateStbReq(stbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag5", 12, &contLen); void* pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag5", 12, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag1", 13, &contLen); void* pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag1", 13, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_OPTION); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_OPTION);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag3", 8, &contLen); void* pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag3", 8, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag3", 20, &contLen); void* pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag3", 20, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 1); EXPECT_EQ(test.GetShowRows(), 1);
@ -640,6 +680,7 @@ TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
} }
@ -653,6 +694,7 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -660,30 +702,35 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbAddColumnReq("1.d7.stb", "tag4", &contLen); void* pReq = BuildAlterStbAddColumnReq("1.d7.stb", "tag4", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DB_NOT_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DB_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbAddColumnReq("1.d6.stb3", "tag4", &contLen); void* pReq = BuildAlterStbAddColumnReq("1.d6.stb3", "tag4", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbAddColumnReq(stbname, "tag3", &contLen); void* pReq = BuildAlterStbAddColumnReq(stbname, "tag3", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbAddColumnReq(stbname, "col1", &contLen); void* pReq = BuildAlterStbAddColumnReq(stbname, "col1", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_ALREADY_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_ALREADY_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -691,6 +738,7 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 1); EXPECT_EQ(test.GetShowRows(), 1);
@ -701,6 +749,7 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
} }
@ -713,30 +762,35 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildCreateStbReq(stbname, &contLen); void* pReq = BuildCreateStbReq(stbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbDropColumnReq(stbname, "col4", &contLen); void* pReq = BuildAlterStbDropColumnReq(stbname, "col4", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbDropColumnReq(stbname, "col1", &contLen); void* pReq = BuildAlterStbDropColumnReq(stbname, "col1", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_ALTER_OPTION); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_ALTER_OPTION);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbDropColumnReq(stbname, "ts", &contLen); void* pReq = BuildAlterStbDropColumnReq(stbname, "ts", &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_ALTER_OPTION); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_ALTER_OPTION);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -744,6 +798,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
@ -751,6 +806,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 1); EXPECT_EQ(test.GetShowRows(), 1);
@ -761,6 +817,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
} }
@ -773,42 +830,49 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildCreateStbReq(stbname, &contLen); void* pReq = BuildCreateStbReq(stbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col5", 12, &contLen, 0); void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col5", 12, &contLen, 0);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "ts", 8, &contLen, 0); void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "ts", 8, &contLen, 0);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_OPTION); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_OPTION);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 8, &contLen, 0); void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 8, &contLen, 0);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", TSDB_MAX_BYTES_PER_ROW, &contLen, 0); void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", TSDB_MAX_BYTES_PER_ROW, &contLen, 0);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES);
rpcFreeCont(pRsp->pCont);
} }
{ {
void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 20, &contLen, 0); void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 20, &contLen, 0);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 1); EXPECT_EQ(test.GetShowRows(), 1);
@ -818,6 +882,7 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col_not_exist", 20, &contLen, 1); void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col_not_exist", 20, &contLen, 1);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST);
rpcFreeCont(pRsp->pCont);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 1); EXPECT_EQ(test.GetShowRows(), 1);
@ -828,5 +893,6 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
} }
} }

22
source/dnode/mnode/sdb/inc/sdb.h
View File

@ -150,7 +150,9 @@ typedef enum {
SDB_IDX = 21, SDB_IDX = 21,
SDB_VIEW = 22, SDB_VIEW = 22,
SDB_STREAM_SEQ = 23, SDB_STREAM_SEQ = 23,
SDB_MAX = 24 SDB_COMPACT = 24,
SDB_COMPACT_DETAIL = 25,
SDB_MAX = 26
} ESdbType; } ESdbType;
typedef struct SSdbRaw { typedef struct SSdbRaw {
@ -170,11 +172,11 @@ typedef struct SSdbRow {
} SSdbRow; } SSdbRow;
typedef struct SSdb { typedef struct SSdb {
SMnode * pMnode; SMnode *pMnode;
SWal * pWal; SWal *pWal;
int64_t sync; int64_t sync;
char * currDir; char *currDir;
char * tmpDir; char *tmpDir;
int64_t commitIndex; int64_t commitIndex;
int64_t commitTerm; int64_t commitTerm;
int64_t commitConfig; int64_t commitConfig;
@ -184,7 +186,7 @@ typedef struct SSdb {
int64_t tableVer[SDB_MAX]; int64_t tableVer[SDB_MAX];
int64_t maxId[SDB_MAX]; int64_t maxId[SDB_MAX];
EKeyType keyTypes[SDB_MAX]; EKeyType keyTypes[SDB_MAX];
SHashObj * hashObjs[SDB_MAX]; SHashObj *hashObjs[SDB_MAX];
TdThreadRwlock locks[SDB_MAX]; TdThreadRwlock locks[SDB_MAX];
SdbInsertFp insertFps[SDB_MAX]; SdbInsertFp insertFps[SDB_MAX];
SdbUpdateFp updateFps[SDB_MAX]; SdbUpdateFp updateFps[SDB_MAX];
@ -199,7 +201,7 @@ typedef struct SSdb {
typedef struct SSdbIter { typedef struct SSdbIter {
TdFilePtr file; TdFilePtr file;
int64_t total; int64_t total;
char * name; char *name;
} SSdbIter; } SSdbIter;
typedef struct { typedef struct {
@ -216,8 +218,8 @@ typedef struct {
typedef struct SSdbOpt { typedef struct SSdbOpt {
const char *path; const char *path;
SMnode * pMnode; SMnode *pMnode;
SWal * pWal; SWal *pWal;
int64_t sync; int64_t sync;
} SSdbOpt; } SSdbOpt;
@ -394,7 +396,7 @@ int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
int32_t sdbGetRawTotalSize(SSdbRaw *pRaw); int32_t sdbGetRawTotalSize(SSdbRaw *pRaw);
SSdbRow *sdbAllocRow(int32_t objSize); SSdbRow *sdbAllocRow(int32_t objSize);
void * sdbGetRowObj(SSdbRow *pRow); void *sdbGetRowObj(SSdbRow *pRow);
void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc); void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc);
int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config); int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config);

6
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -64,7 +64,11 @@ const char *sdbTableName(ESdbType type) {
return "idx"; return "idx";
case SDB_VIEW: case SDB_VIEW:
return "view"; return "view";
default: case SDB_COMPACT:
return "compact";
case SDB_COMPACT_DETAIL:
return "compact_detail";
default:
return "undefine"; return "undefine";
} }
} }

1
source/dnode/snode/src/snode.c
View File

@ -66,6 +66,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
} else { } else {
sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
} }
int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList); int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList);
SReadHandle handle = { SReadHandle handle = {

5
source/dnode/snode/src/snodeInitApi.c
View File

@ -14,8 +14,8 @@
*/ */
#include "storageapi.h" #include "storageapi.h"
#include "tstreamUpdate.h"
#include "streamState.h" #include "streamState.h"
#include "tstreamUpdate.h"
static void initStateStoreAPI(SStateStore* pStore); static void initStateStoreAPI(SStateStore* pStore);
static void initFunctionStateStore(SFunctionStateStore* pStore); static void initFunctionStateStore(SFunctionStateStore* pStore);
@ -100,9 +100,10 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateClose = streamStateClose; pStore->streamStateClose = streamStateClose;
pStore->streamStateBegin = streamStateBegin; pStore->streamStateBegin = streamStateBegin;
pStore->streamStateCommit = streamStateCommit; pStore->streamStateCommit = streamStateCommit;
pStore->streamStateDestroy= streamStateDestroy; pStore->streamStateDestroy = streamStateDestroy;
pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
pStore->streamStateReloadInfo = streamStateReloadInfo; pStore->streamStateReloadInfo = streamStateReloadInfo;
pStore->streamStateCopyBackend = streamStateCopyBackend;
} }
void initFunctionStateStore(SFunctionStateStore* pStore) { void initFunctionStateStore(SFunctionStateStore* pStore) {

Some files were not shown because too many files have changed in this diff Show More